diff --git a/.clang-format b/.clang-format index 01a341ceec6c04935529bb3bc4254a59ec651e93..c24b147cac01868bb984c945843096a3b2e779a1 100644 --- a/.clang-format +++ b/.clang-format @@ -109,6 +109,7 @@ ForEachMacros: - 'css_for_each_child' - 'css_for_each_descendant_post' - 'css_for_each_descendant_pre' + - 'cxl_for_each_cmd' - 'device_for_each_child_node' - 'dma_fence_chain_for_each' - 'do_for_each_ftrace_op' diff --git a/.gitignore b/.gitignore index d01cda8e11779a2fa3e3333fd176e2e77e02fd21..3af66272d6f10c5eccc0e00f3a6ff59a0d3990f1 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,7 @@ *.c.[012]*.* *.dt.yaml *.dtb +*.dtbo *.dtb.S *.dwo *.elf @@ -41,6 +42,7 @@ *.so.dbg *.su *.symtypes +*.symversions *.tab.[ch] *.tar *.xz diff --git a/.mailmap b/.mailmap index ade90fc9140a2787fbb7e5d4799d79e49ba245b0..85b93cdefc87bb0d98fca268fbac125dca87360f 100644 --- a/.mailmap +++ b/.mailmap @@ -174,7 +174,6 @@ Juha Yrjola Juha Yrjola Juha Yrjola Julien Thierry -Kamil Konieczny Kay Sievers Kees Cook Kees Cook @@ -238,6 +237,7 @@ Maxime Ripard Mayuresh Janorkar Michael Buesch Michel Dänzer +Miguel Ojeda Mike Rapoport Mike Rapoport Mike Rapoport diff --git a/CREDITS b/CREDITS index 9add7e6a4fa022ef1b06818bed57392ef3def9b5..cef83b958cbeb79c965e04c0163fd420b194b132 100644 --- a/CREDITS +++ b/CREDITS @@ -1244,10 +1244,10 @@ S: 80050-430 - Curitiba - Paraná S: Brazil N: Oded Gabbay -E: oded.gabbay@gmail.com -D: HabanaLabs and AMD KFD maintainer -S: 12 Shraga Raphaeli -S: Petah-Tikva, 4906418 +E: ogabbay@kernel.org +D: HabanaLabs maintainer +S: 29 Duchifat St. +S: Ra'anana 4372029 S: Israel N: Kumar Gala @@ -2841,14 +2841,11 @@ S: Subiaco, 6008 S: Perth, Western Australia S: Australia -N: Miguel Ojeda Sandonis -E: miguel.ojeda.sandonis@gmail.com -W: http://miguelojeda.es -W: http://jair.lab.fi.uva.es/~migojed/ +N: Miguel Ojeda +E: ojeda@kernel.org +W: https://ojeda.dev D: Author of the ks0108, cfag12864b and cfag12864bfb auxiliary display drivers. D: Maintainer of the auxiliary display drivers tree (drivers/auxdisplay/*) -S: C/ Mieses 20, 9-B -S: Valladolid 47009 S: Spain N: Peter Oruba diff --git a/Documentation/ABI/stable/sysfs-bus-fsl-mc b/Documentation/ABI/stable/sysfs-bus-fsl-mc new file mode 100644 index 0000000000000000000000000000000000000000..58f06c7eeed72086ed7307825b32ed8d9236053e --- /dev/null +++ b/Documentation/ABI/stable/sysfs-bus-fsl-mc @@ -0,0 +1,19 @@ +What: /sys/bus/fsl-mc/rescan +Date: January 2021 +KernelVersion: 5.12 +Contact: Ioana Ciornei +Description: Writing a non-zero value to this attribute will + force a rescan of fsl-mc bus in the system and + synchronize the objects under fsl-mc bus and the + Management Complex firmware. +Users: Userspace drivers and management tools + +What: /sys/bus/fsl-mc/autorescan +Date: January 2021 +KernelVersion: 5.12 +Contact: Ioana Ciornei +Description: Writing a zero value to this attribute will + disable the DPRC IRQs on which automatic rescan + of the fsl-mc bus is performed. A non-zero value + will enable the DPRC IRQs. +Users: Userspace drivers and management tools diff --git a/Documentation/ABI/stable/sysfs-class-tpm b/Documentation/ABI/stable/sysfs-class-tpm index 91ca63ec7581aa72be7d2bb2228d6ab18d92470f..d897ecb9615f356338f24c50abc65184c8667cda 100644 --- a/Documentation/ABI/stable/sysfs-class-tpm +++ b/Documentation/ABI/stable/sysfs-class-tpm @@ -194,3 +194,17 @@ Description: The "tpm_version_major" property shows the TCG spec major version Example output:: 2 + +What: /sys/class/tpm/tpmX/pcr-H/N +Date: March 2021 +KernelVersion: 5.12 +Contact: linux-integrity@vger.kernel.org +Description: produces output in compact hex representation for PCR + number N from hash bank H. N is the numeric value of + the PCR number and H is the crypto string + representation of the hash + + Example output:: + + cat /sys/class/tpm/tpm0/pcr-sha256/7 + 2ED93F199692DC6788EFA6A1FE74514AB9760B2A6CEEAEF6C808C13E4ABB0D42 diff --git a/Documentation/ABI/stable/sysfs-driver-speakup b/Documentation/ABI/stable/sysfs-driver-speakup index 792f58ba327d3bef5c34a4debf340c617a5e8fbb..dc2a6ba1674b87f903553431b493010b0effed29 100644 --- a/Documentation/ABI/stable/sysfs-driver-speakup +++ b/Documentation/ABI/stable/sysfs-driver-speakup @@ -273,7 +273,7 @@ Description: In `/sys/accessibility/speakup` is a directory corresponding to Below is a description of values and parameters for soft synthesizer, which is currently the most commonly used. -What: /sys/accessibility/speakup/soft/caps_start +What: /sys/accessibility/speakup//caps_start KernelVersion: 2.6 Contact: speakup@linux-speakup.org Description: This is the string that is sent to the synthesizer to cause it @@ -281,7 +281,7 @@ Description: This is the string that is sent to the synthesizer to cause it and most others, this causes the pitch of the voice to rise above the currently set pitch. -What: /sys/accessibility/speakup/soft/caps_stop +What: /sys/accessibility/speakup//caps_stop KernelVersion: 2.6 Contact: speakup@linux-speakup.org Description: This is the string sent to the synthesizer to cause it to stop @@ -290,12 +290,12 @@ Description: This is the string sent to the synthesizer to cause it to stop down to the currently set pitch. -What: /sys/accessibility/speakup/soft/delay_time +What: /sys/accessibility/speakup//delay_time KernelVersion: 2.6 Contact: speakup@linux-speakup.org Description: TODO: -What: /sys/accessibility/speakup/soft/direct +What: /sys/accessibility/speakup//direct KernelVersion: 2.6 Contact: speakup@linux-speakup.org Description: Controls if punctuation is spoken by speakup, or by the @@ -306,36 +306,43 @@ Description: Controls if punctuation is spoken by speakup, or by the than". Zero lets speakup speak the punctuation. One lets the synthesizer itself speak punctuation. -What: /sys/accessibility/speakup/soft/freq +What: /sys/accessibility/speakup//freq KernelVersion: 2.6 Contact: speakup@linux-speakup.org Description: Gets or sets the frequency of the speech synthesizer. Range is 0-9. -What: /sys/accessibility/speakup/soft/full_time +What: /sys/accessibility/speakup//flush_time +KernelVersion: 5.12 +Contact: speakup@linux-speakup.org +Description: Gets or sets the timeout to wait for the synthesizer flush to + complete. This can be used when the cable gets faulty and flush + notifications are getting lost. + +What: /sys/accessibility/speakup//full_time KernelVersion: 2.6 Contact: speakup@linux-speakup.org Description: TODO: -What: /sys/accessibility/speakup/soft/jiffy_delta +What: /sys/accessibility/speakup//jiffy_delta KernelVersion: 2.6 Contact: speakup@linux-speakup.org Description: This controls how many jiffys the kernel gives to the synthesizer. Setting this too high can make a system unstable, or even crash it. -What: /sys/accessibility/speakup/soft/pitch +What: /sys/accessibility/speakup//pitch KernelVersion: 2.6 Contact: speakup@linux-speakup.org Description: Gets or sets the pitch of the synthesizer. The range is 0-9. -What: /sys/accessibility/speakup/soft/inflection +What: /sys/accessibility/speakup//inflection KernelVersion: 5.8 Contact: speakup@linux-speakup.org Description: Gets or sets the inflection of the synthesizer, i.e. the pitch range. The range is 0-9. -What: /sys/accessibility/speakup/soft/punct +What: /sys/accessibility/speakup//punct KernelVersion: 2.6 Contact: speakup@linux-speakup.org Description: Gets or sets the amount of punctuation spoken by the @@ -343,13 +350,13 @@ Description: Gets or sets the amount of punctuation spoken by the TODO: How is this related to speakup's punc_level, or reading_punc. -What: /sys/accessibility/speakup/soft/rate +What: /sys/accessibility/speakup//rate KernelVersion: 2.6 Contact: speakup@linux-speakup.org Description: Gets or sets the rate of the synthesizer. Range is from zero slowest, to nine fastest. -What: /sys/accessibility/speakup/soft/tone +What: /sys/accessibility/speakup//tone KernelVersion: 2.6 Contact: speakup@linux-speakup.org Description: Gets or sets the tone of the speech synthesizer. The range for @@ -357,12 +364,12 @@ Description: Gets or sets the tone of the speech synthesizer. The range for difference if using espeak and the espeakup connector. TODO: does espeakup support different tonalities? -What: /sys/accessibility/speakup/soft/trigger_time +What: /sys/accessibility/speakup//trigger_time KernelVersion: 2.6 Contact: speakup@linux-speakup.org Description: TODO: -What: /sys/accessibility/speakup/soft/voice +What: /sys/accessibility/speakup//voice KernelVersion: 2.6 Contact: speakup@linux-speakup.org Description: Gets or sets the voice used by the synthesizer if the @@ -371,7 +378,7 @@ Description: Gets or sets the voice used by the synthesizer if the voices, this parameter will not set the voice when the espeakup connector is used between speakup and espeak. -What: /sys/accessibility/speakup/soft/vol +What: /sys/accessibility/speakup//vol KernelVersion: 2.6 Contact: speakup@linux-speakup.org Description: Gets or sets the volume of the speech synthesizer. Range is 0-9, diff --git a/Documentation/ABI/testing/debugfs-driver-habanalabs b/Documentation/ABI/testing/debugfs-driver-habanalabs index c5d678d3914472514eca90fa67f894845a9d374e..d447a611c41b7707f00ec9c8e925c52013d25f75 100644 --- a/Documentation/ABI/testing/debugfs-driver-habanalabs +++ b/Documentation/ABI/testing/debugfs-driver-habanalabs @@ -1,7 +1,7 @@ What: /sys/kernel/debug/habanalabs/hl/addr Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Sets the device address to be used for read or write through PCI bar, or the device VA of a host mapped memory to be read or written directly from the host. The latter option is allowed @@ -11,7 +11,7 @@ Description: Sets the device address to be used for read or write through What: /sys/kernel/debug/habanalabs/hl/clk_gate Date: May 2020 KernelVersion: 5.8 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Allow the root user to disable/enable in runtime the clock gating mechanism in Gaudi. Due to how Gaudi is built, the clock gating needs to be disabled in order to access the @@ -34,28 +34,28 @@ Description: Allow the root user to disable/enable in runtime the clock What: /sys/kernel/debug/habanalabs/hl/command_buffers Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Displays a list with information about the currently allocated command buffers What: /sys/kernel/debug/habanalabs/hl/command_submission Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Displays a list with information about the currently active command submissions What: /sys/kernel/debug/habanalabs/hl/command_submission_jobs Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Displays a list with detailed information about each JOB (CB) of each active command submission What: /sys/kernel/debug/habanalabs/hl/data32 Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Allows the root user to read or write directly through the device's PCI bar. Writing to this file generates a write transaction while reading from the file generates a read @@ -70,7 +70,7 @@ Description: Allows the root user to read or write directly through the What: /sys/kernel/debug/habanalabs/hl/data64 Date: Jan 2020 KernelVersion: 5.6 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Allows the root user to read or write 64 bit data directly through the device's PCI bar. Writing to this file generates a write transaction while reading from the file generates a read @@ -85,7 +85,7 @@ Description: Allows the root user to read or write 64 bit data directly What: /sys/kernel/debug/habanalabs/hl/device Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Enables the root user to set the device to specific state. Valid values are "disable", "enable", "suspend", "resume". User can read this property to see the valid values @@ -93,28 +93,28 @@ Description: Enables the root user to set the device to specific state. What: /sys/kernel/debug/habanalabs/hl/engines Date: Jul 2019 KernelVersion: 5.3 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Displays the status registers values of the device engines and their derived idle status What: /sys/kernel/debug/habanalabs/hl/i2c_addr Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Sets I2C device address for I2C transaction that is generated by the device's CPU What: /sys/kernel/debug/habanalabs/hl/i2c_bus Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Sets I2C bus address for I2C transaction that is generated by the device's CPU What: /sys/kernel/debug/habanalabs/hl/i2c_data Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Triggers an I2C transaction that is generated by the device's CPU. Writing to this file generates a write transaction while reading from the file generates a read transcation @@ -122,32 +122,32 @@ Description: Triggers an I2C transaction that is generated by the device's What: /sys/kernel/debug/habanalabs/hl/i2c_reg Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Sets I2C register id for I2C transaction that is generated by the device's CPU What: /sys/kernel/debug/habanalabs/hl/led0 Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Sets the state of the first S/W led on the device What: /sys/kernel/debug/habanalabs/hl/led1 Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Sets the state of the second S/W led on the device What: /sys/kernel/debug/habanalabs/hl/led2 Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Sets the state of the third S/W led on the device What: /sys/kernel/debug/habanalabs/hl/mmu Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Displays the hop values and physical address for a given ASID and virtual address. The user should write the ASID and VA into the file and then read the file to get the result. @@ -157,14 +157,14 @@ Description: Displays the hop values and physical address for a given ASID What: /sys/kernel/debug/habanalabs/hl/set_power_state Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Sets the PCI power state. Valid values are "1" for D0 and "2" for D3Hot What: /sys/kernel/debug/habanalabs/hl/userptr Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Displays a list with information about the currently user pointers (user virtual addresses) that are pinned and mapped to DMA addresses @@ -172,13 +172,21 @@ Description: Displays a list with information about the currently user What: /sys/kernel/debug/habanalabs/hl/vm Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Displays a list with information about all the active virtual address mappings per ASID What: /sys/kernel/debug/habanalabs/hl/stop_on_err Date: Mar 2020 KernelVersion: 5.6 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Sets the stop-on_error option for the device engines. Value of "0" is for disable, otherwise enable. + +What: /sys/kernel/debug/habanalabs/hl/dump_security_violations +Date: Jan 2021 +KernelVersion: 5.12 +Contact: ogabbay@kernel.org +Description: Dumps all security violations to dmesg. This will also ack + all security violations meanings those violations will not be + dumped next time user calls this API diff --git a/Documentation/ABI/testing/ima_policy b/Documentation/ABI/testing/ima_policy index e35263f97fc1a7c89c5eccf8a879223d1c81e02a..070779e8d8367e4469f456bc53baa8888367090d 100644 --- a/Documentation/ABI/testing/ima_policy +++ b/Documentation/ABI/testing/ima_policy @@ -29,10 +29,10 @@ Description: option: [[appraise_type=]] [template=] [permit_directio] [appraise_flag=] [keyrings=] base: - func:= [BPRM_CHECK][MMAP_CHECK][CREDS_CHECK][FILE_CHECK]MODULE_CHECK] + func:= [BPRM_CHECK][MMAP_CHECK][CREDS_CHECK][FILE_CHECK][MODULE_CHECK] [FIRMWARE_CHECK] [KEXEC_KERNEL_CHECK] [KEXEC_INITRAMFS_CHECK] - [KEXEC_CMDLINE] [KEY_CHECK] + [KEXEC_CMDLINE] [KEY_CHECK] [CRITICAL_DATA] mask:= [[^]MAY_READ] [[^]MAY_WRITE] [[^]MAY_APPEND] [[^]MAY_EXEC] fsmagic:= hex value @@ -52,6 +52,9 @@ Description: template:= name of a defined IMA template type (eg, ima-ng). Only valid when action is "measure". pcr:= decimal value + label:= [selinux]|[kernel_info]|[data_label] + data_label:= a unique string used for grouping and limiting critical data. + For example, "selinux" to measure critical data for SELinux. default policy: # PROC_SUPER_MAGIC diff --git a/Documentation/ABI/testing/sysfs-bus-coresight-devices-etm4x b/Documentation/ABI/testing/sysfs-bus-coresight-devices-etm4x index 881f0cd99ce4ed7c894ec4bc35c59bf8dc7f1ecc..8e53a32f815056f97220b2092249128c12a848d0 100644 --- a/Documentation/ABI/testing/sysfs-bus-coresight-devices-etm4x +++ b/Documentation/ABI/testing/sysfs-bus-coresight-devices-etm4x @@ -371,6 +371,14 @@ Contact: Mathieu Poirier Description: (Read) Print the content of the Device ID Register (0xFC8). The value is taken directly from the HW. +What: /sys/bus/coresight/devices/etm/mgmt/trcdevarch +Date: January 2021 +KernelVersion: 5.12 +Contact: Mathieu Poirier +Description: (Read) Print the content of the Device Architecture Register + (offset 0xFBC). The value is taken directly read + from the HW. + What: /sys/bus/coresight/devices/etm/mgmt/trcdevtype Date: April 2015 KernelVersion: 4.01 diff --git a/Documentation/ABI/testing/sysfs-bus-cxl b/Documentation/ABI/testing/sysfs-bus-cxl new file mode 100644 index 0000000000000000000000000000000000000000..2fe7490ad6a86b419b14067806f5d7d298d0af1e --- /dev/null +++ b/Documentation/ABI/testing/sysfs-bus-cxl @@ -0,0 +1,26 @@ +What: /sys/bus/cxl/devices/memX/firmware_version +Date: December, 2020 +KernelVersion: v5.12 +Contact: linux-cxl@vger.kernel.org +Description: + (RO) "FW Revision" string as reported by the Identify + Memory Device Output Payload in the CXL-2.0 + specification. + +What: /sys/bus/cxl/devices/memX/ram/size +Date: December, 2020 +KernelVersion: v5.12 +Contact: linux-cxl@vger.kernel.org +Description: + (RO) "Volatile Only Capacity" as bytes. Represents the + identically named field in the Identify Memory Device Output + Payload in the CXL-2.0 specification. + +What: /sys/bus/cxl/devices/memX/pmem/size +Date: December, 2020 +KernelVersion: v5.12 +Contact: linux-cxl@vger.kernel.org +Description: + (RO) "Persistent Only Capacity" as bytes. Represents the + identically named field in the Identify Memory Device Output + Payload in the CXL-2.0 specification. diff --git a/Documentation/ABI/testing/sysfs-bus-dfl-devices-emif b/Documentation/ABI/testing/sysfs-bus-dfl-devices-emif new file mode 100644 index 0000000000000000000000000000000000000000..817d14126d4d0ad23db1243bd6c2b9b52b8388bc --- /dev/null +++ b/Documentation/ABI/testing/sysfs-bus-dfl-devices-emif @@ -0,0 +1,25 @@ +What: /sys/bus/dfl/devices/dfl_dev.X/infX_cal_fail +Date: Oct 2020 +KernelVersion: 5.12 +Contact: Xu Yilun +Description: Read-only. It indicates if the calibration failed on this + memory interface. "1" for calibration failure, "0" for OK. + Format: %u + +What: /sys/bus/dfl/devices/dfl_dev.X/infX_init_done +Date: Oct 2020 +KernelVersion: 5.12 +Contact: Xu Yilun +Description: Read-only. It indicates if the initialization completed on + this memory interface. "1" for initialization complete, "0" + for not yet. + Format: %u + +What: /sys/bus/dfl/devices/dfl_dev.X/infX_clear +Date: Oct 2020 +KernelVersion: 5.12 +Contact: Xu Yilun +Description: Write-only. Writing "1" to this file will zero out all memory + data in this memory interface. Writing of other values is + invalid. + Format: %u diff --git a/Documentation/ABI/testing/sysfs-bus-dfl-devices-n3000-nios b/Documentation/ABI/testing/sysfs-bus-dfl-devices-n3000-nios new file mode 100644 index 0000000000000000000000000000000000000000..5335d742bcaf69286b3e05acfeee7ffc3bcbb4c8 --- /dev/null +++ b/Documentation/ABI/testing/sysfs-bus-dfl-devices-n3000-nios @@ -0,0 +1,47 @@ +What: /sys/bus/dfl/devices/dfl_dev.X/fec_mode +Date: Oct 2020 +KernelVersion: 5.12 +Contact: Xu Yilun +Description: Read-only. Returns the FEC mode of the 25G links of the + ethernet retimers configured by Nios firmware. "rs" for Reed + Solomon FEC, "kr" for Fire Code FEC, "no" for NO FEC. + "not supported" if the FEC mode setting is not supported, this + happens when the Nios firmware version major < 3, or no link is + configured to 25G. + Format: string + +What: /sys/bus/dfl/devices/dfl_dev.X/retimer_A_mode +Date: Oct 2020 +KernelVersion: 5.12 +Contact: Xu Yilun +Description: Read-only. Returns the enumeration value of the working mode of + the retimer A configured by the Nios firmware. The value is + read out from shared registers filled by the Nios firmware. Now + the values could be: + + - "0": Reset + - "1": 4x10G + - "2": 4x25G + - "3": 2x25G + - "4": 2x25G+2x10G + - "5": 1x25G + + If the Nios firmware is updated in future to support more + retimer modes, more enumeration value is expected. + Format: 0x%x + +What: /sys/bus/dfl/devices/dfl_dev.X/retimer_B_mode +Date: Oct 2020 +KernelVersion: 5.12 +Contact: Xu Yilun +Description: Read-only. Returns the enumeration value of the working mode of + the retimer B configured by the Nios firmware. The value format + is the same as retimer_A_mode. + +What: /sys/bus/dfl/devices/dfl_dev.X/nios_fw_version +Date: Oct 2020 +KernelVersion: 5.12 +Contact: Xu Yilun +Description: Read-only. Returns the version of the Nios firmware in the + FPGA. Its format is "major.minor.patch". + Format: %x.%x.%x diff --git a/Documentation/ABI/testing/sysfs-bus-pci-devices-pvpanic b/Documentation/ABI/testing/sysfs-bus-pci-devices-pvpanic new file mode 100644 index 0000000000000000000000000000000000000000..1936f732415571326a24ca147833451917cabdc1 --- /dev/null +++ b/Documentation/ABI/testing/sysfs-bus-pci-devices-pvpanic @@ -0,0 +1,24 @@ +What: /sys/devices/pci0000:00/*/QEMU0001:00/capability +Date: Jan 2021 +Contact: zhenwei pi +Description: + Read-only attribute. Capabilities of pvpanic device which + are supported by QEMU. + + Format: %x. + + Detailed bit definition refers to section + from pvpanic device specification: + https://git.qemu.org/?p=qemu.git;a=blob_plain;f=docs/specs/pvpanic.txt + +What: /sys/devices/pci0000:00/*/QEMU0001:00/events +Date: Jan 2021 +Contact: zhenwei pi +Description: + RW attribute. Set/get which features in-use. This attribute + is used to enable/disable feature(s) of pvpanic device. + Notice that this value should be a subset of capability. + + Format: %x. + + Also refer to pvpanic device specification. diff --git a/Documentation/ABI/testing/sysfs-class-power-ltc4162l b/Documentation/ABI/testing/sysfs-class-power-ltc4162l new file mode 100644 index 0000000000000000000000000000000000000000..ba30db93052bf0d38fd3afc78a9bd66ca25f5115 --- /dev/null +++ b/Documentation/ABI/testing/sysfs-class-power-ltc4162l @@ -0,0 +1,82 @@ +What: /sys/class/power_supply/ltc4162-l/charge_status +Date: Januari 2021 +KernelVersion: 5.11 +Description: + Detailed charge status information as reported by the chip. + + Access: Read + + Valid values: + ilim_reg_active + thermal_reg_active + vin_uvcl_active + iin_limit_active + constant_current + constant_voltage + charger_off + +What: /sys/class/power_supply/ltc4162-l/ibat +Date: Januari 2021 +KernelVersion: 5.11 +Description: + Battery input current as measured by the charger. Negative value + means that the battery is discharging. + + Access: Read + + Valid values: Signed value in microamps + +What: /sys/class/power_supply/ltc4162-l/vbat +Date: Januari 2021 +KernelVersion: 5.11 +Description: + Battery voltage as measured by the charger. + + Access: Read + + Valid values: In microvolts + +What: /sys/class/power_supply/ltc4162-l/vbat_avg +Date: Januari 2021 +KernelVersion: 5.11 +Description: + Battery voltage, averaged over time, as measured by the charger. + + Access: Read + + Valid values: In microvolts + +What: /sys/class/power_supply/ltc4162-l/force_telemetry +Date: Januari 2021 +KernelVersion: 5.11 +Description: + To save battery current, the measurement system is disabled if + the battery is the only source of power. This affects all + voltage, current and temperature measurements. + Write a "1" to this to keep performing telemetry once every few + seconds, even when running on battery (as reported by the online + property, which is "1" when external power is available and "0" + when the system runs on battery). + + Access: Read, Write + + Valid values: 0 (disabled) or 1 (enabled) + +What: /sys/class/power_supply/ltc4162-l/arm_ship_mode +Date: Januari 2021 +KernelVersion: 5.11 +Description: + The charger will normally drain the battery while inactive, + typically drawing about 54 microamps. Write a "1" to this + property to arm a special "ship" mode that extends shelf life + by reducing the leakage to about 2.8 microamps. The chip will + remain in this mode (and no longer respond to I2C commands) + until some external power-supply is attached raising the input + voltage above 1V. It will then automatically revert to "0". + Writing a "0" to the property cancels the "ship" mode request. + The ship mode, when armed, activates once the input voltage + drops below 1V. + + Access: Read, Write + + Valid values: 0 (disable) or 1 (enable) diff --git a/Documentation/ABI/testing/sysfs-devices-memory b/Documentation/ABI/testing/sysfs-devices-memory index 246a45b96d22a921e636a4bb8bd240a4e3e0ee6c..d8b0f80b9e33cf04938e4c6908c189a1115bbc1a 100644 --- a/Documentation/ABI/testing/sysfs-devices-memory +++ b/Documentation/ABI/testing/sysfs-devices-memory @@ -13,21 +13,22 @@ What: /sys/devices/system/memory/memoryX/removable Date: June 2008 Contact: Badari Pulavarty Description: - The file /sys/devices/system/memory/memoryX/removable - indicates whether this memory block is removable or not. - This is useful for a user-level agent to determine - identify removable sections of the memory before attempting - potentially expensive hot-remove memory operation + The file /sys/devices/system/memory/memoryX/removable is a + legacy interface used to indicated whether a memory block is + likely to be offlineable or not. Newer kernel versions return + "1" if and only if the kernel supports memory offlining. Users: hotplug memory remove tools http://www.ibm.com/developerworks/wikis/display/LinuxP/powerpc-utils + lsmem/chmem part of util-linux What: /sys/devices/system/memory/memoryX/phys_device Date: September 2008 Contact: Badari Pulavarty Description: The file /sys/devices/system/memory/memoryX/phys_device - is read-only and is designed to show the name of physical - memory device. Implementation is currently incomplete. + is read-only; it is a legacy interface only ever used on s390x + to expose the covered storage increment. +Users: Legacy s390-tools lsmem/chmem What: /sys/devices/system/memory/memoryX/phys_index Date: September 2008 @@ -43,23 +44,25 @@ Date: September 2008 Contact: Badari Pulavarty Description: The file /sys/devices/system/memory/memoryX/state - is read-write. When read, its contents show the - online/offline state of the memory section. When written, - root can toggle the the online/offline state of a removable - memory section (see removable file description above) - using the following commands:: + is read-write. When read, it returns the online/offline + state of the memory block. When written, root can toggle + the online/offline state of a memory block using the following + commands:: # echo online > /sys/devices/system/memory/memoryX/state # echo offline > /sys/devices/system/memory/memoryX/state - For example, if /sys/devices/system/memory/memory22/removable - contains a value of 1 and - /sys/devices/system/memory/memory22/state contains the - string "online" the following command can be executed by - by root to offline that section:: - - # echo offline > /sys/devices/system/memory/memory22/state - + On newer kernel versions, advanced states can be specified + when onlining to select a target zone: "online_movable" + selects the movable zone. "online_kernel" selects the + applicable kernel zone (DMA, DMA32, or Normal). However, + after successfully setting one of the advanced states, + reading the file will return "online"; the zone information + can be obtained via "valid_zones" instead. + + While onlining is unlikely to fail, there are no guarantees + that offlining will succeed. Offlining is more likely to + succeed if "valid_zones" indicates "Movable". Users: hotplug memory remove tools http://www.ibm.com/developerworks/wikis/display/LinuxP/powerpc-utils @@ -69,8 +72,19 @@ Date: July 2014 Contact: Zhang Zhen Description: The file /sys/devices/system/memory/memoryX/valid_zones is - read-only and is designed to show which zone this memory - block can be onlined to. + read-only. + + For online memory blocks, it returns in which zone memory + provided by a memory block is managed. If multiple zones + apply (not applicable for hotplugged memory), "None" is returned + and the memory block cannot be offlined. + + For offline memory blocks, it returns by which zone memory + provided by a memory block can be managed when onlining. + The first returned zone ("default") will be used when setting + the state of an offline memory block to "online". Only one of + the kernel zones (DMA, DMA32, Normal) is applicable for a single + memory block. What: /sys/devices/system/memoryX/nodeY Date: October 2009 diff --git a/Documentation/ABI/testing/sysfs-devices-xenbus b/Documentation/ABI/testing/sysfs-devices-xenbus new file mode 100644 index 0000000000000000000000000000000000000000..fd796cb4f31573680738d54ba44a81bcb9a2b823 --- /dev/null +++ b/Documentation/ABI/testing/sysfs-devices-xenbus @@ -0,0 +1,41 @@ +What: /sys/devices/*/xenbus/event_channels +Date: February 2021 +Contact: Xen Developers mailing list +Description: + Number of Xen event channels associated with a kernel based + paravirtualized device frontend or backend. + +What: /sys/devices/*/xenbus/events +Date: February 2021 +Contact: Xen Developers mailing list +Description: + Total number of Xen events received for a Xen pv device + frontend or backend. + +What: /sys/devices/*/xenbus/jiffies_eoi_delayed +Date: February 2021 +Contact: Xen Developers mailing list +Description: + Summed up time in jiffies the EOI of an interrupt for a Xen + pv device has been delayed in order to avoid stalls due to + event storms. This value rising is a first sign for a rogue + other end of the pv device. + +What: /sys/devices/*/xenbus/spurious_events +Date: February 2021 +Contact: Xen Developers mailing list +Description: + Number of events received for a Xen pv device which did not + require any action. Too many spurious events in a row will + trigger delayed EOI processing. + +What: /sys/devices/*/xenbus/spurious_threshold +Date: February 2021 +Contact: Xen Developers mailing list +Description: + Controls the tolerated number of subsequent spurious events + before delayed EOI processing is triggered for a Xen pv + device. Default is 1. This can be modified in case the other + end of the pv device is issuing spurious events on a regular + basis and is known not to be malicious on purpose. Raising + the value for such cases can improve pv device performance. diff --git a/Documentation/ABI/testing/sysfs-driver-habanalabs b/Documentation/ABI/testing/sysfs-driver-habanalabs index 169ae4b2a1806da51c545a71234d4f20a69ca709..1f127f71d2b454d6dcce865418aefd58896702bc 100644 --- a/Documentation/ABI/testing/sysfs-driver-habanalabs +++ b/Documentation/ABI/testing/sysfs-driver-habanalabs @@ -1,7 +1,7 @@ What: /sys/class/habanalabs/hl/armcp_kernel_ver Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Version of the Linux kernel running on the device's CPU. Will be DEPRECATED in Linux kernel version 5.10, and be replaced with cpucp_kernel_ver @@ -9,7 +9,7 @@ Description: Version of the Linux kernel running on the device's CPU. What: /sys/class/habanalabs/hl/armcp_ver Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Version of the application running on the device's CPU Will be DEPRECATED in Linux kernel version 5.10, and be replaced with cpucp_ver @@ -17,7 +17,7 @@ Description: Version of the application running on the device's CPU What: /sys/class/habanalabs/hl/clk_max_freq_mhz Date: Jun 2019 KernelVersion: not yet upstreamed -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Allows the user to set the maximum clock frequency, in MHz. The device clock might be set to lower value than the maximum. The user should read the clk_cur_freq_mhz to see the actual @@ -27,52 +27,52 @@ Description: Allows the user to set the maximum clock frequency, in MHz. What: /sys/class/habanalabs/hl/clk_cur_freq_mhz Date: Jun 2019 KernelVersion: not yet upstreamed -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Displays the current frequency, in MHz, of the device clock. This property is valid only for the Gaudi ASIC family What: /sys/class/habanalabs/hl/cpld_ver Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Version of the Device's CPLD F/W What: /sys/class/habanalabs/hl/cpucp_kernel_ver Date: Oct 2020 KernelVersion: 5.10 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Version of the Linux kernel running on the device's CPU What: /sys/class/habanalabs/hl/cpucp_ver Date: Oct 2020 KernelVersion: 5.10 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Version of the application running on the device's CPU What: /sys/class/habanalabs/hl/device_type Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Displays the code name of the device according to its type. The supported values are: "GOYA" What: /sys/class/habanalabs/hl/eeprom Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: A binary file attribute that contains the contents of the on-board EEPROM What: /sys/class/habanalabs/hl/fuse_ver Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Displays the device's version from the eFuse What: /sys/class/habanalabs/hl/hard_reset Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Interface to trigger a hard-reset operation for the device. Hard-reset will reset ALL internal components of the device except for the PCI interface and the internal PLLs @@ -80,14 +80,14 @@ Description: Interface to trigger a hard-reset operation for the device. What: /sys/class/habanalabs/hl/hard_reset_cnt Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Displays how many times the device have undergone a hard-reset operation since the driver was loaded What: /sys/class/habanalabs/hl/high_pll Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Allows the user to set the maximum clock frequency for MME, TPC and IC when the power management profile is set to "automatic". This property is valid only for the Goya ASIC family @@ -95,7 +95,7 @@ Description: Allows the user to set the maximum clock frequency for MME, TPC What: /sys/class/habanalabs/hl/ic_clk Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Allows the user to set the maximum clock frequency, in Hz, of the Interconnect fabric. Writes to this parameter affect the device only when the power management profile is set to "manual" @@ -107,27 +107,27 @@ Description: Allows the user to set the maximum clock frequency, in Hz, of What: /sys/class/habanalabs/hl/ic_clk_curr Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Displays the current clock frequency, in Hz, of the Interconnect fabric. This property is valid only for the Goya ASIC family What: /sys/class/habanalabs/hl/infineon_ver Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Version of the Device's power supply F/W code What: /sys/class/habanalabs/hl/max_power Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Allows the user to set the maximum power consumption of the device in milliwatts. What: /sys/class/habanalabs/hl/mme_clk Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Allows the user to set the maximum clock frequency, in Hz, of the MME compute engine. Writes to this parameter affect the device only when the power management profile is set to "manual" @@ -139,21 +139,21 @@ Description: Allows the user to set the maximum clock frequency, in Hz, of What: /sys/class/habanalabs/hl/mme_clk_curr Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Displays the current clock frequency, in Hz, of the MME compute engine. This property is valid only for the Goya ASIC family What: /sys/class/habanalabs/hl/pci_addr Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Displays the PCI address of the device. This is needed so the user would be able to open a device based on its PCI address What: /sys/class/habanalabs/hl/pm_mng_profile Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Power management profile. Values are "auto", "manual". In "auto" mode, the driver will set the maximum clock frequency to a high value when a user-space process opens the device's file (unless @@ -167,13 +167,13 @@ Description: Power management profile. Values are "auto", "manual". In "auto" What: /sys/class/habanalabs/hl/preboot_btl_ver Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Version of the device's preboot F/W code What: /sys/class/habanalabs/hl/soft_reset Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Interface to trigger a soft-reset operation for the device. Soft-reset will reset only the compute and DMA engines of the device @@ -181,26 +181,26 @@ Description: Interface to trigger a soft-reset operation for the device. What: /sys/class/habanalabs/hl/soft_reset_cnt Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Displays how many times the device have undergone a soft-reset operation since the driver was loaded What: /sys/class/habanalabs/hl/status Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Status of the card: "Operational", "Malfunction", "In reset". What: /sys/class/habanalabs/hl/thermal_ver Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Version of the Device's thermal daemon What: /sys/class/habanalabs/hl/tpc_clk Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Allows the user to set the maximum clock frequency, in Hz, of the TPC compute engines. Writes to this parameter affect the device only when the power management profile is set to "manual" @@ -212,12 +212,12 @@ Description: Allows the user to set the maximum clock frequency, in Hz, of What: /sys/class/habanalabs/hl/tpc_clk_curr Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Displays the current clock frequency, in Hz, of the TPC compute engines. This property is valid only for the Goya ASIC family What: /sys/class/habanalabs/hl/uboot_ver Date: Jan 2019 KernelVersion: 5.1 -Contact: oded.gabbay@gmail.com +Contact: ogabbay@kernel.org Description: Version of the u-boot running on the device's CPU \ No newline at end of file diff --git a/Documentation/ABI/testing/sysfs-driver-input-cros-ec-keyb b/Documentation/ABI/testing/sysfs-driver-input-cros-ec-keyb new file mode 100644 index 0000000000000000000000000000000000000000..c7afc2328045c0cb11145b9db61095557673c568 --- /dev/null +++ b/Documentation/ABI/testing/sysfs-driver-input-cros-ec-keyb @@ -0,0 +1,6 @@ +What: /sys/class/input/input(x)/device/function_row_physmap +Date: January 2021 +Contact: Philip Chen +Description: A space separated list of scancodes for the top row keys, + ordered by the physical positions of the keys, from left + to right. diff --git a/Documentation/ABI/testing/sysfs-driver-intel-m10-bmc b/Documentation/ABI/testing/sysfs-driver-intel-m10-bmc index 979a2d62513ff5c9c25307c980b48a682078209e..9773925138af1a73d5f21d980ddf495f38db844f 100644 --- a/Documentation/ABI/testing/sysfs-driver-intel-m10-bmc +++ b/Documentation/ABI/testing/sysfs-driver-intel-m10-bmc @@ -13,3 +13,24 @@ Contact: Xu Yilun Description: Read only. Returns the firmware version of Intel MAX10 BMC chip. Format: "0x%x". + +What: /sys/bus/spi/devices/.../mac_address +Date: January 2021 +KernelVersion: 5.12 +Contact: Russ Weight +Description: Read only. Returns the first MAC address in a block + of sequential MAC addresses assigned to the board + that is managed by the Intel MAX10 BMC. It is stored in + FLASH storage and is mirrored in the MAX10 BMC register + space. + Format: "%02x:%02x:%02x:%02x:%02x:%02x". + +What: /sys/bus/spi/devices/.../mac_count +Date: January 2021 +KernelVersion: 5.12 +Contact: Russ Weight +Description: Read only. Returns the number of sequential MAC + addresses assigned to the board managed by the Intel + MAX10 BMC. This value is stored in FLASH and is mirrored + in the MAX10 BMC register space. + Format: "%u". diff --git a/Documentation/ABI/testing/sysfs-driver-ufs b/Documentation/ABI/testing/sysfs-driver-ufs index 75ccc5c62b3c498a940dafb12014b56cdfe156c2..d1bc23cb6a9d8d68acb234fd43fb46e1435dd2d3 100644 --- a/Documentation/ABI/testing/sysfs-driver-ufs +++ b/Documentation/ABI/testing/sysfs-driver-ufs @@ -1161,3 +1161,14 @@ Description: This entry shows the configured size of WriteBooster buffer. 0400h corresponds to 4GB. The file is read only. + +What: /sys/bus/platform/drivers/ufshcd/*/wb_on +Date: January 2021 +Contact: Bean Huo +Description: This node is used to set or display whether UFS WriteBooster is + enabled. Echo 0 to this file to disable UFS WriteBooster or 1 to + enable it. The WriteBooster is enabled after power-on/reset, + however, it will be disabled/enable while CLK scaling down/up + (if the platform supports UFSHCD_CAP_CLK_SCALING). For a + platform that doesn't support UFSHCD_CAP_CLK_SCALING, we can + disable/enable WriteBooster through this sysfs node. diff --git a/Documentation/ABI/testing/sysfs-firmware-acpi b/Documentation/ABI/testing/sysfs-firmware-acpi index b16d30a71709f771136c58cb0ad939e3cece1a8d..819939d858c9a83a15b54ec3389054bc461d6734 100644 --- a/Documentation/ABI/testing/sysfs-firmware-acpi +++ b/Documentation/ABI/testing/sysfs-firmware-acpi @@ -1,3 +1,46 @@ +What: /sys/firmware/acpi/fpdt/ +Date: Jan 2021 +Contact: Zhang Rui +Description: + ACPI Firmware Performance Data Table (FPDT) provides + information for firmware performance data for system boot, + S3 suspend and S3 resume. This sysfs entry contains the + performance data retrieved from the FPDT. + + boot: + firmware_start_ns: Timer value logged at the beginning + of firmware image execution. In nanoseconds. + bootloader_load_ns: Timer value logged just prior to + loading the OS boot loader into memory. + In nanoseconds. + bootloader_launch_ns: Timer value logged just prior to + launching the currently loaded OS boot loader + image. In nanoseconds. + exitbootservice_start_ns: Timer value logged at the + point when the OS loader calls the + ExitBootServices function for UEFI compatible + firmware. In nanoseconds. + exitbootservice_end_ns: Timer value logged at the point + just prior to the OS loader gaining control + back from the ExitBootServices function for + UEFI compatible firmware. In nanoseconds. + suspend: + suspend_start_ns: Timer value recorded at the previous + OS write to SLP_TYP upon entry to S3. In + nanoseconds. + suspend_end_ns: Timer value recorded at the previous + firmware write to SLP_TYP used to trigger + hardware entry to S3. In nanoseconds. + resume: + resume_count: A count of the number of S3 resume cycles + since the last full boot sequence. + resume_avg_ns: Average timer value of all resume cycles + logged since the last full boot sequence, + including the most recent resume. In nanoseconds. + resume_prev_ns: Timer recorded at the end of the previous + platform runtime firmware S3 resume, just prior to + handoff to the OS waking vector. In nanoseconds. + What: /sys/firmware/acpi/bgrt/ Date: January 2012 Contact: Matthew Garrett diff --git a/Documentation/ABI/testing/sysfs-firmware-sfi b/Documentation/ABI/testing/sysfs-firmware-sfi deleted file mode 100644 index 5210e0f06ddbe9982420cdf0da7eb57907630184..0000000000000000000000000000000000000000 --- a/Documentation/ABI/testing/sysfs-firmware-sfi +++ /dev/null @@ -1,15 +0,0 @@ -What: /sys/firmware/sfi/tables/ -Date: May 2010 -Contact: Len Brown -Description: - SFI defines a number of small static memory tables - so the kernel can get platform information from firmware. - - The tables are defined in the latest SFI specification: - http://simplefirmware.org/documentation - - While the tables are used by the kernel, user-space - can observe them this way:: - - # cd /sys/firmware/sfi/tables - # cat $TABLENAME > $TABLENAME.bin diff --git a/Documentation/ABI/testing/sysfs-platform-ideapad-laptop b/Documentation/ABI/testing/sysfs-platform-ideapad-laptop index fd2ac02bc5bd8b6fae8314a4598304f3a8fe7bd5..4989ab266682c6da43c29a88d7f11cce13cccffb 100644 --- a/Documentation/ABI/testing/sysfs-platform-ideapad-laptop +++ b/Documentation/ABI/testing/sysfs-platform-ideapad-laptop @@ -1,11 +1,11 @@ -What: /sys/devices/platform/ideapad/camera_power +What: /sys/bus/platform/devices/VPC2004:*/camera_power Date: Dec 2010 KernelVersion: 2.6.37 Contact: "Ike Panhc " Description: Control the power of camera module. 1 means on, 0 means off. -What: /sys/devices/platform/ideapad/fan_mode +What: /sys/bus/platform/devices/VPC2004:*/fan_mode Date: June 2012 KernelVersion: 3.6 Contact: "Maxim Mikityanskiy " @@ -18,7 +18,7 @@ Description: * 2 -> Dust Cleaning * 4 -> Efficient Thermal Dissipation Mode -What: /sys/devices/platform/ideapad/touchpad +What: /sys/bus/platform/devices/VPC2004:*/touchpad Date: May 2017 KernelVersion: 4.13 Contact: "Ritesh Raj Sarraf " @@ -27,7 +27,16 @@ Description: * 1 -> Switched On * 0 -> Switched Off -What: /sys/bus/pci/devices///VPC2004:00/fn_lock +What: /sys/bus/platform/devices/VPC2004:*/conservation_mode +Date: Aug 2017 +KernelVersion: 4.14 +Contact: platform-driver-x86@vger.kernel.org +Description: + Controls whether the conservation mode is enabled or not. + This feature limits the maximum battery charge percentage to + around 50-60% in order to prolong the lifetime of the battery. + +What: /sys/bus/platform/devices/VPC2004:*/fn_lock Date: May 2018 KernelVersion: 4.18 Contact: "Oleg Keri " @@ -41,3 +50,12 @@ Description: # echo "0" > \ /sys/bus/pci/devices/0000:00:1f.0/PNP0C09:00/VPC2004:00/fn_lock + +What: /sys/bus/platform/devices/VPC2004:*/usb_charging +Date: Feb 2021 +KernelVersion: 5.12 +Contact: platform-driver-x86@vger.kernel.org +Description: + Controls whether the "always on USB charging" feature is + enabled or not. This feature enables charging USB devices + even if the computer is not turned on. diff --git a/Documentation/ABI/testing/sysfs-platform-kim b/Documentation/ABI/testing/sysfs-platform-kim index a7f81de68046167220459f4e0eef4a53defec572..6a52d6d2b60183cae869c1421a53f0cdc85f13f4 100644 --- a/Documentation/ABI/testing/sysfs-platform-kim +++ b/Documentation/ABI/testing/sysfs-platform-kim @@ -7,7 +7,7 @@ Description: is connected. example: "/dev/ttyS0". The device name flows down to architecture specific board - initialization file from the SFI/ATAGS bootloader + initialization file from the ATAGS bootloader firmware. The name exposed is read from the user-space dameon and opens the device when install is requested. diff --git a/Documentation/ABI/testing/sysfs-platform_profile b/Documentation/ABI/testing/sysfs-platform_profile index 9d6b89b66cca7f0368830b7f14731872abd42aab..dae9c89419053b56b9e115afa16e8c23e246e864 100644 --- a/Documentation/ABI/testing/sysfs-platform_profile +++ b/Documentation/ABI/testing/sysfs-platform_profile @@ -5,13 +5,17 @@ Description: This file contains a space-separated list of profiles supported for Drivers must use the following standard profile-names: - ============ ============================================ - low-power Low power consumption - cool Cooler operation - quiet Quieter operation - balanced Balance between low power consumption and performance - performance High performance operation - ============ ============================================ + ==================== ======================================== + low-power Low power consumption + cool Cooler operation + quiet Quieter operation + balanced Balance between low power consumption + and performance + balanced-performance Balance between performance and low + power consumption with a slight bias + towards performance + performance High performance operation + ==================== ======================================== Userspace may expect drivers to offer more than one of these standard profile names. diff --git a/Documentation/PCI/endpoint/function/binding/pci-ntb.rst b/Documentation/PCI/endpoint/function/binding/pci-ntb.rst new file mode 100644 index 0000000000000000000000000000000000000000..40253d3d5163ba918fa6017d470a4d47b3518a30 --- /dev/null +++ b/Documentation/PCI/endpoint/function/binding/pci-ntb.rst @@ -0,0 +1,38 @@ +.. SPDX-License-Identifier: GPL-2.0 + +========================== +PCI NTB Endpoint Function +========================== + +1) Create a subdirectory to pci_epf_ntb directory in configfs. + +Standard EPF Configurable Fields: + +================ =========================================================== +vendorid should be 0x104c +deviceid should be 0xb00d for TI's J721E SoC +revid don't care +progif_code don't care +subclass_code should be 0x00 +baseclass_code should be 0x5 +cache_line_size don't care +subsys_vendor_id don't care +subsys_id don't care +interrupt_pin don't care +msi_interrupts don't care +msix_interrupts don't care +================ =========================================================== + +2) Create a subdirectory to directory created in 1 + +NTB EPF specific configurable fields: + +================ =========================================================== +db_count Number of doorbells; default = 4 +mw1 size of memory window1 +mw2 size of memory window2 +mw3 size of memory window3 +mw4 size of memory window4 +num_mws Number of memory windows; max = 4 +spad_count Number of scratchpad registers; default = 64 +================ =========================================================== diff --git a/Documentation/PCI/endpoint/index.rst b/Documentation/PCI/endpoint/index.rst index 4ca7439fbfc9b07aa1b96e7b208ac5d88f8b4a73..38ea1f604b6d3287e80be6d6f916c62007d7cbd7 100644 --- a/Documentation/PCI/endpoint/index.rst +++ b/Documentation/PCI/endpoint/index.rst @@ -11,5 +11,8 @@ PCI Endpoint Framework pci-endpoint-cfs pci-test-function pci-test-howto + pci-ntb-function + pci-ntb-howto function/binding/pci-test + function/binding/pci-ntb diff --git a/Documentation/PCI/endpoint/pci-endpoint-cfs.rst b/Documentation/PCI/endpoint/pci-endpoint-cfs.rst index 1bbd81ed06c8f7321060e78fcaf748f36efee807..696f8eeb4738880086ac54652bcc10e4467e36d9 100644 --- a/Documentation/PCI/endpoint/pci-endpoint-cfs.rst +++ b/Documentation/PCI/endpoint/pci-endpoint-cfs.rst @@ -68,6 +68,16 @@ created) ... subsys_vendor_id ... subsys_id ... interrupt_pin + ... primary/ + ... / + ... secondary/ + ... / + +If an EPF device has to be associated with 2 EPCs (like in the case of +Non-transparent bridge), symlink of endpoint controller connected to primary +interface should be added in 'primary' directory and symlink of endpoint +controller connected to secondary interface should be added in 'secondary' +directory. EPC Device ========== diff --git a/Documentation/PCI/endpoint/pci-ntb-function.rst b/Documentation/PCI/endpoint/pci-ntb-function.rst new file mode 100644 index 0000000000000000000000000000000000000000..3b9d836a4924f8cc4ffbf44fd769758d6828f5a7 --- /dev/null +++ b/Documentation/PCI/endpoint/pci-ntb-function.rst @@ -0,0 +1,348 @@ +.. SPDX-License-Identifier: GPL-2.0 + +================= +PCI NTB Function +================= + +:Author: Kishon Vijay Abraham I + +PCI Non-Transparent Bridges (NTB) allow two host systems to communicate +with each other by exposing each host as a device to the other host. +NTBs typically support the ability to generate interrupts on the remote +machine, expose memory ranges as BARs, and perform DMA. They also support +scratchpads, which are areas of memory within the NTB that are accessible +from both machines. + +PCI NTB Function allows two different systems (or hosts) to communicate +with each other by configuring the endpoint instances in such a way that +transactions from one system are routed to the other system. + +In the below diagram, PCI NTB function configures the SoC with multiple +PCI Endpoint (EP) instances in such a way that transactions from one EP +controller are routed to the other EP controller. Once PCI NTB function +configures the SoC with multiple EP instances, HOST1 and HOST2 can +communicate with each other using SoC as a bridge. + +.. code-block:: text + + +-------------+ +-------------+ + | | | | + | HOST1 | | HOST2 | + | | | | + +------^------+ +------^------+ + | | + | | + +---------|-------------------------------------------------|---------+ + | +------v------+ +------v------+ | + | | | | | | + | | EP | | EP | | + | | CONTROLLER1 | | CONTROLLER2 | | + | | <-----------------------------------> | | + | | | | | | + | | | | | | + | | | SoC With Multiple EP Instances | | | + | | | (Configured using NTB Function) | | | + | +-------------+ +-------------+ | + +---------------------------------------------------------------------+ + +Constructs used for Implementing NTB +==================================== + + 1) Config Region + 2) Self Scratchpad Registers + 3) Peer Scratchpad Registers + 4) Doorbell (DB) Registers + 5) Memory Window (MW) + + +Config Region: +-------------- + +Config Region is a construct that is specific to NTB implemented using NTB +Endpoint Function Driver. The host and endpoint side NTB function driver will +exchange information with each other using this region. Config Region has +Control/Status Registers for configuring the Endpoint Controller. Host can +write into this region for configuring the outbound Address Translation Unit +(ATU) and to indicate the link status. Endpoint can indicate the status of +commands issued by host in this region. Endpoint can also indicate the +scratchpad offset and number of memory windows to the host using this region. + +The format of Config Region is given below. All the fields here are 32 bits. + +.. code-block:: text + + +------------------------+ + | COMMAND | + +------------------------+ + | ARGUMENT | + +------------------------+ + | STATUS | + +------------------------+ + | TOPOLOGY | + +------------------------+ + | ADDRESS (LOWER 32) | + +------------------------+ + | ADDRESS (UPPER 32) | + +------------------------+ + | SIZE | + +------------------------+ + | NO OF MEMORY WINDOW | + +------------------------+ + | MEMORY WINDOW1 OFFSET | + +------------------------+ + | SPAD OFFSET | + +------------------------+ + | SPAD COUNT | + +------------------------+ + | DB ENTRY SIZE | + +------------------------+ + | DB DATA | + +------------------------+ + | : | + +------------------------+ + | : | + +------------------------+ + | DB DATA | + +------------------------+ + + + COMMAND: + + NTB function supports three commands: + + CMD_CONFIGURE_DOORBELL (0x1): Command to configure doorbell. Before + invoking this command, the host should allocate and initialize + MSI/MSI-X vectors (i.e., initialize the MSI/MSI-X Capability in the + Endpoint). The endpoint on receiving this command will configure + the outbound ATU such that transactions to Doorbell BAR will be routed + to the MSI/MSI-X address programmed by the host. The ARGUMENT + register should be populated with number of DBs to configure (in the + lower 16 bits) and if MSI or MSI-X should be configured (BIT 16). + + CMD_CONFIGURE_MW (0x2): Command to configure memory window (MW). The + host invokes this command after allocating a buffer that can be + accessed by remote host. The allocated address should be programmed + in the ADDRESS register (64 bit), the size should be programmed in + the SIZE register and the memory window index should be programmed + in the ARGUMENT register. The endpoint on receiving this command + will configure the outbound ATU such that transactions to MW BAR + are routed to the address provided by the host. + + CMD_LINK_UP (0x3): Command to indicate an NTB application is + bound to the EP device on the host side. Once the endpoint + receives this command from both the hosts, the endpoint will + raise a LINK_UP event to both the hosts to indicate the host + NTB applications can start communicating with each other. + + ARGUMENT: + + The value of this register is based on the commands issued in + command register. See COMMAND section for more information. + + TOPOLOGY: + + Set to NTB_TOPO_B2B_USD for Primary interface + Set to NTB_TOPO_B2B_DSD for Secondary interface + + ADDRESS/SIZE: + + Address and Size to be used while configuring the memory window. + See "CMD_CONFIGURE_MW" for more info. + + MEMORY WINDOW1 OFFSET: + + Memory Window 1 and Doorbell registers are packed together in the + same BAR. The initial portion of the region will have doorbell + registers and the latter portion of the region is for memory window 1. + This register will specify the offset of the memory window 1. + + NO OF MEMORY WINDOW: + + Specifies the number of memory windows supported by the NTB device. + + SPAD OFFSET: + + Self scratchpad region and config region are packed together in the + same BAR. The initial portion of the region will have config region + and the latter portion of the region is for self scratchpad. This + register will specify the offset of the self scratchpad registers. + + SPAD COUNT: + + Specifies the number of scratchpad registers supported by the NTB + device. + + DB ENTRY SIZE: + + Used to determine the offset within the DB BAR that should be written + in order to raise doorbell. EPF NTB can use either MSI or MSI-X to + ring doorbell (MSI-X support will be added later). MSI uses same + address for all the interrupts and MSI-X can provide different + addresses for different interrupts. The MSI/MSI-X address is provided + by the host and the address it gives is based on the MSI/MSI-X + implementation supported by the host. For instance, ARM platform + using GIC ITS will have the same MSI-X address for all the interrupts. + In order to support all the combinations and use the same mechanism + for both MSI and MSI-X, EPF NTB allocates a separate region in the + Outbound Address Space for each of the interrupts. This region will + be mapped to the MSI/MSI-X address provided by the host. If a host + provides the same address for all the interrupts, all the regions + will be translated to the same address. If a host provides different + addresses, the regions will be translated to different addresses. This + will ensure there is no difference while raising the doorbell. + + DB DATA: + + EPF NTB supports 32 interrupts, so there are 32 DB DATA registers. + This holds the MSI/MSI-X data that has to be written to MSI address + for raising doorbell interrupt. This will be populated by EPF NTB + while invoking CMD_CONFIGURE_DOORBELL. + +Scratchpad Registers: +--------------------- + + Each host has its own register space allocated in the memory of NTB endpoint + controller. They are both readable and writable from both sides of the bridge. + They are used by applications built over NTB and can be used to pass control + and status information between both sides of a device. + + Scratchpad registers has 2 parts + 1) Self Scratchpad: Host's own register space + 2) Peer Scratchpad: Remote host's register space. + +Doorbell Registers: +------------------- + + Doorbell Registers are used by the hosts to interrupt each other. + +Memory Window: +-------------- + + Actual transfer of data between the two hosts will happen using the + memory window. + +Modeling Constructs: +==================== + +There are 5 or more distinct regions (config, self scratchpad, peer +scratchpad, doorbell, one or more memory windows) to be modeled to achieve +NTB functionality. At least one memory window is required while more than +one is permitted. All these regions should be mapped to BARs for hosts to +access these regions. + +If one 32-bit BAR is allocated for each of these regions, the scheme would +look like this: + +====== =============== +BAR NO CONSTRUCTS USED +====== =============== +BAR0 Config Region +BAR1 Self Scratchpad +BAR2 Peer Scratchpad +BAR3 Doorbell +BAR4 Memory Window 1 +BAR5 Memory Window 2 +====== =============== + +However if we allocate a separate BAR for each of the regions, there would not +be enough BARs for all the regions in a platform that supports only 64-bit +BARs. + +In order to be supported by most of the platforms, the regions should be +packed and mapped to BARs in a way that provides NTB functionality and +also makes sure the host doesn't access any region that it is not supposed +to. + +The following scheme is used in EPF NTB Function: + +====== =============================== +BAR NO CONSTRUCTS USED +====== =============================== +BAR0 Config Region + Self Scratchpad +BAR1 Peer Scratchpad +BAR2 Doorbell + Memory Window 1 +BAR3 Memory Window 2 +BAR4 Memory Window 3 +BAR5 Memory Window 4 +====== =============================== + +With this scheme, for the basic NTB functionality 3 BARs should be sufficient. + +Modeling Config/Scratchpad Region: +---------------------------------- + +.. code-block:: text + + +-----------------+------->+------------------+ +-----------------+ + | BAR0 | | CONFIG REGION | | BAR0 | + +-----------------+----+ +------------------+<-------+-----------------+ + | BAR1 | | |SCRATCHPAD REGION | | BAR1 | + +-----------------+ +-->+------------------+<-------+-----------------+ + | BAR2 | Local Memory | BAR2 | + +-----------------+ +-----------------+ + | BAR3 | | BAR3 | + +-----------------+ +-----------------+ + | BAR4 | | BAR4 | + +-----------------+ +-----------------+ + | BAR5 | | BAR5 | + +-----------------+ +-----------------+ + EP CONTROLLER 1 EP CONTROLLER 2 + +Above diagram shows Config region + Scratchpad region for HOST1 (connected to +EP controller 1) allocated in local memory. The HOST1 can access the config +region and scratchpad region (self scratchpad) using BAR0 of EP controller 1. +The peer host (HOST2 connected to EP controller 2) can also access this +scratchpad region (peer scratchpad) using BAR1 of EP controller 2. This +diagram shows the case where Config region and Scratchpad regions are allocated +for HOST1, however the same is applicable for HOST2. + +Modeling Doorbell/Memory Window 1: +---------------------------------- + +.. code-block:: text + + +-----------------+ +----->+----------------+-----------+-----------------+ + | BAR0 | | | Doorbell 1 +-----------> MSI-X ADDRESS 1 | + +-----------------+ | +----------------+ +-----------------+ + | BAR1 | | | Doorbell 2 +---------+ | | + +-----------------+----+ +----------------+ | | | + | BAR2 | | Doorbell 3 +-------+ | +-----------------+ + +-----------------+----+ +----------------+ | +-> MSI-X ADDRESS 2 | + | BAR3 | | | Doorbell 4 +-----+ | +-----------------+ + +-----------------+ | |----------------+ | | | | + | BAR4 | | | | | | +-----------------+ + +-----------------+ | | MW1 +---+ | +-->+ MSI-X ADDRESS 3|| + | BAR5 | | | | | | +-----------------+ + +-----------------+ +----->-----------------+ | | | | + EP CONTROLLER 1 | | | | +-----------------+ + | | | +---->+ MSI-X ADDRESS 4 | + +----------------+ | +-----------------+ + EP CONTROLLER 2 | | | + (OB SPACE) | | | + +-------> MW1 | + | | + | | + +-----------------+ + | | + | | + | | + | | + | | + +-----------------+ + PCI Address Space + (Managed by HOST2) + +Above diagram shows how the doorbell and memory window 1 is mapped so that +HOST1 can raise doorbell interrupt on HOST2 and also how HOST1 can access +buffers exposed by HOST2 using memory window1 (MW1). Here doorbell and +memory window 1 regions are allocated in EP controller 2 outbound (OB) address +space. Allocating and configuring BARs for doorbell and memory window1 +is done during the initialization phase of NTB endpoint function driver. +Mapping from EP controller 2 OB space to PCI address space is done when HOST2 +sends CMD_CONFIGURE_MW/CMD_CONFIGURE_DOORBELL. + +Modeling Optional Memory Windows: +--------------------------------- + +This is modeled the same was as MW1 but each of the additional memory windows +is mapped to separate BARs. diff --git a/Documentation/PCI/endpoint/pci-ntb-howto.rst b/Documentation/PCI/endpoint/pci-ntb-howto.rst new file mode 100644 index 0000000000000000000000000000000000000000..1884bf29caba4bf178ce53b0abf372074576ec18 --- /dev/null +++ b/Documentation/PCI/endpoint/pci-ntb-howto.rst @@ -0,0 +1,161 @@ +.. SPDX-License-Identifier: GPL-2.0 + +=================================================================== +PCI Non-Transparent Bridge (NTB) Endpoint Function (EPF) User Guide +=================================================================== + +:Author: Kishon Vijay Abraham I + +This document is a guide to help users use pci-epf-ntb function driver +and ntb_hw_epf host driver for NTB functionality. The list of steps to +be followed in the host side and EP side is given below. For the hardware +configuration and internals of NTB using configurable endpoints see +Documentation/PCI/endpoint/pci-ntb-function.rst + +Endpoint Device +=============== + +Endpoint Controller Devices +--------------------------- + +For implementing NTB functionality at least two endpoint controller devices +are required. + +To find the list of endpoint controller devices in the system:: + + # ls /sys/class/pci_epc/ + 2900000.pcie-ep 2910000.pcie-ep + +If PCI_ENDPOINT_CONFIGFS is enabled:: + + # ls /sys/kernel/config/pci_ep/controllers + 2900000.pcie-ep 2910000.pcie-ep + + +Endpoint Function Drivers +------------------------- + +To find the list of endpoint function drivers in the system:: + + # ls /sys/bus/pci-epf/drivers + pci_epf_ntb pci_epf_ntb + +If PCI_ENDPOINT_CONFIGFS is enabled:: + + # ls /sys/kernel/config/pci_ep/functions + pci_epf_ntb pci_epf_ntb + + +Creating pci-epf-ntb Device +---------------------------- + +PCI endpoint function device can be created using the configfs. To create +pci-epf-ntb device, the following commands can be used:: + + # mount -t configfs none /sys/kernel/config + # cd /sys/kernel/config/pci_ep/ + # mkdir functions/pci_epf_ntb/func1 + +The "mkdir func1" above creates the pci-epf-ntb function device that will +be probed by pci_epf_ntb driver. + +The PCI endpoint framework populates the directory with the following +configurable fields:: + + # ls functions/pci_epf_ntb/func1 + baseclass_code deviceid msi_interrupts pci-epf-ntb.0 + progif_code secondary subsys_id vendorid + cache_line_size interrupt_pin msix_interrupts primary + revid subclass_code subsys_vendor_id + +The PCI endpoint function driver populates these entries with default values +when the device is bound to the driver. The pci-epf-ntb driver populates +vendorid with 0xffff and interrupt_pin with 0x0001:: + + # cat functions/pci_epf_ntb/func1/vendorid + 0xffff + # cat functions/pci_epf_ntb/func1/interrupt_pin + 0x0001 + + +Configuring pci-epf-ntb Device +------------------------------- + +The user can configure the pci-epf-ntb device using its configfs entry. In order +to change the vendorid and the deviceid, the following +commands can be used:: + + # echo 0x104c > functions/pci_epf_ntb/func1/vendorid + # echo 0xb00d > functions/pci_epf_ntb/func1/deviceid + +In order to configure NTB specific attributes, a new sub-directory to func1 +should be created:: + + # mkdir functions/pci_epf_ntb/func1/pci_epf_ntb.0/ + +The NTB function driver will populate this directory with various attributes +that can be configured by the user:: + + # ls functions/pci_epf_ntb/func1/pci_epf_ntb.0/ + db_count mw1 mw2 mw3 mw4 num_mws + spad_count + +A sample configuration for NTB function is given below:: + + # echo 4 > functions/pci_epf_ntb/func1/pci_epf_ntb.0/db_count + # echo 128 > functions/pci_epf_ntb/func1/pci_epf_ntb.0/spad_count + # echo 2 > functions/pci_epf_ntb/func1/pci_epf_ntb.0/num_mws + # echo 0x100000 > functions/pci_epf_ntb/func1/pci_epf_ntb.0/mw1 + # echo 0x100000 > functions/pci_epf_ntb/func1/pci_epf_ntb.0/mw2 + +Binding pci-epf-ntb Device to EP Controller +-------------------------------------------- + +NTB function device should be attached to two PCI endpoint controllers +connected to the two hosts. Use the 'primary' and 'secondary' entries +inside NTB function device to attach one PCI endpoint controller to +primary interface and the other PCI endpoint controller to the secondary +interface:: + + # ln -s controllers/2900000.pcie-ep/ functions/pci-epf-ntb/func1/primary + # ln -s controllers/2910000.pcie-ep/ functions/pci-epf-ntb/func1/secondary + +Once the above step is completed, both the PCI endpoint controllers are ready to +establish a link with the host. + + +Start the Link +-------------- + +In order for the endpoint device to establish a link with the host, the _start_ +field should be populated with '1'. For NTB, both the PCI endpoint controllers +should establish link with the host:: + + # echo 1 > controllers/2900000.pcie-ep/start + # echo 1 > controllers/2910000.pcie-ep/start + + +RootComplex Device +================== + +lspci Output +------------ + +Note that the devices listed here correspond to the values populated in +"Creating pci-epf-ntb Device" section above:: + + # lspci + 0000:00:00.0 PCI bridge: Texas Instruments Device b00d + 0000:01:00.0 RAM memory: Texas Instruments Device b00d + + +Using ntb_hw_epf Device +----------------------- + +The host side software follows the standard NTB software architecture in Linux. +All the existing client side NTB utilities like NTB Transport Client and NTB +Netdev, NTB Ping Pong Test Client and NTB Tool Test Client can be used with NTB +function device. + +For more information on NTB see +:doc:`Non-Transparent Bridge <../../driver-api/ntb>` diff --git a/Documentation/RCU/RTFP.txt b/Documentation/RCU/RTFP.txt index 9bccf16736f78d3b9b365cafeeb489700ef782b2..3b0876c773552ff7209c34079df40fbf9e338b46 100644 --- a/Documentation/RCU/RTFP.txt +++ b/Documentation/RCU/RTFP.txt @@ -683,7 +683,7 @@ Orran Krieger and Rusty Russell and Dipankar Sarma and Maneesh Soni" ,month="October" ,year="2001" ,note="Available: -\url{http://lkml.org/lkml/2001/10/13/105} +\url{https://lore.kernel.org/r/Pine.LNX.4.33.0110131015410.8707-100000@penguin.transmeta.com} [Viewed August 21, 2004]" ,annotation={ } @@ -826,7 +826,7 @@ Symposium on Distributed Computing} ,month="October" ,year="2002" ,note="Available: -\url{https://lkml.org/lkml/2002/10/24/262} +\url{https://lore.kernel.org/r/3DB86B05.447E7410@us.ibm.com} [Viewed February 15, 2014]" ,annotation={ Mingming Cao's patch to introduce RCU to SysV IPC. @@ -839,7 +839,7 @@ Symposium on Distributed Computing} ,month="March" ,year="2003" ,note="Available: -\url{http://lkml.org/lkml/2003/3/9/205} +\url{https://lore.kernel.org/r/Pine.LNX.4.44.0303091831560.2129-100000@home.transmeta.com} [Viewed March 13, 2006]" ,annotation={ Linus suggests replacing brlock with RCU and/or seqlocks: @@ -1036,15 +1036,15 @@ Add per-cpu batch counter" ,annotation={ RCU runs reasonably on a 512-CPU SGI using Manfred Spraul's patches, which may be found at: - https://lkml.org/lkml/2004/5/20/49 (split vars into cachelines) - https://lkml.org/lkml/2004/5/22/114 (cpu_quiet() patch) - https://lkml.org/lkml/2004/5/25/24 (0/5) - https://lkml.org/lkml/2004/5/25/23 (1/5) - https://lkml.org/lkml/2004/5/25/265 (works for Jack) - https://lkml.org/lkml/2004/5/25/20 (2/5) - https://lkml.org/lkml/2004/5/25/22 (3/5) - https://lkml.org/lkml/2004/5/25/19 (4/5) - https://lkml.org/lkml/2004/5/25/21 (5/5) + https://lore.kernel.org/r/40AC9823.6020709@colorfullife.com (split vars into cachelines) + https://lore.kernel.org/r/Pine.LNX.4.44.0405222141260.11106-100000@dbl.q-ag.de (cpu_quiet() patch) + https://lore.kernel.org/r/200405250535.i4P5ZJo8017583@dbl.q-ag.de (0/5) + https://lore.kernel.org/r/200405250535.i4P5ZKAQ017591@dbl.q-ag.de (1/5) + https://lore.kernel.org/r/20040525203215.GB5127@sgi.com (works for Jack) + https://lore.kernel.org/r/200405250535.i4P5ZLiR017599@dbl.q-ag.de (2/5) + https://lore.kernel.org/r/200405250535.i4P5ZMFt017607@dbl.q-ag.de (3/5) + https://lore.kernel.org/r/200405250535.i4P5ZN6g017615@dbl.q-ag.de (4/5) + https://lore.kernel.org/r/200405250535.i4P5ZO7I017623@dbl.q-ag.de (5/5) } } @@ -1106,7 +1106,7 @@ Oregon Health and Sciences University" ,month="August" ,year="2004" ,note="Available: -\url{http://lkml.org/lkml/2004/8/6/237} +\url{https://lore.kernel.org/r/20040807192424.GF3936@in.ibm.com} [Viewed June 8, 2010]" ,annotation={ Introduce rcu_dereference(). @@ -1119,7 +1119,7 @@ Oregon Health and Sciences University" ,month="August" ,year="2004" ,note="Available: -\url{http://lkml.org/lkml/2004/8/30/87} +\url{https://lore.kernel.org/r/1093873222.984.12.camel@new.localdomain} [Viewed February 17, 2005]" ,annotation={ Uses active code in rcu_read_lock() and rcu_read_unlock() to @@ -1186,7 +1186,7 @@ Oregon Health and Sciences University" ,month="October" ,year="2004" ,note="Available: -\url{http://lkml.org/lkml/2004/10/23/241} +\url{https://lore.kernel.org/r/20041023202723.GA1930@us.ibm.com} [Viewed June 8, 2010]" ,annotation={ Introduce rcu_assign_pointer(). @@ -1203,7 +1203,7 @@ Oregon Health and Sciences University" ,annotation={ James Morris posts Kaigai Kohei's patch to LKML. [Viewed December 10, 2004] - Kaigai's patch is at https://lkml.org/lkml/2004/9/27/52 + Kaigai's patch is at https://lore.kernel.org/r/200409271057.i8RAvcA1007873@mailsv.bs1.fc.nec.co.jp } } @@ -1241,7 +1241,7 @@ Oregon Health and Sciences University" ,year="2005" ,day="17" ,note="Available: -\url{http://lkml.org/lkml/2005/3/17/199} +\url{https://lore.kernel.org/r/20050318002026.GA2693@us.ibm.com} [Viewed September 5, 2005]" ,annotation={ First posting showing how RCU can be safely adapted for @@ -1256,7 +1256,7 @@ Oregon Health and Sciences University" ,year="2005" ,day="18" ,note="Available: -\url{http://lkml.org/lkml/2005/3/18/122} +\url{https://lore.kernel.org/r/Pine.OSF.4.05.10503181336310.2466-100000@da410.phys.au.dk} [Viewed March 30, 2006]" ,annotation={ Esben Neilsen suggests read-side suppression of grace-period @@ -1302,7 +1302,7 @@ Data Structures" ,month="May" ,year="2005" ,note="Available: -\url{http://lkml.org/lkml/2005/5/9/185} +\url{https://lore.kernel.org/r/20050510012444.GA3011@us.ibm.com} [Viewed May 13, 2005]" ,annotation={ First publication of working lock-based deferred free patches @@ -1385,7 +1385,7 @@ Data Structures" ,day="1" ,year="2005" ,note="Available: -\url{http://lkml.org/lkml/2005/8/1/155} +\url{https://lore.kernel.org/r/20050801171137.GA1754@us.ibm.com} [Viewed March 14, 2006]" ,annotation={ First operating counter-based realtime RCU patch posted to LKML. @@ -1399,7 +1399,7 @@ Data Structures" ,day="8" ,year="2005" ,note="Available: -\url{http://lkml.org/lkml/2005/8/8/108} +\url{https://lore.kernel.org/r/20050808144216.GA1307@us.ibm.com} [Viewed March 14, 2006]" ,annotation={ First operating counter-based realtime RCU patch posted to LKML, @@ -1415,7 +1415,7 @@ Data Structures" ,day="1" ,year="2005" ,note="Available: -\url{http://lkml.org/lkml/2005/10/1/70} +\url{https://lore.kernel.org/r/20051001182056.GA1613@us.ibm.com} [Viewed March 14, 2006]" ,annotation={ First rcutorture patch. @@ -1429,7 +1429,7 @@ Data Structures" ,day="6" ,year="2006" ,note="Available: -\url{https://lkml.org/lkml/2006/1/7/22} +\url{https://lore.kernel.org/r/20060106.231054.43576567.davem@davemloft.net} [Viewed February 29, 2012]" ,annotation={ David Miller's view on hashed arrays of locks: used to really @@ -1464,7 +1464,7 @@ Distributed Processing Symposium" ,day="20" ,year="2006" ,note="Available: -\url{http://lkml.org/lkml/2006/6/20/238} +\url{https://lore.kernel.org/r/20060408134707.22479.33814.sendpatchset@linux.site} [Viewed March 25, 2008]" ,annotation={ RCU-protected radix tree. @@ -1554,7 +1554,7 @@ Revised: ,day="28" ,year="2006" ,note="Available: -\url{http://lkml.org/lkml/2006/9/28/160} +\url{https://lore.kernel.org/r/20060928142616.GA20185@infradead.org} [Viewed March 27, 2008]" } @@ -1593,7 +1593,7 @@ Revised: ,year="2006" ,day=26 ,note="Available: -\url{http://lkml.org/lkml/2006/10/26/73} +\url{https://lore.kernel.org/r/20061026105731.GE11803@in.ibm.com} [Viewed January 26, 2009]" ,annotation={ RCU-based reader-writer lock that allows readers to proceed with @@ -1612,12 +1612,12 @@ Revised: ,year="2006" ,day=17 ,note="Available: -\url{http://lkml.org/lkml/2006/11/17/56} +\url{https://lore.kernel.org/r/20061117092925.GT7164@kernel.dk} [Viewed May 28, 2007]" ,annotation={ SRCU's grace periods are too slow for Jens, even after a factor-of-three speedup. - Sped-up version of SRCU at http://lkml.org/lkml/2006/11/17/359. + Sped-up version of SRCU at https://lore.kernel.org/r/20061118002845.GF2632@us.ibm.com. } } @@ -1629,7 +1629,7 @@ Revised: ,year="2006" ,day=19 ,note="Available: -\url{http://lkml.org/lkml/2006/11/19/69} +\url{https://lore.kernel.org/r/20061119190027.GA3676@oleg} [Viewed May 28, 2007]" ,annotation={ First cut of QRCU. Expanded/corrected versions followed. @@ -1644,7 +1644,7 @@ Revised: ,year="2006" ,day=30 ,note="Available: -\url{http://lkml.org/lkml/2006/11/29/330} +\url{https://lore.kernel.org/r/20061130015714.GC1350@oleg} [Viewed November 26, 2008]" ,annotation={ Expanded/corrected version of QRCU. @@ -1709,7 +1709,7 @@ Revised: ,year="2007" ,day=3 ,note="Available: -\url{http://lkml.org/lkml/2007/1/3/112} +\url{https://lore.kernel.org/r/20070103152738.GA16063@localdomain} [Viewed May 28, 2007]" ,annotation={ Patch for list_splice_rcu(). @@ -1737,7 +1737,7 @@ Revised: ,year="2007" ,day=28 ,note="Available: -\url{http://lkml.org/lkml/2007/1/28/34} +\url{https://lore.kernel.org/r/20070128120509.719287000@programming.kicks-ass.net} [Viewed March 27, 2008]" ,annotation={ RCU-like implementation for frequent updaters and rare readers(!). @@ -1767,7 +1767,7 @@ Revised: ,year="2007" ,day=24 ,note="Available: -\url{http://lkml.org/lkml/2007/2/25/18} +\url{https://lore.kernel.org/r/20070225062349.GA17468@linux.vnet.ibm.com} [Viewed March 27, 2008]" ,annotation={ Patch for QRCU supplying lock-free fast path. @@ -1846,7 +1846,7 @@ Revised: ,annotation={ LWN article describing Promela and spin, and also using Oleg Nesterov's QRCU as an example (with Paul McKenney's fastpath). - Merged patch at: http://lkml.org/lkml/2007/2/25/18 + Merged patch at: https://lore.kernel.org/r/20070225062349.GA17468@linux.vnet.ibm.com } } @@ -1885,7 +1885,7 @@ Revised: ,day="10" ,year="2007" ,note="Available: -\url{http://lkml.org/lkml/2007/9/10/213} +\url{https://lore.kernel.org/r/20070910183004.GA3299@linux.vnet.ibm.com} [Viewed October 25, 2007]" ,annotation={ Final patch for preemptable RCU to -rt. (Later patches were @@ -1933,7 +1933,7 @@ Revised: ,day="20" ,year="2007" ,note="Available: -\url{http://lkml.org/lkml/2007/12/20/244} +\url{https://lore.kernel.org/r/20071220142540.GB22523@Krystal} [Viewed March 27, 2008]" ,annotation={ Request for call_rcu_sched() and rcu_barrier_sched(). @@ -2013,7 +2013,7 @@ Revised: ,day="29" ,year="2008" ,note="Available: -\url{http://lkml.org/lkml/2008/1/29/208} +\url{https://lore.kernel.org/r/Pine.LNX.4.58.0801291113350.20371@gandalf.stny.rr.com} [Viewed March 27, 2008]" ,annotation={ Patch that prevents preemptible RCU from unnecessarily waking @@ -2028,7 +2028,7 @@ Revised: ,day="1" ,year="2008" ,note="Available: -\url{http://lkml.org/lkml/2008/2/2/255} +\url{https://lore.kernel.org/r/20080202214124.GA28612@linux.vnet.ibm.com} [Viewed October 18, 2008]" ,annotation={ Explanation of compilers violating dependency ordering. @@ -2088,7 +2088,7 @@ lot of {Linux} into your technology!!!" ,day="3" ,year="2008" ,note="Available: -\url{http://lkml.org/lkml/2008/6/2/539} +\url{https://lore.kernel.org/r/4844BE83.5010401@cn.fujitsu.com} [Viewed December 10, 2008]" ,annotation={ Updated RCU classic algorithm. Introduced multi-tailed list @@ -2122,7 +2122,7 @@ lot of {Linux} into your technology!!!" ,day="21" ,year="2008" ,note="Available: -\url{http://lkml.org/lkml/2008/8/21/336} +\url{https://lore.kernel.org/r/48AD8969.7060900@colorfullife.com} [Viewed December 8, 2008]" ,annotation={ State-based RCU. One key thing that this patch does is to @@ -2137,7 +2137,7 @@ lot of {Linux} into your technology!!!" ,day="6" ,year="2008" ,note="Available: -\url{http://lkml.org/lkml/2008/9/6/86} +\url{https://lore.kernel.org/r/48C2B1D2.5070801@colorfullife.com} [Viewed December 8, 2008]" ,annotation={ Manfred notes a fix required to my attempt to separate irq @@ -2183,7 +2183,7 @@ lot of {Linux} into your technology!!!" ,day="14" ,year="2009" ,note="Available: -\url{http://lkml.org/lkml/2009/1/14/449} +\url{https://lore.kernel.org/r/20090114202044.GJ6734@linux.vnet.ibm.com} [Viewed January 15, 2009]" ,annotation={ Small-footprint implementation of RCU for uniprocessor @@ -2218,7 +2218,7 @@ lot of {Linux} into your technology!!!" git://lttng.org/userspace-rcu.git http://lttng.org/cgi-bin/gitweb.cgi?p=userspace-rcu.git http://lttng.org/urcu - http://lkml.org/lkml/2009/2/5/572 + https://lore.kernel.org/r/20090206030543.GB8560@Krystal } } @@ -2258,7 +2258,7 @@ lot of {Linux} into your technology!!!" ,day="25" ,year="2009" ,note="Available: -\url{http://lkml.org/lkml/2009/6/25/306} +\url{https://lore.kernel.org/r/20090625160706.GA9467@linux.vnet.ibm.com} [Viewed August 16, 2009]" ,annotation={ First posting of expedited RCU to be accepted into -tip. @@ -2272,7 +2272,7 @@ lot of {Linux} into your technology!!!" ,day="23" ,year="2009" ,note="Available: -\url{http://lkml.org/lkml/2009/7/23/294} +\url{https://lore.kernel.org/r/20090724001429.GA17374@linux.vnet.ibm.com} [Viewed August 15, 2009]" ,annotation={ First posting of simple and fast preemptable RCU. @@ -2350,7 +2350,7 @@ lot of {Linux} into your technology!!!" ,month="December" ,year="2009" ,note="Available: -\url{http://lkml.org/lkml/2009/10/18/129} +\url{https://lore.kernel.org/r/20091018232918.GA7385@Krystal} [Viewed December 29, 2009]" ,annotation={ Mathieu proposed defer_rcu() with fixed-size per-thread pool @@ -2518,7 +2518,7 @@ lot of {Linux} into your technology!!!" ,month="January" ,year="2011" ,note="Available: -\url{https://lkml.org/lkml/2011/1/18/322} +\url{https://lore.kernel.org/r/AANLkTimajU0x1v6y3rH2+jr-bZ=tNLs1S_agXdGGAa3S@mail.gmail.com} [Viewed March 4, 2011]" ,annotation={ "The RCU-based name lookup is at the other end of the spectrum - the diff --git a/Documentation/accounting/cgroupstats.rst b/Documentation/accounting/cgroupstats.rst index b9afc48f4ea24d664a117faedfb51198982e5c79..85186e7d40351742d0b8f5011a1c8341dd7fad94 100644 --- a/Documentation/accounting/cgroupstats.rst +++ b/Documentation/accounting/cgroupstats.rst @@ -3,8 +3,8 @@ Control Groupstats ================== Control Groupstats is inspired by the discussion at -http://lkml.org/lkml/2007/4/11/187 and implements per cgroup statistics as -suggested by Andrew Morton in http://lkml.org/lkml/2007/4/11/263. +https://lore.kernel.org/r/461CF883.2030308@sw.ru and implements per cgroup statistics as +suggested by Andrew Morton in https://lore.kernel.org/r/20070411114927.1277d7c9.akpm@linux-foundation.org. Per cgroup statistics infrastructure re-uses code from the taskstats interface. A new set of cgroup operations are registered with commands diff --git a/Documentation/admin-guide/README.rst b/Documentation/admin-guide/README.rst index 261b7b4cca1f6691b5d0285b450dad8cd97bd93f..35314b63008c258209d4454a86692fed462e3c46 100644 --- a/Documentation/admin-guide/README.rst +++ b/Documentation/admin-guide/README.rst @@ -226,10 +226,11 @@ Configuring the kernel all module options to built in (=y) options. You can also preserve modules by LMC_KEEP. - "make kvmconfig" Enable additional options for kvm guest kernel support. + "make kvm_guest.config" Enable additional options for kvm guest kernel + support. - "make xenconfig" Enable additional options for xen dom0 guest kernel - support. + "make xen.config" Enable additional options for xen dom0 guest kernel + support. "make tinyconfig" Configure the tiniest possible kernel. diff --git a/Documentation/admin-guide/auxdisplay/cfag12864b.rst b/Documentation/admin-guide/auxdisplay/cfag12864b.rst index 18c2865bd322303b2c8abd91f1f91988f0a9454a..da385d851acc889395718ce889e42431867652c6 100644 --- a/Documentation/admin-guide/auxdisplay/cfag12864b.rst +++ b/Documentation/admin-guide/auxdisplay/cfag12864b.rst @@ -3,7 +3,7 @@ cfag12864b LCD Driver Documentation =================================== :License: GPLv2 -:Author & Maintainer: Miguel Ojeda Sandonis +:Author & Maintainer: Miguel Ojeda :Date: 2006-10-27 diff --git a/Documentation/admin-guide/auxdisplay/ks0108.rst b/Documentation/admin-guide/auxdisplay/ks0108.rst index c0b7faf73136129b6d86873763b534281f82316f..a7d3fe5093730ae45ec3bd78aef8e72df2d1586a 100644 --- a/Documentation/admin-guide/auxdisplay/ks0108.rst +++ b/Documentation/admin-guide/auxdisplay/ks0108.rst @@ -3,7 +3,7 @@ ks0108 LCD Controller Driver Documentation ========================================== :License: GPLv2 -:Author & Maintainer: Miguel Ojeda Sandonis +:Author & Maintainer: Miguel Ojeda :Date: 2006-10-27 diff --git a/Documentation/admin-guide/cgroup-v1/memory.rst b/Documentation/admin-guide/cgroup-v1/memory.rst index 52688ae3446137d0ea4704971b4eb32f6e134214..0936412e044eed0b1a1af5231441f5841cac61b5 100644 --- a/Documentation/admin-guide/cgroup-v1/memory.rst +++ b/Documentation/admin-guide/cgroup-v1/memory.rst @@ -963,21 +963,21 @@ References 2. Singh, Balbir. Memory Controller (RSS Control), http://lwn.net/Articles/222762/ 3. Emelianov, Pavel. Resource controllers based on process cgroups - http://lkml.org/lkml/2007/3/6/198 + https://lore.kernel.org/r/45ED7DEC.7010403@sw.ru 4. Emelianov, Pavel. RSS controller based on process cgroups (v2) - http://lkml.org/lkml/2007/4/9/78 + https://lore.kernel.org/r/461A3010.90403@sw.ru 5. Emelianov, Pavel. RSS controller based on process cgroups (v3) - http://lkml.org/lkml/2007/5/30/244 + https://lore.kernel.org/r/465D9739.8070209@openvz.org 6. Menage, Paul. Control Groups v10, http://lwn.net/Articles/236032/ 7. Vaidyanathan, Srinivasan, Control Groups: Pagecache accounting and control subsystem (v3), http://lwn.net/Articles/235534/ 8. Singh, Balbir. RSS controller v2 test results (lmbench), - http://lkml.org/lkml/2007/5/17/232 + https://lore.kernel.org/r/464C95D4.7070806@linux.vnet.ibm.com 9. Singh, Balbir. RSS controller v2 AIM9 results - http://lkml.org/lkml/2007/5/18/1 + https://lore.kernel.org/r/464D267A.50107@linux.vnet.ibm.com 10. Singh, Balbir. Memory controller v6 test results, - http://lkml.org/lkml/2007/8/19/36 + https://lore.kernel.org/r/20070819094658.654.84837.sendpatchset@balbir-laptop 11. Singh, Balbir. Memory controller introduction (v6), - http://lkml.org/lkml/2007/8/17/69 + https://lore.kernel.org/r/20070817084228.26003.12568.sendpatchset@balbir-laptop 12. Corbet, Jonathan, Controlling memory use in cgroups, http://lwn.net/Articles/243795/ diff --git a/Documentation/admin-guide/cgroup-v2.rst b/Documentation/admin-guide/cgroup-v2.rst index 1de8695c264b25acbad1f6920a556862cfdc4b7d..64c62b979f2fddb70fd342c744ba5d76bcf69063 100644 --- a/Documentation/admin-guide/cgroup-v2.rst +++ b/Documentation/admin-guide/cgroup-v2.rst @@ -1,3 +1,5 @@ +.. _cgroup-v2: + ================ Control Group v2 ================ @@ -172,7 +174,6 @@ disabling controllers in v1 and make them always available in v2. cgroup v2 currently supports the following mount options. nsdelegate - Consider cgroup namespaces as delegation boundaries. This option is system wide and can only be set on mount or modified through remount from the init namespace. The mount option is @@ -180,7 +181,6 @@ cgroup v2 currently supports the following mount options. Delegation section for details. memory_localevents - Only populate memory.events with data for the current cgroup, and not any subtrees. This is legacy behaviour, the default behaviour without this option is to include subtree counts. @@ -189,7 +189,6 @@ cgroup v2 currently supports the following mount options. option is ignored on non-init namespace mounts. memory_recursiveprot - Recursively apply memory.min and memory.low protection to entire subtrees, without requiring explicit downward propagation into leaf cgroups. This allows protecting entire @@ -786,7 +785,6 @@ Core Interface Files All cgroup core files are prefixed with "cgroup." cgroup.type - A read-write single value file which exists on non-root cgroups. @@ -954,6 +952,8 @@ All cgroup core files are prefixed with "cgroup." Controllers =========== +.. _cgroup-v2-cpu: + CPU --- @@ -1259,9 +1259,9 @@ PAGE_SIZE multiple when read back. can show up in the middle. Don't rely on items remaining in a fixed position; use the keys to look up specific values! - If the entry has no per-node counter(or not show in the - mempry.numa_stat). We use 'npn'(non-per-node) as the tag - to indicate that it will not show in the mempry.numa_stat. + If the entry has no per-node counter (or not show in the + memory.numa_stat). We use 'npn' (non-per-node) as the tag + to indicate that it will not show in the memory.numa_stat. anon Amount of memory used in anonymous mappings such as @@ -1277,11 +1277,11 @@ PAGE_SIZE multiple when read back. pagetables Amount of memory allocated for page tables. - percpu(npn) + percpu (npn) Amount of memory used for storing per-cpu kernel data structures. - sock(npn) + sock (npn) Amount of memory used in network transmission buffers shmem @@ -1299,6 +1299,10 @@ PAGE_SIZE multiple when read back. Amount of cached filesystem data that was modified and is currently being written back to disk + swapcached + Amount of swap cached in memory. The swapcache is accounted + against both memory and swap usage. + anon_thp Amount of memory used in anonymous mappings backed by transparent hugepages @@ -1329,7 +1333,7 @@ PAGE_SIZE multiple when read back. Part of "slab" that cannot be reclaimed on memory pressure. - slab(npn) + slab (npn) Amount of memory used for storing in-kernel data structures. @@ -1357,39 +1361,39 @@ PAGE_SIZE multiple when read back. workingset_nodereclaim Number of times a shadow node has been reclaimed - pgfault(npn) + pgfault (npn) Total number of page faults incurred - pgmajfault(npn) + pgmajfault (npn) Number of major page faults incurred - pgrefill(npn) + pgrefill (npn) Amount of scanned pages (in an active LRU list) - pgscan(npn) + pgscan (npn) Amount of scanned pages (in an inactive LRU list) - pgsteal(npn) + pgsteal (npn) Amount of reclaimed pages - pgactivate(npn) + pgactivate (npn) Amount of pages moved to the active LRU list - pgdeactivate(npn) + pgdeactivate (npn) Amount of pages moved to the inactive LRU list - pglazyfree(npn) + pglazyfree (npn) Amount of pages postponed to be freed under memory pressure - pglazyfreed(npn) + pglazyfreed (npn) Amount of reclaimed lazyfree pages - thp_fault_alloc(npn) + thp_fault_alloc (npn) Number of transparent hugepages which were allocated to satisfy a page fault. This counter is not present when CONFIG_TRANSPARENT_HUGEPAGE is not set. - thp_collapse_alloc(npn) + thp_collapse_alloc (npn) Number of transparent hugepages which were allocated to allow collapsing an existing range of pages. This counter is not present when CONFIG_TRANSPARENT_HUGEPAGE is not set. @@ -1558,7 +1562,7 @@ IO Interface Files 8:0 rbytes=90430464 wbytes=299008000 rios=8950 wios=1252 dbytes=50331648 dios=3021 io.cost.qos - A read-write nested-keyed file with exists only on the root + A read-write nested-keyed file which exists only on the root cgroup. This file configures the Quality of Service of the IO cost @@ -1613,7 +1617,7 @@ IO Interface Files automatic mode can be restored by setting "ctrl" to "auto". io.cost.model - A read-write nested-keyed file with exists only on the root + A read-write nested-keyed file which exists only on the root cgroup. This file configures the cost model of the IO cost model based @@ -2000,10 +2004,12 @@ Cpuset Interface Files cpuset-enabled cgroups. This flag is owned by the parent cgroup and is not delegatable. - It accepts only the following input values when written to. + It accepts only the following input values when written to. - "root" - a partition root - "member" - a non-root member of a partition + ======== ================================ + "root" a partition root + "member" a non-root member of a partition + ======== ================================ When set to be a partition root, the current cgroup is the root of a new partition or scheduling domain that comprises @@ -2044,9 +2050,11 @@ Cpuset Interface Files root to change. On read, the "cpuset.sched.partition" file can show the following values. - "member" Non-root member of a partition - "root" Partition root - "root invalid" Invalid partition root + ============== ============================== + "member" Non-root member of a partition + "root" Partition root + "root invalid" Invalid partition root + ============== ============================== It is a partition root if the first 2 partition root conditions above are true and at least one CPU from "cpuset.cpus" is @@ -2090,7 +2098,7 @@ If the program returns 0, the attempt fails with -EPERM, otherwise it succeeds. An example of BPF_CGROUP_DEVICE program may be found in the kernel -source tree in the tools/testing/selftests/bpf/dev_cgroup.c file. +source tree in the tools/testing/selftests/bpf/progs/dev_cgroup.c file. RDMA @@ -2219,7 +2227,7 @@ Without cgroup namespace, the "/proc/$PID/cgroup" file shows the complete path of the cgroup of a process. In a container setup where a set of cgroups and namespaces are intended to isolate processes the "/proc/$PID/cgroup" file may leak potential system level information -to the isolated processes. For Example:: +to the isolated processes. For example:: # cat /proc/self/cgroup 0::/batchjobs/container_id1 diff --git a/Documentation/admin-guide/cifs/authors.rst b/Documentation/admin-guide/cifs/authors.rst index b02d6dd6c070eb0770bff460e3eef51def58a1c1..5c1d2f0fa7d11333298db2184069eb522886c713 100644 --- a/Documentation/admin-guide/cifs/authors.rst +++ b/Documentation/admin-guide/cifs/authors.rst @@ -5,10 +5,10 @@ Authors Original Author --------------- -Steve French (sfrench@samba.org) +Steve French (smfrench@gmail.com, sfrench@samba.org) The author wishes to express his appreciation and thanks to: -Andrew Tridgell (Samba team) for his early suggestions about smb/cifs VFS +Andrew Tridgell (Samba team) for his early suggestions about SMB/CIFS VFS improvements. Thanks to IBM for allowing me time and test resources to pursue this project, to Jim McDonough from IBM (and the Samba Team) for his help, to the IBM Linux JFS team for explaining many esoteric Linux filesystem features. @@ -51,7 +51,7 @@ Patch Contributors - Ronnie Sahlberg (for SMB3 xattr work, bug fixes, and lots of great work on compounding) - Shirish Pargaonkar (for many ACL patches over the years) - Sachin Prabhu (many bug fixes, including for reconnect, copy offload and security) -- Paulo Alcantara +- Paulo Alcantara (for some excellent work in DFS, and in booting from SMB3) - Long Li (some great work on RDMA, SMB Direct) diff --git a/Documentation/admin-guide/cifs/changes.rst b/Documentation/admin-guide/cifs/changes.rst index 71f2ecb62299316f1799c9f9c769b9a3a610fda3..3147bbae9c43f21a41c5e110404821b5c2027e62 100644 --- a/Documentation/admin-guide/cifs/changes.rst +++ b/Documentation/admin-guide/cifs/changes.rst @@ -3,6 +3,7 @@ Changes ======= See https://wiki.samba.org/index.php/LinuxCIFSKernel for summary -information (that may be easier to read than parsing the output of -"git log fs/cifs") about fixes/improvements to CIFS/SMB2/SMB3 support (changes +information about fixes/improvements to CIFS/SMB2/SMB3 support (changes to cifs.ko module) by kernel version (and cifs internal module version). +This may be easier to read than parsing the output of "git log fs/cifs" +by release. diff --git a/Documentation/admin-guide/cifs/introduction.rst b/Documentation/admin-guide/cifs/introduction.rst index cc2851d93d17057fc3614ea9f503ed5e2ab69020..53ea62906aa5b127a5fb915eb429aca4900d5a2e 100644 --- a/Documentation/admin-guide/cifs/introduction.rst +++ b/Documentation/admin-guide/cifs/introduction.rst @@ -7,19 +7,19 @@ Introduction protocol which was the successor to the Server Message Block (SMB) protocol, the native file sharing mechanism for most early PC operating systems. New and improved versions of CIFS are now - called SMB2 and SMB3. Use of SMB3 (and later, including SMB3.1.1) - is strongly preferred over using older dialects like CIFS due to - security reasons. All modern dialects, including the most recent, - SMB3.1.1 are supported by the CIFS VFS module. The SMB3 protocol - is implemented and supported by all major file servers - such as all modern versions of Windows (including Windows 2016 - Server), as well as by Samba (which provides excellent - CIFS/SMB2/SMB3 server support and tools for Linux and many other - operating systems). Apple systems also support SMB3 well, as - do most Network Attached Storage vendors, so this network - filesystem client can mount to a wide variety of systems. - It also supports mounting to the cloud (for example - Microsoft Azure), including the necessary security features. + called SMB2 and SMB3. Use of SMB3 (and later, including SMB3.1.1 + the most current dialect) is strongly preferred over using older + dialects like CIFS due to security reasons. All modern dialects, + including the most recent, SMB3.1.1, are supported by the CIFS VFS + module. The SMB3 protocol is implemented and supported by all major + file servers such as Windows (including Windows 2019 Server), as + well as by Samba (which provides excellent CIFS/SMB2/SMB3 server + support and tools for Linux and many other operating systems). + Apple systems also support SMB3 well, as do most Network Attached + Storage vendors, so this network filesystem client can mount to a + wide variety of systems. It also supports mounting to the cloud + (for example Microsoft Azure), including the necessary security + features. The intent of this module is to provide the most advanced network file system function for SMB3 compliant servers, including advanced @@ -27,8 +27,8 @@ Introduction POSIX compliance, secure per-user session establishment, encryption, high performance safe distributed caching (leases/oplocks), optional packet signing, large files, Unicode support and other internationalization - improvements. Since both Samba server and this filesystem client support - the CIFS Unix extensions (and in the future SMB3 POSIX extensions), + improvements. Since both Samba server and this filesystem client support the + CIFS Unix extensions, and the Linux client also suppors SMB3 POSIX extensions, the combination can provide a reasonable alternative to other network and cluster file systems for fileserving in some Linux to Linux environments, not just in Linux to Windows (or Linux to Mac) environments. diff --git a/Documentation/admin-guide/cifs/todo.rst b/Documentation/admin-guide/cifs/todo.rst index 25f11576e7b91120aa2480e6a0d6ccc1f4fe3a7f..2646ed2e2d3e32751d3eacfc89a87b73bd2d79e4 100644 --- a/Documentation/admin-guide/cifs/todo.rst +++ b/Documentation/admin-guide/cifs/todo.rst @@ -13,24 +13,26 @@ is a partial list of the known problems and missing features: a) SMB3 (and SMB3.1.1) missing optional features: - - multichannel (started), integration with RDMA - - directory leases (improved metadata caching), started (root dir only) + - multichannel (partially integrated), integration of multichannel with RDMA + - directory leases (improved metadata caching). Currently only implemented for root dir - T10 copy offload ie "ODX" (copy chunk, and "Duplicate Extents" ioctl currently the only two server side copy mechanisms supported) b) improved sparse file support (fiemap and SEEK_HOLE are implemented - but additional features would be supportable by the protocol). + but additional features would be supportable by the protocol such + as FALLOC_FL_COLLAPSE_RANGE and FALLOC_FL_INSERT_RANGE) c) Directory entry caching relies on a 1 second timer, rather than using Directory Leases, currently only the root file handle is cached longer + by leveraging Directory Leases -d) quota support (needs minor kernel change since quota calls - to make it to network filesystems or deviceless filesystems) +d) quota support (needs minor kernel change since quota calls otherwise + won't make it to network filesystems or deviceless filesystems). e) Additional use cases can be optimized to use "compounding" (e.g. open/query/close and open/setinfo/close) to reduce the number of roundtrips to the server and improve performance. Various cases - (stat, statfs, create, unlink, mkdir) already have been improved by + (stat, statfs, create, unlink, mkdir, xattrs) already have been improved by using compounding but more can be done. In addition we could significantly reduce redundant opens by using deferred close (with handle caching leases) and better using reference counters on file @@ -60,7 +62,9 @@ k) Add tools to take advantage of more smb3 specific ioctls and features metadata attributes easier from tools (e.g. extending what was done in smb-info tool). -l) encrypted file support +l) encrypted file support (currently the attribute showing the file is + encrypted on the server is reported, but changing the attribute is not + supported). m) improved stats gathering tools (perhaps integration with nfsometer?) to extend and make easier to use what is currently in /proc/fs/cifs/Stats @@ -69,14 +73,13 @@ n) Add support for claims based ACLs ("DAC") o) mount helper GUI (to simplify the various configuration options on mount) -p) Add support for witness protocol (perhaps ioctl to cifs.ko from user space - tool listening on witness protocol RPC) to allow for notification of share - move, server failover, and server adapter changes. And also improve other - failover scenarios, e.g. when client knows multiple DFS entries point to - different servers, and the server we are connected to has gone down. +p) Expand support for witness protocol to allow for notification of share + move, and server network adapter changes. Currently only notifications by + the witness protocol for server move is supported by the Linux client. q) Allow mount.cifs to be more verbose in reporting errors with dialect - or unsupported feature errors. + or unsupported feature errors. This would now be easier due to the + implementation of the new mount API. r) updating cifs documentation, and user guide. @@ -87,11 +90,10 @@ t) split cifs and smb3 support into separate modules so legacy (and less secure) CIFS dialect can be disabled in environments that don't need it and simplify the code. -v) POSIX Extensions for SMB3.1.1 (started, create and mkdir support added - so far). +v) Additional testing of POSIX Extensions for SMB3.1.1 w) Add support for additional strong encryption types, and additional spnego - authentication mechanisms (see MS-SMB2) + authentication mechanisms (see MS-SMB2). GCM-256 is now partially implemented. x) Finish support for SMB3.1.1 compression diff --git a/Documentation/admin-guide/cifs/usage.rst b/Documentation/admin-guide/cifs/usage.rst index b6d9f02bc12b92e079f7d619f87e2cabd886fc75..13783dc68ab78bb7d09ee3ba1a5060a74589dbcf 100644 --- a/Documentation/admin-guide/cifs/usage.rst +++ b/Documentation/admin-guide/cifs/usage.rst @@ -83,7 +83,7 @@ and encrypted shares and stronger signing and authentication algorithms. There are additional mount options that may be helpful for SMB3 to get improved POSIX behavior (NB: can use vers=3.0 to force only SMB3, never 2.1): - ``mfsymlinks`` and ``cifsacl`` and ``idsfromsid`` + ``mfsymlinks`` and either ``cifsacl`` or ``modefromsid`` (usually with ``idsfromsid``) Allowing User Mounts ==================== diff --git a/Documentation/admin-guide/cpu-load.rst b/Documentation/admin-guide/cpu-load.rst index f3ada90e9ca8822f9d27312d3ff0461f90d80a1f..21a984337080afb36f2f9bd46c24da6771617f74 100644 --- a/Documentation/admin-guide/cpu-load.rst +++ b/Documentation/admin-guide/cpu-load.rst @@ -107,7 +107,7 @@ will lead to quite erratic information inside ``/proc/stat``:: References ---------- -- http://lkml.org/lkml/2007/2/12/6 +- https://lore.kernel.org/r/loom.20070212T063225-663@post.gmane.org - Documentation/filesystems/proc.rst (1.8) diff --git a/Documentation/admin-guide/device-mapper/dm-crypt.rst b/Documentation/admin-guide/device-mapper/dm-crypt.rst index 1a6753b76dbb4f50fdeee43082473009a1a3480b..aa2d04d95df6cdfc43a2a6320b294376c8953950 100644 --- a/Documentation/admin-guide/device-mapper/dm-crypt.rst +++ b/Documentation/admin-guide/device-mapper/dm-crypt.rst @@ -67,7 +67,7 @@ Parameters:: the value passed in . - Either 'logon', 'user' or 'encrypted' kernel key type. + Either 'logon', 'user', 'encrypted' or 'trusted' kernel key type. The kernel keyring key description crypt target should look for diff --git a/Documentation/admin-guide/device-mapper/dm-integrity.rst b/Documentation/admin-guide/device-mapper/dm-integrity.rst index 2cc5488acbd97a1efda5152a4c6e80299246d2a2..8db172efa272936501e788680fe0362d238dcb32 100644 --- a/Documentation/admin-guide/device-mapper/dm-integrity.rst +++ b/Documentation/admin-guide/device-mapper/dm-integrity.rst @@ -143,8 +143,8 @@ recalculate journal_crypt:algorithm(:key) (the key is optional) Encrypt the journal using given algorithm to make sure that the attacker can't read the journal. You can use a block cipher here - (such as "cbc(aes)") or a stream cipher (for example "chacha20", - "salsa20" or "ctr(aes)"). + (such as "cbc(aes)") or a stream cipher (for example "chacha20" + or "ctr(aes)"). The journal contains history of last writes to the block device, an attacker reading the journal could see the last sector numbers @@ -186,6 +186,17 @@ fix_padding space-efficient. If this option is not present, large padding is used - that is for compatibility with older kernels. +fix_hmac + Improve security of internal_hash and journal_mac: + + - the section number is mixed to the mac, so that an attacker can't + copy sectors from one journal section to another journal section + - the superblock is protected by journal_mac + - a 16-byte salt stored in the superblock is mixed to the mac, so + that the attacker can't detect that two disks have the same hmac + key and also to disallow the attacker to move sectors from one + disk to another + legacy_recalculate Allow recalculating of volumes with HMAC keys. This is disabled by default for security reasons - an attacker could modify the volume, diff --git a/Documentation/admin-guide/kernel-parameters.rst b/Documentation/admin-guide/kernel-parameters.rst index 682ab28b5c94b3d292d085173adcb1f2ebce6194..1132796a8d96e7636fc28f0804519849dec3987f 100644 --- a/Documentation/admin-guide/kernel-parameters.rst +++ b/Documentation/admin-guide/kernel-parameters.rst @@ -60,7 +60,7 @@ Note that for the special case of a range one can split the range into equal sized groups and for each group use some amount from the beginning of that group: - -cpu number>:/ + -:/ For example one can add to the command line following parameter: diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 07f9372b494aa134740facf4d2821a489545333c..04545725f187ffe787f3a226a328b1c951251f16 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -606,7 +606,7 @@ kernel/dma/contiguous.c cma_pernuma=nn[MG] - [ARM64,KNL] + [ARM64,KNL,CMA] Sets the size of kernel per-numa memory area for contiguous memory allocations. A value of 0 disables per-numa CMA altogether. And If this option is not @@ -1434,6 +1434,11 @@ to enforce probe and suspend/resume ordering. rpm -- Like "on", but also use to order runtime PM. + fw_devlink.strict= + [KNL] Treat all inferred dependencies as mandatory + dependencies. This only applies for fw_devlink=on|rpm. + Format: + gamecon.map[2|3]= [HW,JOY] Multisystem joystick and NES/SNES/PSX pad support via parallel port (up to 5 devices per port) @@ -1525,12 +1530,12 @@ hpet_mmap= [X86, HPET_MMAP] Allow userspace to mmap HPET registers. Default set by CONFIG_HPET_MMAP_DEFAULT. - hugetlb_cma= [HW] The size of a cma area used for allocation + hugetlb_cma= [HW,CMA] The size of a CMA area used for allocation of gigantic hugepages. Format: nn[KMGTPE] - Reserve a cma area of given size and allocate gigantic - hugepages using the cma allocator. If enabled, the + Reserve a CMA area of given size and allocate gigantic + hugepages using the CMA allocator. If enabled, the boot-time allocation of gigantic hugepages is skipped. hugepages= [HW] Number of HugeTLB pages to allocate at boot. @@ -1674,6 +1679,12 @@ In such case C2/C3 won't be used again. idle=nomwait: Disable mwait for CPU C-states + idxd.sva= [HW] + Format: + Allow force disabling of Shared Virtual Memory (SVA) + support for the idxd driver. By default it is set to + true (1). + ieee754= [MIPS] Select IEEE Std 754 conformance mode Format: { strict | legacy | 2008 | relaxed } Default: strict @@ -1747,7 +1758,7 @@ ima_policy= [IMA] The builtin policies to load during IMA setup. Format: "tcb | appraise_tcb | secure_boot | - fail_securely" + fail_securely | critical_data" The "tcb" policy measures all programs exec'd, files mmap'd for exec, and all files opened with the read @@ -1766,6 +1777,9 @@ filesystems with the SB_I_UNVERIFIABLE_SIGNATURE flag. + The "critical_data" policy measures kernel integrity + critical data. + ima_tcb [IMA] Deprecated. Use ima_policy= instead. Load a policy which meets the needs of the Trusted Computing Base. This means IMA will measure all @@ -3270,9 +3284,14 @@ parameter, xsave area per process might occupy more memory on xsaves enabled systems. - nohlt [BUGS=ARM,SH] Tells the kernel that the sleep(SH) or - wfi(ARM) instruction doesn't work correctly and not to - use it. This is also useful when using JTAG debugger. + nohlt [ARM,ARM64,MICROBLAZE,SH] Forces the kernel to busy wait + in do_idle() and not use the arch_cpu_idle() + implementation; requires CONFIG_GENERIC_IDLE_POLL_SETUP + to be effective. This is useful on platforms where the + sleep(SH) or wfi(ARM,ARM64) instructions do not work + correctly or when doing power measurements to evalute + the impact of the sleep instructions. This is also + useful when using JTAG debugger. no_file_caps Tells the kernel not to honor file capabilities. The only way then for a file to be executed with privilege @@ -3285,6 +3304,21 @@ in certain environments such as networked servers or real-time systems. + no_hash_pointers + Force pointers printed to the console or buffers to be + unhashed. By default, when a pointer is printed via %p + format string, that pointer is "hashed", i.e. obscured + by hashing the pointer value. This is a security feature + that hides actual kernel addresses from unprivileged + users, but it also makes debugging the kernel more + difficult since unequal pointers can no longer be + compared. However, if this command-line option is + specified, then all normal pointers will have their true + value printed. Pointers printed via %pK may still be + hashed. This option should only be specified when + debugging the kernel. Please do not use on production + kernels. + nohibernate [HIBERNATION] Disable hibernation and resume. nohz= [KNL] Boottime enable/disable dynamic ticks @@ -4870,14 +4904,6 @@ last alloc / free. For more information see Documentation/vm/slub.rst. - slub_memcg_sysfs= [MM, SLUB] - Determines whether to enable sysfs directories for - memory cgroup sub-caches. 1 to enable, 0 to disable. - The default is determined by CONFIG_SLUB_MEMCG_SYSFS_ON. - Enabling this can lead to a very high number of debug - directories and files being created under - /sys/kernel/slub. - slub_max_order= [MM, SLUB] Determines the maximum allowed order for slabs. A high setting may cause OOMs due to memory @@ -5156,6 +5182,12 @@ growing up) the main stack are reserved for no other mapping. Default value is 256 pages. + stack_depot_disable= [KNL] + Setting this to true through kernel command line will + disable the stack depot thereby saving the static memory + consumed by the stack hash table. By default this is set + to false. + stacktrace [FTRACE] Enabled the stack tracer on boot up. @@ -5956,12 +5988,6 @@ default x2apic cluster mode on platforms supporting x2apic. - x86_intel_mid_timer= [X86-32,APBT] - Choose timer option for x86 Intel MID platform. - Two valid options are apbt timer only and lapic timer - plus one apbt timer for broadcast timer. - x86_intel_mid_timer=apbt_only | lapic_and_apbt - xen_512gb_limit [KNL,X86-64,XEN] Restricts the kernel running paravirtualized under Xen to use only up to 512 GB of RAM. The reason to do so is diff --git a/Documentation/admin-guide/kernel-per-CPU-kthreads.rst b/Documentation/admin-guide/kernel-per-CPU-kthreads.rst index dc36aeb65d0af6c7c1d789f0b1775a1531c428ca..531f689311f24745531a6d835a4bee920014bbfe 100644 --- a/Documentation/admin-guide/kernel-per-CPU-kthreads.rst +++ b/Documentation/admin-guide/kernel-per-CPU-kthreads.rst @@ -273,7 +273,7 @@ To reduce its OS jitter, do any of the following: However, there is an RFC patch from Christoph Lameter (based on an earlier one from Gilad Ben-Yossef) that reduces or even eliminates vmstat overhead for some - workloads at https://lkml.org/lkml/2013/9/4/379. + workloads at https://lore.kernel.org/r/00000140e9dfd6bd-40db3d4f-c1be-434f-8132-7820f81bb586-000000@email.amazonses.com. e. If running on high-end powerpc servers, build with CONFIG_PPC_RTAS_DAEMON=n. This prevents the RTAS daemon from running on each CPU every second or so. diff --git a/Documentation/admin-guide/laptops/thinkpad-acpi.rst b/Documentation/admin-guide/laptops/thinkpad-acpi.rst index 5fe1ade88c177170fd6af1364d19b5c396eaef77..91fd6846ce17ca032309d376d58f8f0a9e39e412 100644 --- a/Documentation/admin-guide/laptops/thinkpad-acpi.rst +++ b/Documentation/admin-guide/laptops/thinkpad-acpi.rst @@ -51,6 +51,7 @@ detailed description): - UWB enable and disable - LCD Shadow (PrivacyGuard) enable and disable - Lap mode sensor + - Setting keyboard language A compatibility table by model and feature is maintained on the web site, http://ibm-acpi.sf.net/. I appreciate any success or failure @@ -1466,6 +1467,30 @@ Sysfs notes rfkill controller switch "tpacpi_uwb_sw": refer to Documentation/driver-api/rfkill.rst for details. + +Setting keyboard language +------------------------- + +sysfs: keyboard_lang + +This feature is used to set keyboard language to ECFW using ASL interface. +Fewer thinkpads models like T580 , T590 , T15 Gen 1 etc.. has "=", "(', +")" numeric keys, which are not displaying correctly, when keyboard language +is other than "english". This is because the default keyboard language in ECFW +is set as "english". Hence using this sysfs, user can set the correct keyboard +language to ECFW and then these key's will work correctly. + +Example of command to set keyboard language is mentioned below:: + + echo jp > /sys/devices/platform/thinkpad_acpi/keyboard_lang + +Text corresponding to keyboard layout to be set in sysfs are: be(Belgian), +cz(Czech), da(Danish), de(German), en(English), es(Spain), et(Estonian), +fr(French), fr-ch(French(Switzerland)), hu(Hungarian), it(Italy), jp (Japan), +nl(Dutch), nn(Norway), pl(Polish), pt(portugese), sl(Slovenian), sv(Sweden), +tr(Turkey) + + Adaptive keyboard ----------------- diff --git a/Documentation/admin-guide/mm/memory-hotplug.rst b/Documentation/admin-guide/mm/memory-hotplug.rst index 5c4432c96c4b6d76a6e4c5245e6657a9e2e67382..5307f90738aa9a072209293ee4464ab3592cfafb 100644 --- a/Documentation/admin-guide/mm/memory-hotplug.rst +++ b/Documentation/admin-guide/mm/memory-hotplug.rst @@ -160,16 +160,16 @@ Under each memory block, you can see 5 files: "online_movable", "online", "offline" command which will be performed on all sections in the block. -``phys_device`` read-only: designed to show the name of physical memory - device. This is not well implemented now. -``removable`` read-only: contains an integer value indicating - whether the memory block is removable or not - removable. A value of 1 indicates that the memory - block is removable and a value of 0 indicates that - it is not removable. A memory block is removable only if - every section in the block is removable. -``valid_zones`` read-only: designed to show which zones this memory block - can be onlined to. +``phys_device`` read-only: legacy interface only ever used on s390x to + expose the covered storage increment. +``removable`` read-only: legacy interface that indicated whether a memory + block was likely to be offlineable or not. Newer kernel + versions return "1" if and only if the kernel supports + memory offlining. +``valid_zones`` read-only: designed to show by which zone memory provided by + a memory block is managed, and to show by which zone memory + provided by an offline memory block could be managed when + onlining. The first column shows it`s default zone. diff --git a/Documentation/admin-guide/perf-security.rst b/Documentation/admin-guide/perf-security.rst index 904e4eb37f99c7a2e84126f5a690bce047dd3622..34aa334320cad3d0f632429670cc31e727709dca 100644 --- a/Documentation/admin-guide/perf-security.rst +++ b/Documentation/admin-guide/perf-security.rst @@ -72,7 +72,7 @@ monitoring and observability operations, thus, bypass *scope* permissions checks in the kernel. CAP_PERFMON implements the principle of least privilege [13]_ (POSIX 1003.1e: 2.2.2.39) for performance monitoring and observability operations in the kernel and provides a secure approach to -perfomance monitoring and observability in the system. +performance monitoring and observability in the system. For backward compatibility reasons the access to perf_events monitoring and observability operations is also open for CAP_SYS_ADMIN privileged diff --git a/Documentation/admin-guide/spkguide.txt b/Documentation/admin-guide/spkguide.txt index 5ff6a0fe87d1e5b06c910a8c9fb2eee20d9c5b4a..977ab3f5a0a8878dea8204657ef223315c459d1d 100644 --- a/Documentation/admin-guide/spkguide.txt +++ b/Documentation/admin-guide/spkguide.txt @@ -1033,7 +1033,9 @@ speakup + keypad 3, you would hear: The speakup key is depressed, so the name of the key state is speakup. This part of the message comes from the states collection. -14.2. Loading Your Own Messages +14.2. Changing language + +14.2.1. Loading Your Own Messages The files under the i18n subdirectory all follow the same format. They consist of lines, with one message per line. @@ -1066,8 +1068,50 @@ echo '1 azul' > /speakup/i18n/colors The next time that Speakup says message 1 from the colors group, it will say "azul", rather than "blue." +14.2.2. Choose a language + In the future, translations into various languages will be made available, -and most users will just load the files necessary for their language. +and most users will just load the files necessary for their language. So far, +only French language is available beyond native Canadian English language. + +French is only available after you are logged in. + +Canadian English is the default language. To toggle another language, +download the source of Speakup and untar it in your home directory. The +following command should let you do this: + +tar xvjf speakup-.tar.bz2 + +where is the version number of the application. + +Next, change to the newly created directory, then into the tools/ directory, and +run the script speakup_setlocale. You are asked the language that you want to +use. Type the number associated to your language (e.g. fr for French) then press +Enter. Needed files are copied in the i18n directory. + +Note: the speakupconf must be installed on your system so that settings are saved. +Otherwise, you will have an error: your language will be loaded but you will +have to run the script again every time Speakup restarts. +See section 16.1. for information about speakupconf. + +You will have to repeat these steps for any change of locale, i.e. if you wish +change the speakup's language or charset (iso-8859-15 ou UTF-8). + +If you wish store the settings, note that at your next login, you will need to +do: + +speakup load + +Alternatively, you can add the above line to your file +~/.bashrc or ~/.bash_profile. + +If your system administrator ran himself the script, all the users will be able +to change from English to the language choosed by root and do directly +speakupconf load (or add this to the ~/.bashrc or +~/.bash_profile file). If there are several languages to handle, the +administrator (or every user) will have to run the first steps until speakupconf +save, choosing the appropriate language, in every user's home directory. Every +user will then be able to do speakupconf load, Speakup will load his own settings. 14.3. No Support for Non-Western-European Languages diff --git a/Documentation/admin-guide/sysctl/fs.rst b/Documentation/admin-guide/sysctl/fs.rst index f48277a0a850222c268f778980d88078fc562ccb..2a501c9ddc556eeb2620616e6bc0aa717316503a 100644 --- a/Documentation/admin-guide/sysctl/fs.rst +++ b/Documentation/admin-guide/sysctl/fs.rst @@ -380,5 +380,5 @@ This configuration option sets the maximum number of "watches" that are allowed for each user. Each "watch" costs roughly 90 bytes on a 32bit kernel, and roughly 160 bytes on a 64bit one. -The current default value for max_user_watches is the 1/32 of the available -low memory, divided for the "watch" cost in bytes. +The current default value for max_user_watches is the 1/25 (4%) of the +available low memory, divided for the "watch" cost in bytes. diff --git a/Documentation/admin-guide/sysctl/vm.rst b/Documentation/admin-guide/sysctl/vm.rst index e35a3f2fb006adb1fd9c7e70a4005da4f834d98b..586cd4b8642842c0aafac352fed248521ef2d6c0 100644 --- a/Documentation/admin-guide/sysctl/vm.rst +++ b/Documentation/admin-guide/sysctl/vm.rst @@ -983,11 +983,11 @@ that benefit from having their data cached, zone_reclaim_mode should be left disabled as the caching effect is likely to be more important than data locality. -zone_reclaim may be enabled if it's known that the workload is partitioned -such that each partition fits within a NUMA node and that accessing remote -memory would cause a measurable performance reduction. The page allocator -will then reclaim easily reusable pages (those page cache pages that are -currently not used) before allocating off node pages. +Consider enabling one or more zone_reclaim mode bits if it's known that the +workload is partitioned such that each partition fits within a NUMA node +and that accessing remote memory would cause a measurable performance +reduction. The page allocator will take additional actions before +allocating off node pages. Allowing zone reclaim to write out pages stops processes that are writing large amounts of data from dirtying pages on other nodes. Zone diff --git a/Documentation/admin-guide/xfs.rst b/Documentation/admin-guide/xfs.rst index 6178153d33200e8ebf5179b3fb8cdea497e74c46..5422407a96d7a3cfe877273d50d9e2b26ee478c9 100644 --- a/Documentation/admin-guide/xfs.rst +++ b/Documentation/admin-guide/xfs.rst @@ -284,6 +284,9 @@ The following sysctls are available for the XFS filesystem: removes unused preallocation from clean inodes and releases the unused space back to the free pool. + fs.xfs.speculative_cow_prealloc_lifetime + This is an alias for speculative_prealloc_lifetime. + fs.xfs.error_level (Min: 0 Default: 3 Max: 11) A volume knob for error reporting when internal errors occur. This will generate detailed messages & backtraces for filesystem @@ -356,12 +359,13 @@ The following sysctls are available for the XFS filesystem: Deprecated Sysctls ================== -=========================== ================ - Name Removal Schedule -=========================== ================ -fs.xfs.irix_sgid_inherit September 2025 -fs.xfs.irix_symlink_mode September 2025 -=========================== ================ +=========================================== ================ + Name Removal Schedule +=========================================== ================ +fs.xfs.irix_sgid_inherit September 2025 +fs.xfs.irix_symlink_mode September 2025 +fs.xfs.speculative_cow_prealloc_lifetime September 2025 +=========================================== ================ Removed Sysctls diff --git a/Documentation/arm/booting.rst b/Documentation/arm/booting.rst index a2263451dc2c4085b328007fd8acb9db523e9032..5974e37b3d2029eab0eb92a45818cf5f976e0690 100644 --- a/Documentation/arm/booting.rst +++ b/Documentation/arm/booting.rst @@ -128,7 +128,7 @@ it. The recommended placement is in the first 16KiB of RAM. The boot loader must load a device tree image (dtb) into system ram at a 64bit aligned address and initialize it with the boot data. The -dtb format is documented in Documentation/devicetree/booting-without-of.rst. +dtb format is documented at https://www.devicetree.org/specifications/. The kernel will look for the dtb magic value of 0xd00dfeed at the dtb physical address to determine if a dtb has been passed instead of a tagged list. diff --git a/Documentation/arm/index.rst b/Documentation/arm/index.rst index a2e9e1bba7b930ba7885101a05c09d68729657a2..b4bea32472b662a1eaf339350f6ac8a96a0a9102 100644 --- a/Documentation/arm/index.rst +++ b/Documentation/arm/index.rst @@ -33,7 +33,7 @@ SoC-specific documents ixp4xx - marvel + marvell microchip netwinder diff --git a/Documentation/arm/marvel.rst b/Documentation/arm/marvell.rst similarity index 97% rename from Documentation/arm/marvel.rst rename to Documentation/arm/marvell.rst index 16ab2eb085b862b4684e4c2a4c33313a647690f3..94cd7338359428b515616ec55052b03ccfda8477 100644 --- a/Documentation/arm/marvel.rst +++ b/Documentation/arm/marvell.rst @@ -127,7 +127,7 @@ EBU Armada family - 88F6828 Armada 388 - Product infos: http://www.marvell.com/embedded-processors/armada-38x/ - - Functional Spec: https://marvellcorp.wufoo.com/forms/marvell-armada-38x-functional-specifications/ + - Functional Spec: http://www.marvell.com/content/dam/marvell/en/public-collateral/embedded-processors/marvell-embedded-processors-armada-38x-functional-specifications-2015-11.pdf Core: ARM Cortex-A9 @@ -183,7 +183,10 @@ EBU Armada family ARMv8 http://www.marvell.com/embedded-processors/armada-3700/ Product Brief: - http://www.marvell.com/embedded-processors/assets/PB-88F3700-FNL.pdf + http://www.marvell.com/content/dam/marvell/en/public-collateral/embedded-processors/marvell-embedded-processors-armada-37xx-product-brief-2016-01.pdf + + Hardware Spec: + http://www.marvell.com/content/dam/marvell/en/public-collateral/embedded-processors/marvell-embedded-processors-armada-37xx-hardware-specifications-2019-09.pdf Device tree files: arch/arm64/boot/dts/marvell/armada-37* diff --git a/Documentation/block/inline-encryption.rst b/Documentation/block/inline-encryption.rst index e75151e467d391653ef198f2032f526786b5b2c1..7f9b40d6b416b11c23ee0e2b806a2b12fadc8098 100644 --- a/Documentation/block/inline-encryption.rst +++ b/Documentation/block/inline-encryption.rst @@ -182,8 +182,9 @@ API presented to device drivers A :c:type:``struct blk_keyslot_manager`` should be set up by device drivers in the ``request_queue`` of the device. The device driver needs to call -``blk_ksm_init`` on the ``blk_keyslot_manager``, which specifying the number of -keyslots supported by the hardware. +``blk_ksm_init`` (or its resource-managed variant ``devm_blk_ksm_init``) on the +``blk_keyslot_manager``, while specifying the number of keyslots supported by +the hardware. The device driver also needs to tell the KSM how to actually manipulate the IE hardware in the device to do things like programming the crypto key into @@ -202,10 +203,9 @@ needs each and every of its keyslots to be reprogrammed with the key it "should have" at the point in time when the function is called. This is useful e.g. if a device loses all its keys on runtime power down/up. -``blk_ksm_destroy`` should be called to free up all resources used by a keyslot -manager upon ``blk_ksm_init``, once the ``blk_keyslot_manager`` is no longer -needed. - +If the driver used ``blk_ksm_init`` instead of ``devm_blk_ksm_init``, then +``blk_ksm_destroy`` should be called to free up all resources used by a +``blk_keyslot_manager`` once it is no longer needed. Layered Devices =============== diff --git a/Documentation/conf.py b/Documentation/conf.py index 6a767294887e940b4fbf79f6d31cd183e65f4222..fd65168c10f84f67d62848f3d420dfa4ca5e6ee1 100644 --- a/Documentation/conf.py +++ b/Documentation/conf.py @@ -31,7 +31,7 @@ from load_config import loadConfig # -- General configuration ------------------------------------------------ # If your documentation needs a minimal Sphinx version, state it here. -needs_sphinx = '1.3' +needs_sphinx = '1.7' # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom @@ -49,8 +49,7 @@ extensions = ['kerneldoc', 'rstFlatTable', 'kernel_include', if major >= 3: sys.stderr.write('''WARNING: The kernel documentation build process support for Sphinx v3.0 and above is brand new. Be prepared for - possible issues in the generated output. - ''') + possible issues in the generated output.\n''') if (major > 3) or (minor > 0 or patch >= 2): # Sphinx c function parser is more pedantic with regards to type # checking. Due to that, having macros at c:function cause problems. @@ -112,19 +111,12 @@ if major >= 3: else: extensions.append('cdomain') - if major == 1 and minor < 7: - sys.stderr.write('WARNING: Sphinx 1.7 or greater will be required as of ' - 'the 5.12 release\n') # Ensure that autosectionlabel will produce unique names autosectionlabel_prefix_document = True autosectionlabel_maxdepth = 2 -# The name of the math extension changed on Sphinx 1.4 -if (major == 1 and minor > 3) or (major > 1): - extensions.append("sphinx.ext.imgmath") -else: - extensions.append("sphinx.ext.pngmath") +extensions.append("sphinx.ext.imgmath") # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] @@ -375,71 +367,9 @@ if cjk_cmd.find("Noto Sans CJK SC") >= 0: ''' # Fix reference escape troubles with Sphinx 1.4.x -if major == 1 and minor > 3: +if major == 1: latex_elements['preamble'] += '\\renewcommand*{\\DUrole}[2]{ #2 }\n' -if major == 1 and minor <= 4: - latex_elements['preamble'] += '\\usepackage[margin=0.5in, top=1in, bottom=1in]{geometry}' -elif major == 1 and (minor > 5 or (minor == 5 and patch >= 3)): - latex_elements['sphinxsetup'] = 'hmargin=0.5in, vmargin=1in' - latex_elements['preamble'] += '\\fvset{fontsize=auto}\n' - -# Customize notice background colors on Sphinx < 1.6: -if major == 1 and minor < 6: - latex_elements['preamble'] += ''' - \\usepackage{ifthen} - - % Put notes in color and let them be inside a table - \\definecolor{NoteColor}{RGB}{204,255,255} - \\definecolor{WarningColor}{RGB}{255,204,204} - \\definecolor{AttentionColor}{RGB}{255,255,204} - \\definecolor{ImportantColor}{RGB}{192,255,204} - \\definecolor{OtherColor}{RGB}{204,204,204} - \\newlength{\\mynoticelength} - \\makeatletter\\newenvironment{coloredbox}[1]{% - \\setlength{\\fboxrule}{1pt} - \\setlength{\\fboxsep}{7pt} - \\setlength{\\mynoticelength}{\\linewidth} - \\addtolength{\\mynoticelength}{-2\\fboxsep} - \\addtolength{\\mynoticelength}{-2\\fboxrule} - \\begin{lrbox}{\\@tempboxa}\\begin{minipage}{\\mynoticelength}}{\\end{minipage}\\end{lrbox}% - \\ifthenelse% - {\\equal{\\py@noticetype}{note}}% - {\\colorbox{NoteColor}{\\usebox{\\@tempboxa}}}% - {% - \\ifthenelse% - {\\equal{\\py@noticetype}{warning}}% - {\\colorbox{WarningColor}{\\usebox{\\@tempboxa}}}% - {% - \\ifthenelse% - {\\equal{\\py@noticetype}{attention}}% - {\\colorbox{AttentionColor}{\\usebox{\\@tempboxa}}}% - {% - \\ifthenelse% - {\\equal{\\py@noticetype}{important}}% - {\\colorbox{ImportantColor}{\\usebox{\\@tempboxa}}}% - {\\colorbox{OtherColor}{\\usebox{\\@tempboxa}}}% - }% - }% - }% - }\\makeatother - - \\makeatletter - \\renewenvironment{notice}[2]{% - \\def\\py@noticetype{#1} - \\begin{coloredbox}{#1} - \\bf\\it - \\par\\strong{#2} - \\csname py@noticestart@#1\\endcsname - } - { - \\csname py@noticeend@\\py@noticetype\\endcsname - \\end{coloredbox} - } - \\makeatother - - ''' - # With Sphinx 1.6, it is possible to change the Bg color directly # by using: # \definecolor{sphinxnoteBgColor}{RGB}{204,255,255} diff --git a/Documentation/core-api/dma-api.rst b/Documentation/core-api/dma-api.rst index 75cb757bbff00a28826329aa6316a73b7f1576cb..e6d23f117308dfb621c679815ac941d1c8551c6d 100644 --- a/Documentation/core-api/dma-api.rst +++ b/Documentation/core-api/dma-api.rst @@ -528,16 +528,14 @@ an I/O device, you should not be using this part of the API. :: - void * - dma_alloc_noncoherent(struct device *dev, size_t size, - dma_addr_t *dma_handle, enum dma_data_direction dir, - gfp_t gfp) + struct page * + dma_alloc_pages(struct device *dev, size_t size, dma_addr_t *dma_handle, + enum dma_data_direction dir, gfp_t gfp) -This routine allocates a region of bytes of consistent memory. It -returns a pointer to the allocated region (in the processor's virtual address -space) or NULL if the allocation failed. The returned memory may or may not -be in the kernel direct mapping. Drivers must not call virt_to_page on -the returned memory region. +This routine allocates a region of bytes of non-coherent memory. It +returns a pointer to first struct page for the region, or NULL if the +allocation failed. The resulting struct page can be used for everything a +struct page is suitable for. It also returns a which may be cast to an unsigned integer the same width as the bus and given to the device as the DMA address base of @@ -558,51 +556,33 @@ reused. :: void - dma_free_noncoherent(struct device *dev, size_t size, void *cpu_addr, + dma_free_pages(struct device *dev, size_t size, struct page *page, dma_addr_t dma_handle, enum dma_data_direction dir) -Free a region of memory previously allocated using dma_alloc_noncoherent(). -dev, size and dma_handle and dir must all be the same as those passed into -dma_alloc_noncoherent(). cpu_addr must be the virtual address returned by -dma_alloc_noncoherent(). +Free a region of memory previously allocated using dma_alloc_pages(). +dev, size, dma_handle and dir must all be the same as those passed into +dma_alloc_pages(). page must be the pointer returned by dma_alloc_pages(). :: - struct page * - dma_alloc_pages(struct device *dev, size_t size, dma_addr_t *dma_handle, - enum dma_data_direction dir, gfp_t gfp) - -This routine allocates a region of bytes of non-coherent memory. It -returns a pointer to first struct page for the region, or NULL if the -allocation failed. The resulting struct page can be used for everything a -struct page is suitable for. - -It also returns a which may be cast to an unsigned integer the -same width as the bus and given to the device as the DMA address base of -the region. - -The dir parameter specified if data is read and/or written by the device, -see dma_map_single() for details. - -The gfp parameter allows the caller to specify the ``GFP_`` flags (see -kmalloc()) for the allocation, but rejects flags used to specify a memory -zone such as GFP_DMA or GFP_HIGHMEM. + void * + dma_alloc_noncoherent(struct device *dev, size_t size, + dma_addr_t *dma_handle, enum dma_data_direction dir, + gfp_t gfp) -Before giving the memory to the device, dma_sync_single_for_device() needs -to be called, and before reading memory written by the device, -dma_sync_single_for_cpu(), just like for streaming DMA mappings that are -reused. +This routine is a convenient wrapper around dma_alloc_pages that returns the +kernel virtual address for the allocated memory instead of the page structure. :: void - dma_free_pages(struct device *dev, size_t size, struct page *page, + dma_free_noncoherent(struct device *dev, size_t size, void *cpu_addr, dma_addr_t dma_handle, enum dma_data_direction dir) -Free a region of memory previously allocated using dma_alloc_pages(). -dev, size and dma_handle and dir must all be the same as those passed into -dma_alloc_noncoherent(). page must be the pointer returned by -dma_alloc_pages(). +Free a region of memory previously allocated using dma_alloc_noncoherent(). +dev, size, dma_handle and dir must all be the same as those passed into +dma_alloc_noncoherent(). cpu_addr must be the virtual address returned by +dma_alloc_noncoherent(). :: diff --git a/Documentation/core-api/mm-api.rst b/Documentation/core-api/mm-api.rst index 2adffb3f7914d42d740e099eb44b4b1028d3ef36..201b5423303bbb27189d63ef5d97d0700d81df5f 100644 --- a/Documentation/core-api/mm-api.rst +++ b/Documentation/core-api/mm-api.rst @@ -19,11 +19,8 @@ User Space Memory Access Memory Allocation Controls ========================== -Functions which need to allocate memory often use GFP flags to express -how that memory should be allocated. The GFP acronym stands for "get -free pages", the underlying memory allocation function. Not every GFP -flag is allowed to every function which may allocate memory. Most -users will want to use a plain ``GFP_KERNEL``. +.. kernel-doc:: include/linux/gfp.h + :internal: .. kernel-doc:: include/linux/gfp.h :doc: Page mobility and placement hints diff --git a/Documentation/crypto/api-skcipher.rst b/Documentation/crypto/api-skcipher.rst index 1aaf8985894bb34317882210d3f73302d6e85417..04d6cc5357c810b7c11cb3353f06af020b7d4b42 100644 --- a/Documentation/crypto/api-skcipher.rst +++ b/Documentation/crypto/api-skcipher.rst @@ -28,8 +28,8 @@ Symmetric Key Cipher Request Handle Single Block Cipher API ----------------------- -.. kernel-doc:: include/linux/crypto.h +.. kernel-doc:: include/crypto/internal/cipher.h :doc: Single Block Cipher API -.. kernel-doc:: include/linux/crypto.h +.. kernel-doc:: include/crypto/internal/cipher.h :functions: crypto_alloc_cipher crypto_free_cipher crypto_has_cipher crypto_cipher_blocksize crypto_cipher_setkey crypto_cipher_encrypt_one crypto_cipher_decrypt_one diff --git a/Documentation/dev-tools/index.rst b/Documentation/dev-tools/index.rst index f7809c7b1ba9e2999c8a4af28097bfc0aa1783c5..1b1cf4f5c9d9038efca1243a78fe47035864f1b1 100644 --- a/Documentation/dev-tools/index.rst +++ b/Documentation/dev-tools/index.rst @@ -22,6 +22,7 @@ whole; patches welcome! ubsan kmemleak kcsan + kfence gdb-kernel-debugging kgdb kselftest diff --git a/Documentation/dev-tools/kasan.rst b/Documentation/dev-tools/kasan.rst index a248ac3941bed10022329c14d15e29f157a0ec57..ddf4239a58908cef17c30b716d86277bd49911c4 100644 --- a/Documentation/dev-tools/kasan.rst +++ b/Documentation/dev-tools/kasan.rst @@ -147,16 +147,15 @@ negative values to distinguish between different kinds of inaccessible memory like redzones or freed memory (see mm/kasan/kasan.h). In the report above the arrows point to the shadow byte 03, which means that -the accessed address is partially accessible. - -For tag-based KASAN this last report section shows the memory tags around the -accessed address (see `Implementation details`_ section). +the accessed address is partially accessible. For tag-based KASAN modes this +last report section shows the memory tags around the accessed address +(see the `Implementation details`_ section). Boot parameters ~~~~~~~~~~~~~~~ -Hardware tag-based KASAN mode (see the section about different mode below) is -intended for use in production as a security mitigation. Therefore it supports +Hardware tag-based KASAN mode (see the section about various modes below) is +intended for use in production as a security mitigation. Therefore, it supports boot parameters that allow to disable KASAN competely or otherwise control particular KASAN features. @@ -166,7 +165,8 @@ particular KASAN features. traces collection (default: ``on``). - ``kasan.fault=report`` or ``=panic`` controls whether to only print a KASAN - report or also panic the kernel (default: ``report``). + report or also panic the kernel (default: ``report``). Note, that tag + checking gets disabled after the first reported bug. For developers ~~~~~~~~~~~~~~ @@ -289,6 +289,16 @@ reserved to tag freed memory regions. Hardware tag-based KASAN currently only supports tagging of kmem_cache_alloc/kmalloc and page_alloc memory. +If the hardware doesn't support MTE (pre ARMv8.5), hardware tag-based KASAN +won't be enabled. In this case all boot parameters are ignored. + +Note, that enabling CONFIG_KASAN_HW_TAGS always results in in-kernel TBI being +enabled. Even when kasan.mode=off is provided, or when the hardware doesn't +support MTE (but supports TBI). + +Hardware tag-based KASAN only reports the first found bug. After that MTE tag +checking gets disabled. + What memory accesses are sanitised by KASAN? -------------------------------------------- @@ -352,17 +362,17 @@ unmapped. This will require changes in arch-specific code. This allows ``VMAP_STACK`` support on x86, and can simplify support of architectures that do not have a fixed module region. -CONFIG_KASAN_KUNIT_TEST & CONFIG_TEST_KASAN_MODULE --------------------------------------------------- +CONFIG_KASAN_KUNIT_TEST and CONFIG_KASAN_MODULE_TEST +---------------------------------------------------- -KASAN tests consist on two parts: +KASAN tests consist of two parts: 1. Tests that are integrated with the KUnit Test Framework. Enabled with ``CONFIG_KASAN_KUNIT_TEST``. These tests can be run and partially verified automatically in a few different ways, see the instructions below. 2. Tests that are currently incompatible with KUnit. Enabled with -``CONFIG_TEST_KASAN_MODULE`` and can only be run as a module. These tests can +``CONFIG_KASAN_MODULE_TEST`` and can only be run as a module. These tests can only be verified manually, by loading the kernel module and inspecting the kernel log for KASAN reports. diff --git a/Documentation/dev-tools/kfence.rst b/Documentation/dev-tools/kfence.rst new file mode 100644 index 0000000000000000000000000000000000000000..fdf04e741ea575d02786319c49c373a53e7a55b2 --- /dev/null +++ b/Documentation/dev-tools/kfence.rst @@ -0,0 +1,298 @@ +.. SPDX-License-Identifier: GPL-2.0 +.. Copyright (C) 2020, Google LLC. + +Kernel Electric-Fence (KFENCE) +============================== + +Kernel Electric-Fence (KFENCE) is a low-overhead sampling-based memory safety +error detector. KFENCE detects heap out-of-bounds access, use-after-free, and +invalid-free errors. + +KFENCE is designed to be enabled in production kernels, and has near zero +performance overhead. Compared to KASAN, KFENCE trades performance for +precision. The main motivation behind KFENCE's design, is that with enough +total uptime KFENCE will detect bugs in code paths not typically exercised by +non-production test workloads. One way to quickly achieve a large enough total +uptime is when the tool is deployed across a large fleet of machines. + +Usage +----- + +To enable KFENCE, configure the kernel with:: + + CONFIG_KFENCE=y + +To build a kernel with KFENCE support, but disabled by default (to enable, set +``kfence.sample_interval`` to non-zero value), configure the kernel with:: + + CONFIG_KFENCE=y + CONFIG_KFENCE_SAMPLE_INTERVAL=0 + +KFENCE provides several other configuration options to customize behaviour (see +the respective help text in ``lib/Kconfig.kfence`` for more info). + +Tuning performance +~~~~~~~~~~~~~~~~~~ + +The most important parameter is KFENCE's sample interval, which can be set via +the kernel boot parameter ``kfence.sample_interval`` in milliseconds. The +sample interval determines the frequency with which heap allocations will be +guarded by KFENCE. The default is configurable via the Kconfig option +``CONFIG_KFENCE_SAMPLE_INTERVAL``. Setting ``kfence.sample_interval=0`` +disables KFENCE. + +The KFENCE memory pool is of fixed size, and if the pool is exhausted, no +further KFENCE allocations occur. With ``CONFIG_KFENCE_NUM_OBJECTS`` (default +255), the number of available guarded objects can be controlled. Each object +requires 2 pages, one for the object itself and the other one used as a guard +page; object pages are interleaved with guard pages, and every object page is +therefore surrounded by two guard pages. + +The total memory dedicated to the KFENCE memory pool can be computed as:: + + ( #objects + 1 ) * 2 * PAGE_SIZE + +Using the default config, and assuming a page size of 4 KiB, results in +dedicating 2 MiB to the KFENCE memory pool. + +Note: On architectures that support huge pages, KFENCE will ensure that the +pool is using pages of size ``PAGE_SIZE``. This will result in additional page +tables being allocated. + +Error reports +~~~~~~~~~~~~~ + +A typical out-of-bounds access looks like this:: + + ================================================================== + BUG: KFENCE: out-of-bounds read in test_out_of_bounds_read+0xa3/0x22b + + Out-of-bounds read at 0xffffffffb672efff (1B left of kfence-#17): + test_out_of_bounds_read+0xa3/0x22b + kunit_try_run_case+0x51/0x85 + kunit_generic_run_threadfn_adapter+0x16/0x30 + kthread+0x137/0x160 + ret_from_fork+0x22/0x30 + + kfence-#17 [0xffffffffb672f000-0xffffffffb672f01f, size=32, cache=kmalloc-32] allocated by task 507: + test_alloc+0xf3/0x25b + test_out_of_bounds_read+0x98/0x22b + kunit_try_run_case+0x51/0x85 + kunit_generic_run_threadfn_adapter+0x16/0x30 + kthread+0x137/0x160 + ret_from_fork+0x22/0x30 + + CPU: 4 PID: 107 Comm: kunit_try_catch Not tainted 5.8.0-rc6+ #7 + Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.13.0-1 04/01/2014 + ================================================================== + +The header of the report provides a short summary of the function involved in +the access. It is followed by more detailed information about the access and +its origin. Note that, real kernel addresses are only shown when using the +kernel command line option ``no_hash_pointers``. + +Use-after-free accesses are reported as:: + + ================================================================== + BUG: KFENCE: use-after-free read in test_use_after_free_read+0xb3/0x143 + + Use-after-free read at 0xffffffffb673dfe0 (in kfence-#24): + test_use_after_free_read+0xb3/0x143 + kunit_try_run_case+0x51/0x85 + kunit_generic_run_threadfn_adapter+0x16/0x30 + kthread+0x137/0x160 + ret_from_fork+0x22/0x30 + + kfence-#24 [0xffffffffb673dfe0-0xffffffffb673dfff, size=32, cache=kmalloc-32] allocated by task 507: + test_alloc+0xf3/0x25b + test_use_after_free_read+0x76/0x143 + kunit_try_run_case+0x51/0x85 + kunit_generic_run_threadfn_adapter+0x16/0x30 + kthread+0x137/0x160 + ret_from_fork+0x22/0x30 + + freed by task 507: + test_use_after_free_read+0xa8/0x143 + kunit_try_run_case+0x51/0x85 + kunit_generic_run_threadfn_adapter+0x16/0x30 + kthread+0x137/0x160 + ret_from_fork+0x22/0x30 + + CPU: 4 PID: 109 Comm: kunit_try_catch Tainted: G W 5.8.0-rc6+ #7 + Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.13.0-1 04/01/2014 + ================================================================== + +KFENCE also reports on invalid frees, such as double-frees:: + + ================================================================== + BUG: KFENCE: invalid free in test_double_free+0xdc/0x171 + + Invalid free of 0xffffffffb6741000: + test_double_free+0xdc/0x171 + kunit_try_run_case+0x51/0x85 + kunit_generic_run_threadfn_adapter+0x16/0x30 + kthread+0x137/0x160 + ret_from_fork+0x22/0x30 + + kfence-#26 [0xffffffffb6741000-0xffffffffb674101f, size=32, cache=kmalloc-32] allocated by task 507: + test_alloc+0xf3/0x25b + test_double_free+0x76/0x171 + kunit_try_run_case+0x51/0x85 + kunit_generic_run_threadfn_adapter+0x16/0x30 + kthread+0x137/0x160 + ret_from_fork+0x22/0x30 + + freed by task 507: + test_double_free+0xa8/0x171 + kunit_try_run_case+0x51/0x85 + kunit_generic_run_threadfn_adapter+0x16/0x30 + kthread+0x137/0x160 + ret_from_fork+0x22/0x30 + + CPU: 4 PID: 111 Comm: kunit_try_catch Tainted: G W 5.8.0-rc6+ #7 + Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.13.0-1 04/01/2014 + ================================================================== + +KFENCE also uses pattern-based redzones on the other side of an object's guard +page, to detect out-of-bounds writes on the unprotected side of the object. +These are reported on frees:: + + ================================================================== + BUG: KFENCE: memory corruption in test_kmalloc_aligned_oob_write+0xef/0x184 + + Corrupted memory at 0xffffffffb6797ff9 [ 0xac . . . . . . ] (in kfence-#69): + test_kmalloc_aligned_oob_write+0xef/0x184 + kunit_try_run_case+0x51/0x85 + kunit_generic_run_threadfn_adapter+0x16/0x30 + kthread+0x137/0x160 + ret_from_fork+0x22/0x30 + + kfence-#69 [0xffffffffb6797fb0-0xffffffffb6797ff8, size=73, cache=kmalloc-96] allocated by task 507: + test_alloc+0xf3/0x25b + test_kmalloc_aligned_oob_write+0x57/0x184 + kunit_try_run_case+0x51/0x85 + kunit_generic_run_threadfn_adapter+0x16/0x30 + kthread+0x137/0x160 + ret_from_fork+0x22/0x30 + + CPU: 4 PID: 120 Comm: kunit_try_catch Tainted: G W 5.8.0-rc6+ #7 + Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.13.0-1 04/01/2014 + ================================================================== + +For such errors, the address where the corruption occurred as well as the +invalidly written bytes (offset from the address) are shown; in this +representation, '.' denote untouched bytes. In the example above ``0xac`` is +the value written to the invalid address at offset 0, and the remaining '.' +denote that no following bytes have been touched. Note that, real values are +only shown if the kernel was booted with ``no_hash_pointers``; to avoid +information disclosure otherwise, '!' is used instead to denote invalidly +written bytes. + +And finally, KFENCE may also report on invalid accesses to any protected page +where it was not possible to determine an associated object, e.g. if adjacent +object pages had not yet been allocated:: + + ================================================================== + BUG: KFENCE: invalid read in test_invalid_access+0x26/0xe0 + + Invalid read at 0xffffffffb670b00a: + test_invalid_access+0x26/0xe0 + kunit_try_run_case+0x51/0x85 + kunit_generic_run_threadfn_adapter+0x16/0x30 + kthread+0x137/0x160 + ret_from_fork+0x22/0x30 + + CPU: 4 PID: 124 Comm: kunit_try_catch Tainted: G W 5.8.0-rc6+ #7 + Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.13.0-1 04/01/2014 + ================================================================== + +DebugFS interface +~~~~~~~~~~~~~~~~~ + +Some debugging information is exposed via debugfs: + +* The file ``/sys/kernel/debug/kfence/stats`` provides runtime statistics. + +* The file ``/sys/kernel/debug/kfence/objects`` provides a list of objects + allocated via KFENCE, including those already freed but protected. + +Implementation Details +---------------------- + +Guarded allocations are set up based on the sample interval. After expiration +of the sample interval, the next allocation through the main allocator (SLAB or +SLUB) returns a guarded allocation from the KFENCE object pool (allocation +sizes up to PAGE_SIZE are supported). At this point, the timer is reset, and +the next allocation is set up after the expiration of the interval. To "gate" a +KFENCE allocation through the main allocator's fast-path without overhead, +KFENCE relies on static branches via the static keys infrastructure. The static +branch is toggled to redirect the allocation to KFENCE. + +KFENCE objects each reside on a dedicated page, at either the left or right +page boundaries selected at random. The pages to the left and right of the +object page are "guard pages", whose attributes are changed to a protected +state, and cause page faults on any attempted access. Such page faults are then +intercepted by KFENCE, which handles the fault gracefully by reporting an +out-of-bounds access, and marking the page as accessible so that the faulting +code can (wrongly) continue executing (set ``panic_on_warn`` to panic instead). + +To detect out-of-bounds writes to memory within the object's page itself, +KFENCE also uses pattern-based redzones. For each object page, a redzone is set +up for all non-object memory. For typical alignments, the redzone is only +required on the unguarded side of an object. Because KFENCE must honor the +cache's requested alignment, special alignments may result in unprotected gaps +on either side of an object, all of which are redzoned. + +The following figure illustrates the page layout:: + + ---+-----------+-----------+-----------+-----------+-----------+--- + | xxxxxxxxx | O : | xxxxxxxxx | : O | xxxxxxxxx | + | xxxxxxxxx | B : | xxxxxxxxx | : B | xxxxxxxxx | + | x GUARD x | J : RED- | x GUARD x | RED- : J | x GUARD x | + | xxxxxxxxx | E : ZONE | xxxxxxxxx | ZONE : E | xxxxxxxxx | + | xxxxxxxxx | C : | xxxxxxxxx | : C | xxxxxxxxx | + | xxxxxxxxx | T : | xxxxxxxxx | : T | xxxxxxxxx | + ---+-----------+-----------+-----------+-----------+-----------+--- + +Upon deallocation of a KFENCE object, the object's page is again protected and +the object is marked as freed. Any further access to the object causes a fault +and KFENCE reports a use-after-free access. Freed objects are inserted at the +tail of KFENCE's freelist, so that the least recently freed objects are reused +first, and the chances of detecting use-after-frees of recently freed objects +is increased. + +Interface +--------- + +The following describes the functions which are used by allocators as well as +page handling code to set up and deal with KFENCE allocations. + +.. kernel-doc:: include/linux/kfence.h + :functions: is_kfence_address + kfence_shutdown_cache + kfence_alloc kfence_free __kfence_free + kfence_ksize kfence_object_start + kfence_handle_page_fault + +Related Tools +------------- + +In userspace, a similar approach is taken by `GWP-ASan +`_. GWP-ASan also relies on guard pages and +a sampling strategy to detect memory unsafety bugs at scale. KFENCE's design is +directly influenced by GWP-ASan, and can be seen as its kernel sibling. Another +similar but non-sampling approach, that also inspired the name "KFENCE", can be +found in the userspace `Electric Fence Malloc Debugger +`_. + +In the kernel, several tools exist to debug memory access errors, and in +particular KASAN can detect all bug classes that KFENCE can detect. While KASAN +is more precise, relying on compiler instrumentation, this comes at a +performance cost. + +It is worth highlighting that KASAN and KFENCE are complementary, with +different target environments. For instance, KASAN is the better debugging-aid, +where test cases or reproducers exists: due to the lower chance to detect the +error, it would require more effort using KFENCE to debug. Deployments at scale +that cannot afford to enable KASAN, however, would benefit from using KFENCE to +discover bugs due to code paths not exercised by test cases or fuzzers. diff --git a/Documentation/dev-tools/kunit/index.rst b/Documentation/dev-tools/kunit/index.rst index c234a3ab3c34fc492dc4067095259d8c61986e20..848478838347532c939421828017ed14b3d7c1d7 100644 --- a/Documentation/dev-tools/kunit/index.rst +++ b/Documentation/dev-tools/kunit/index.rst @@ -13,6 +13,7 @@ KUnit - Unit Testing for the Linux Kernel api/index style faq + tips What is KUnit? ============== @@ -88,6 +89,7 @@ How do I use it? ================ * :doc:`start` - for new users of KUnit +* :doc:`tips` - for short examples of best practices * :doc:`usage` - for a more detailed explanation of KUnit features * :doc:`api/index` - for the list of KUnit APIs used for testing * :doc:`kunit-tool` - for more information on the kunit_tool helper script diff --git a/Documentation/dev-tools/kunit/start.rst b/Documentation/dev-tools/kunit/start.rst index 454f307813ea6689d5ee80954e5d32792a0c8627..0e65cabe08eb9dadb78dfce990b6250d871e5f9d 100644 --- a/Documentation/dev-tools/kunit/start.rst +++ b/Documentation/dev-tools/kunit/start.rst @@ -196,8 +196,9 @@ Now add the following to ``drivers/misc/Kconfig``: .. code-block:: kconfig config MISC_EXAMPLE_TEST - bool "Test for my example" + tristate "Test for my example" if !KUNIT_ALL_TESTS depends on MISC_EXAMPLE && KUNIT=y + default KUNIT_ALL_TESTS and the following to ``drivers/misc/Makefile``: @@ -233,5 +234,7 @@ Congrats! You just wrote your first KUnit test! Next Steps ========== -* Check out the :doc:`usage` page for a more +* Check out the :doc:`tips` page for tips on + writing idiomatic KUnit tests. +* Optional: see the :doc:`usage` page for a more in-depth explanation of KUnit. diff --git a/Documentation/dev-tools/kunit/tips.rst b/Documentation/dev-tools/kunit/tips.rst new file mode 100644 index 0000000000000000000000000000000000000000..a6ca0af140985302073ecb8181d11813e6a21604 --- /dev/null +++ b/Documentation/dev-tools/kunit/tips.rst @@ -0,0 +1,115 @@ +.. SPDX-License-Identifier: GPL-2.0 + +============================ +Tips For Writing KUnit Tests +============================ + +Exiting early on failed expectations +------------------------------------ + +``KUNIT_EXPECT_EQ`` and friends will mark the test as failed and continue +execution. In some cases, it's unsafe to continue and you can use the +``KUNIT_ASSERT`` variant to exit on failure. + +.. code-block:: c + + void example_test_user_alloc_function(struct kunit *test) + { + void *object = alloc_some_object_for_me(); + + /* Make sure we got a valid pointer back. */ + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, object); + do_something_with_object(object); + } + +Allocating memory +----------------- + +Where you would use ``kzalloc``, you should prefer ``kunit_kzalloc`` instead. +KUnit will ensure the memory is freed once the test completes. + +This is particularly useful since it lets you use the ``KUNIT_ASSERT_EQ`` +macros to exit early from a test without having to worry about remembering to +call ``kfree``. + +Example: + +.. code-block:: c + + void example_test_allocation(struct kunit *test) + { + char *buffer = kunit_kzalloc(test, 16, GFP_KERNEL); + /* Ensure allocation succeeded. */ + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, buffer); + + KUNIT_ASSERT_STREQ(test, buffer, ""); + } + + +Testing static functions +------------------------ + +If you don't want to expose functions or variables just for testing, one option +is to conditionally ``#include`` the test file at the end of your .c file, e.g. + +.. code-block:: c + + /* In my_file.c */ + + static int do_interesting_thing(); + + #ifdef CONFIG_MY_KUNIT_TEST + #include "my_kunit_test.c" + #endif + +Injecting test-only code +------------------------ + +Similarly to the above, it can be useful to add test-specific logic. + +.. code-block:: c + + /* In my_file.h */ + + #ifdef CONFIG_MY_KUNIT_TEST + /* Defined in my_kunit_test.c */ + void test_only_hook(void); + #else + void test_only_hook(void) { } + #endif + +TODO(dlatypov@google.com): add an example of using ``current->kunit_test`` in +such a hook when it's not only updated for ``CONFIG_KASAN=y``. + +Customizing error messages +-------------------------- + +Each of the ``KUNIT_EXPECT`` and ``KUNIT_ASSERT`` macros have a ``_MSG`` variant. +These take a format string and arguments to provide additional context to the automatically generated error messages. + +.. code-block:: c + + char some_str[41]; + generate_sha1_hex_string(some_str); + + /* Before. Not easy to tell why the test failed. */ + KUNIT_EXPECT_EQ(test, strlen(some_str), 40); + + /* After. Now we see the offending string. */ + KUNIT_EXPECT_EQ_MSG(test, strlen(some_str), 40, "some_str='%s'", some_str); + +Alternatively, one can take full control over the error message by using ``KUNIT_FAIL()``, e.g. + +.. code-block:: c + + /* Before */ + KUNIT_EXPECT_EQ(test, some_setup_function(), 0); + + /* After: full control over the failure message. */ + if (some_setup_function()) + KUNIT_FAIL(test, "Failed to setup thing for testing"); + +Next Steps +========== +* Optional: see the :doc:`usage` page for a more + in-depth explanation of KUnit. diff --git a/Documentation/devicetree/bindings/Makefile b/Documentation/devicetree/bindings/Makefile index 8f2b054bec5a0778b9025eeda172530515d13434..780e5618ec0ae2fc7942c86a802d9cc98119494b 100644 --- a/Documentation/devicetree/bindings/Makefile +++ b/Documentation/devicetree/bindings/Makefile @@ -10,7 +10,7 @@ DT_SCHEMA_MIN_VERSION = 2020.8.1 PHONY += check_dtschema_version check_dtschema_version: @{ echo $(DT_SCHEMA_MIN_VERSION); \ - $(DT_DOC_CHECKER) --version 2>/dev/null || echo 0; } | sort -VC || \ + $(DT_DOC_CHECKER) --version 2>/dev/null || echo 0; } | sort -Vc >/dev/null || \ { echo "ERROR: dtschema minimum version is v$(DT_SCHEMA_MIN_VERSION)" >&2; false; } quiet_cmd_extract_ex = DTEX $@ @@ -78,10 +78,10 @@ $(obj)/processed-schema.json: $(DT_SCHEMA_FILES) check_dtschema_version FORCE endif -extra-$(CHECK_DT_BINDING) += processed-schema-examples.json -extra-$(CHECK_DTBS) += processed-schema.json -extra-$(CHECK_DT_BINDING) += $(patsubst $(src)/%.yaml,%.example.dts, $(DT_SCHEMA_FILES)) -extra-$(CHECK_DT_BINDING) += $(patsubst $(src)/%.yaml,%.example.dt.yaml, $(DT_SCHEMA_FILES)) +always-$(CHECK_DT_BINDING) += processed-schema-examples.json +always-$(CHECK_DTBS) += processed-schema.json +always-$(CHECK_DT_BINDING) += $(patsubst $(src)/%.yaml,%.example.dts, $(DT_SCHEMA_FILES)) +always-$(CHECK_DT_BINDING) += $(patsubst $(src)/%.yaml,%.example.dt.yaml, $(DT_SCHEMA_FILES)) # Hack: avoid 'Argument list too long' error for 'make clean'. Remove most of # build artifacts here before they are processed by scripts/Makefile.clean diff --git a/Documentation/devicetree/bindings/arm/atmel-sysregs.txt b/Documentation/devicetree/bindings/arm/atmel-sysregs.txt index 67719f15eb4c8ea89d1246d844b4f272bf2c7cf3..807264a78edce85171e04d23327d90b6259ccd1c 100644 --- a/Documentation/devicetree/bindings/arm/atmel-sysregs.txt +++ b/Documentation/devicetree/bindings/arm/atmel-sysregs.txt @@ -91,7 +91,8 @@ SHDWC SAMA5D2-Compatible Shutdown Controller 1) shdwc node required properties: -- compatible: should be "atmel,sama5d2-shdwc" or "microchip,sam9x60-shdwc". +- compatible: should be "atmel,sama5d2-shdwc", "microchip,sam9x60-shdwc" or + "microchip,sama7g5-shdwc" - reg: should contain registers location and length - clocks: phandle to input clock. - #address-cells: should be one. The cell is the wake-up input index. @@ -103,7 +104,7 @@ optional properties: microseconds. It's usually a board-related property. - atmel,wakeup-rtc-timer: boolean to enable Real-Time Clock wake-up. -optional microchip,sam9x60-shdwc properties: +optional microchip,sam9x60-shdwc or microchip,sama7g5-shdwc properties: - atmel,wakeup-rtt-timer: boolean to enable Real-time Timer Wake-up. The node contains child nodes for each wake-up input that the platform uses. diff --git a/Documentation/devicetree/bindings/arm/coresight.txt b/Documentation/devicetree/bindings/arm/coresight.txt index d711676b4a51ebaa3caa91e05e363b87b6dd9474..7f9c1ca8748766b66f9dbfa41e3734522a490863 100644 --- a/Documentation/devicetree/bindings/arm/coresight.txt +++ b/Documentation/devicetree/bindings/arm/coresight.txt @@ -34,9 +34,12 @@ its hardware characteristcs. Program Flow Trace Macrocell: "arm,coresight-etm3x", "arm,primecell"; - - Embedded Trace Macrocell (version 4.x): + - Embedded Trace Macrocell (version 4.x), with memory mapped access. "arm,coresight-etm4x", "arm,primecell"; + - Embedded Trace Macrocell (version 4.x), with system register access only. + "arm,coresight-etm4x-sysreg"; + - Coresight programmable Replicator : "arm,coresight-dynamic-replicator", "arm,primecell"; diff --git a/Documentation/devicetree/bindings/arm/marvell/ap80x-system-controller.txt b/Documentation/devicetree/bindings/arm/marvell/ap80x-system-controller.txt index e31511255d8e3b6aeb3215f996a42c07445d7b02..052a967c1f28e2271b082a5cb8927c57d57eb1bb 100644 --- a/Documentation/devicetree/bindings/arm/marvell/ap80x-system-controller.txt +++ b/Documentation/devicetree/bindings/arm/marvell/ap80x-system-controller.txt @@ -80,6 +80,11 @@ Required properties: - offset: offset address inside the syscon block +Optional properties: + +- marvell,pwm-offset: offset address of PWM duration control registers inside + the syscon block + Example: ap_syscon: system-controller@6f4000 { compatible = "syscon", "simple-mfd"; @@ -101,6 +106,9 @@ ap_syscon: system-controller@6f4000 { gpio-controller; #gpio-cells = <2>; gpio-ranges = <&ap_pinctrl 0 0 19>; + marvell,pwm-offset = <0x10c0>; + #pwm-cells = <2>; + clocks = <&ap_clk 3>; }; }; diff --git a/Documentation/devicetree/bindings/arm/rockchip.yaml b/Documentation/devicetree/bindings/arm/rockchip.yaml index 8a2dd9f1cff25e45357e1999a1c2c87bbef8c2e6..c3036f95c7bcee1b5d2c9f08a33fe08a0fb81b59 100644 --- a/Documentation/devicetree/bindings/arm/rockchip.yaml +++ b/Documentation/devicetree/bindings/arm/rockchip.yaml @@ -132,6 +132,7 @@ properties: - enum: - friendlyarm,nanopc-t4 - friendlyarm,nanopi-m4 + - friendlyarm,nanopi-m4b - friendlyarm,nanopi-neo4 - const: rockchip,rk3399 diff --git a/Documentation/devicetree/bindings/arm/socionext/socionext,uniphier-system-cache.yaml b/Documentation/devicetree/bindings/arm/socionext/socionext,uniphier-system-cache.yaml index 2e765bb3e6f67797d1fc9254df4fc577f4db6140..7ca5375f278f35e8035334481f5e61dbe3764c15 100644 --- a/Documentation/devicetree/bindings/arm/socionext/socionext,uniphier-system-cache.yaml +++ b/Documentation/devicetree/bindings/arm/socionext/socionext,uniphier-system-cache.yaml @@ -30,8 +30,8 @@ properties: Interrupts can be used to notify the completion of cache operations. The number of interrupts should match to the number of CPU cores. The specified interrupts correspond to CPU0, CPU1, ... in this order. - minItems: 1 - maxItems: 4 + minItems: 1 + maxItems: 4 cache-unified: true diff --git a/Documentation/devicetree/bindings/ata/sata_highbank.yaml b/Documentation/devicetree/bindings/ata/sata_highbank.yaml index 5e2a2394e600ee7bff46a0eda24284f202717f6e..ce75d77e92891cf8a49079c287fa0b296d795ad5 100644 --- a/Documentation/devicetree/bindings/ata/sata_highbank.yaml +++ b/Documentation/devicetree/bindings/ata/sata_highbank.yaml @@ -61,6 +61,7 @@ properties: maxItems: 8 calxeda,sgpio-gpio: + maxItems: 3 description: | phandle-gpio bank, bit offset, and default on or off, which indicates that the driver supports SGPIO indicator lights using the indicated diff --git a/Documentation/devicetree/bindings/auxdisplay/holtek,ht16k33.yaml b/Documentation/devicetree/bindings/auxdisplay/holtek,ht16k33.yaml new file mode 100644 index 0000000000000000000000000000000000000000..64ffff460026040f45f6e8164c539ca26168712b --- /dev/null +++ b/Documentation/devicetree/bindings/auxdisplay/holtek,ht16k33.yaml @@ -0,0 +1,77 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/auxdisplay/holtek,ht16k33.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Holtek HT16K33 RAM mapping 16*8 LED controller with keyscan + +maintainers: + - Robin van der Gracht + +allOf: + - $ref: "/schemas/input/matrix-keymap.yaml#" + +properties: + compatible: + const: holtek,ht16k33 + + reg: + maxItems: 1 + + refresh-rate-hz: + maxItems: 1 + description: Display update interval in Hertz + + interrupts: + maxItems: 1 + + debounce-delay-ms: + maxItems: 1 + description: Debouncing interval time in milliseconds + + linux,keymap: true + + linux,no-autorepeat: + description: Disable keyrepeat + + default-brightness-level: + minimum: 1 + maximum: 16 + default: 16 + description: Initial brightness level + +required: + - compatible + - reg + - refresh-rate-hz + +additionalProperties: false + +examples: + - | + #include + #include + i2c1 { + #address-cells = <1>; + #size-cells = <0>; + + ht16k33: ht16k33@70 { + compatible = "holtek,ht16k33"; + reg = <0x70>; + refresh-rate-hz = <20>; + interrupt-parent = <&gpio4>; + interrupts = <5 (IRQ_TYPE_LEVEL_HIGH | IRQ_TYPE_EDGE_RISING)>; + debounce-delay-ms = <50>; + linux,keymap = , + , + , + , + , + , + , + , + , + ; + }; + }; diff --git a/Documentation/devicetree/bindings/clock/adi,axi-clkgen.yaml b/Documentation/devicetree/bindings/clock/adi,axi-clkgen.yaml index 0d06387184d68d3c36d32fcb30d52751ad69fc39..983033fe5b177c2cb536e660634d39b1123a8b5b 100644 --- a/Documentation/devicetree/bindings/clock/adi,axi-clkgen.yaml +++ b/Documentation/devicetree/bindings/clock/adi,axi-clkgen.yaml @@ -20,6 +20,7 @@ properties: compatible: enum: - adi,axi-clkgen-2.00.a + - adi,zynqmp-axi-clkgen-2.00.a clocks: description: diff --git a/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-ccu.yaml b/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-ccu.yaml index 3b45344ed75818ce35bf6ce5fc03ce59bd6b684f..a27025cd390999de887cc2358b770442150bbce8 100644 --- a/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-ccu.yaml +++ b/Documentation/devicetree/bindings/clock/allwinner,sun4i-a10-ccu.yaml @@ -41,6 +41,8 @@ properties: - allwinner,sun50i-h5-ccu - allwinner,sun50i-h6-ccu - allwinner,sun50i-h6-r-ccu + - allwinner,sun50i-h616-ccu + - allwinner,sun50i-h616-r-ccu - allwinner,suniv-f1c100s-ccu - nextthing,gr8-ccu @@ -82,6 +84,7 @@ if: - allwinner,sun50i-a64-r-ccu - allwinner,sun50i-a100-r-ccu - allwinner,sun50i-h6-r-ccu + - allwinner,sun50i-h616-r-ccu then: properties: @@ -100,6 +103,7 @@ else: enum: - allwinner,sun50i-a100-ccu - allwinner,sun50i-h6-ccu + - allwinner,sun50i-h616-ccu then: properties: diff --git a/Documentation/devicetree/bindings/clock/allwinner,sun9i-a80-usb-clocks.yaml b/Documentation/devicetree/bindings/clock/allwinner,sun9i-a80-usb-clks.yaml similarity index 95% rename from Documentation/devicetree/bindings/clock/allwinner,sun9i-a80-usb-clocks.yaml rename to Documentation/devicetree/bindings/clock/allwinner,sun9i-a80-usb-clks.yaml index fa0ee03a527f3cd5690d821b0123a37151fbba36..6532fb6821bc1dc4a7a7a42628bbf5e2be7a9305 100644 --- a/Documentation/devicetree/bindings/clock/allwinner,sun9i-a80-usb-clocks.yaml +++ b/Documentation/devicetree/bindings/clock/allwinner,sun9i-a80-usb-clks.yaml @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0+ %YAML 1.2 --- -$id: http://devicetree.org/schemas/clock/allwinner,sun9i-a80-usb-clocks.yaml# +$id: http://devicetree.org/schemas/clock/allwinner,sun9i-a80-usb-clks.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# title: Allwinner A80 USB Clock Controller Device Tree Bindings @@ -18,7 +18,7 @@ properties: const: 1 compatible: - const: allwinner,sun9i-a80-usb-clocks + const: allwinner,sun9i-a80-usb-clks reg: maxItems: 1 diff --git a/Documentation/devicetree/bindings/clock/arm,syscon-icst.yaml b/Documentation/devicetree/bindings/clock/arm,syscon-icst.yaml index eb241587efd178c191ad0d8709a75ccddc5af85f..118c5543e037b15d84906eb175dfff2505e50f96 100644 --- a/Documentation/devicetree/bindings/clock/arm,syscon-icst.yaml +++ b/Documentation/devicetree/bindings/clock/arm,syscon-icst.yaml @@ -66,8 +66,8 @@ properties: - arm,syscon-icst525-integratorcp-cm-mem - arm,integrator-cm-auxosc - arm,versatile-cm-auxosc - - arm,impd-vco1 - - arm,impd-vco2 + - arm,impd1-vco1 + - arm,impd1-vco2 clocks: description: Parent clock for the ICST VCO diff --git a/Documentation/devicetree/bindings/clock/canaan,k210-clk.yaml b/Documentation/devicetree/bindings/clock/canaan,k210-clk.yaml index 565ca468cb448f02024546f85ed56161d97cfe9d..7f5cf4001f762baaa538cd654c541e6cd29e0a38 100644 --- a/Documentation/devicetree/bindings/clock/canaan,k210-clk.yaml +++ b/Documentation/devicetree/bindings/clock/canaan,k210-clk.yaml @@ -22,6 +22,7 @@ properties: const: canaan,k210-clk clocks: + maxItems: 1 description: Phandle of the SoC 26MHz fixed-rate oscillator clock. diff --git a/Documentation/devicetree/bindings/clock/csr,atlas7-car.txt b/Documentation/devicetree/bindings/clock/csr,atlas7-car.txt deleted file mode 100644 index 54d6d135833942fe241e4fcaaf5ea1211a9d483b..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/clock/csr,atlas7-car.txt +++ /dev/null @@ -1,55 +0,0 @@ -* Clock and reset bindings for CSR atlas7 - -Required properties: -- compatible: Should be "sirf,atlas7-car" -- reg: Address and length of the register set -- #clock-cells: Should be <1> -- #reset-cells: Should be <1> - -The clock consumer should specify the desired clock by having the clock -ID in its "clocks" phandle cell. -The ID list atlas7_clks defined in drivers/clk/sirf/clk-atlas7.c - -The reset consumer should specify the desired reset by having the reset -ID in its "reset" phandle cell. -The ID list atlas7_reset_unit defined in drivers/clk/sirf/clk-atlas7.c - -Examples: Clock and reset controller node: - -car: clock-controller@18620000 { - compatible = "sirf,atlas7-car"; - reg = <0x18620000 0x1000>; - #clock-cells = <1>; - #reset-cells = <1>; -}; - -Examples: Consumers using clock or reset: - -timer@10dc0000 { - compatible = "sirf,macro-tick"; - reg = <0x10dc0000 0x1000>; - clocks = <&car 54>; - interrupts = <0 0 0>, - <0 1 0>, - <0 2 0>, - <0 49 0>, - <0 50 0>, - <0 51 0>; -}; - -uart1: uart@18020000 { - cell-index = <1>; - compatible = "sirf,macro-uart"; - reg = <0x18020000 0x1000>; - clocks = <&clks 95>; - interrupts = <0 18 0>; - fifosize = <32>; -}; - -vpp@13110000 { - compatible = "sirf,prima2-vpp"; - reg = <0x13110000 0x10000>; - interrupts = <0 31 0>; - clocks = <&car 85>; - resets = <&car 29>; -}; diff --git a/Documentation/devicetree/bindings/clock/idt,versaclock5.yaml b/Documentation/devicetree/bindings/clock/idt,versaclock5.yaml index 2ac1131fd9222a866c4866eaa3505b822230f458..c268debe5b8d58cd04530e7c8cde17a638ad149d 100644 --- a/Documentation/devicetree/bindings/clock/idt,versaclock5.yaml +++ b/Documentation/devicetree/bindings/clock/idt,versaclock5.yaml @@ -59,6 +59,12 @@ properties: minItems: 1 maxItems: 2 + idt,xtal-load-femtofarads: + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 9000 + maximum: 22760 + description: Optional load capacitor for XTAL1 and XTAL2 + patternProperties: "^OUT[1-4]$": type: object diff --git a/Documentation/devicetree/bindings/clock/imx27-clock.yaml b/Documentation/devicetree/bindings/clock/imx27-clock.yaml index a75365453dbce252d72e191b920b9daf98d2513b..160268f2448740b6f9188db9a623803643c474b6 100644 --- a/Documentation/devicetree/bindings/clock/imx27-clock.yaml +++ b/Documentation/devicetree/bindings/clock/imx27-clock.yaml @@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml# title: Clock bindings for Freescale i.MX27 maintainers: - - Fabio Estevam + - Fabio Estevam description: | The clock consumer should specify the desired clock by having the clock diff --git a/Documentation/devicetree/bindings/clock/imx31-clock.yaml b/Documentation/devicetree/bindings/clock/imx31-clock.yaml index a25a374b3b2aab1767e6b649a5af280d0697738a..d2336261c922a007a7a67e90136e380ac2bd7176 100644 --- a/Documentation/devicetree/bindings/clock/imx31-clock.yaml +++ b/Documentation/devicetree/bindings/clock/imx31-clock.yaml @@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml# title: Clock bindings for Freescale i.MX31 maintainers: - - Fabio Estevam + - Fabio Estevam description: | The clock consumer should specify the desired clock by having the clock diff --git a/Documentation/devicetree/bindings/clock/imx5-clock.yaml b/Documentation/devicetree/bindings/clock/imx5-clock.yaml index 90775c2669b84ba123d9551a58894a9d274ff4d5..b1740d7abe686c455fa51f1cb2a2c4d559b9ed98 100644 --- a/Documentation/devicetree/bindings/clock/imx5-clock.yaml +++ b/Documentation/devicetree/bindings/clock/imx5-clock.yaml @@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml# title: Clock bindings for Freescale i.MX5 maintainers: - - Fabio Estevam + - Fabio Estevam description: | The clock consumer should specify the desired clock by having the clock diff --git a/Documentation/devicetree/bindings/clock/intel,easic-n5x.yaml b/Documentation/devicetree/bindings/clock/intel,easic-n5x.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8f45976e946ee3451eca7e76c17421c24359c96a --- /dev/null +++ b/Documentation/devicetree/bindings/clock/intel,easic-n5x.yaml @@ -0,0 +1,46 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/clock/intel,easic-n5x.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Intel SoCFPGA eASIC N5X platform clock controller binding + +maintainers: + - Dinh Nguyen + +description: + The Intel eASIC N5X Clock controller is an integrated clock controller, which + generates and supplies to all modules. + +properties: + compatible: + const: intel,easic-n5x-clkmgr + + '#clock-cells': + const: 1 + + reg: + maxItems: 1 + + clocks: + maxItems: 1 + +required: + - compatible + - reg + - clocks + - '#clock-cells' + +additionalProperties: false + +examples: + # Clock controller node: + - | + clkmgr: clock-controller@ffd10000 { + compatible = "intel,easic-n5x-clkmgr"; + reg = <0xffd10000 0x1000>; + clocks = <&osc1>; + #clock-cells = <1>; + }; +... diff --git a/Documentation/devicetree/bindings/clock/mstar,msc313-mpll.yaml b/Documentation/devicetree/bindings/clock/mstar,msc313-mpll.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0df5d75d4ebc5615eb40ca8a71bd847672359c3c --- /dev/null +++ b/Documentation/devicetree/bindings/clock/mstar,msc313-mpll.yaml @@ -0,0 +1,46 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/clock/mstar,msc313-mpll.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: MStar/Sigmastar MSC313 MPLL + +maintainers: + - Daniel Palmer + +description: | + The MStar/SigmaStar MSC313 and later ARMv7 chips have an MPLL block that + takes the external xtal input and multiplies it to create a high + frequency clock and divides that down into a number of clocks that + peripherals use. + +properties: + compatible: + const: mstar,msc313-mpll + + "#clock-cells": + const: 1 + + clocks: + maxItems: 1 + + reg: + maxItems: 1 + +required: + - compatible + - "#clock-cells" + - clocks + - reg + +additionalProperties: false + +examples: + - | + mpll@206000 { + compatible = "mstar,msc313-mpll"; + reg = <0x206000 0x200>; + #clock-cells = <1>; + clocks = <&xtal>; + }; diff --git a/Documentation/devicetree/bindings/clock/prima2-clock.txt b/Documentation/devicetree/bindings/clock/prima2-clock.txt deleted file mode 100644 index 5016979c0f786bb3a01814d4f0c70b7d88be27ed..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/clock/prima2-clock.txt +++ /dev/null @@ -1,73 +0,0 @@ -* Clock bindings for CSR SiRFprimaII - -Required properties: -- compatible: Should be "sirf,prima2-clkc" -- reg: Address and length of the register set -- interrupts: Should contain clock controller interrupt -- #clock-cells: Should be <1> - -The clock consumer should specify the desired clock by having the clock -ID in its "clocks" phandle cell. The following is a full list of prima2 -clocks and IDs. - - Clock ID - --------------------------- - rtc 0 - osc 1 - pll1 2 - pll2 3 - pll3 4 - mem 5 - sys 6 - security 7 - dsp 8 - gps 9 - mf 10 - io 11 - cpu 12 - uart0 13 - uart1 14 - uart2 15 - tsc 16 - i2c0 17 - i2c1 18 - spi0 19 - spi1 20 - pwmc 21 - efuse 22 - pulse 23 - dmac0 24 - dmac1 25 - nand 26 - audio 27 - usp0 28 - usp1 29 - usp2 30 - vip 31 - gfx 32 - mm 33 - lcd 34 - vpp 35 - mmc01 36 - mmc23 37 - mmc45 38 - usbpll 39 - usb0 40 - usb1 41 - -Examples: - -clks: clock-controller@88000000 { - compatible = "sirf,prima2-clkc"; - reg = <0x88000000 0x1000>; - interrupts = <3>; - #clock-cells = <1>; -}; - -i2c0: i2c@b00e0000 { - cell-index = <0>; - compatible = "sirf,prima2-i2c"; - reg = <0xb00e0000 0x10000>; - interrupts = <24>; - clocks = <&clks 17>; -}; diff --git a/Documentation/devicetree/bindings/clock/qcom,a7pll.yaml b/Documentation/devicetree/bindings/clock/qcom,a7pll.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8666e995725f6f866416beadc76851777b39c4d5 --- /dev/null +++ b/Documentation/devicetree/bindings/clock/qcom,a7pll.yaml @@ -0,0 +1,51 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/clock/qcom,a7pll.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Qualcomm A7 PLL Binding + +maintainers: + - Manivannan Sadhasivam + +description: + The A7 PLL on the Qualcomm platforms like SDX55 is used to provide high + frequency clock to the CPU. + +properties: + compatible: + enum: + - qcom,sdx55-a7pll + + reg: + maxItems: 1 + + '#clock-cells': + const: 0 + + clocks: + items: + - description: board XO clock + + clock-names: + items: + - const: bi_tcxo + +required: + - compatible + - reg + - '#clock-cells' + +additionalProperties: false + +examples: + - | + #include + a7pll: clock@17808000 { + compatible = "qcom,sdx55-a7pll"; + reg = <0x17808000 0x1000>; + clocks = <&rpmhcc RPMH_CXO_CLK>; + clock-names = "bi_tcxo"; + #clock-cells = <0>; + }; diff --git a/Documentation/devicetree/bindings/clock/qcom,gcc-sc7280.yaml b/Documentation/devicetree/bindings/clock/qcom,gcc-sc7280.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5693b8997570bd4adb9b4c68be710aea2402c9d1 --- /dev/null +++ b/Documentation/devicetree/bindings/clock/qcom,gcc-sc7280.yaml @@ -0,0 +1,92 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/clock/qcom,gcc-sc7280.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Qualcomm Global Clock & Reset Controller Binding for SC7280 + +maintainers: + - Taniya Das + +description: | + Qualcomm global clock control module which supports the clocks, resets and + power domains on SC7280. + + See also: + - dt-bindings/clock/qcom,gcc-sc7280.h + +properties: + compatible: + const: qcom,gcc-sc7280 + + clocks: + items: + - description: Board XO source + - description: Board active XO source + - description: Sleep clock source + - description: PCIE-0 pipe clock source + - description: PCIE-1 pipe clock source + - description: USF phy rx symbol 0 clock source + - description: USF phy rx symbol 1 clock source + - description: USF phy tx symbol 0 clock source + - description: USB30 phy wrapper pipe clock source + + clock-names: + items: + - const: bi_tcxo + - const: bi_tcxo_ao + - const: sleep_clk + - const: pcie_0_pipe_clk + - const: pcie_1_pipe_clk + - const: ufs_phy_rx_symbol_0_clk + - const: ufs_phy_rx_symbol_1_clk + - const: ufs_phy_tx_symbol_0_clk + - const: usb3_phy_wrapper_gcc_usb30_pipe_clk + + '#clock-cells': + const: 1 + + '#reset-cells': + const: 1 + + '#power-domain-cells': + const: 1 + + reg: + maxItems: 1 + +required: + - compatible + - clocks + - clock-names + - reg + - '#clock-cells' + - '#reset-cells' + - '#power-domain-cells' + +additionalProperties: false + +examples: + - | + #include + clock-controller@100000 { + compatible = "qcom,gcc-sc7280"; + reg = <0x00100000 0x1f0000>; + clocks = <&rpmhcc RPMH_CXO_CLK>, + <&rpmhcc RPMH_CXO_CLK_A>, + <&sleep_clk>, + <&pcie_0_pipe_clk>, <&pcie_1_pipe_clk>, + <&ufs_phy_rx_symbol_0_clk>, <&ufs_phy_rx_symbol_1_clk>, + <&ufs_phy_tx_symbol_0_clk>, + <&usb3_phy_wrapper_gcc_usb30_pipe_clk>; + + clock-names = "bi_tcxo", "bi_tcxo_ao", "sleep_clk", "pcie_0_pipe_clk", + "pcie_1_pipe_clk", "ufs_phy_rx_symbol_0_clk", + "ufs_phy_rx_symbol_1_clk", "ufs_phy_tx_symbol_0_clk", + "usb3_phy_wrapper_gcc_usb30_pipe_clk"; + #clock-cells = <1>; + #reset-cells = <1>; + #power-domain-cells = <1>; + }; +... diff --git a/Documentation/devicetree/bindings/clock/qcom,gcc-sc8180x.yaml b/Documentation/devicetree/bindings/clock/qcom,gcc-sc8180x.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f03ef96e57fa8d8a5b832a070d29db601877fec2 --- /dev/null +++ b/Documentation/devicetree/bindings/clock/qcom,gcc-sc8180x.yaml @@ -0,0 +1,76 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/clock/qcom,gcc-sc8180x.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Qualcomm Global Clock & Reset Controller Binding for SC8180x + +maintainers: + - Bjorn Andersson + +description: | + Qualcomm global clock control module which supports the clocks, resets and + power domains on SC8180x. + + See also: + - dt-bindings/clock/qcom,gcc-sc8180x.h + +properties: + compatible: + const: qcom,gcc-sc8180x + + clocks: + items: + - description: Board XO source + - description: Board active XO source + - description: Sleep clock source + + clock-names: + items: + - const: bi_tcxo + - const: bi_tcxo_ao + - const: sleep_clk + + '#clock-cells': + const: 1 + + '#reset-cells': + const: 1 + + '#power-domain-cells': + const: 1 + + reg: + maxItems: 1 + + protected-clocks: + description: + Protected clock specifier list as per common clock binding. + +required: + - compatible + - clocks + - clock-names + - reg + - '#clock-cells' + - '#reset-cells' + - '#power-domain-cells' + +additionalProperties: false + +examples: + - | + #include + clock-controller@100000 { + compatible = "qcom,gcc-sc8180x"; + reg = <0x00100000 0x1f0000>; + clocks = <&rpmhcc RPMH_CXO_CLK>, + <&rpmhcc RPMH_CXO_CLK_A>, + <&sleep_clk>; + clock-names = "bi_tcxo", "bi_tcxo_ao", "sleep_clk"; + #clock-cells = <1>; + #reset-cells = <1>; + #power-domain-cells = <1>; + }; +... diff --git a/Documentation/devicetree/bindings/clock/qcom,gcc-sm8350.yaml b/Documentation/devicetree/bindings/clock/qcom,gcc-sm8350.yaml new file mode 100644 index 0000000000000000000000000000000000000000..78f35832aa41916392fd430779e17fb6550b772b --- /dev/null +++ b/Documentation/devicetree/bindings/clock/qcom,gcc-sm8350.yaml @@ -0,0 +1,96 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/clock/qcom,gcc-sm8350.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Qualcomm Global Clock & Reset Controller Binding for SM8350 + +maintainers: + - Vinod Koul + +description: | + Qualcomm global clock control module which supports the clocks, resets and + power domains on SM8350. + + See also: + - dt-bindings/clock/qcom,gcc-sm8350.h + +properties: + compatible: + const: qcom,gcc-sm8350 + + clocks: + items: + - description: Board XO source + - description: Sleep clock source + - description: PLL test clock source (Optional clock) + - description: PCIE 0 Pipe clock source (Optional clock) + - description: PCIE 1 Pipe clock source (Optional clock) + - description: UFS card Rx symbol 0 clock source (Optional clock) + - description: UFS card Rx symbol 1 clock source (Optional clock) + - description: UFS card Tx symbol 0 clock source (Optional clock) + - description: UFS phy Rx symbol 0 clock source (Optional clock) + - description: UFS phy Rx symbol 1 clock source (Optional clock) + - description: UFS phy Tx symbol 0 clock source (Optional clock) + - description: USB3 phy wrapper pipe clock source (Optional clock) + - description: USB3 phy sec pipe clock source (Optional clock) + minItems: 2 + maxItems: 13 + + clock-names: + items: + - const: bi_tcxo + - const: sleep_clk + - const: core_bi_pll_test_se # Optional clock + - const: pcie_0_pipe_clk # Optional clock + - const: pcie_1_pipe_clk # Optional clock + - const: ufs_card_rx_symbol_0_clk # Optional clock + - const: ufs_card_rx_symbol_1_clk # Optional clock + - const: ufs_card_tx_symbol_0_clk # Optional clock + - const: ufs_phy_rx_symbol_0_clk # Optional clock + - const: ufs_phy_rx_symbol_1_clk # Optional clock + - const: ufs_phy_tx_symbol_0_clk # Optional clock + - const: usb3_phy_wrapper_gcc_usb30_pipe_clk # Optional clock + - const: usb3_uni_phy_sec_gcc_usb30_pipe_clk # Optional clock + minItems: 2 + maxItems: 13 + + '#clock-cells': + const: 1 + + '#reset-cells': + const: 1 + + '#power-domain-cells': + const: 1 + + reg: + maxItems: 1 + +required: + - compatible + - clocks + - clock-names + - reg + - '#clock-cells' + - '#reset-cells' + - '#power-domain-cells' + +additionalProperties: false + +examples: + - | + #include + clock-controller@100000 { + compatible = "qcom,gcc-sm8350"; + reg = <0x00100000 0x1f0000>; + clocks = <&rpmhcc RPMH_CXO_CLK>, + <&sleep_clk>; + clock-names = "bi_tcxo", "sleep_clk"; + #clock-cells = <1>; + #reset-cells = <1>; + #power-domain-cells = <1>; + }; + +... diff --git a/Documentation/devicetree/bindings/clock/qcom,gpucc-sdm660.yaml b/Documentation/devicetree/bindings/clock/qcom,gpucc-sdm660.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3f70eb59aae3f4f9c96f286ddcaa419228d6f47f --- /dev/null +++ b/Documentation/devicetree/bindings/clock/qcom,gpucc-sdm660.yaml @@ -0,0 +1,76 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/clock/qcom,gpucc-sdm660.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Qualcomm Graphics Clock & Reset Controller Binding for SDM630 and SDM660 + +maintainers: + - AngeloGioacchino Del Regno + +description: | + Qualcomm graphics clock control module which supports the clocks, resets and + power domains on SDM630 and SDM660. + + See also dt-bindings/clock/qcom,gpucc-sdm660.h. + +properties: + compatible: + enum: + - qcom,gpucc-sdm630 + - qcom,gpucc-sdm660 + + clocks: + items: + - description: Board XO source + - description: GPLL0 main gpu branch + - description: GPLL0 divider gpu branch + + clock-names: + items: + - const: xo + - const: gcc_gpu_gpll0_clk + - const: gcc_gpu_gpll0_div_clk + + '#clock-cells': + const: 1 + + '#reset-cells': + const: 1 + + '#power-domain-cells': + const: 1 + + reg: + maxItems: 1 + +required: + - compatible + - reg + - clocks + - clock-names + - '#clock-cells' + - '#reset-cells' + - '#power-domain-cells' + +additionalProperties: false + +examples: + - | + #include + #include + + clock-controller@5065000 { + compatible = "qcom,gpucc-sdm660"; + reg = <0x05065000 0x9038>; + clocks = <&rpmcc RPM_SMD_XO_CLK_SRC>, + <&gcc GCC_GPU_GPLL0_CLK>, + <&gcc GCC_GPU_GPLL0_DIV_CLK>; + clock-names = "xo", "gcc_gpu_gpll0_clk", + "gcc_gpu_gpll0_div_clk"; + #clock-cells = <1>; + #power-domain-cells = <1>; + #reset-cells = <1>; + }; +... diff --git a/Documentation/devicetree/bindings/clock/qcom,mmcc.yaml b/Documentation/devicetree/bindings/clock/qcom,mmcc.yaml index af32dee14fc67e0da20fc15163b629188b10d284..8b0b1c56f354df9f5fbe3c0dab90e30efa7fd56c 100644 --- a/Documentation/devicetree/bindings/clock/qcom,mmcc.yaml +++ b/Documentation/devicetree/bindings/clock/qcom,mmcc.yaml @@ -24,6 +24,8 @@ properties: - qcom,mmcc-msm8974 - qcom,mmcc-msm8996 - qcom,mmcc-msm8998 + - qcom,mmcc-sdm630 + - qcom,mmcc-sdm660 clocks: items: diff --git a/Documentation/devicetree/bindings/clock/qcom,rpmhcc.yaml b/Documentation/devicetree/bindings/clock/qcom,rpmhcc.yaml index 12c9cbc0ebf92979ffb0793e17c0e827970cbf08..9ea0b3f5a4f2562c9d17e8f44150a298d4b63c34 100644 --- a/Documentation/devicetree/bindings/clock/qcom,rpmhcc.yaml +++ b/Documentation/devicetree/bindings/clock/qcom,rpmhcc.yaml @@ -18,6 +18,8 @@ properties: compatible: enum: - qcom,sc7180-rpmh-clk + - qcom,sc7280-rpmh-clk + - qcom,sc8180x-rpmh-clk - qcom,sdm845-rpmh-clk - qcom,sdx55-rpmh-clk - qcom,sm8150-rpmh-clk diff --git a/Documentation/devicetree/bindings/clock/renesas,rcar-usb2-clock-sel.yaml b/Documentation/devicetree/bindings/clock/renesas,rcar-usb2-clock-sel.yaml index 5be1229b3d6ecce7b6c0f46fec9043dee02fff8f..6eaabb4d82ec21cbb9bd939dc32a2c1b24d54e7f 100644 --- a/Documentation/devicetree/bindings/clock/renesas,rcar-usb2-clock-sel.yaml +++ b/Documentation/devicetree/bindings/clock/renesas,rcar-usb2-clock-sel.yaml @@ -35,6 +35,9 @@ properties: compatible: items: - enum: + - renesas,r8a774a1-rcar-usb2-clock-sel # RZ/G2M + - renesas,r8a774b1-rcar-usb2-clock-sel # RZ/G2N + - renesas,r8a774e1-rcar-usb2-clock-sel # RZ/G2H - renesas,r8a7795-rcar-usb2-clock-sel # R-Car H3 - renesas,r8a7796-rcar-usb2-clock-sel # R-Car M3-W - renesas,r8a77961-rcar-usb2-clock-sel # R-Car M3-W+ diff --git a/Documentation/devicetree/bindings/clock/silabs,si570.txt b/Documentation/devicetree/bindings/clock/silabs,si570.txt index 901935e929d2fd0a608d1ee38000c3d9c2af926c..5dda17df1ac5e8a6f1678d2fea33a677c4ab9387 100644 --- a/Documentation/devicetree/bindings/clock/silabs,si570.txt +++ b/Documentation/devicetree/bindings/clock/silabs,si570.txt @@ -28,6 +28,8 @@ Optional properties: - clock-frequency: Output frequency to generate. This defines the output frequency set during boot. It can be reprogrammed during runtime through the common clock framework. + - silabs,skip-recall: Do not perform NVM->RAM recall operation. It will rely + on hardware loading of RAM from NVM at power on. Example: si570: clock-generator@5d { diff --git a/Documentation/devicetree/bindings/clock/ste-u300-syscon-clock.txt b/Documentation/devicetree/bindings/clock/ste-u300-syscon-clock.txt deleted file mode 100644 index 7cafcb98ead79b89b1dd3938de7ff109c7128ef2..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/clock/ste-u300-syscon-clock.txt +++ /dev/null @@ -1,80 +0,0 @@ -Clock bindings for ST-Ericsson U300 System Controller Clocks - -Bindings for the gated system controller clocks: - -Required properties: -- compatible: must be "stericsson,u300-syscon-clk" -- #clock-cells: must be <0> -- clock-type: specifies the type of clock: - 0 = slow clock - 1 = fast clock - 2 = rest/remaining clock -- clock-id: specifies the clock in the type range - -Optional properties: -- clocks: parent clock(s) - -The available clocks per type are as follows: - -Type: ID: Clock: -------------------- -0 0 Slow peripheral bridge clock -0 1 UART0 clock -0 4 GPIO clock -0 6 RTC clock -0 7 Application timer clock -0 8 Access timer clock - -1 0 Fast peripheral bridge clock -1 1 I2C bus 0 clock -1 2 I2C bus 1 clock -1 5 MMC interface peripheral (silicon) clock -1 6 SPI clock - -2 3 CPU clock -2 4 DMA controller clock -2 5 External Memory Interface (EMIF) clock -2 6 NAND flask interface clock -2 8 XGAM graphics engine clock -2 9 Shared External Memory Interface (SEMI) clock -2 10 AHB Subsystem Bridge clock -2 12 Interrupt controller clock - -Example: - -gpio_clk: gpio_clk@13M { - #clock-cells = <0>; - compatible = "stericsson,u300-syscon-clk"; - clock-type = <0>; /* Slow */ - clock-id = <4>; - clocks = <&slow_clk>; -}; - -gpio: gpio@c0016000 { - compatible = "stericsson,gpio-coh901"; - (...) - clocks = <&gpio_clk>; -}; - - -Bindings for the MMC/SD card clock: - -Required properties: -- compatible: must be "stericsson,u300-syscon-mclk" -- #clock-cells: must be <0> - -Optional properties: -- clocks: parent clock(s) - -mmc_mclk: mmc_mclk { - #clock-cells = <0>; - compatible = "stericsson,u300-syscon-mclk"; - clocks = <&mmc_pclk>; -}; - -mmcsd: mmcsd@c0001000 { - compatible = "arm,pl18x", "arm,primecell"; - clocks = <&mmc_pclk>, <&mmc_mclk>; - clock-names = "apb_pclk", "mclk"; - (...) -}; diff --git a/Documentation/devicetree/bindings/clock/tango4-clock.txt b/Documentation/devicetree/bindings/clock/tango4-clock.txt deleted file mode 100644 index 19c580a7bda258adb15a49314bdf45cccba4b4e9..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/clock/tango4-clock.txt +++ /dev/null @@ -1,23 +0,0 @@ -* Sigma Designs Tango4 Clock Generator - -The Tango4 clock generator outputs cpu_clk and sys_clk (the latter is used -for RAM and various peripheral devices). The clock binding described here -is applicable to all Tango4 SoCs. - -Required Properties: - -- compatible: should be "sigma,tango4-clkgen". -- reg: physical base address of the device and length of memory mapped region. -- clocks: phandle of the input clock (crystal oscillator). -- clock-output-names: should be "cpuclk" and "sysclk". -- #clock-cells: should be set to 1. - -Example: - - clkgen: clkgen@10000 { - compatible = "sigma,tango4-clkgen"; - reg = <0x10000 0x40>; - clocks = <&xtal>; - clock-output-names = "cpuclk", "sysclk"; - #clock-cells = <1>; - }; diff --git a/Documentation/devicetree/bindings/clock/zx296702-clk.txt b/Documentation/devicetree/bindings/clock/zx296702-clk.txt deleted file mode 100644 index 5c91c9e4f1beb47f35feb64361f3de23bcc35389..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/clock/zx296702-clk.txt +++ /dev/null @@ -1,34 +0,0 @@ -Device Tree Clock bindings for ZTE zx296702 - -This binding uses the common clock binding[1]. - -[1] Documentation/devicetree/bindings/clock/clock-bindings.txt - -Required properties: -- compatible : shall be one of the following: - "zte,zx296702-topcrm-clk": - zx296702 top clock selection, divider and gating - - "zte,zx296702-lsp0crpm-clk" and - "zte,zx296702-lsp1crpm-clk": - zx296702 device level clock selection and gating - -- reg: Address and length of the register set - -The clock consumer should specify the desired clock by having the clock -ID in its "clocks" phandle cell. See include/dt-bindings/clock/zx296702-clock.h -for the full list of zx296702 clock IDs. - - -topclk: topcrm@09800000 { - compatible = "zte,zx296702-topcrm-clk"; - reg = <0x09800000 0x1000>; - #clock-cells = <1>; -}; - -uart0: serial@09405000 { - compatible = "zte,zx296702-uart"; - reg = <0x09405000 0x1000>; - interrupts = ; - clocks = <&lsp1clk ZX296702_UART0_PCLK>; -}; diff --git a/Documentation/devicetree/bindings/clock/zx296718-clk.txt b/Documentation/devicetree/bindings/clock/zx296718-clk.txt deleted file mode 100644 index 3a46bf0b2540cf7a6c6b565cc0cd47f2e9a0bdfa..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/clock/zx296718-clk.txt +++ /dev/null @@ -1,37 +0,0 @@ -Device Tree Clock bindings for ZTE zx296718 - -This binding uses the common clock binding[1]. - -[1] Documentation/devicetree/bindings/clock/clock-bindings.txt - -Required properties: -- compatible : shall be one of the following: - "zte,zx296718-topcrm": - zx296718 top clock selection, divider and gating - - "zte,zx296718-lsp0crm" and - "zte,zx296718-lsp1crm": - zx296718 device level clock selection and gating - - "zte,zx296718-audiocrm": - zx296718 audio clock selection, divider and gating - -- reg: Address and length of the register set - -The clock consumer should specify the desired clock by having the clock -ID in its "clocks" phandle cell. See include/dt-bindings/clock/zx296718-clock.h -for the full list of zx296718 clock IDs. - - -topclk: topcrm@1461000 { - compatible = "zte,zx296718-topcrm-clk"; - reg = <0x01461000 0x1000>; - #clock-cells = <1>; -}; - -usbphy0:usb-phy0 { - compatible = "zte,zx296718-usb-phy"; - #phy-cells = <0>; - clocks = <&topclk USB20_PHY_CLK>; - clock-names = "phyclk"; -}; diff --git a/Documentation/devicetree/bindings/connector/usb-connector.yaml b/Documentation/devicetree/bindings/connector/usb-connector.yaml index ce11b202719956f73966cac49ac8c16f10920334..b6daedd6251633624ac4676eedefa73d18a385d7 100644 --- a/Documentation/devicetree/bindings/connector/usb-connector.yaml +++ b/Documentation/devicetree/bindings/connector/usb-connector.yaml @@ -155,22 +155,23 @@ properties: power dual role. ports: - description: OF graph bindings (specified in bindings/graph.txt) that model - any data bus to the connector unless the bus is between parent node and - the connector. Since a single connector can have multiple data buses every - bus has an assigned OF graph port number as described below. - type: object + $ref: /schemas/graph.yaml#/properties/ports + description: OF graph bindings modeling any data bus to the connector + unless the bus is between parent node and the connector. Since a single + connector can have multiple data buses every bus has an assigned OF graph + port number as described below. + properties: port@0: - type: object + $ref: /schemas/graph.yaml#/properties/port description: High Speed (HS), present in all connectors. port@1: - type: object + $ref: /schemas/graph.yaml#/properties/port description: Super Speed (SS), present in SS capable connectors. port@2: - type: object + $ref: /schemas/graph.yaml#/properties/port description: Sideband Use (SBU), present in USB-C. This describes the alternate mode connection of which SBU is a part. diff --git a/Documentation/devicetree/bindings/crypto/allwinner,sun8i-ce.yaml b/Documentation/devicetree/bindings/crypto/allwinner,sun8i-ce.yaml index 7a60d84289cc90a0a2f302c55b4122a526ded2ba..6ab07eba7778244fc088d750a7810196e872b25e 100644 --- a/Documentation/devicetree/bindings/crypto/allwinner,sun8i-ce.yaml +++ b/Documentation/devicetree/bindings/crypto/allwinner,sun8i-ce.yaml @@ -46,8 +46,7 @@ properties: if: properties: compatible: - items: - const: allwinner,sun50i-h6-crypto + const: allwinner,sun50i-h6-crypto then: properties: clocks: diff --git a/Documentation/devicetree/bindings/crypto/intel,keembay-ocs-hcu.yaml b/Documentation/devicetree/bindings/crypto/intel,keembay-ocs-hcu.yaml new file mode 100644 index 0000000000000000000000000000000000000000..acb92706d28096b5e5b3e8ecd34cf07c1ebe88f1 --- /dev/null +++ b/Documentation/devicetree/bindings/crypto/intel,keembay-ocs-hcu.yaml @@ -0,0 +1,46 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/crypto/intel,keembay-ocs-hcu.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Intel Keem Bay OCS HCU Device Tree Bindings + +maintainers: + - Declan Murphy + - Daniele Alessandrelli + +description: + The Intel Keem Bay Offload and Crypto Subsystem (OCS) Hash Control Unit (HCU) + provides hardware-accelerated hashing and HMAC. + +properties: + compatible: + const: intel,keembay-ocs-hcu + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + + clocks: + maxItems: 1 + +required: + - compatible + - reg + - interrupts + - clocks + +additionalProperties: false + +examples: + - | + #include + crypto@3000b000 { + compatible = "intel,keembay-ocs-hcu"; + reg = <0x3000b000 0x1000>; + interrupts = ; + clocks = <&scmi_clk 94>; + }; diff --git a/Documentation/devicetree/bindings/crypto/samsung-slimsss.yaml b/Documentation/devicetree/bindings/crypto/samsung-slimsss.yaml index 7743eae049ab5895ad5dd292dcf3ba646d9fbc1a..676950bb7b37df340ff29e8b937b4ed8c8bb3690 100644 --- a/Documentation/devicetree/bindings/crypto/samsung-slimsss.yaml +++ b/Documentation/devicetree/bindings/crypto/samsung-slimsss.yaml @@ -8,7 +8,6 @@ title: Samsung Exynos SoC SlimSSS (Slim Security SubSystem) module maintainers: - Krzysztof Kozlowski - - Kamil Konieczny description: |+ The SlimSSS module in Exynos5433 SoC supports the following: diff --git a/Documentation/devicetree/bindings/crypto/samsung-sss.yaml b/Documentation/devicetree/bindings/crypto/samsung-sss.yaml index cf1c47a81d7f9ecb5ef6dce363a84c994264a6ab..6d62b0e42fc90fa0f9cfc97f726226ca683b6dd6 100644 --- a/Documentation/devicetree/bindings/crypto/samsung-sss.yaml +++ b/Documentation/devicetree/bindings/crypto/samsung-sss.yaml @@ -8,7 +8,6 @@ title: Samsung Exynos SoC SSS (Security SubSystem) module maintainers: - Krzysztof Kozlowski - - Kamil Konieczny description: |+ The SSS module in S5PV210 SoC supports the following: diff --git a/Documentation/devicetree/bindings/crypto/ti,sa2ul.yaml b/Documentation/devicetree/bindings/crypto/ti,sa2ul.yaml index 1465c9ebaf938c6d2ea2288e7b0f697dd6f7bba7..1d48ac712b23c0268db1ad961802fc0a37deb6a2 100644 --- a/Documentation/devicetree/bindings/crypto/ti,sa2ul.yaml +++ b/Documentation/devicetree/bindings/crypto/ti,sa2ul.yaml @@ -66,7 +66,7 @@ examples: #include main_crypto: crypto@4e00000 { - compatible = "ti,j721-sa2ul"; + compatible = "ti,j721e-sa2ul"; reg = <0x4e00000 0x1200>; power-domains = <&k3_pds 264 TI_SCI_PD_EXCLUSIVE>; dmas = <&main_udmap 0xc000>, <&main_udmap 0x4000>, diff --git a/Documentation/devicetree/bindings/display/allwinner,sun4i-a10-display-backend.yaml b/Documentation/devicetree/bindings/display/allwinner,sun4i-a10-display-backend.yaml index 86057d5410650e400eac24cee01dfdbaf91fdb57..12a7df0e38b2f1f35b1b8de234b5d67ada5e0761 100644 --- a/Documentation/devicetree/bindings/display/allwinner,sun4i-a10-display-backend.yaml +++ b/Documentation/devicetree/bindings/display/allwinner,sun4i-a10-display-backend.yaml @@ -84,36 +84,23 @@ properties: const: dma-mem ports: - type: object - description: | - A ports node with endpoint definitions as defined in - Documentation/devicetree/bindings/media/video-interfaces.txt. + $ref: /schemas/graph.yaml#/properties/ports properties: - "#address-cells": - const: 1 - - "#size-cells": - const: 0 - port@0: - type: object - description: | + $ref: /schemas/graph.yaml#/properties/port + description: Input endpoints of the controller. port@1: - type: object - description: | + $ref: /schemas/graph.yaml#/properties/port + description: Output endpoints of the controller. required: - - "#address-cells" - - "#size-cells" - port@0 - port@1 - additionalProperties: false - required: - compatible - reg diff --git a/Documentation/devicetree/bindings/display/allwinner,sun4i-a10-display-frontend.yaml b/Documentation/devicetree/bindings/display/allwinner,sun4i-a10-display-frontend.yaml index 3eb1c2bbf4e7f976c195f13dba51a507ae8ec641..055157fbf3bf935fdceb1cebf90191f86d1595c7 100644 --- a/Documentation/devicetree/bindings/display/allwinner,sun4i-a10-display-frontend.yaml +++ b/Documentation/devicetree/bindings/display/allwinner,sun4i-a10-display-frontend.yaml @@ -57,35 +57,22 @@ properties: maxItems: 1 ports: - type: object - description: | - A ports node with endpoint definitions as defined in - Documentation/devicetree/bindings/media/video-interfaces.txt. + $ref: /schemas/graph.yaml#/properties/ports properties: - "#address-cells": - const: 1 - - "#size-cells": - const: 0 - port@0: - type: object + $ref: /schemas/graph.yaml#/properties/port description: | Input endpoints of the controller. port@1: - type: object + $ref: /schemas/graph.yaml#/properties/port description: | Output endpoints of the controller. required: - - "#address-cells" - - "#size-cells" - port@1 - additionalProperties: false - required: - compatible - reg diff --git a/Documentation/devicetree/bindings/display/allwinner,sun4i-a10-hdmi.yaml b/Documentation/devicetree/bindings/display/allwinner,sun4i-a10-hdmi.yaml index 75e6479397a5023d781379be1fe760271bde7e35..7f11452539f413e17627d5c098cef77ecc4388df 100644 --- a/Documentation/devicetree/bindings/display/allwinner,sun4i-a10-hdmi.yaml +++ b/Documentation/devicetree/bindings/display/allwinner,sun4i-a10-hdmi.yaml @@ -76,37 +76,24 @@ properties: - const: audio-tx ports: - type: object - description: | - A ports node with endpoint definitions as defined in - Documentation/devicetree/bindings/media/video-interfaces.txt. + $ref: /schemas/graph.yaml#/properties/ports properties: - "#address-cells": - const: 1 - - "#size-cells": - const: 0 - port@0: - type: object + $ref: /schemas/graph.yaml#/properties/port description: | Input endpoints of the controller. port@1: - type: object + $ref: /schemas/graph.yaml#/properties/port description: | Output endpoints of the controller. Usually an HDMI connector. required: - - "#address-cells" - - "#size-cells" - port@0 - port@1 - additionalProperties: false - required: - compatible - reg diff --git a/Documentation/devicetree/bindings/display/allwinner,sun4i-a10-tcon.yaml b/Documentation/devicetree/bindings/display/allwinner,sun4i-a10-tcon.yaml index 4c15a2644a7cfc216e9f4be9e6a542d5de3c7c94..c13faf3e6581ef101a7a2b26fa0db14769edb764 100644 --- a/Documentation/devicetree/bindings/display/allwinner,sun4i-a10-tcon.yaml +++ b/Documentation/devicetree/bindings/display/allwinner,sun4i-a10-tcon.yaml @@ -115,31 +115,24 @@ properties: - const: lvds ports: - type: object - description: | - A ports node with endpoint definitions as defined in - Documentation/devicetree/bindings/media/video-interfaces.txt. + $ref: /schemas/graph.yaml#/properties/ports properties: - "#address-cells": - const: 1 - - "#size-cells": - const: 0 - port@0: - type: object + $ref: /schemas/graph.yaml#/properties/port description: | Input endpoints of the controller. port@1: - type: object + $ref: /schemas/graph.yaml#/$defs/port-base + unevaluatedProperties: false description: | Output endpoints of the controller. patternProperties: "^endpoint(@[0-9])$": - type: object + $ref: /schemas/graph.yaml#/$defs/endpoint-base + unevaluatedProperties: false properties: allwinner,tcon-channel: @@ -156,16 +149,10 @@ properties: property is not present, the endpoint number will be used as the channel number. - unevaluatedProperties: true - required: - - "#address-cells" - - "#size-cells" - port@0 - port@1 - additionalProperties: false - required: - compatible - reg diff --git a/Documentation/devicetree/bindings/display/allwinner,sun4i-a10-tv-encoder.yaml b/Documentation/devicetree/bindings/display/allwinner,sun4i-a10-tv-encoder.yaml index 6009324be967ef517a64b99c0efed92d7862f932..afc0ed799e0ed20200c87735769a27c827c27f3c 100644 --- a/Documentation/devicetree/bindings/display/allwinner,sun4i-a10-tv-encoder.yaml +++ b/Documentation/devicetree/bindings/display/allwinner,sun4i-a10-tv-encoder.yaml @@ -24,11 +24,9 @@ properties: maxItems: 1 port: - type: object + $ref: /schemas/graph.yaml#/properties/port description: - A port node with endpoint definitions as defined in - Documentation/devicetree/bindings/media/video-interfaces.txt. The - first port should be the input endpoint, usually coming from the + The first port should be the input endpoint, usually coming from the associated TCON. required: diff --git a/Documentation/devicetree/bindings/display/allwinner,sun6i-a31-drc.yaml b/Documentation/devicetree/bindings/display/allwinner,sun6i-a31-drc.yaml index 0c1ce55940e1641cfb34530823bbcef307b2376f..71cce5687580612c9cc8a79470b64754a125533d 100644 --- a/Documentation/devicetree/bindings/display/allwinner,sun6i-a31-drc.yaml +++ b/Documentation/devicetree/bindings/display/allwinner,sun6i-a31-drc.yaml @@ -46,36 +46,23 @@ properties: maxItems: 1 ports: - type: object - description: | - A ports node with endpoint definitions as defined in - Documentation/devicetree/bindings/media/video-interfaces.txt. + $ref: /schemas/graph.yaml#/properties/ports properties: - "#address-cells": - const: 1 - - "#size-cells": - const: 0 - port@0: - type: object + $ref: /schemas/graph.yaml#/properties/port description: | Input endpoints of the controller. port@1: - type: object + $ref: /schemas/graph.yaml#/properties/port description: | Output endpoints of the controller. required: - - "#address-cells" - - "#size-cells" - port@0 - port@1 - additionalProperties: false - required: - compatible - reg diff --git a/Documentation/devicetree/bindings/display/allwinner,sun6i-a31-mipi-dsi.yaml b/Documentation/devicetree/bindings/display/allwinner,sun6i-a31-mipi-dsi.yaml index 7aa330dabc446e8823bd81ffeef7f3dd5fb1cb21..a738d7c12a97a481d1cbd97b4ef266413baad0ce 100644 --- a/Documentation/devicetree/bindings/display/allwinner,sun6i-a31-mipi-dsi.yaml +++ b/Documentation/devicetree/bindings/display/allwinner,sun6i-a31-mipi-dsi.yaml @@ -47,11 +47,9 @@ properties: const: dphy port: - type: object + $ref: /schemas/graph.yaml#/properties/port description: - A port node with endpoint definitions as defined in - Documentation/devicetree/bindings/media/video-interfaces.txt. That - port should be the input endpoint, usually coming from the + The port should be the input endpoint, usually coming from the associated TCON. required: diff --git a/Documentation/devicetree/bindings/display/allwinner,sun8i-a83t-de2-mixer.yaml b/Documentation/devicetree/bindings/display/allwinner,sun8i-a83t-de2-mixer.yaml index c040eef5651820d1aef3579382b2cc297740ab96..4f91eec26de9dcf95ed613f12c85994d4a4adae1 100644 --- a/Documentation/devicetree/bindings/display/allwinner,sun8i-a83t-de2-mixer.yaml +++ b/Documentation/devicetree/bindings/display/allwinner,sun8i-a83t-de2-mixer.yaml @@ -43,35 +43,22 @@ properties: maxItems: 1 ports: - type: object - description: | - A ports node with endpoint definitions as defined in - Documentation/devicetree/bindings/media/video-interfaces.txt. + $ref: /schemas/graph.yaml#/properties/ports properties: - "#address-cells": - const: 1 - - "#size-cells": - const: 0 - port@0: - type: object + $ref: /schemas/graph.yaml#/properties/port description: | Input endpoints of the controller. port@1: - type: object + $ref: /schemas/graph.yaml#/properties/port description: | Output endpoints of the controller. required: - - "#address-cells" - - "#size-cells" - port@1 - additionalProperties: false - required: - compatible - reg diff --git a/Documentation/devicetree/bindings/display/allwinner,sun8i-a83t-dw-hdmi.yaml b/Documentation/devicetree/bindings/display/allwinner,sun8i-a83t-dw-hdmi.yaml index fa4769a0b26ec9f838fff75ddc82c0662ecbf2ca..b3e9992525c2ae4f5a0c2ac306074857faa34997 100644 --- a/Documentation/devicetree/bindings/display/allwinner,sun8i-a83t-dw-hdmi.yaml +++ b/Documentation/devicetree/bindings/display/allwinner,sun8i-a83t-dw-hdmi.yaml @@ -93,38 +93,25 @@ properties: The VCC power supply of the controller ports: - type: object - description: | - A ports node with endpoint definitions as defined in - Documentation/devicetree/bindings/media/video-interfaces.txt. + $ref: /schemas/graph.yaml#/properties/ports properties: - "#address-cells": - const: 1 - - "#size-cells": - const: 0 - port@0: - type: object + $ref: /schemas/graph.yaml#/properties/port description: | Input endpoints of the controller. Usually the associated TCON. port@1: - type: object + $ref: /schemas/graph.yaml#/properties/port description: | Output endpoints of the controller. Usually an HDMI connector. required: - - "#address-cells" - - "#size-cells" - port@0 - port@1 - additionalProperties: false - required: - compatible - reg diff --git a/Documentation/devicetree/bindings/display/allwinner,sun8i-r40-tcon-top.yaml b/Documentation/devicetree/bindings/display/allwinner,sun8i-r40-tcon-top.yaml index b98ca609824b92a2e3bfef82251be8b1aadd4221..ec21e8bf2767316ece8d222404a9a6dc61602df2 100644 --- a/Documentation/devicetree/bindings/display/allwinner,sun8i-r40-tcon-top.yaml +++ b/Documentation/devicetree/bindings/display/allwinner,sun8i-r40-tcon-top.yaml @@ -80,141 +80,45 @@ properties: maxItems: 1 ports: - type: object - description: | - A ports node with endpoint definitions as defined in - Documentation/devicetree/bindings/media/video-interfaces.txt. - All ports should have only one endpoint connected to - remote endpoint. + $ref: /schemas/graph.yaml#/properties/ports properties: - "#address-cells": - const: 1 - - "#size-cells": - const: 0 - port@0: - type: object + $ref: /schemas/graph.yaml#/properties/port description: | Input endpoint for Mixer 0 mux. port@1: - type: object + $ref: /schemas/graph.yaml#/properties/port description: | Output endpoint for Mixer 0 mux - properties: - "#address-cells": - const: 1 - - "#size-cells": - const: 0 - - reg: true - - patternProperties: - "^endpoint@[0-9]$": - type: object - - properties: - reg: - description: | - ID of the target TCON - - required: - - reg - - required: - - "#address-cells" - - "#size-cells" - - additionalProperties: false - port@2: - type: object + $ref: /schemas/graph.yaml#/properties/port description: | Input endpoint for Mixer 1 mux. port@3: - type: object + $ref: /schemas/graph.yaml#/properties/port description: | Output endpoint for Mixer 1 mux - properties: - "#address-cells": - const: 1 - - "#size-cells": - const: 0 - - reg: true - - patternProperties: - "^endpoint@[0-9]$": - type: object - - properties: - reg: - description: | - ID of the target TCON - - required: - - reg - - required: - - "#address-cells" - - "#size-cells" - - additionalProperties: false - port@4: - type: object + $ref: /schemas/graph.yaml#/properties/port description: | Input endpoint for HDMI mux. - properties: - "#address-cells": - const: 1 - - "#size-cells": - const: 0 - - reg: true - - patternProperties: - "^endpoint@[0-9]$": - type: object - - properties: - reg: - description: | - ID of the target TCON - - required: - - reg - - required: - - "#address-cells" - - "#size-cells" - - additionalProperties: false - port@5: - type: object + $ref: /schemas/graph.yaml#/properties/port description: | Output endpoint for HDMI mux required: - - "#address-cells" - - "#size-cells" - port@0 - port@1 - port@4 - port@5 - additionalProperties: false - required: - "#clock-cells" - compatible diff --git a/Documentation/devicetree/bindings/display/allwinner,sun9i-a80-deu.yaml b/Documentation/devicetree/bindings/display/allwinner,sun9i-a80-deu.yaml index 96de41d32b3eb741cbbcb5267f34f1ad052d76c9..637372ec4614d42a1211532c628f35d8754f5a88 100644 --- a/Documentation/devicetree/bindings/display/allwinner,sun9i-a80-deu.yaml +++ b/Documentation/devicetree/bindings/display/allwinner,sun9i-a80-deu.yaml @@ -40,36 +40,23 @@ properties: maxItems: 1 ports: - type: object - description: | - A ports node with endpoint definitions as defined in - Documentation/devicetree/bindings/media/video-interfaces.txt. + $ref: /schemas/graph.yaml#/properties/ports properties: - "#address-cells": - const: 1 - - "#size-cells": - const: 0 - port@0: - type: object + $ref: /schemas/graph.yaml#/properties/port description: | Input endpoints of the controller. port@1: - type: object + $ref: /schemas/graph.yaml#/properties/port description: | Output endpoints of the controller. required: - - "#address-cells" - - "#size-cells" - port@0 - port@1 - additionalProperties: false - required: - compatible - reg diff --git a/Documentation/devicetree/bindings/display/amlogic,meson-dw-hdmi.yaml b/Documentation/devicetree/bindings/display/amlogic,meson-dw-hdmi.yaml index 0da42ab8fd3a52b2f9fef7b1ff47919a273c3003..cf5a208f2f1056215e8dfdd57a8eff65138855ef 100644 --- a/Documentation/devicetree/bindings/display/amlogic,meson-dw-hdmi.yaml +++ b/Documentation/devicetree/bindings/display/amlogic,meson-dw-hdmi.yaml @@ -81,12 +81,12 @@ properties: description: phandle to an external 5V regulator to power the HDMI logic port@0: - type: object + $ref: /schemas/graph.yaml#/properties/port description: A port node pointing to the VENC Input port node. port@1: - type: object + $ref: /schemas/graph.yaml#/properties/port description: A port node pointing to the TMDS Output port node. diff --git a/Documentation/devicetree/bindings/display/amlogic,meson-vpu.yaml b/Documentation/devicetree/bindings/display/amlogic,meson-vpu.yaml index a8d202c9d004c67f7009e17d8928cbb415e45355..851cb078121731aff6c86a21dfabd93177727988 100644 --- a/Documentation/devicetree/bindings/display/amlogic,meson-vpu.yaml +++ b/Documentation/devicetree/bindings/display/amlogic,meson-vpu.yaml @@ -83,12 +83,12 @@ properties: description: phandle to the associated power domain port@0: - type: object + $ref: /schemas/graph.yaml#/properties/port description: A port node pointing to the CVBS VDAC port node. port@1: - type: object + $ref: /schemas/graph.yaml#/properties/port description: A port node pointing to the HDMI-TX port node. diff --git a/Documentation/devicetree/bindings/display/brcm,bcm2711-hdmi.yaml b/Documentation/devicetree/bindings/display/brcm,bcm2711-hdmi.yaml index 7ce06f9f9f8ee70d8a3d2d817af4724539eada01..57324a5f0271e1b8ca371bd15a4fbfa9bf8c9f05 100644 --- a/Documentation/devicetree/bindings/display/brcm,bcm2711-hdmi.yaml +++ b/Documentation/devicetree/bindings/display/brcm,bcm2711-hdmi.yaml @@ -53,6 +53,24 @@ properties: - const: audio - const: cec + interrupts: + items: + - description: CEC TX interrupt + - description: CEC RX interrupt + - description: CEC stuck at low interrupt + - description: Wake-up interrupt + - description: Hotplug connected interrupt + - description: Hotplug removed interrupt + + interrupt-names: + items: + - const: cec-tx + - const: cec-rx + - const: cec-low + - const: wakeup + - const: hpd-connected + - const: hpd-removed + ddc: allOf: - $ref: /schemas/types.yaml#/definitions/phandle @@ -60,6 +78,7 @@ properties: Phandle of the I2C controller used for DDC EDID probing hpd-gpios: + maxItems: 1 description: > The GPIO pin for the HDMI hotplug detect (if it doesn't appear as an interrupt/status bit in the HDMI controller itself) @@ -90,7 +109,7 @@ required: - resets - ddc -additionalProperties: false +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/display/brcm,bcm2835-dpi.yaml b/Documentation/devicetree/bindings/display/brcm,bcm2835-dpi.yaml index 5c1024bbc1b30c61d8f0b58e85e69187e7a1cc5f..c9ad0ecc9b6d73acca6cbf5aec65f2b1281a5673 100644 --- a/Documentation/devicetree/bindings/display/brcm,bcm2835-dpi.yaml +++ b/Documentation/devicetree/bindings/display/brcm,bcm2835-dpi.yaml @@ -27,10 +27,9 @@ properties: - const: pixel port: - type: object - description: > - Port node with a single endpoint connecting to the panel, as - defined in Documentation/devicetree/bindings/media/video-interfaces.txt. + $ref: /schemas/graph.yaml#/properties/port + description: + Port node with a single endpoint connecting to the panel. required: - compatible diff --git a/Documentation/devicetree/bindings/display/brcm,bcm2835-dsi0.yaml b/Documentation/devicetree/bindings/display/brcm,bcm2835-dsi0.yaml index eb44e072b6e59d24a8f25b3b880363c88423d3da..55c60919991f7d171fd26af3ce0fa4a61a572f7b 100644 --- a/Documentation/devicetree/bindings/display/brcm,bcm2835-dsi0.yaml +++ b/Documentation/devicetree/bindings/display/brcm,bcm2835-dsi0.yaml @@ -18,6 +18,7 @@ properties: compatible: enum: + - brcm,bcm2711-dsi1 - brcm,bcm2835-dsi0 - brcm,bcm2835-dsi1 diff --git a/Documentation/devicetree/bindings/display/brcm,bcm2835-hdmi.yaml b/Documentation/devicetree/bindings/display/brcm,bcm2835-hdmi.yaml index f54b4e4808f075180d7326063fe2dbbaadf87436..031e35e76db2a812c0894b1375e5280e4ff7aef4 100644 --- a/Documentation/devicetree/bindings/display/brcm,bcm2835-hdmi.yaml +++ b/Documentation/devicetree/bindings/display/brcm,bcm2835-hdmi.yaml @@ -37,6 +37,7 @@ properties: Phandle of the I2C controller used for DDC EDID probing hpd-gpios: + maxItems: 1 description: > The GPIO pin for the HDMI hotplug detect (if it doesn't appear as an interrupt/status bit in the HDMI controller itself) diff --git a/Documentation/devicetree/bindings/display/brcm,bcm2835-hvs.yaml b/Documentation/devicetree/bindings/display/brcm,bcm2835-hvs.yaml index e826ab0adb75dc87665a9550c67439449420adfc..2e8566f47e63c0ea30b7855c0fd8a163e940a935 100644 --- a/Documentation/devicetree/bindings/display/brcm,bcm2835-hvs.yaml +++ b/Documentation/devicetree/bindings/display/brcm,bcm2835-hvs.yaml @@ -36,7 +36,7 @@ if: properties: compatible: contains: - const: brcm,bcm2711-hvs" + const: brcm,bcm2711-hvs then: required: diff --git a/Documentation/devicetree/bindings/display/bridge/analogix,anx7625.yaml b/Documentation/devicetree/bindings/display/bridge/analogix,anx7625.yaml index 9392b5502a3293cce3601be417d4d9aee6796792..c789784efe3062fcee91adcdd7b80d64b6418f10 100644 --- a/Documentation/devicetree/bindings/display/bridge/analogix,anx7625.yaml +++ b/Documentation/devicetree/bindings/display/bridge/analogix,anx7625.yaml @@ -35,16 +35,16 @@ properties: maxItems: 1 ports: - type: object + $ref: /schemas/graph.yaml#/properties/ports properties: port@0: - type: object + $ref: /schemas/graph.yaml#/properties/port description: Video port for MIPI DSI input. port@1: - type: object + $ref: /schemas/graph.yaml#/properties/port description: Video port for panel or connector. diff --git a/Documentation/devicetree/bindings/display/bridge/analogix,anx7814.yaml b/Documentation/devicetree/bindings/display/bridge/analogix,anx7814.yaml index 3ba477aefdd79466869fade87d51b70f61568e12..8e13f27b28edc0c64684c883869ba4885ee479fd 100644 --- a/Documentation/devicetree/bindings/display/bridge/analogix,anx7814.yaml +++ b/Documentation/devicetree/bindings/display/bridge/analogix,anx7814.yaml @@ -42,31 +42,18 @@ properties: description: Regulator for 1.0V digital core power. ports: - type: object - description: - A node containing input and output port nodes with endpoint - definitions as documented in - Documentation/devicetree/bindings/media/video-interfaces.txt - Documentation/devicetree/bindings/graph.txt + $ref: /schemas/graph.yaml#/properties/ports properties: port@0: - type: object + $ref: /schemas/graph.yaml#/properties/port description: Video port for HDMI input. - properties: - reg: - const: 0 - port@1: - type: object + $ref: /schemas/graph.yaml#/properties/port description: Video port for SlimPort, DisplayPort, eDP or MyDP output. - properties: - reg: - const: 1 - required: - port@0 - port@1 diff --git a/Documentation/devicetree/bindings/display/bridge/anx6345.yaml b/Documentation/devicetree/bindings/display/bridge/anx6345.yaml index fccd63521a8ca3fa68ed45d42c39184accabbe3b..1c0406c38fe549ac5fbc1d98950363262496dec3 100644 --- a/Documentation/devicetree/bindings/display/bridge/anx6345.yaml +++ b/Documentation/devicetree/bindings/display/bridge/anx6345.yaml @@ -32,31 +32,23 @@ properties: description: Regulator for 2.5V digital core power. ports: - type: object + $ref: /schemas/graph.yaml#/properties/ports properties: - '#address-cells': - const: 1 - - '#size-cells': - const: 0 - port@0: - type: object - description: | + $ref: /schemas/graph.yaml#/properties/port + description: Video port for LVTTL input port@1: - type: object - description: | + $ref: /schemas/graph.yaml#/properties/port + description: Video port for eDP output (panel or connector). May be omitted if EDID works reliably. required: - port@0 - additionalProperties: false - required: - compatible - reg diff --git a/Documentation/devicetree/bindings/display/bridge/cdns,mhdp8546.yaml b/Documentation/devicetree/bindings/display/bridge/cdns,mhdp8546.yaml index 74d675fc6e7ba65b56f2572e834019a18871f4cb..63427878715e2ba56246e309178a0047b85396a4 100644 --- a/Documentation/devicetree/bindings/display/bridge/cdns,mhdp8546.yaml +++ b/Documentation/devicetree/bindings/display/bridge/cdns,mhdp8546.yaml @@ -57,47 +57,37 @@ properties: maxItems: 1 ports: - type: object - description: - Ports as described in Documentation/devicetree/bindings/graph.txt. + $ref: /schemas/graph.yaml#/properties/ports properties: - '#address-cells': - const: 1 - - '#size-cells': - const: 0 - port@0: - type: object + $ref: /schemas/graph.yaml#/properties/port description: First input port representing the DP bridge input. port@1: - type: object + $ref: /schemas/graph.yaml#/properties/port description: Second input port representing the DP bridge input. port@2: - type: object + $ref: /schemas/graph.yaml#/properties/port description: Third input port representing the DP bridge input. port@3: - type: object + $ref: /schemas/graph.yaml#/properties/port description: Fourth input port representing the DP bridge input. port@4: - type: object + $ref: /schemas/graph.yaml#/properties/port description: Output port representing the DP bridge output. required: - port@0 - port@4 - - '#address-cells' - - '#size-cells' allOf: - if: diff --git a/Documentation/devicetree/bindings/display/bridge/chrontel,ch7033.yaml b/Documentation/devicetree/bindings/display/bridge/chrontel,ch7033.yaml index 9f38f55fc990421ba118383773da7bb324060ab1..bb6289c7d375e79928edcc1f8d649f9849754919 100644 --- a/Documentation/devicetree/bindings/display/bridge/chrontel,ch7033.yaml +++ b/Documentation/devicetree/bindings/display/bridge/chrontel,ch7033.yaml @@ -19,16 +19,16 @@ properties: description: I2C address of the device ports: - type: object + $ref: /schemas/graph.yaml#/properties/ports properties: port@0: - type: object + $ref: /schemas/graph.yaml#/properties/port description: | Video port for RGB input. port@1: - type: object + $ref: /schemas/graph.yaml#/properties/port description: | DVI port, should be connected to a node compatible with the dvi-connector binding. diff --git a/Documentation/devicetree/bindings/display/bridge/intel,keembay-dsi.yaml b/Documentation/devicetree/bindings/display/bridge/intel,keembay-dsi.yaml index 35c9dfd866501a04ae317ddee260d7c78ea02203..dcb1336ee2a56e398ce1df321294693db3dc2a3f 100644 --- a/Documentation/devicetree/bindings/display/bridge/intel,keembay-dsi.yaml +++ b/Documentation/devicetree/bindings/display/bridge/intel,keembay-dsi.yaml @@ -35,29 +35,21 @@ properties: - const: clk_mipi_cfg ports: - type: object + $ref: /schemas/graph.yaml#/properties/ports properties: - '#address-cells': - const: 1 - - '#size-cells': - const: 0 - port@0: - type: object + $ref: /schemas/graph.yaml#/properties/port description: MIPI DSI input port. port@1: - type: object + $ref: /schemas/graph.yaml#/properties/port description: DSI output port. required: - port@0 - port@1 - additionalProperties: false - required: - compatible - reg diff --git a/Documentation/devicetree/bindings/display/bridge/ite,it6505.yaml b/Documentation/devicetree/bindings/display/bridge/ite,it6505.yaml index 02cfc0a3b550289b157a546946a87e85e73bdf3f..833d11b2303a7d11a4f831522c16feb6991e7070 100644 --- a/Documentation/devicetree/bindings/display/bridge/ite,it6505.yaml +++ b/Documentation/devicetree/bindings/display/bridge/ite,it6505.yaml @@ -53,7 +53,7 @@ properties: description: extcon specifier for the Power Delivery port: - type: object + $ref: /schemas/graph.yaml#/properties/port description: A port node pointing to DPI host port node required: diff --git a/Documentation/devicetree/bindings/display/bridge/lontium,lt9611.yaml b/Documentation/devicetree/bindings/display/bridge/lontium,lt9611.yaml index 7a1c89b995e215bb095451ab5daaf54fa4825a36..5b9d36f7af304848ff93168d1b7764ee85339fd6 100644 --- a/Documentation/devicetree/bindings/display/bridge/lontium,lt9611.yaml +++ b/Documentation/devicetree/bindings/display/bridge/lontium,lt9611.yaml @@ -38,82 +38,26 @@ properties: description: Regulator for 3.3V IO power. ports: - type: object + $ref: /schemas/graph.yaml#/properties/ports properties: - "#address-cells": - const: 1 - - "#size-cells": - const: 0 - port@0: - type: object - description: | + $ref: /schemas/graph.yaml#/properties/port + description: Primary MIPI port-1 for MIPI input - properties: - reg: - const: 0 - - patternProperties: - "^endpoint(@[0-9])$": - type: object - additionalProperties: false - - properties: - remote-endpoint: - $ref: /schemas/types.yaml#/definitions/phandle - - required: - - reg - port@1: - type: object - description: | + $ref: /schemas/graph.yaml#/properties/port + description: Additional MIPI port-2 for MIPI input, used in combination with primary MIPI port-1 to drive higher resolution displays - properties: - reg: - const: 1 - - patternProperties: - "^endpoint(@[0-9])$": - type: object - additionalProperties: false - - properties: - remote-endpoint: - $ref: /schemas/types.yaml#/definitions/phandle - - required: - - reg - port@2: - type: object - description: | + $ref: /schemas/graph.yaml#/properties/port + description: HDMI port for HDMI output - properties: - reg: - const: 2 - - patternProperties: - "^endpoint(@[0-9])$": - type: object - additionalProperties: false - - properties: - remote-endpoint: - $ref: /schemas/types.yaml#/definitions/phandle - - required: - - reg - required: - - "#address-cells" - - "#size-cells" - port@0 - port@2 diff --git a/Documentation/devicetree/bindings/display/bridge/lvds-codec.yaml b/Documentation/devicetree/bindings/display/bridge/lvds-codec.yaml index 66a14d60ce1d54b2aafd4594aa140980b93918f9..304a1367faaa728b29b3f121700d5024af17182c 100644 --- a/Documentation/devicetree/bindings/display/bridge/lvds-codec.yaml +++ b/Documentation/devicetree/bindings/display/bridge/lvds-codec.yaml @@ -45,25 +45,17 @@ properties: - thine,thc63lvdm83d # For the THC63LVDM83D LVDS serializer ports: - type: object - description: | - This device has two video ports. Their connections are modeled using the - OF graph bindings specified in Documentation/devicetree/bindings/graph.txt - properties: - '#address-cells': - const: 1 - - '#size-cells': - const: 0 + $ref: /schemas/graph.yaml#/properties/ports + properties: port@0: - type: object + $ref: /schemas/graph.yaml#/properties/port description: | For LVDS encoders, port 0 is the parallel input For LVDS decoders, port 0 is the LVDS input port@1: - type: object + $ref: /schemas/graph.yaml#/properties/port description: | For LVDS encoders, port 1 is the LVDS output For LVDS decoders, port 1 is the parallel output @@ -72,8 +64,6 @@ properties: - port@0 - port@1 - additionalProperties: false - powerdown-gpios: description: The GPIO used to control the power down line of this device. diff --git a/Documentation/devicetree/bindings/display/bridge/nwl-dsi.yaml b/Documentation/devicetree/bindings/display/bridge/nwl-dsi.yaml index a125b2dd3a2f11bb7e80e4ccd5ca225489fcf263..350fb8f400f02265b0cc1c89ff7484db5c36c95c 100644 --- a/Documentation/devicetree/bindings/display/bridge/nwl-dsi.yaml +++ b/Documentation/devicetree/bindings/display/bridge/nwl-dsi.yaml @@ -84,40 +84,23 @@ properties: - const: pclk ports: - type: object - description: - A node containing DSI input & output port nodes with endpoint - definitions as documented in - Documentation/devicetree/bindings/graph.txt. + $ref: /schemas/graph.yaml#/properties/ports + properties: port@0: - type: object + $ref: /schemas/graph.yaml#/$defs/port-base description: Input port node to receive pixel data from the display controller. Exactly one endpoint must be specified. properties: - '#address-cells': - const: 1 - - '#size-cells': - const: 0 - endpoint@0: + $ref: /schemas/graph.yaml#/properties/endpoint description: sub-node describing the input from LCDIF - type: object endpoint@1: + $ref: /schemas/graph.yaml#/properties/endpoint description: sub-node describing the input from DCSS - type: object - - reg: - const: 0 - - required: - - '#address-cells' - - '#size-cells' - - reg oneOf: - required: @@ -125,28 +108,18 @@ properties: - required: - endpoint@1 - additionalProperties: false + unevaluatedProperties: false port@1: - type: object + $ref: /schemas/graph.yaml#/properties/port description: DSI output port node to the panel or the next bridge in the chain - '#address-cells': - const: 1 - - '#size-cells': - const: 0 - required: - - '#address-cells' - - '#size-cells' - port@0 - port@1 - additionalProperties: false - required: - '#address-cells' - '#size-cells' diff --git a/Documentation/devicetree/bindings/display/bridge/ps8640.yaml b/Documentation/devicetree/bindings/display/bridge/ps8640.yaml index 763c7909473efe7bf3aa62ed6818eda598380fca..fce82b605c8bf4e959c4e56e3390945f433e0dbe 100644 --- a/Documentation/devicetree/bindings/display/bridge/ps8640.yaml +++ b/Documentation/devicetree/bindings/display/bridge/ps8640.yaml @@ -41,34 +41,22 @@ properties: description: Regulator for 3.3V digital core power. ports: - type: object - description: - A node containing DSI input & output port nodes with endpoint - definitions as documented in - Documentation/devicetree/bindings/media/video-interfaces.txt - Documentation/devicetree/bindings/graph.txt - properties: - '#address-cells': - const: 1 - - '#size-cells': - const: 0 + $ref: /schemas/graph.yaml#/properties/ports + properties: port@0: - type: object - description: | + $ref: /schemas/graph.yaml#/properties/port + description: Video port for DSI input port@1: - type: object - description: | + $ref: /schemas/graph.yaml#/properties/port + description: Video port for eDP output (panel or connector). required: - port@0 - additionalProperties: false - required: - compatible - reg diff --git a/Documentation/devicetree/bindings/display/bridge/renesas,lvds.yaml b/Documentation/devicetree/bindings/display/bridge/renesas,lvds.yaml index e5b163951b919fed5dd54aa579909cabb4546551..acfc327f70a709ef82a3c6368aa87c913b29bdde 100644 --- a/Documentation/devicetree/bindings/display/bridge/renesas,lvds.yaml +++ b/Documentation/devicetree/bindings/display/bridge/renesas,lvds.yaml @@ -49,33 +49,21 @@ properties: maxItems: 1 ports: - type: object - description: | - This device has two video ports. Their connections are modelled using the - OF graph bindings specified in Documentation/devicetree/bindings/graph.txt. - Each port shall have a single endpoint. + $ref: /schemas/graph.yaml#/properties/ports properties: - '#address-cells': - const: 1 - - '#size-cells': - const: 0 - port@0: - type: object + $ref: /schemas/graph.yaml#/properties/port description: Parallel RGB input port port@1: - type: object + $ref: /schemas/graph.yaml#/properties/port description: LVDS output port required: - port@0 - port@1 - additionalProperties: false - power-domains: maxItems: 1 @@ -83,9 +71,9 @@ properties: $ref: /schemas/types.yaml#/definitions/phandle description: phandle to the companion LVDS encoder. This property is mandatory - for the first LVDS encoder on D3 and E3 SoCs, and shall point to - the second encoder to be used as a companion in dual-link mode. It - shall not be set for any other LVDS encoder. + for the first LVDS encoder on R-Car D3 and E3, and RZ/G2E SoCs, and shall + point to the second encoder to be used as a companion in dual-link mode. + It shall not be set for any other LVDS encoder. required: - compatible diff --git a/Documentation/devicetree/bindings/display/bridge/simple-bridge.yaml b/Documentation/devicetree/bindings/display/bridge/simple-bridge.yaml index 64e8a1c24b402102ab03b028efc5ece510e27cf5..6c7b577fd471111d15621cbcb0ce29b8b20a8fbe 100644 --- a/Documentation/devicetree/bindings/display/bridge/simple-bridge.yaml +++ b/Documentation/devicetree/bindings/display/bridge/simple-bridge.yaml @@ -30,31 +30,21 @@ properties: - ti,ths8135 ports: - type: object - description: | - This device has two video ports. Their connections are modeled using the - OF graph bindings specified in Documentation/devicetree/bindings/graph.txt. - properties: - '#address-cells': - const: 1 - - '#size-cells': - const: 0 + $ref: /schemas/graph.yaml#/properties/ports + properties: port@0: - type: object + $ref: /schemas/graph.yaml#/properties/port description: The bridge input port@1: - type: object + $ref: /schemas/graph.yaml#/properties/port description: The bridge output required: - port@0 - port@1 - additionalProperties: false - enable-gpios: maxItems: 1 description: GPIO controlling bridge enable diff --git a/Documentation/devicetree/bindings/display/bridge/snps,dw-mipi-dsi.yaml b/Documentation/devicetree/bindings/display/bridge/snps,dw-mipi-dsi.yaml index e42cb610f545c112f068766a7a96d8a32c6f9dce..3c3e51af154bc5a0eea9caa93203997a1f8150cf 100644 --- a/Documentation/devicetree/bindings/display/bridge/snps,dw-mipi-dsi.yaml +++ b/Documentation/devicetree/bindings/display/bridge/snps,dw-mipi-dsi.yaml @@ -47,14 +47,15 @@ properties: const: apb ports: - type: object + $ref: /schemas/graph.yaml#/properties/ports properties: port@0: - type: object + $ref: /schemas/graph.yaml#/properties/port description: Input node to receive pixel data. + port@1: - type: object + $ref: /schemas/graph.yaml#/properties/port description: DSI output node to panel. required: diff --git a/Documentation/devicetree/bindings/display/bridge/thine,thc63lvd1024.yaml b/Documentation/devicetree/bindings/display/bridge/thine,thc63lvd1024.yaml index 3d5ce08a5792726e71dea33154939237726d0023..8ae382429d2bff1efe1acc24e33fcd6c22b9aefe 100644 --- a/Documentation/devicetree/bindings/display/bridge/thine,thc63lvd1024.yaml +++ b/Documentation/devicetree/bindings/display/bridge/thine,thc63lvd1024.yaml @@ -25,46 +25,41 @@ properties: const: thine,thc63lvd1024 ports: - type: object + $ref: /schemas/graph.yaml#/properties/ports description: | - This device has four video ports. Their connections are modeled using the - OF graph bindings specified in Documentation/devicetree/bindings/graph.txt. + The device can operate in single or dual input and output modes. - The device can operate in single-link mode or dual-link mode. In - single-link mode, all pixels are received on port@0, and port@1 shall not - contain any endpoint. In dual-link mode, even-numbered pixels are - received on port@0 and odd-numbered pixels on port@1, and both port@0 and - port@1 shall contain endpoints. + When operating in single input mode, all pixels are received on port@0, + and port@1 shall not contain any endpoint. In dual input mode, + even-numbered pixels are received on port@0 and odd-numbered pixels on + port@1, and both port@0 and port@1 shall contain endpoints. - properties: - '#address-cells': - const: 1 - - '#size-cells': - const: 0 + When operating in single output mode all pixels are output from the first + CMOS/TTL port and port@3 shall not contain any endpoint. In dual output + mode pixels are output from both CMOS/TTL ports and both port@2 and + port@3 shall contain endpoints. + properties: port@0: - type: object + $ref: /schemas/graph.yaml#/properties/port description: First LVDS input port port@1: - type: object + $ref: /schemas/graph.yaml#/properties/port description: Second LVDS input port port@2: - type: object + $ref: /schemas/graph.yaml#/properties/port description: First digital CMOS/TTL parallel output port@3: - type: object + $ref: /schemas/graph.yaml#/properties/port description: Second digital CMOS/TTL parallel output required: - port@0 - port@2 - additionalProperties: false - oe-gpios: maxItems: 1 description: Output enable GPIO signal, pin name "OE", active high. diff --git a/Documentation/devicetree/bindings/display/bridge/ti,sn65dsi86.yaml b/Documentation/devicetree/bindings/display/bridge/ti,sn65dsi86.yaml index f8622bd0f61ee62d232af4e5417f30e8fb564d61..26932d2e86aba5b5b791a4183a042c446fdce3a6 100644 --- a/Documentation/devicetree/bindings/display/bridge/ti,sn65dsi86.yaml +++ b/Documentation/devicetree/bindings/display/bridge/ti,sn65dsi86.yaml @@ -71,54 +71,26 @@ properties: description: See ../../pwm/pwm.yaml for description of the cell formats. ports: - type: object - additionalProperties: false + $ref: /schemas/graph.yaml#/properties/ports properties: - "#address-cells": - const: 1 - - "#size-cells": - const: 0 - port@0: - type: object - additionalProperties: false - + $ref: /schemas/graph.yaml#/properties/port description: Video port for MIPI DSI input - properties: - reg: - const: 0 - - endpoint: - type: object - additionalProperties: false - properties: - remote-endpoint: true - - required: - - reg - port@1: - type: object - additionalProperties: false - + $ref: /schemas/graph.yaml#/$defs/port-base + unevaluatedProperties: false description: Video port for eDP output (panel or connector). properties: - reg: - const: 1 - endpoint: - type: object - additionalProperties: false + $ref: /schemas/graph.yaml#/$defs/endpoint-base + unevaluatedProperties: false properties: - remote-endpoint: true - data-lanes: oneOf: - minItems: 1 @@ -171,12 +143,7 @@ properties: dependencies: lane-polarities: [data-lanes] - required: - - reg - required: - - "#address-cells" - - "#size-cells" - port@0 - port@1 diff --git a/Documentation/devicetree/bindings/display/bridge/ti,tfp410.yaml b/Documentation/devicetree/bindings/display/bridge/ti,tfp410.yaml index 605831c1e8366eb045b04460cfeaaa4ecec04b87..4c5dd8ec2951b1836c01a399ca82f81fb145ad69 100644 --- a/Documentation/devicetree/bindings/display/bridge/ti,tfp410.yaml +++ b/Documentation/devicetree/bindings/display/bridge/ti,tfp410.yaml @@ -31,23 +31,18 @@ properties: maximum: 7 ports: - description: - A node containing input and output port nodes with endpoint - definitions as documented in - Documentation/devicetree/bindings/media/video-interfaces.txt - type: object + $ref: /schemas/graph.yaml#/properties/ports properties: port@0: + $ref: /schemas/graph.yaml#/$defs/port-base + unevaluatedProperties: false description: DPI input port. - type: object properties: - reg: - const: 0 - endpoint: - type: object + $ref: /schemas/graph.yaml#/$defs/endpoint-base + unevaluatedProperties: false properties: pclk-sample: @@ -67,15 +62,8 @@ properties: default: 24 port@1: + $ref: /schemas/graph.yaml#/properties/port description: DVI output port. - type: object - - properties: - reg: - const: 1 - - endpoint: - type: object required: - port@0 diff --git a/Documentation/devicetree/bindings/display/bridge/toshiba,tc358762.yaml b/Documentation/devicetree/bindings/display/bridge/toshiba,tc358762.yaml index 195025e6803cdd4804d7962a59f8247f29f0eba4..5216c27fc0ada58881235761037162c5873ed0c6 100644 --- a/Documentation/devicetree/bindings/display/bridge/toshiba,tc358762.yaml +++ b/Documentation/devicetree/bindings/display/bridge/toshiba,tc358762.yaml @@ -25,62 +25,20 @@ properties: description: Regulator for 1.2V internal core power. ports: - type: object + $ref: /schemas/graph.yaml#/properties/ports properties: - "#address-cells": - const: 1 - - "#size-cells": - const: 0 - port@0: - type: object - additionalProperties: false - - description: | + $ref: /schemas/graph.yaml#/properties/port + description: Video port for MIPI DSI input - properties: - reg: - const: 0 - - patternProperties: - endpoint: - type: object - additionalProperties: false - - properties: - remote-endpoint: true - - required: - - reg - port@1: - type: object - additionalProperties: false - - description: | + $ref: /schemas/graph.yaml#/properties/port + description: Video port for MIPI DPI output (panel or connector). - properties: - reg: - const: 1 - - patternProperties: - endpoint: - type: object - additionalProperties: false - - properties: - remote-endpoint: true - - required: - - reg - required: - - "#address-cells" - - "#size-cells" - port@0 - port@1 diff --git a/Documentation/devicetree/bindings/display/bridge/toshiba,tc358768.yaml b/Documentation/devicetree/bindings/display/bridge/toshiba,tc358768.yaml index c036a75db8f7ce29391ee5ae464c1262bd608c5b..eacfe716508327fd2b791b2c9aa0d75a3e175192 100644 --- a/Documentation/devicetree/bindings/display/bridge/toshiba,tc358768.yaml +++ b/Documentation/devicetree/bindings/display/bridge/toshiba,tc358768.yaml @@ -42,65 +42,30 @@ properties: const: refclk ports: - type: object + $ref: /schemas/graph.yaml#/properties/ports properties: - "#address-cells": - const: 1 - - "#size-cells": - const: 0 - port@0: - type: object - additionalProperties: false - + $ref: /schemas/graph.yaml#/$defs/port-base + unevaluatedProperties: false description: | Video port for RGB input properties: - reg: - const: 0 - - patternProperties: endpoint: - type: object - additionalProperties: false + $ref: /schemas/graph.yaml#/$defs/endpoint-base + unevaluatedProperties: false properties: data-lines: enum: [ 16, 18, 24 ] - remote-endpoint: true - - required: - - reg - port@1: - type: object - additionalProperties: false - + $ref: /schemas/graph.yaml#/properties/port description: | Video port for DSI output (panel or connector). - properties: - reg: - const: 1 - - patternProperties: - endpoint: - type: object - additionalProperties: false - - properties: - remote-endpoint: true - - required: - - reg - required: - - "#address-cells" - - "#size-cells" - port@0 - port@1 @@ -156,4 +121,3 @@ examples: }; }; }; - diff --git a/Documentation/devicetree/bindings/display/bridge/toshiba,tc358775.yaml b/Documentation/devicetree/bindings/display/bridge/toshiba,tc358775.yaml index b5959cc78b8d34e1ce51772b8dbc29532750d653..10471c6c1ff91b2681cec910da55ab1bdf2bdf9a 100644 --- a/Documentation/devicetree/bindings/display/bridge/toshiba,tc358775.yaml +++ b/Documentation/devicetree/bindings/display/bridge/toshiba,tc358775.yaml @@ -42,31 +42,22 @@ properties: description: Hardware reset, Low active ports: - type: object - description: - A node containing input and output port nodes with endpoint definitions - as documented in - Documentation/devicetree/bindings/media/video-interfaces.txt - properties: - "#address-cells": - const: 1 - - "#size-cells": - const: 0 + $ref: /schemas/graph.yaml#/properties/ports + properties: port@0: - type: object + $ref: /schemas/graph.yaml#/properties/port description: | DSI Input. The remote endpoint phandle should be a reference to a valid mipi_dsi_host device node. port@1: - type: object + $ref: /schemas/graph.yaml#/properties/port description: | Video port for LVDS output (panel or connector). port@2: - type: object + $ref: /schemas/graph.yaml#/properties/port description: | Video port for Dual link LVDS output (panel or connector). diff --git a/Documentation/devicetree/bindings/display/connector/analog-tv-connector.yaml b/Documentation/devicetree/bindings/display/connector/analog-tv-connector.yaml index eebe88fed999e9f269a9a1a732d1e5c61f3c5940..a31ca2d52b868f16e04d06e5e3ca7e2f38571ad4 100644 --- a/Documentation/devicetree/bindings/display/connector/analog-tv-connector.yaml +++ b/Documentation/devicetree/bindings/display/connector/analog-tv-connector.yaml @@ -25,6 +25,7 @@ properties: $ref: /schemas/types.yaml#/definitions/uint32 port: + $ref: /schemas/graph.yaml#/properties/port description: Connection to controller providing analog TV signals required: diff --git a/Documentation/devicetree/bindings/display/connector/dp-connector.yaml b/Documentation/devicetree/bindings/display/connector/dp-connector.yaml new file mode 100644 index 0000000000000000000000000000000000000000..22792a79e7ce5177113f17ab472048c220dbb3c3 --- /dev/null +++ b/Documentation/devicetree/bindings/display/connector/dp-connector.yaml @@ -0,0 +1,55 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/display/connector/dp-connector.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: DisplayPort Connector + +maintainers: + - Tomi Valkeinen + +properties: + compatible: + const: dp-connector + + label: true + + type: + enum: + - full-size + - mini + + hpd-gpios: + description: A GPIO line connected to HPD + maxItems: 1 + + dp-pwr-supply: + description: Power supply for the DP_PWR pin + + port: + $ref: /schemas/graph.yaml#/properties/port + description: Connection to controller providing DP signals + +required: + - compatible + - type + - port + +additionalProperties: false + +examples: + - | + connector { + compatible = "dp-connector"; + label = "dp0"; + type = "full-size"; + + port { + dp_connector_in: endpoint { + remote-endpoint = <&dp_out>; + }; + }; + }; + +... diff --git a/Documentation/devicetree/bindings/display/connector/dvi-connector.yaml b/Documentation/devicetree/bindings/display/connector/dvi-connector.yaml index 71cb9220fa593e500df65b09c67e50d4611b91ac..93eb14294e6888d8cf22043a80b18146deb8b5c5 100644 --- a/Documentation/devicetree/bindings/display/connector/dvi-connector.yaml +++ b/Documentation/devicetree/bindings/display/connector/dvi-connector.yaml @@ -36,6 +36,7 @@ properties: description: the connector has pins for DVI dual-link port: + $ref: /schemas/graph.yaml#/properties/port description: Connection to controller providing DVI signals required: diff --git a/Documentation/devicetree/bindings/display/connector/hdmi-connector.yaml b/Documentation/devicetree/bindings/display/connector/hdmi-connector.yaml index 14d7128af592a98fadd532d2266dec3f81a0641c..83c0d008265b349f2e6ba529bbfe9b418c2d79f4 100644 --- a/Documentation/devicetree/bindings/display/connector/hdmi-connector.yaml +++ b/Documentation/devicetree/bindings/display/connector/hdmi-connector.yaml @@ -37,6 +37,7 @@ properties: maxItems: 1 port: + $ref: /schemas/graph.yaml#/properties/port description: Connection to controller providing HDMI signals required: diff --git a/Documentation/devicetree/bindings/display/connector/vga-connector.yaml b/Documentation/devicetree/bindings/display/connector/vga-connector.yaml index 5782c4bb3252ba43bde59e1b514d8b83210b3b4f..25f86800200052ad78db03c4258142361876540c 100644 --- a/Documentation/devicetree/bindings/display/connector/vga-connector.yaml +++ b/Documentation/devicetree/bindings/display/connector/vga-connector.yaml @@ -20,6 +20,7 @@ properties: $ref: /schemas/types.yaml#/definitions/phandle port: + $ref: /schemas/graph.yaml#/properties/port description: Connection to controller providing VGA signals required: diff --git a/Documentation/devicetree/bindings/display/ht16k33.txt b/Documentation/devicetree/bindings/display/ht16k33.txt deleted file mode 100644 index d5a8b070b46779a410062a624ba9af0b498aaf18..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/display/ht16k33.txt +++ /dev/null @@ -1,40 +0,0 @@ -Holtek ht16k33 RAM mapping 16*8 LED controller driver with keyscan -------------------------------------------------------------------------------- - -Required properties: -- compatible: "holtek,ht16k33" -- reg: I2C slave address of the chip. -- interrupts: Interrupt specification for the key pressed interrupt. -- refresh-rate-hz: Display update interval in HZ. -- debounce-delay-ms: Debouncing interval time in milliseconds. -- linux,keymap: The keymap for keys as described in the binding - document (devicetree/bindings/input/matrix-keymap.txt). - -Optional properties: -- linux,no-autorepeat: Disable keyrepeat. -- default-brightness-level: Initial brightness level [0-15] (default: 15). - -Example: - -&i2c1 { - ht16k33: ht16k33@70 { - compatible = "holtek,ht16k33"; - reg = <0x70>; - refresh-rate-hz = <20>; - debounce-delay-ms = <50>; - interrupt-parent = <&gpio4>; - interrupts = <5 (IRQ_TYPE_LEVEL_HIGH | IRQ_TYPE_EDGE_RISING)>; - linux,keymap = < - MATRIX_KEY(2, 0, KEY_F6) - MATRIX_KEY(3, 0, KEY_F8) - MATRIX_KEY(4, 0, KEY_F10) - MATRIX_KEY(5, 0, KEY_F4) - MATRIX_KEY(6, 0, KEY_F2) - MATRIX_KEY(2, 1, KEY_F5) - MATRIX_KEY(3, 1, KEY_F7) - MATRIX_KEY(4, 1, KEY_F9) - MATRIX_KEY(5, 1, KEY_F3) - MATRIX_KEY(6, 1, KEY_F1) - >; - }; -}; diff --git a/Documentation/devicetree/bindings/display/imx/nxp,imx8mq-dcss.yaml b/Documentation/devicetree/bindings/display/imx/nxp,imx8mq-dcss.yaml index f1f25aa794d935bb6dbbe8bdd4ebb80d589a9fb6..0091df9dd73b72974b003bb825b78161da79f077 100644 --- a/Documentation/devicetree/bindings/display/imx/nxp,imx8mq-dcss.yaml +++ b/Documentation/devicetree/bindings/display/imx/nxp,imx8mq-dcss.yaml @@ -74,7 +74,7 @@ properties: - description: Must be 400 MHz port: - type: object + $ref: /schemas/graph.yaml#/properties/port description: A port node pointing to the input port of a HDMI/DP or MIPI display bridge. diff --git a/Documentation/devicetree/bindings/display/ingenic,ipu.yaml b/Documentation/devicetree/bindings/display/ingenic,ipu.yaml index 12064a8e7a92a09de004d31a8abd1abd2794bc3b..e679f48a38862f27c811a5687464092a5abbfae8 100644 --- a/Documentation/devicetree/bindings/display/ingenic,ipu.yaml +++ b/Documentation/devicetree/bindings/display/ingenic,ipu.yaml @@ -31,9 +31,8 @@ properties: clock-names: const: ipu -patternProperties: - "^ports?$": - description: OF graph bindings (specified in bindings/graph.txt). + port: + $ref: /schemas/graph.yaml#/properties/port required: - compatible diff --git a/Documentation/devicetree/bindings/display/ingenic,lcd.yaml b/Documentation/devicetree/bindings/display/ingenic,lcd.yaml index 768050f30dbaf1287bc2fd8ae32574670cfd8e4b..50d2b0a50e8ac7b400a8228f6432c619c8574b83 100644 --- a/Documentation/devicetree/bindings/display/ingenic,lcd.yaml +++ b/Documentation/devicetree/bindings/display/ingenic,lcd.yaml @@ -39,18 +39,18 @@ properties: minItems: 1 port: - description: OF graph bindings (specified in bindings/graph.txt). + $ref: /schemas/graph.yaml#/properties/port ports: - description: OF graph bindings (specified in bindings/graph.txt). - type: object + $ref: /schemas/graph.yaml#/properties/ports + properties: port@0: - type: object + $ref: /schemas/graph.yaml#/properties/port description: DPI output, to interface with TFT panels. port@8: - type: object + $ref: /schemas/graph.yaml#/properties/port description: Link to the Image Processing Unit (IPU). (See ingenic,ipu.yaml). diff --git a/Documentation/devicetree/bindings/display/intel,keembay-display.yaml b/Documentation/devicetree/bindings/display/intel,keembay-display.yaml index 0a697d45c2ad9c9864883ce3b7c0ee854dc8e05a..bc6622b010cabf282b8c58ca140a3bd4945ea8aa 100644 --- a/Documentation/devicetree/bindings/display/intel,keembay-display.yaml +++ b/Documentation/devicetree/bindings/display/intel,keembay-display.yaml @@ -36,7 +36,7 @@ properties: maxItems: 1 port: - type: object + $ref: /schemas/graph.yaml#/properties/port description: Display output node to DSI. required: diff --git a/Documentation/devicetree/bindings/display/mediatek/mediatek,disp.txt b/Documentation/devicetree/bindings/display/mediatek/mediatek,disp.txt index ed76332ec01e830dd0607a000afb6c4c34e5dedd..93b160df3eec4b684c15602cd86a34c2fe047aef 100644 --- a/Documentation/devicetree/bindings/display/mediatek/mediatek,disp.txt +++ b/Documentation/devicetree/bindings/display/mediatek/mediatek,disp.txt @@ -37,13 +37,14 @@ Required properties (all function blocks): "mediatek,-disp-aal" - adaptive ambient light controller "mediatek,-disp-gamma" - gamma correction "mediatek,-disp-merge" - merge streams from two RDMA sources + "mediatek,-disp-postmask" - control round corner for display frame "mediatek,-disp-split" - split stream to two encoders "mediatek,-disp-ufoe" - data compression engine "mediatek,-dsi" - DSI controller, see mediatek,dsi.txt "mediatek,-dpi" - DPI controller, see mediatek,dpi.txt "mediatek,-disp-mutex" - display mutex "mediatek,-disp-od" - overdrive - the supported chips are mt2701, mt7623, mt2712, mt8167 and mt8173. + the supported chips are mt2701, mt7623, mt2712, mt8167, mt8173, mt8183 and mt8192. - reg: Physical base address and length of the function block register space - interrupts: The interrupt signal from the function block (required, except for merge and split function blocks). @@ -66,6 +67,14 @@ Required properties (DMA function blocks): argument, see Documentation/devicetree/bindings/iommu/mediatek,iommu.txt for details. +Optional properties (RDMA function blocks): +- mediatek,rdma-fifo-size: rdma fifo size may be different even in same SOC, add this + property to the corresponding rdma + the value is the Max value which defined in hardware data sheet. + mediatek,rdma-fifo-size of mt8173-rdma0 is 8K + mediatek,rdma-fifo-size of mt8183-rdma0 is 5K + mediatek,rdma-fifo-size of mt8183-rdma1 is 2K + Examples: mmsys: clock-controller@14000000 { @@ -103,6 +112,7 @@ rdma0: rdma@1400e000 { clocks = <&mmsys CLK_MM_DISP_RDMA0>; iommus = <&iommu M4U_PORT_DISP_RDMA0>; mediatek,larb = <&larb0>; + mediatek,rdma-fifosize = <8192>; }; rdma1: rdma@1400f000 { diff --git a/Documentation/devicetree/bindings/display/mediatek/mediatek,dsi.txt b/Documentation/devicetree/bindings/display/mediatek/mediatek,dsi.txt index f06f24d405a5aab3a9e0ef06154103082aa10ad7..8238a86686bec7ef85be27be2888763ddaf3eb1b 100644 --- a/Documentation/devicetree/bindings/display/mediatek/mediatek,dsi.txt +++ b/Documentation/devicetree/bindings/display/mediatek/mediatek,dsi.txt @@ -22,23 +22,7 @@ Required properties: MIPI TX Configuration Module ============================ -The MIPI TX configuration module controls the MIPI D-PHY. - -Required properties: -- compatible: "mediatek,-mipi-tx" -- the supported chips are mt2701, 7623, mt8173 and mt8183. -- reg: Physical base address and length of the controller's registers -- clocks: PLL reference clock -- clock-output-names: name of the output clock line to the DSI encoder -- #clock-cells: must be <0>; -- #phy-cells: must be <0>. - -Optional properties: -- drive-strength-microamp: adjust driving current, should be 3000 ~ 6000. And - the step is 200. -- nvmem-cells: A phandle to the calibration data provided by a nvmem device. If - unspecified default values shall be used. -- nvmem-cell-names: Should be "calibration-data" +See phy/mediatek,dsi-phy.yaml Example: diff --git a/Documentation/devicetree/bindings/display/mediatek/mediatek,hdmi.txt b/Documentation/devicetree/bindings/display/mediatek/mediatek,hdmi.txt index 6b1c586403e44cdc29a9da779ac162fa6e13cbba..b284ca51b9135308c61c55ff2adfb62ba4e14038 100644 --- a/Documentation/devicetree/bindings/display/mediatek/mediatek,hdmi.txt +++ b/Documentation/devicetree/bindings/display/mediatek/mediatek,hdmi.txt @@ -53,23 +53,7 @@ Required properties: HDMI PHY ======== - -The HDMI PHY serializes the HDMI encoder's three channel 10-bit parallel -output and drives the HDMI pads. - -Required properties: -- compatible: "mediatek,-hdmi-phy" -- the supported chips are mt2701, mt7623 and mt8173 -- reg: Physical base address and length of the module's registers -- clocks: PLL reference clock -- clock-names: must contain "pll_ref" -- clock-output-names: must be "hdmitx_dig_cts" on mt8173 -- #phy-cells: must be <0> -- #clock-cells: must be <0> - -Optional properties: -- mediatek,ibias: TX DRV bias current for <1.65Gbps, defaults to 0xa -- mediatek,ibias_up: TX DRV bias current for >1.65Gbps, defaults to 0x1c +See phy/mediatek,hdmi-phy.yaml Example: diff --git a/Documentation/devicetree/bindings/display/panel/advantech,idk-2121wr.yaml b/Documentation/devicetree/bindings/display/panel/advantech,idk-2121wr.yaml index 6b7fddc80c412702c706c367db005ddcdb1fe71f..67682fe77f1014216144d935c51db2424ba5bcb7 100644 --- a/Documentation/devicetree/bindings/display/panel/advantech,idk-2121wr.yaml +++ b/Documentation/devicetree/bindings/display/panel/advantech,idk-2121wr.yaml @@ -37,34 +37,33 @@ properties: panel-timing: true ports: - type: object + $ref: /schemas/graph.yaml#/properties/ports + properties: port@0: - type: object + $ref: /schemas/graph.yaml#/$defs/port-base + unevaluatedProperties: false description: The sink for odd pixels. properties: - reg: - const: 0 - dual-lvds-odd-pixels: true required: - - reg - dual-lvds-odd-pixels port@1: - type: object + $ref: /schemas/graph.yaml#/$defs/port-base + unevaluatedProperties: false description: The sink for even pixels. properties: - reg: - const: 1 - dual-lvds-even-pixels: true required: - - reg - dual-lvds-even-pixels + required: + - port@0 + - port@1 + additionalProperties: false required: diff --git a/Documentation/devicetree/bindings/display/panel/jdi,lt070me05000.yaml b/Documentation/devicetree/bindings/display/panel/jdi,lt070me05000.yaml index b8b9435e464cab4f8dd285d071a51dce5a32a7cd..4f92365e888aeefb1730f8942ebfc3dafce747d1 100644 --- a/Documentation/devicetree/bindings/display/panel/jdi,lt070me05000.yaml +++ b/Documentation/devicetree/bindings/display/panel/jdi,lt070me05000.yaml @@ -30,6 +30,7 @@ properties: power supply for LCM (1.8V) dcdc-en-gpios: + maxItems: 1 description: | phandle of the gpio for power ic line Power IC supply enable, High active diff --git a/Documentation/devicetree/bindings/display/panel/mantix,mlaf057we51-x.yaml b/Documentation/devicetree/bindings/display/panel/mantix,mlaf057we51-x.yaml index 51f423297ec8447b46575016d8b428644ba4822d..a4b8569ab81c959732faad79afabfb466e4d7ab7 100644 --- a/Documentation/devicetree/bindings/display/panel/mantix,mlaf057we51-x.yaml +++ b/Documentation/devicetree/bindings/display/panel/mantix,mlaf057we51-x.yaml @@ -20,6 +20,7 @@ properties: compatible: enum: - mantix,mlaf057we51-x + - ys,ys57pss36bh5gq port: true reg: @@ -37,7 +38,8 @@ properties: reset-gpios: true - 'mantix,tp-rstn-gpios': + mantix,tp-rstn-gpios: + maxItems: 1 description: second reset line that triggers DSI config load backlight: true diff --git a/Documentation/devicetree/bindings/display/panel/novatek,nt36672a.yaml b/Documentation/devicetree/bindings/display/panel/novatek,nt36672a.yaml index 2f5df1d235aea8a30a5d9273a4e3c7142fcaa07a..ef4c0a24512dc81ee193240754211090c3414e7a 100644 --- a/Documentation/devicetree/bindings/display/panel/novatek,nt36672a.yaml +++ b/Documentation/devicetree/bindings/display/panel/novatek,nt36672a.yaml @@ -30,6 +30,7 @@ properties: panel. The novatek,nt36672a compatible shall always be provided as a fallback. reset-gpios: + maxItems: 1 description: phandle of gpio for reset line - This should be 8mA, gpio can be configured using mux, pinctrl, pinctrl-names (active high) diff --git a/Documentation/devicetree/bindings/display/panel/panel-common.yaml b/Documentation/devicetree/bindings/display/panel/panel-common.yaml index cd6dc5461721d7504a0e4722f3550f7f36f798a8..5b38dc89cb21bc9e90df321609cf81b6395ac460 100644 --- a/Documentation/devicetree/bindings/display/panel/panel-common.yaml +++ b/Documentation/devicetree/bindings/display/panel/panel-common.yaml @@ -68,16 +68,7 @@ properties: # Connectivity port: - type: object - - ports: - type: object - description: - Panels receive video data through one or multiple connections. While - the nature of those connections is specific to the panel type, the - connectivity is expressed in a standard fashion using ports as specified - in the device graph bindings defined in - Documentation/devicetree/bindings/graph.txt. + $ref: /schemas/graph.yaml#/properties/port ddc-i2c-bus: $ref: /schemas/types.yaml#/definitions/phandle diff --git a/Documentation/devicetree/bindings/display/panel/panel-simple-dsi.yaml b/Documentation/devicetree/bindings/display/panel/panel-simple-dsi.yaml index 72e4b6d4d5e1e517153575b9f15ee40c7fbf30af..fbd71669248f71c4b6b49d0b802405e019b812dd 100644 --- a/Documentation/devicetree/bindings/display/panel/panel-simple-dsi.yaml +++ b/Documentation/devicetree/bindings/display/panel/panel-simple-dsi.yaml @@ -35,6 +35,8 @@ properties: - boe,tv080wum-nl0 # Innolux P079ZCA 7.85" 768x1024 TFT LCD panel - innolux,p079zca + # Khadas TS050 5" 1080x1920 LCD panel + - khadas,ts050 # Kingdisplay KD097D04 9.7" 1536x2048 TFT LCD panel - kingdisplay,kd097d04 # LG ACX467AKM-7 4.95" 1080×1920 LCD Panel diff --git a/Documentation/devicetree/bindings/display/panel/panel-simple.yaml b/Documentation/devicetree/bindings/display/panel/panel-simple.yaml index 3207608d1178315f3379254263b87f2d153ebb3a..62b0d54d87b7f0e2c6c32c3cd7320da1640a9bad 100644 --- a/Documentation/devicetree/bindings/display/panel/panel-simple.yaml +++ b/Documentation/devicetree/bindings/display/panel/panel-simple.yaml @@ -76,6 +76,8 @@ properties: # BOE OPTOELECTRONICS TECHNOLOGY 10.1" WXGA TFT LCD panel - boe,nv101wxmn51 # BOE NV133FHM-N61 13.3" FHD (1920x1080) TFT LCD Panel + - boe,nv110wtm-n61 + # BOE NV110WTM-N61 11.0" 2160x1440 TFT LCD Panel - boe,nv133fhm-n61 # BOE NV133FHM-N62 13.3" FHD (1920x1080) TFT LCD Panel - boe,nv133fhm-n62 diff --git a/Documentation/devicetree/bindings/display/panel/samsung,s6e63m0.yaml b/Documentation/devicetree/bindings/display/panel/samsung,s6e63m0.yaml index 1dab80ae1d0a12e9499315706f33af2ccac84ccb..ea58df49263ac3bd08bc55a42a2fd1c374ad9b70 100644 --- a/Documentation/devicetree/bindings/display/panel/samsung,s6e63m0.yaml +++ b/Documentation/devicetree/bindings/display/panel/samsung,s6e63m0.yaml @@ -11,6 +11,7 @@ maintainers: allOf: - $ref: panel-common.yaml# + - $ref: /schemas/leds/backlight/common.yaml# properties: compatible: @@ -19,6 +20,8 @@ properties: reg: true reset-gpios: true port: true + default-brightness: true + max-brightness: true vdd3-supply: description: VDD regulator @@ -31,7 +34,6 @@ required: - reset-gpios - vdd3-supply - vci-supply - - port unevaluatedProperties: false diff --git a/Documentation/devicetree/bindings/display/rockchip/rockchip,rk3066-hdmi.yaml b/Documentation/devicetree/bindings/display/rockchip/rockchip,rk3066-hdmi.yaml index 4110d003ce1f594748f049ec6beb38f0d6087582..008c144257cbb0cc2e2e3933f987eff7f81b390c 100644 --- a/Documentation/devicetree/bindings/display/rockchip/rockchip,rk3066-hdmi.yaml +++ b/Documentation/devicetree/bindings/display/rockchip/rockchip,rk3066-hdmi.yaml @@ -43,34 +43,24 @@ properties: This soc uses GRF regs to switch the HDMI TX input between vop0 and vop1. ports: - type: object + $ref: /schemas/graph.yaml#/properties/ports properties: - "#address-cells": - const: 1 - - "#size-cells": - const: 0 - port@0: - type: object + $ref: /schemas/graph.yaml#/properties/port description: Port node with two endpoints, numbered 0 and 1, connected respectively to vop0 and vop1. port@1: - type: object + $ref: /schemas/graph.yaml#/properties/port description: Port node with one endpoint connected to a hdmi-connector node. required: - - "#address-cells" - - "#size-cells" - port@0 - port@1 - additionalProperties: false - required: - compatible - reg diff --git a/Documentation/devicetree/bindings/display/rockchip/rockchip-vop.yaml b/Documentation/devicetree/bindings/display/rockchip/rockchip-vop.yaml index ed8148e26e24df8b0c4405ead1edd050087591e1..6f43d885c9b30892ccdd7d3414fc7224f577962f 100644 --- a/Documentation/devicetree/bindings/display/rockchip/rockchip-vop.yaml +++ b/Documentation/devicetree/bindings/display/rockchip/rockchip-vop.yaml @@ -70,10 +70,7 @@ properties: - const: dclk port: - type: object - description: - A port node with endpoint definitions as defined in - Documentation/devicetree/bindings/media/video-interfaces.txt. + $ref: /schemas/graph.yaml#/properties/port assigned-clocks: maxItems: 2 diff --git a/Documentation/devicetree/bindings/display/st,stm32-dsi.yaml b/Documentation/devicetree/bindings/display/st,stm32-dsi.yaml index 327a14d85df85e6fba3c9bcdd1405e68434ba3b6..679daed4124ebc1ccc10a2cacc7744153e49cbd8 100644 --- a/Documentation/devicetree/bindings/display/st,stm32-dsi.yaml +++ b/Documentation/devicetree/bindings/display/st,stm32-dsi.yaml @@ -51,20 +51,16 @@ properties: Phandle of the regulator that provides the supply voltage. ports: - type: object - description: - A node containing DSI input & output port nodes with endpoint - definitions as documented in - Documentation/devicetree/bindings/media/video-interfaces.txt - Documentation/devicetree/bindings/graph.txt + $ref: /schemas/graph.yaml#/properties/ports + properties: port@0: - type: object + $ref: /schemas/graph.yaml#/properties/port description: DSI input port node, connected to the ltdc rgb output port. port@1: - type: object + $ref: /schemas/graph.yaml#/properties/port description: DSI output port node, connected to a panel or a bridge input port" diff --git a/Documentation/devicetree/bindings/display/st,stm32-ltdc.yaml b/Documentation/devicetree/bindings/display/st,stm32-ltdc.yaml index bf8ad916e9b0a76c495b11ea787c52f7de56d1d1..d54f9ca207afaa9b4d851b93421fc5c9f5ffd343 100644 --- a/Documentation/devicetree/bindings/display/st,stm32-ltdc.yaml +++ b/Documentation/devicetree/bindings/display/st,stm32-ltdc.yaml @@ -35,15 +35,13 @@ properties: maxItems: 1 port: - type: object - description: - "Video port for DPI RGB output. + $ref: /schemas/graph.yaml#/properties/port + description: | + Video port for DPI RGB output. ltdc has one video port with up to 2 endpoints: - for external dpi rgb panel or bridge, using gpios. - for internal dpi input of the MIPI DSI host controller. Note: These 2 endpoints cannot be activated simultaneously. - Please refer to the bindings defined in - Documentation/devicetree/bindings/media/video-interfaces.txt." required: - compatible diff --git a/Documentation/devicetree/bindings/display/ste,mcde.txt b/Documentation/devicetree/bindings/display/ste,mcde.txt deleted file mode 100644 index 4c33c692bd5f58b6dd01b826b03c29d2cb4e3cfb..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/display/ste,mcde.txt +++ /dev/null @@ -1,104 +0,0 @@ -ST-Ericsson Multi Channel Display Engine MCDE - -The ST-Ericsson MCDE is a display controller with support for compositing -and displaying several channels memory resident graphics data on DSI or -LCD displays or bridges. It is used in the ST-Ericsson U8500 platform. - -Required properties: - -- compatible: must be: - "ste,mcde" -- reg: register base for the main MCDE control registers, should be - 0x1000 in size -- interrupts: the interrupt line for the MCDE -- epod-supply: a phandle to the EPOD regulator -- vana-supply: a phandle to the analog voltage regulator -- clocks: an array of the MCDE clocks in this strict order: - MCDECLK (main MCDE clock), LCDCLK (LCD clock), PLLDSI - (HDMI clock), DSI0ESCLK (DSI0 energy save clock), - DSI1ESCLK (DSI1 energy save clock), DSI2ESCLK (DSI2 energy - save clock) -- clock-names: must be the following array: - "mcde", "lcd", "hdmi" - to match the required clock inputs above. -- #address-cells: should be <1> (for the DSI hosts that will be children) -- #size-cells: should be <1> (for the DSI hosts that will be children) -- ranges: this should always be stated - -Required subnodes: - -The devicetree must specify subnodes for the DSI host adapters. -These must have the following characteristics: - -- compatible: must be: - "ste,mcde-dsi" -- reg: must specify the register range for the DSI host -- vana-supply: phandle to the VANA voltage regulator -- clocks: phandles to the high speed and low power (energy save) clocks - the high speed clock is not present on the third (dsi2) block, so it - should only have the "lp" clock -- clock-names: "hs" for the high speed clock and "lp" for the low power - (energy save) clock -- #address-cells: should be <1> -- #size-cells: should be <0> - -Display panels and bridges will appear as children on the DSI hosts, and -the displays are connected to the DSI hosts using the common binding -for video transmitter interfaces; see -Documentation/devicetree/bindings/media/video-interfaces.txt - -If a DSI host is unused (not connected) it will have no children defined. - -Example: - -mcde@a0350000 { - compatible = "ste,mcde"; - reg = <0xa0350000 0x1000>; - interrupts = ; - epod-supply = <&db8500_b2r2_mcde_reg>; - vana-supply = <&ab8500_ldo_ana_reg>; - clocks = <&prcmu_clk PRCMU_MCDECLK>, /* Main MCDE clock */ - <&prcmu_clk PRCMU_LCDCLK>, /* LCD clock */ - <&prcmu_clk PRCMU_PLLDSI>; /* HDMI clock */ - clock-names = "mcde", "lcd", "hdmi"; - #address-cells = <1>; - #size-cells = <1>; - ranges; - - dsi0: dsi@a0351000 { - compatible = "ste,mcde-dsi"; - reg = <0xa0351000 0x1000>; - vana-supply = <&ab8500_ldo_ana_reg>; - clocks = <&prcmu_clk PRCMU_DSI0CLK>, <&prcmu_clk PRCMU_DSI0ESCCLK>; - clock-names = "hs", "lp"; - #address-cells = <1>; - #size-cells = <0>; - - panel { - compatible = "samsung,s6d16d0"; - reg = <0>; - vdd1-supply = <&ab8500_ldo_aux1_reg>; - reset-gpios = <&gpio2 1 GPIO_ACTIVE_LOW>; - }; - - }; - dsi1: dsi@a0352000 { - compatible = "ste,mcde-dsi"; - reg = <0xa0352000 0x1000>; - vana-supply = <&ab8500_ldo_ana_reg>; - clocks = <&prcmu_clk PRCMU_DSI1CLK>, <&prcmu_clk PRCMU_DSI1ESCCLK>; - clock-names = "hs", "lp"; - #address-cells = <1>; - #size-cells = <0>; - }; - dsi2: dsi@a0353000 { - compatible = "ste,mcde-dsi"; - reg = <0xa0353000 0x1000>; - vana-supply = <&ab8500_ldo_ana_reg>; - /* This DSI port only has the Low Power / Energy Save clock */ - clocks = <&prcmu_clk PRCMU_DSI2ESCCLK>; - clock-names = "lp"; - #address-cells = <1>; - #size-cells = <0>; - }; -}; diff --git a/Documentation/devicetree/bindings/display/ste,mcde.yaml b/Documentation/devicetree/bindings/display/ste,mcde.yaml new file mode 100644 index 0000000000000000000000000000000000000000..de0c678b3c29406e365ca5382c39dcfe06d9db44 --- /dev/null +++ b/Documentation/devicetree/bindings/display/ste,mcde.yaml @@ -0,0 +1,168 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/display/ste,mcde.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: ST-Ericsson Multi Channel Display Engine MCDE + +maintainers: + - Linus Walleij + +properties: + compatible: + const: ste,mcde + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + + clocks: + description: an array of the MCDE clocks + items: + - description: MCDECLK (main MCDE clock) + - description: LCDCLK (LCD clock) + - description: PLLDSI (HDMI clock) + + clock-names: + items: + - const: mcde + - const: lcd + - const: hdmi + + resets: + maxItems: 1 + + epod-supply: + description: a phandle to the EPOD regulator + + vana-supply: + description: a phandle to the analog voltage regulator + + port: + $ref: /schemas/graph.yaml#/properties/port + description: + A DPI port node + + "#address-cells": + const: 1 + + "#size-cells": + const: 1 + + ranges: true + +patternProperties: + "^dsi@[0-9a-f]+$": + description: subnodes for the three DSI host adapters + type: object + allOf: + - $ref: dsi-controller.yaml# + properties: + compatible: + const: ste,mcde-dsi + + reg: + maxItems: 1 + + vana-supply: + description: a phandle to the analog voltage regulator + + clocks: + description: phandles to the high speed and low power (energy save) clocks + the high speed clock is not present on the third (dsi2) block, so it + should only have the "lp" clock + minItems: 1 + maxItems: 2 + + clock-names: + oneOf: + - items: + - const: hs + - const: lp + - items: + - const: lp + + required: + - compatible + - reg + - vana-supply + - clocks + - clock-names + + unevaluatedProperties: false + +required: + - compatible + - reg + - interrupts + - clocks + - clock-names + - epod-supply + - vana-supply + +additionalProperties: false + +examples: + - | + #include + #include + #include + #include + + mcde@a0350000 { + compatible = "ste,mcde"; + reg = <0xa0350000 0x1000>; + interrupts = ; + epod-supply = <&db8500_b2r2_mcde_reg>; + vana-supply = <&ab8500_ldo_ana_reg>; + clocks = <&prcmu_clk PRCMU_MCDECLK>, + <&prcmu_clk PRCMU_LCDCLK>, + <&prcmu_clk PRCMU_PLLDSI>; + clock-names = "mcde", "lcd", "hdmi"; + #address-cells = <1>; + #size-cells = <1>; + ranges; + + dsi0: dsi@a0351000 { + compatible = "ste,mcde-dsi"; + reg = <0xa0351000 0x1000>; + vana-supply = <&ab8500_ldo_ana_reg>; + clocks = <&prcmu_clk PRCMU_DSI0CLK>, <&prcmu_clk PRCMU_DSI0ESCCLK>; + clock-names = "hs", "lp"; + #address-cells = <1>; + #size-cells = <0>; + + panel@0 { + compatible = "samsung,s6d16d0"; + reg = <0>; + vdd1-supply = <&ab8500_ldo_aux1_reg>; + reset-gpios = <&gpio2 1 GPIO_ACTIVE_LOW>; + }; + }; + + dsi1: dsi@a0352000 { + compatible = "ste,mcde-dsi"; + reg = <0xa0352000 0x1000>; + vana-supply = <&ab8500_ldo_ana_reg>; + clocks = <&prcmu_clk PRCMU_DSI1CLK>, <&prcmu_clk PRCMU_DSI1ESCCLK>; + clock-names = "hs", "lp"; + #address-cells = <1>; + #size-cells = <0>; + }; + + dsi2: dsi@a0353000 { + compatible = "ste,mcde-dsi"; + reg = <0xa0353000 0x1000>; + vana-supply = <&ab8500_ldo_ana_reg>; + /* This DSI port only has the Low Power / Energy Save clock */ + clocks = <&prcmu_clk PRCMU_DSI2ESCCLK>; + clock-names = "lp"; + #address-cells = <1>; + #size-cells = <0>; + }; + }; + +... diff --git a/Documentation/devicetree/bindings/display/ti/ti,am65x-dss.yaml b/Documentation/devicetree/bindings/display/ti/ti,am65x-dss.yaml index 4dc30738ee576cc579f14294e72723aa0cba9886..781c1868b0b8f629851d7f3eabf54282e88ab918 100644 --- a/Documentation/devicetree/bindings/display/ti/ti,am65x-dss.yaml +++ b/Documentation/devicetree/bindings/display/ti/ti,am65x-dss.yaml @@ -74,30 +74,19 @@ properties: type: boolean ports: - type: object - description: - Ports as described in Documentation/devicetree/bindings/graph.txt - properties: - "#address-cells": - const: 1 - - "#size-cells": - const: 0 + $ref: /schemas/graph.yaml#/properties/ports + properties: port@0: - type: object + $ref: /schemas/graph.yaml#/properties/port description: The DSS OLDI output port node form video port 1 port@1: - type: object + $ref: /schemas/graph.yaml#/properties/port description: The DSS DPI output port node from video port 2 - required: - - "#address-cells" - - "#size-cells" - ti,am65x-oldi-io-ctrl: $ref: "/schemas/types.yaml#/definitions/phandle-array" maxItems: 1 diff --git a/Documentation/devicetree/bindings/display/ti/ti,j721e-dss.yaml b/Documentation/devicetree/bindings/display/ti/ti,j721e-dss.yaml index c9a947d55fa4a5999c45c5430dbbb5ba237ad085..2986f9acc9f00615fe0460fd1c16239ba78b22f1 100644 --- a/Documentation/devicetree/bindings/display/ti/ti,j721e-dss.yaml +++ b/Documentation/devicetree/bindings/display/ti/ti,j721e-dss.yaml @@ -107,40 +107,29 @@ properties: type: boolean ports: - type: object - description: - Ports as described in Documentation/devicetree/bindings/graph.txt - properties: - "#address-cells": - const: 1 - - "#size-cells": - const: 0 + $ref: /schemas/graph.yaml#/properties/ports + properties: port@0: - type: object + $ref: /schemas/graph.yaml#/properties/port description: The output port node form video port 1 port@1: - type: object + $ref: /schemas/graph.yaml#/properties/port description: The output port node from video port 2 port@2: - type: object + $ref: /schemas/graph.yaml#/properties/port description: The output port node from video port 3 port@3: - type: object + $ref: /schemas/graph.yaml#/properties/port description: The output port node from video port 4 - required: - - "#address-cells" - - "#size-cells" - max-memory-bandwidth: $ref: /schemas/types.yaml#/definitions/uint32 description: diff --git a/Documentation/devicetree/bindings/display/ti/ti,k2g-dss.yaml b/Documentation/devicetree/bindings/display/ti/ti,k2g-dss.yaml index 8f87b82c66951e1216aa549fc461db36d22115b8..7ce7bbad57803ca70a66fab61e8636ae6849a1cd 100644 --- a/Documentation/devicetree/bindings/display/ti/ti,k2g-dss.yaml +++ b/Documentation/devicetree/bindings/display/ti/ti,k2g-dss.yaml @@ -54,9 +54,8 @@ properties: description: phandle to the associated power domain port: - type: object + $ref: /schemas/graph.yaml#/properties/port description: - Port as described in Documentation/devicetree/bindings/graph.txt. The DSS DPI output port node max-memory-bandwidth: diff --git a/Documentation/devicetree/bindings/dma/ingenic,dma.yaml b/Documentation/devicetree/bindings/dma/ingenic,dma.yaml index 6a2043721b95f4dab79e5bdd41ad92608a987b54..ac4d59494fc8e033036eda935b53d4592f25616d 100644 --- a/Documentation/devicetree/bindings/dma/ingenic,dma.yaml +++ b/Documentation/devicetree/bindings/dma/ingenic,dma.yaml @@ -17,6 +17,8 @@ properties: enum: - ingenic,jz4740-dma - ingenic,jz4725b-dma + - ingenic,jz4760-dma + - ingenic,jz4760b-dma - ingenic,jz4770-dma - ingenic,jz4780-dma - ingenic,x1000-dma diff --git a/Documentation/devicetree/bindings/dma/intel,ldma.yaml b/Documentation/devicetree/bindings/dma/intel,ldma.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a5c4be7835937613bf72fc2d4caa64fedbefee64 --- /dev/null +++ b/Documentation/devicetree/bindings/dma/intel,ldma.yaml @@ -0,0 +1,116 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/dma/intel,ldma.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Lightning Mountain centralized DMA controllers. + +maintainers: + - chuanhua.lei@intel.com + - mallikarjunax.reddy@intel.com + +allOf: + - $ref: "dma-controller.yaml#" + +properties: + compatible: + enum: + - intel,lgm-cdma + - intel,lgm-dma2tx + - intel,lgm-dma1rx + - intel,lgm-dma1tx + - intel,lgm-dma0tx + - intel,lgm-dma3 + - intel,lgm-toe-dma30 + - intel,lgm-toe-dma31 + + reg: + maxItems: 1 + + "#dma-cells": + const: 3 + description: + The first cell is the peripheral's DMA request line. + The second cell is the peripheral's (port) number corresponding to the channel. + The third cell is the burst length of the channel. + + dma-channels: + minimum: 1 + maximum: 16 + + dma-channel-mask: + maxItems: 1 + + clocks: + maxItems: 1 + + resets: + maxItems: 1 + + reset-names: + items: + - const: ctrl + + interrupts: + maxItems: 1 + + intel,dma-poll-cnt: + $ref: /schemas/types.yaml#/definitions/uint32 + description: + DMA descriptor polling counter is used to control the poling mechanism + for the descriptor fetching for all channels. + + intel,dma-byte-en: + type: boolean + description: + DMA byte enable is only valid for DMA write(RX). + Byte enable(1) means DMA write will be based on the number of dwords + instead of the whole burst. + + intel,dma-drb: + type: boolean + description: + DMA descriptor read back to make sure data and desc synchronization. + + intel,dma-dburst-wr: + type: boolean + description: + Enable RX dynamic burst write. When it is enabled, the DMA does RX dynamic burst; + if it is disabled, the DMA RX will still support programmable fixed burst size of 2,4,8,16. + It only applies to RX DMA and memcopy DMA. + +required: + - compatible + - reg + +additionalProperties: false + +examples: + - | + dma0: dma-controller@e0e00000 { + compatible = "intel,lgm-cdma"; + reg = <0xe0e00000 0x1000>; + #dma-cells = <3>; + dma-channels = <16>; + dma-channel-mask = <0xFFFF>; + interrupt-parent = <&ioapic1>; + interrupts = <82 1>; + resets = <&rcu0 0x30 0>; + reset-names = "ctrl"; + clocks = <&cgu0 80>; + intel,dma-poll-cnt = <4>; + intel,dma-byte-en; + intel,dma-drb; + }; + - | + dma3: dma-controller@ec800000 { + compatible = "intel,lgm-dma3"; + reg = <0xec800000 0x1000>; + clocks = <&cgu0 71>; + resets = <&rcu0 0x10 9>; + #dma-cells = <3>; + intel,dma-poll-cnt = <16>; + intel,dma-byte-en; + intel,dma-dburst-wr; + }; diff --git a/Documentation/devicetree/bindings/dma/owl-dma.yaml b/Documentation/devicetree/bindings/dma/owl-dma.yaml index 256d62af2c64b9eef49b724f031aa1b12d6ad21a..93b4847554fbc85d316961736bb749b7c93ce96c 100644 --- a/Documentation/devicetree/bindings/dma/owl-dma.yaml +++ b/Documentation/devicetree/bindings/dma/owl-dma.yaml @@ -8,8 +8,8 @@ title: Actions Semi Owl SoCs DMA controller description: | The OWL DMA is a general-purpose direct memory access controller capable of - supporting 10 and 12 independent DMA channels for S700 and S900 SoCs - respectively. + supporting 10 independent DMA channels for the Actions Semi S700 SoC and 12 + independent DMA channels for the S500 and S900 SoC variants. maintainers: - Manivannan Sadhasivam @@ -20,8 +20,9 @@ allOf: properties: compatible: enum: - - actions,s900-dma + - actions,s500-dma - actions,s700-dma + - actions,s900-dma reg: maxItems: 1 diff --git a/Documentation/devicetree/bindings/dma/renesas,rcar-dmac.yaml b/Documentation/devicetree/bindings/dma/renesas,rcar-dmac.yaml index c07eb6f2fc8d2f127c2854f927b9365e8b943f5a..7f2a54bc732d3a19dc4dbd3c722a825535da3242 100644 --- a/Documentation/devicetree/bindings/dma/renesas,rcar-dmac.yaml +++ b/Documentation/devicetree/bindings/dma/renesas,rcar-dmac.yaml @@ -14,34 +14,37 @@ allOf: properties: compatible: - items: - - enum: - - renesas,dmac-r8a7742 # RZ/G1H - - renesas,dmac-r8a7743 # RZ/G1M - - renesas,dmac-r8a7744 # RZ/G1N - - renesas,dmac-r8a7745 # RZ/G1E - - renesas,dmac-r8a77470 # RZ/G1C - - renesas,dmac-r8a774a1 # RZ/G2M - - renesas,dmac-r8a774b1 # RZ/G2N - - renesas,dmac-r8a774c0 # RZ/G2E - - renesas,dmac-r8a774e1 # RZ/G2H - - renesas,dmac-r8a7790 # R-Car H2 - - renesas,dmac-r8a7791 # R-Car M2-W - - renesas,dmac-r8a7792 # R-Car V2H - - renesas,dmac-r8a7793 # R-Car M2-N - - renesas,dmac-r8a7794 # R-Car E2 - - renesas,dmac-r8a7795 # R-Car H3 - - renesas,dmac-r8a7796 # R-Car M3-W - - renesas,dmac-r8a77961 # R-Car M3-W+ - - renesas,dmac-r8a77965 # R-Car M3-N - - renesas,dmac-r8a77970 # R-Car V3M - - renesas,dmac-r8a77980 # R-Car V3H - - renesas,dmac-r8a77990 # R-Car E3 - - renesas,dmac-r8a77995 # R-Car D3 - - const: renesas,rcar-dmac - - reg: - maxItems: 1 + oneOf: + - items: + - enum: + - renesas,dmac-r8a7742 # RZ/G1H + - renesas,dmac-r8a7743 # RZ/G1M + - renesas,dmac-r8a7744 # RZ/G1N + - renesas,dmac-r8a7745 # RZ/G1E + - renesas,dmac-r8a77470 # RZ/G1C + - renesas,dmac-r8a774a1 # RZ/G2M + - renesas,dmac-r8a774b1 # RZ/G2N + - renesas,dmac-r8a774c0 # RZ/G2E + - renesas,dmac-r8a774e1 # RZ/G2H + - renesas,dmac-r8a7790 # R-Car H2 + - renesas,dmac-r8a7791 # R-Car M2-W + - renesas,dmac-r8a7792 # R-Car V2H + - renesas,dmac-r8a7793 # R-Car M2-N + - renesas,dmac-r8a7794 # R-Car E2 + - renesas,dmac-r8a7795 # R-Car H3 + - renesas,dmac-r8a7796 # R-Car M3-W + - renesas,dmac-r8a77961 # R-Car M3-W+ + - renesas,dmac-r8a77965 # R-Car M3-N + - renesas,dmac-r8a77970 # R-Car V3M + - renesas,dmac-r8a77980 # R-Car V3H + - renesas,dmac-r8a77990 # R-Car E3 + - renesas,dmac-r8a77995 # R-Car D3 + - const: renesas,rcar-dmac + + - items: + - const: renesas,dmac-r8a779a0 # R-Car V3U + + reg: true interrupts: minItems: 9 @@ -110,6 +113,23 @@ required: - power-domains - resets +if: + properties: + compatible: + contains: + enum: + - renesas,dmac-r8a779a0 +then: + properties: + reg: + items: + - description: Base register block + - description: Channel register block +else: + properties: + reg: + maxItems: 1 + additionalProperties: false examples: diff --git a/Documentation/devicetree/bindings/dma/sirfsoc-dma.txt b/Documentation/devicetree/bindings/dma/sirfsoc-dma.txt deleted file mode 100644 index ccd52d6a231a2b41a010d375a7c395b6c6da0f42..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/dma/sirfsoc-dma.txt +++ /dev/null @@ -1,44 +0,0 @@ -* CSR SiRFSoC DMA controller - -See dma.txt first - -Required properties: -- compatible: Should be "sirf,prima2-dmac", "sirf,atlas7-dmac" or - "sirf,atlas7-dmac-v2" -- reg: Should contain DMA registers location and length. -- interrupts: Should contain one interrupt shared by all channel -- #dma-cells: must be <1>. used to represent the number of integer - cells in the dmas property of client device. -- clocks: clock required - -Example: - -Controller: -dmac0: dma-controller@b00b0000 { - compatible = "sirf,prima2-dmac"; - reg = <0xb00b0000 0x10000>; - interrupts = <12>; - clocks = <&clks 24>; - #dma-cells = <1>; -}; - - -Client: -Fill the specific dma request line in dmas. In the below example, spi0 read -channel request line is 9 of the 2nd dma controller, while write channel uses -4 of the 2nd dma controller; spi1 read channel request line is 12 of the 1st -dma controller, while write channel uses 13 of the 1st dma controller: - -spi0: spi@b00d0000 { - compatible = "sirf,prima2-spi"; - dmas = <&dmac1 9>, - <&dmac1 4>; - dma-names = "rx", "tx"; -}; - -spi1: spi@b0170000 { - compatible = "sirf,prima2-spi"; - dmas = <&dmac0 12>, - <&dmac0 13>; - dma-names = "rx", "tx"; -}; diff --git a/Documentation/devicetree/bindings/dma/snps,dw-axi-dmac.txt b/Documentation/devicetree/bindings/dma/snps,dw-axi-dmac.txt deleted file mode 100644 index dbe160400adcdafffdb6618e5f18e89616d732b9..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/dma/snps,dw-axi-dmac.txt +++ /dev/null @@ -1,39 +0,0 @@ -Synopsys DesignWare AXI DMA Controller - -Required properties: -- compatible: "snps,axi-dma-1.01a" -- reg: Address range of the DMAC registers. This should include - all of the per-channel registers. -- interrupt: Should contain the DMAC interrupt number. -- dma-channels: Number of channels supported by hardware. -- snps,dma-masters: Number of AXI masters supported by the hardware. -- snps,data-width: Maximum AXI data width supported by hardware. - (0 - 8bits, 1 - 16bits, 2 - 32bits, ..., 6 - 512bits) -- snps,priority: Priority of channel. Array size is equal to the number of - dma-channels. Priority value must be programmed within [0:dma-channels-1] - range. (0 - minimum priority) -- snps,block-size: Maximum block size supported by the controller channel. - Array size is equal to the number of dma-channels. - -Optional properties: -- snps,axi-max-burst-len: Restrict master AXI burst length by value specified - in this property. If this property is missing the maximum AXI burst length - supported by DMAC is used. [1:256] - -Example: - -dmac: dma-controller@80000 { - compatible = "snps,axi-dma-1.01a"; - reg = <0x80000 0x400>; - clocks = <&core_clk>, <&cfgr_clk>; - clock-names = "core-clk", "cfgr-clk"; - interrupt-parent = <&intc>; - interrupts = <27>; - - dma-channels = <4>; - snps,dma-masters = <2>; - snps,data-width = <3>; - snps,block-size = <4096 4096 4096 4096>; - snps,priority = <0 1 2 3>; - snps,axi-max-burst-len = <16>; -}; diff --git a/Documentation/devicetree/bindings/dma/snps,dw-axi-dmac.yaml b/Documentation/devicetree/bindings/dma/snps,dw-axi-dmac.yaml new file mode 100644 index 0000000000000000000000000000000000000000..79e241498e2532ce0695e6a97f473e089cd6f5a9 --- /dev/null +++ b/Documentation/devicetree/bindings/dma/snps,dw-axi-dmac.yaml @@ -0,0 +1,126 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/dma/snps,dw-axi-dmac.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Synopsys DesignWare AXI DMA Controller + +maintainers: + - Eugeniy Paltsev + - Jee Heng Sia + +description: + Synopsys DesignWare AXI DMA Controller DT Binding + +allOf: + - $ref: "dma-controller.yaml#" + +properties: + compatible: + enum: + - snps,axi-dma-1.01a + - intel,kmb-axi-dma + + reg: + minItems: 1 + items: + - description: Address range of the DMAC registers + - description: Address range of the DMAC APB registers + + reg-names: + items: + - const: axidma_ctrl_regs + - const: axidma_apb_regs + + interrupts: + maxItems: 1 + + clocks: + items: + - description: Bus Clock + - description: Module Clock + + clock-names: + items: + - const: core-clk + - const: cfgr-clk + + '#dma-cells': + const: 1 + + dma-channels: + minimum: 1 + maximum: 8 + + snps,dma-masters: + description: | + Number of AXI masters supported by the hardware. + $ref: /schemas/types.yaml#/definitions/uint32 + enum: [1, 2] + + snps,data-width: + description: | + AXI data width supported by hardware. + (0 - 8bits, 1 - 16bits, 2 - 32bits, ..., 6 - 512bits) + $ref: /schemas/types.yaml#/definitions/uint32 + enum: [0, 1, 2, 3, 4, 5, 6] + + snps,priority: + description: | + Channel priority specifier associated with the DMA channels. + $ref: /schemas/types.yaml#/definitions/uint32-array + minItems: 1 + maxItems: 8 + + snps,block-size: + description: | + Channel block size specifier associated with the DMA channels. + $ref: /schemas/types.yaml#/definitions/uint32-array + minItems: 1 + maxItems: 8 + + snps,axi-max-burst-len: + description: | + Restrict master AXI burst length by value specified in this property. + If this property is missing the maximum AXI burst length supported by + DMAC is used. + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 1 + maximum: 256 + +required: + - compatible + - reg + - clocks + - clock-names + - interrupts + - '#dma-cells' + - dma-channels + - snps,dma-masters + - snps,data-width + - snps,priority + - snps,block-size + +additionalProperties: false + +examples: + - | + #include + #include + /* example with snps,dw-axi-dmac */ + dmac: dma-controller@80000 { + compatible = "snps,axi-dma-1.01a"; + reg = <0x80000 0x400>; + clocks = <&core_clk>, <&cfgr_clk>; + clock-names = "core-clk", "cfgr-clk"; + interrupt-parent = <&intc>; + interrupts = <27>; + #dma-cells = <1>; + dma-channels = <4>; + snps,dma-masters = <2>; + snps,data-width = <3>; + snps,block-size = <4096 4096 4096 4096>; + snps,priority = <0 1 2 3>; + snps,axi-max-burst-len = <16>; + }; diff --git a/Documentation/devicetree/bindings/dma/ste-coh901318.txt b/Documentation/devicetree/bindings/dma/ste-coh901318.txt deleted file mode 100644 index 091ad057e9cfd7fc28e164eb0d077d30b53268fe..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/dma/ste-coh901318.txt +++ /dev/null @@ -1,32 +0,0 @@ -ST-Ericsson COH 901 318 DMA Controller - -This is a DMA controller which has begun as a fork of the -ARM PL08x PrimeCell VHDL code. - -Required properties: -- compatible: should be "stericsson,coh901318" -- reg: register locations and length -- interrupts: the single DMA IRQ -- #dma-cells: must be set to <1>, as the channels on the - COH 901 318 are simple and identified by a single number -- dma-channels: the number of DMA channels handled - -Example: - -dmac: dma-controller@c00020000 { - compatible = "stericsson,coh901318"; - reg = <0xc0020000 0x1000>; - interrupt-parent = <&vica>; - interrupts = <2>; - #dma-cells = <1>; - dma-channels = <40>; -}; - -Consumers example: - -uart0: serial@c0013000 { - compatible = "..."; - (...) - dmas = <&dmac 17 &dmac 18>; - dma-names = "tx", "rx"; -}; diff --git a/Documentation/devicetree/bindings/dma/zxdma.txt b/Documentation/devicetree/bindings/dma/zxdma.txt deleted file mode 100644 index 0ab80f69e566ef7c66001486a7cd5acea5685df8..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/dma/zxdma.txt +++ /dev/null @@ -1,38 +0,0 @@ -* ZTE ZX296702 DMA controller - -Required properties: -- compatible: Should be "zte,zx296702-dma" -- reg: Should contain DMA registers location and length. -- interrupts: Should contain one interrupt shared by all channel -- #dma-cells: see dma.txt, should be 1, para number -- dma-channels: physical channels supported -- dma-requests: virtual channels supported, each virtual channel - have specific request line -- clocks: clock required - -Example: - -Controller: - dma: dma-controller@09c00000{ - compatible = "zte,zx296702-dma"; - reg = <0x09c00000 0x1000>; - clocks = <&topclk ZX296702_DMA_ACLK>; - interrupts = ; - #dma-cells = <1>; - dma-channels = <24>; - dma-requests = <24>; - }; - -Client: -Use specific request line passing from dmax -For example, spdif0 tx channel request line is 4 - spdif0: spdif0@b004000 { - #sound-dai-cells = <0>; - compatible = "zte,zx296702-spdif"; - reg = <0x0b004000 0x1000>; - clocks = <&lsp0clk ZX296702_SPDIF0_DIV>; - clock-names = "tx"; - interrupts = ; - dmas = <&dma 4>; - dma-names = "tx"; - } diff --git a/Documentation/devicetree/bindings/dsp/fsl,dsp.yaml b/Documentation/devicetree/bindings/dsp/fsl,dsp.yaml index 4cc0112301536201e48fc8b6b6f39d068997037e..7afc9f2be13ab90fceca8b949ad6b0058cb6aabf 100644 --- a/Documentation/devicetree/bindings/dsp/fsl,dsp.yaml +++ b/Documentation/devicetree/bindings/dsp/fsl,dsp.yaml @@ -21,7 +21,7 @@ properties: - fsl,imx8mp-dsp reg: - description: Should contain register location and length + maxItems: 1 clocks: items: diff --git a/Documentation/devicetree/bindings/eeprom/at24.yaml b/Documentation/devicetree/bindings/eeprom/at24.yaml index d5117c638b75c76c472bf0a1ff4f00d415533567..021d8ae42da318e4ed1504be26916246d1d0c0c5 100644 --- a/Documentation/devicetree/bindings/eeprom/at24.yaml +++ b/Documentation/devicetree/bindings/eeprom/at24.yaml @@ -95,9 +95,6 @@ properties: pattern: spd$ # These are special cases that don't conform to the above pattern. # Each requires a standard at24 model as fallback. - - items: - - const: rohm,br24t01 - - const: atmel,24c01 - items: - const: nxp,se97b - const: atmel,24c02 @@ -113,6 +110,12 @@ properties: - items: - const: renesas,r1ex24128 - const: atmel,24c128 + - items: + - const: rohm,br24g01 + - const: atmel,24c01 + - items: + - const: rohm,br24t01 + - const: atmel,24c01 label: description: Descriptive name of the EEPROM. diff --git a/Documentation/devicetree/bindings/eeprom/at25.yaml b/Documentation/devicetree/bindings/eeprom/at25.yaml index 121a601db22e40ffb8eb9e87be1eef05bee78350..6a2dc8b3ed144fe971d43d8805327d9e0a12b3b6 100644 --- a/Documentation/devicetree/bindings/eeprom/at25.yaml +++ b/Documentation/devicetree/bindings/eeprom/at25.yaml @@ -39,8 +39,7 @@ properties: - const: atmel,at25 reg: - description: - Chip select number. + maxItems: 1 spi-max-frequency: true diff --git a/Documentation/devicetree/bindings/extcon/extcon-ptn5150.yaml b/Documentation/devicetree/bindings/extcon/extcon-ptn5150.yaml index 4b0f414486d281cd132a203b33f6d15276d4011c..d5cfa32ea52dd5858801e86da488f6d845b9ab54 100644 --- a/Documentation/devicetree/bindings/extcon/extcon-ptn5150.yaml +++ b/Documentation/devicetree/bindings/extcon/extcon-ptn5150.yaml @@ -19,6 +19,7 @@ properties: const: nxp,ptn5150 int-gpios: + maxItems: 1 deprecated: true description: GPIO pin (input) connected to the PTN5150's INTB pin. @@ -31,6 +32,7 @@ properties: maxItems: 1 vbus-gpios: + maxItems: 1 description: GPIO pin (output) used to control VBUS. If skipped, no such control takes place. diff --git a/Documentation/devicetree/bindings/gpio/gpio-atlas7.txt b/Documentation/devicetree/bindings/gpio/gpio-atlas7.txt deleted file mode 100644 index d7e123fc90b574bfc3d46c292394a50dde2853e4..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/gpio/gpio-atlas7.txt +++ /dev/null @@ -1,50 +0,0 @@ -CSR SiRFatlas7 GPIO controller bindings - -Required properties: -- compatible : "sirf,atlas7-gpio" -- reg : Address range of the pinctrl registers -- interrupts : Interrupts used by every GPIO group -- gpio-banks : How many gpio banks on this controller -- gpio-controller : Indicates this device is a GPIO controller -- interrupt-controller : Marks the device node as an interrupt controller - -The GPIO controller also acts as an interrupt controller. It uses the default -two cells specifier as described in Documentation/devicetree/bindings/ -interrupt-controller/interrupts.txt. - -Example: - - gpio_0: gpio_mediam@17040000 { - compatible = "sirf,atlas7-gpio"; - reg = <0x17040000 0x1000>; - interrupts = <0 13 0>, <0 14 0>; - - #gpio-cells = <2>; - #interrupt-cells = <2>; - - gpio-controller; - interrupt-controller; - - gpio-banks = <2>; - gpio-ranges = <&pinctrl 0 0 0>, - <&pinctrl 32 0 0>; - gpio-ranges-group-names = "lvds_gpio_grp", - "uart_nand_gpio_grp"; - }; - - leds { - compatible = "gpio-leds"; - - led1 { - gpios = <&gpio_1 15 0>; - ... - }; - - led2 { - gpios = <&gpio_2 34 0>; - ... - }; - }; - -Please refer to gpio.txt in this directory for details of the common -gpio properties used by devices. diff --git a/Documentation/devicetree/bindings/gpio/gpio-davinci.txt b/Documentation/devicetree/bindings/gpio/gpio-davinci.txt index cd91d61eac31492bcb8ce34d92a9a3912105bb1b..696ea46227d1ad9edc748f1060b0a3db39d6a592 100644 --- a/Documentation/devicetree/bindings/gpio/gpio-davinci.txt +++ b/Documentation/devicetree/bindings/gpio/gpio-davinci.txt @@ -7,6 +7,7 @@ Required Properties: "ti,k2g-gpio", "ti,keystone-gpio": for 66AK2G "ti,am654-gpio", "ti,keystone-gpio": for TI K3 AM654 "ti,j721e-gpio", "ti,keystone-gpio": for J721E SoCs + "ti,am64-gpio", "ti,keystone-gpio": for AM64 SoCs - reg: Physical base address of the controller and the size of memory mapped registers. diff --git a/Documentation/devicetree/bindings/gpio/gpio-pca95xx.yaml b/Documentation/devicetree/bindings/gpio/gpio-pca95xx.yaml index f5ee23c2df605de2797dcfb01090c3427c57fc8c..b6a6e742b66d34a072a2bf8ae101ea7bc4f430af 100644 --- a/Documentation/devicetree/bindings/gpio/gpio-pca95xx.yaml +++ b/Documentation/devicetree/bindings/gpio/gpio-pca95xx.yaml @@ -32,6 +32,7 @@ properties: - maxim,max7327 - nxp,pca6416 - nxp,pca9505 + - nxp,pca9506 - nxp,pca9534 - nxp,pca9535 - nxp,pca9536 @@ -70,7 +71,7 @@ properties: gpio-line-names: minItems: 1 - maxItems: 32 + maxItems: 40 interrupts: maxItems: 1 @@ -81,6 +82,7 @@ properties: const: 2 reset-gpios: + maxItems: 1 description: GPIO specification for the RESET input. This is an active low signal to the PCA953x. Not valid for Maxim MAX732x devices. diff --git a/Documentation/devicetree/bindings/gpio/gpio-stericsson-coh901.txt b/Documentation/devicetree/bindings/gpio/gpio-stericsson-coh901.txt deleted file mode 100644 index fd665b44d76741a763559be360e8d7b236aab4cb..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/gpio/gpio-stericsson-coh901.txt +++ /dev/null @@ -1,7 +0,0 @@ -ST-Ericsson COH 901 571/3 GPIO controller - -Required properties: -- compatible: Compatible property value should be "stericsson,gpio-coh901" -- reg: Physical base address of the controller and length of memory mapped - region. -- interrupts: the 0...n interrupts assigned to the different GPIO ports/banks. diff --git a/Documentation/devicetree/bindings/gpio/mstar,msc313-gpio.yaml b/Documentation/devicetree/bindings/gpio/mstar,msc313-gpio.yaml index 1f2ef408bb436ed29961a04de358752253406816..fe1e1c63ffe3fc8773a8d7939173cc7e40fb867e 100644 --- a/Documentation/devicetree/bindings/gpio/mstar,msc313-gpio.yaml +++ b/Documentation/devicetree/bindings/gpio/mstar,msc313-gpio.yaml @@ -46,7 +46,7 @@ examples: #include gpio: gpio@207800 { - compatible = "mstar,msc313e-gpio"; + compatible = "mstar,msc313-gpio"; #gpio-cells = <2>; reg = <0x207800 0x200>; gpio-controller; diff --git a/Documentation/devicetree/bindings/gpio/renesas,rcar-gpio.yaml b/Documentation/devicetree/bindings/gpio/renesas,rcar-gpio.yaml index 5026662e45081b6c2c1959db44a8a4427d572400..f2541739ee3b9f7cd4873a312c61a41fd384d7db 100644 --- a/Documentation/devicetree/bindings/gpio/renesas,rcar-gpio.yaml +++ b/Documentation/devicetree/bindings/gpio/renesas,rcar-gpio.yaml @@ -48,6 +48,9 @@ properties: - renesas,gpio-r8a77995 # R-Car D3 - const: renesas,rcar-gen3-gpio # R-Car Gen3 or RZ/G2 + - items: + - const: renesas,gpio-r8a779a0 # R-Car V3U + reg: maxItems: 1 diff --git a/Documentation/devicetree/bindings/gpio/sifive,gpio.yaml b/Documentation/devicetree/bindings/gpio/sifive,gpio.yaml index a0efd8dc25384ec7db714212c8547aa51651b362..c2902aac251452050895a2f0cfb753b5f47ce9cc 100644 --- a/Documentation/devicetree/bindings/gpio/sifive,gpio.yaml +++ b/Documentation/devicetree/bindings/gpio/sifive,gpio.yaml @@ -13,7 +13,10 @@ maintainers: properties: compatible: items: - - const: sifive,fu540-c000-gpio + - enum: + - sifive,fu540-c000-gpio + - sifive,fu740-c000-gpio + - canaan,k210-gpiohs - const: sifive,gpio0 reg: @@ -21,9 +24,9 @@ properties: interrupts: description: - interrupt mapping one per GPIO. Maximum 16 GPIOs. + Interrupt mapping, one per GPIO. Maximum 32 GPIOs. minItems: 1 - maxItems: 16 + maxItems: 32 interrupt-controller: true @@ -36,6 +39,14 @@ properties: "#gpio-cells": const: 2 + ngpios: + description: + The number of GPIOs available on the controller implementation. + It is 16 for the SiFive SoCs and 32 for the Canaan K210. + minimum: 1 + maximum: 32 + default: 16 + gpio-controller: true required: @@ -44,10 +55,20 @@ required: - interrupts - interrupt-controller - "#interrupt-cells" - - clocks - "#gpio-cells" - gpio-controller +if: + properties: + compatible: + contains: + enum: + - sifive,fu540-c000-gpio + - sifive,fu740-c000-gpio +then: + required: + - clocks + additionalProperties: false examples: diff --git a/Documentation/devicetree/bindings/gpio/toshiba,gpio-visconti.yaml b/Documentation/devicetree/bindings/gpio/toshiba,gpio-visconti.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9ad470e0195375551bdcfc8b62d05f72b9f55a35 --- /dev/null +++ b/Documentation/devicetree/bindings/gpio/toshiba,gpio-visconti.yaml @@ -0,0 +1,70 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/gpio/toshiba,gpio-visconti.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Toshiba Visconti ARM SoCs GPIO controller + +maintainers: + - Nobuhiro Iwamatsu + +properties: + compatible: + items: + - const: toshiba,gpio-tmpv7708 + + reg: + maxItems: 1 + + "#gpio-cells": + const: 2 + + gpio-ranges: true + + gpio-controller: true + + interrupt-controller: true + + "#interrupt-cells": + const: 2 + + interrupts: + description: + interrupt mapping one per GPIO. + minItems: 16 + maxItems: 16 + +required: + - compatible + - reg + - "#gpio-cells" + - gpio-ranges + - gpio-controller + - interrupt-controller + - "#interrupt-cells" + - interrupt-parent + +additionalProperties: false + +examples: + - | + #include + #include + + soc { + #address-cells = <2>; + #size-cells = <2>; + + gpio: gpio@28020000 { + compatible = "toshiba,gpio-tmpv7708"; + reg = <0 0x28020000 0 0x1000>; + #gpio-cells = <0x2>; + gpio-ranges = <&pmux 0 0 32>; + gpio-controller; + interrupt-controller; + #interrupt-cells = <2>; + interrupt-parent = <&gic>; + }; + }; +... diff --git a/Documentation/devicetree/bindings/gpio/zx296702-gpio.txt b/Documentation/devicetree/bindings/gpio/zx296702-gpio.txt deleted file mode 100644 index 0dab156fcf41775fad139313d64c70fa2f65a052..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/gpio/zx296702-gpio.txt +++ /dev/null @@ -1,24 +0,0 @@ -ZTE ZX296702 GPIO controller - -Required properties: -- compatible : "zte,zx296702-gpio" -- #gpio-cells : Should be two. The first cell is the pin number and the - second cell is used to specify optional parameters: - - bit 0 specifies polarity (0 for normal, 1 for inverted) -- gpio-controller : Marks the device node as a GPIO controller. -- interrupts : Interrupt mapping for GPIO IRQ. -- gpio-ranges : Interaction with the PINCTRL subsystem. - -gpio1: gpio@b008040 { - compatible = "zte,zx296702-gpio"; - reg = <0xb008040 0x40>; - gpio-controller; - #gpio-cells = <2>; - gpio-ranges = < &pmx0 0 54 2 &pmx0 2 59 14>; - interrupts = ; - interrupt-parent = <&intc>; - interrupt-controller; - #interrupt-cells = <2>; - clock-names = "gpio_pclk"; - clocks = <&lsp0clk ZX296702_GPIO_CLK>; -}; diff --git a/Documentation/devicetree/bindings/gpu/brcm,bcm-v3d.txt b/Documentation/devicetree/bindings/gpu/brcm,bcm-v3d.txt deleted file mode 100644 index b2df82b4462563b81cff00b28235fff8b74efaca..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/gpu/brcm,bcm-v3d.txt +++ /dev/null @@ -1,33 +0,0 @@ -Broadcom V3D GPU - -Only the Broadcom V3D 3.x and newer GPUs are covered by this binding. -For V3D 2.x, see brcm,bcm-vc4.txt. - -Required properties: -- compatible: Should be "brcm,7268-v3d" or "brcm,7278-v3d" -- reg: Physical base addresses and lengths of the register areas -- reg-names: Names for the register areas. The "hub" and "core0" - register areas are always required. The "gca" register area - is required if the GCA cache controller is present. The - "bridge" register area is required if an external reset - controller is not present. -- interrupts: The interrupt numbers. The first interrupt is for the hub, - while the following interrupts are separate interrupt lines - for the cores (if they don't share the hub's interrupt). - See bindings/interrupt-controller/interrupts.txt - -Optional properties: -- clocks: The core clock the unit runs on -- resets: The reset line for v3d, if not using a mapping of the bridge - See bindings/reset/reset.txt - -v3d { - compatible = "brcm,7268-v3d"; - reg = <0xf1204000 0x100>, - <0xf1200000 0x4000>, - <0xf1208000 0x4000>, - <0xf1204100 0x100>; - reg-names = "bridge", "hub", "core0", "gca"; - interrupts = <0 78 4>, - <0 77 4>; -}; diff --git a/Documentation/devicetree/bindings/gpu/brcm,bcm-v3d.yaml b/Documentation/devicetree/bindings/gpu/brcm,bcm-v3d.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9d72264fa90a99ce8e9bff306a692425fae2d544 --- /dev/null +++ b/Documentation/devicetree/bindings/gpu/brcm,bcm-v3d.yaml @@ -0,0 +1,75 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/gpu/brcm,bcm-v3d.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Broadcom V3D GPU Bindings + +maintainers: + - Eric Anholt + - Nicolas Saenz Julienne + +properties: + $nodename: + pattern: '^gpu@[a-f0-9]+$' + + compatible: + enum: + - brcm,7268-v3d + - brcm,7278-v3d + + reg: + items: + - description: hub register (required) + - description: core0 register (required) + - description: GCA cache controller register (if GCA controller present) + - description: bridge register (if no external reset controller) + minItems: 2 + + reg-names: + items: + - const: hub + - const: core0 + - enum: [ bridge, gca ] + - enum: [ bridge, gca ] + minItems: 2 + maxItems: 4 + + interrupts: + items: + - description: hub interrupt (required) + - description: core interrupts (if it doesn't share the hub's interrupt) + minItems: 1 + + clocks: + maxItems: 1 + + resets: + maxItems: 1 + + power-domains: + maxItems: 1 + +required: + - compatible + - reg + - reg-names + - interrupts + +additionalProperties: false + +examples: + - | + gpu@f1200000 { + compatible = "brcm,7268-v3d"; + reg = <0xf1200000 0x4000>, + <0xf1208000 0x4000>, + <0xf1204000 0x100>, + <0xf1204100 0x100>; + reg-names = "hub", "core0", "bridge", "gca"; + interrupts = <0 78 4>, + <0 77 4>; + }; + +... diff --git a/Documentation/devicetree/bindings/graph.txt b/Documentation/devicetree/bindings/graph.txt index 0415e2c53ba072e8db49427d476714ee15f5221a..14733b5cb61e2f08f850e710298f22b6f0104025 100644 --- a/Documentation/devicetree/bindings/graph.txt +++ b/Documentation/devicetree/bindings/graph.txt @@ -1,128 +1 @@ -Common bindings for device graphs - -General concept ---------------- - -The hierarchical organisation of the device tree is well suited to describe -control flow to devices, but there can be more complex connections between -devices that work together to form a logical compound device, following an -arbitrarily complex graph. -There already is a simple directed graph between devices tree nodes using -phandle properties pointing to other nodes to describe connections that -can not be inferred from device tree parent-child relationships. The device -tree graph bindings described herein abstract more complex devices that can -have multiple specifiable ports, each of which can be linked to one or more -ports of other devices. - -These common bindings do not contain any information about the direction or -type of the connections, they just map their existence. Specific properties -may be described by specialized bindings depending on the type of connection. - -To see how this binding applies to video pipelines, for example, see -Documentation/devicetree/bindings/media/video-interfaces.txt. -Here the ports describe data interfaces, and the links between them are -the connecting data buses. A single port with multiple connections can -correspond to multiple devices being connected to the same physical bus. - -Organisation of ports and endpoints ------------------------------------ - -Ports are described by child 'port' nodes contained in the device node. -Each port node contains an 'endpoint' subnode for each remote device port -connected to this port. If a single port is connected to more than one -remote device, an 'endpoint' child node must be provided for each link. -If more than one port is present in a device node or there is more than one -endpoint at a port, or a port node needs to be associated with a selected -hardware interface, a common scheme using '#address-cells', '#size-cells' -and 'reg' properties is used to number the nodes. - -device { - ... - #address-cells = <1>; - #size-cells = <0>; - - port@0 { - #address-cells = <1>; - #size-cells = <0>; - reg = <0>; - - endpoint@0 { - reg = <0>; - ... - }; - endpoint@1 { - reg = <1>; - ... - }; - }; - - port@1 { - reg = <1>; - - endpoint { ... }; - }; -}; - -All 'port' nodes can be grouped under an optional 'ports' node, which -allows to specify #address-cells, #size-cells properties for the 'port' -nodes independently from any other child device nodes a device might -have. - -device { - ... - ports { - #address-cells = <1>; - #size-cells = <0>; - - port@0 { - ... - endpoint@0 { ... }; - endpoint@1 { ... }; - }; - - port@1 { ... }; - }; -}; - -Links between endpoints ------------------------ - -Each endpoint should contain a 'remote-endpoint' phandle property that points -to the corresponding endpoint in the port of the remote device. In turn, the -remote endpoint should contain a 'remote-endpoint' property. If it has one, it -must not point to anything other than the local endpoint. Two endpoints with -their 'remote-endpoint' phandles pointing at each other form a link between the -containing ports. - -device-1 { - port { - device_1_output: endpoint { - remote-endpoint = <&device_2_input>; - }; - }; -}; - -device-2 { - port { - device_2_input: endpoint { - remote-endpoint = <&device_1_output>; - }; - }; -}; - -Required properties -------------------- - -If there is more than one 'port' or more than one 'endpoint' node or 'reg' -property present in the port and/or endpoint nodes then the following -properties are required in a relevant parent node: - - - #address-cells : number of cells required to define port/endpoint - identifier, should be 1. - - #size-cells : should be zero. - -Optional endpoint properties ----------------------------- - -- remote-endpoint: phandle to an 'endpoint' subnode of a remote device node. - +This file has moved to graph.yaml in dt-schema repo diff --git a/Documentation/devicetree/bindings/hwlock/ti,omap-hwspinlock.yaml b/Documentation/devicetree/bindings/hwlock/ti,omap-hwspinlock.yaml index ac35491a6f65c6c346e8123d624d21377268cf63..ae1b37dbee7573c9d0bbae91a1185e28881d7cc1 100644 --- a/Documentation/devicetree/bindings/hwlock/ti,omap-hwspinlock.yaml +++ b/Documentation/devicetree/bindings/hwlock/ti,omap-hwspinlock.yaml @@ -13,6 +13,7 @@ properties: compatible: enum: - ti,omap4-hwspinlock # for OMAP44xx, OMAP54xx, AM33xx, AM43xx, DRA7xx SoCs + - ti,am64-hwspinlock # for K3 AM64x SoCs - ti,am654-hwspinlock # for K3 AM65x, J721E and J7200 SoCs reg: diff --git a/Documentation/devicetree/bindings/hwmon/ti,tps23861.yaml b/Documentation/devicetree/bindings/hwmon/ti,tps23861.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3bc8e73dfbf09f4d9d8f43152822dd9eb40341af --- /dev/null +++ b/Documentation/devicetree/bindings/hwmon/ti,tps23861.yaml @@ -0,0 +1,51 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +%YAML 1.2 +--- + +$id: http://devicetree.org/schemas/hwmon/ti,tps23861.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: TI TPS23861 PoE PSE + +maintainers: + - Robert Marko + +description: | + The TPS23861 is a IEEE 802.3at Quad Port Power-over-Ethernet PSE Controller. + + Datasheets: + https://www.ti.com/lit/gpn/tps23861 + + +properties: + compatible: + enum: + - ti,tps23861 + + reg: + maxItems: 1 + + shunt-resistor-micro-ohms: + description: The value of curent sense resistor in microohms. + default: 255000 + minimum: 250000 + maximum: 255000 + +required: + - compatible + - reg + +additionalProperties: false + +examples: + - | + i2c { + #address-cells = <1>; + #size-cells = <0>; + + tps23861@30 { + compatible = "ti,tps23861"; + reg = <0x30>; + shunt-resistor-micro-ohms = <255000>; + }; + }; diff --git a/Documentation/devicetree/bindings/i2c/i2c-sirf.txt b/Documentation/devicetree/bindings/i2c/i2c-sirf.txt deleted file mode 100644 index 2701eefb00f77c9f64d79c63efed43c4b2cec239..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/i2c/i2c-sirf.txt +++ /dev/null @@ -1,19 +0,0 @@ -I2C for SiRFprimaII platforms - -Required properties : -- compatible : Must be "sirf,prima2-i2c" -- reg: physical base address of the controller and length of memory mapped - region. -- interrupts: interrupt number to the cpu. - -Optional properties: -- clock-frequency : Constains desired I2C/HS-I2C bus clock frequency in Hz. - The absence of the property indicates the default frequency 100 kHz. - -Examples : - -i2c0: i2c@b00e0000 { - compatible = "sirf,prima2-i2c"; - reg = <0xb00e0000 0x10000>; - interrupts = <24>; -}; diff --git a/Documentation/devicetree/bindings/i2c/i2c-stu300.txt b/Documentation/devicetree/bindings/i2c/i2c-stu300.txt deleted file mode 100644 index bd81a482634f3416241196270ee8ad4b4308fc0b..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/i2c/i2c-stu300.txt +++ /dev/null @@ -1,15 +0,0 @@ -ST Microelectronics DDC I2C - -Required properties : -- compatible : Must be "st,ddci2c" -- reg: physical base address of the controller and length of memory mapped - region. -- interrupts: interrupt number to the cpu. -- #address-cells = <1>; -- #size-cells = <0>; - -Optional properties: -- Child nodes conforming to i2c bus binding - -Examples : - diff --git a/Documentation/devicetree/bindings/i2c/i2c-zx2967.txt b/Documentation/devicetree/bindings/i2c/i2c-zx2967.txt deleted file mode 100644 index cb806d1ae4c9d525ba76f713c1a797d7eb7ce1e6..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/i2c/i2c-zx2967.txt +++ /dev/null @@ -1,22 +0,0 @@ -ZTE zx2967 I2C controller - -Required properties: - - compatible: must be "zte,zx296718-i2c" - - reg: physical address and length of the device registers - - interrupts: a single interrupt specifier - - clocks: clock for the device - - #address-cells: should be <1> - - #size-cells: should be <0> - - clock-frequency: the desired I2C bus clock frequency. - -Examples: - - i2c@112000 { - compatible = "zte,zx296718-i2c"; - reg = <0x00112000 0x1000>; - interrupts = ; - clocks = <&osc24m>; - #address-cells = <1> - #size-cells = <0>; - clock-frequency = <1600000>; - }; diff --git a/Documentation/devicetree/bindings/i2c/marvell,mv64xxx-i2c.yaml b/Documentation/devicetree/bindings/i2c/marvell,mv64xxx-i2c.yaml index 5b5ae402f97abf3a72f94621c70308eb6e275683..eb72dd571def9ddb0b9875a7e01b9f185c317aae 100644 --- a/Documentation/devicetree/bindings/i2c/marvell,mv64xxx-i2c.yaml +++ b/Documentation/devicetree/bindings/i2c/marvell,mv64xxx-i2c.yaml @@ -18,21 +18,14 @@ properties: - const: allwinner,sun4i-a10-i2c - const: allwinner,sun6i-a31-i2c - items: - - const: allwinner,sun8i-a23-i2c + - enum: + - allwinner,sun8i-a23-i2c + - allwinner,sun8i-a83t-i2c + - allwinner,sun50i-a64-i2c + - allwinner,sun50i-a100-i2c + - allwinner,sun50i-h6-i2c + - allwinner,sun50i-h616-i2c - const: allwinner,sun6i-a31-i2c - - items: - - const: allwinner,sun8i-a83t-i2c - - const: allwinner,sun6i-a31-i2c - - items: - - const: allwinner,sun50i-a64-i2c - - const: allwinner,sun6i-a31-i2c - - items: - - const: allwinner,sun50i-a100-i2c - - const: allwinner,sun6i-a31-i2c - - items: - - const: allwinner,sun50i-h6-i2c - - const: allwinner,sun6i-a31-i2c - - const: marvell,mv64xxx-i2c - const: marvell,mv78230-i2c - const: marvell,mv78230-a0-i2c diff --git a/Documentation/devicetree/bindings/i2c/nuvoton,npcm7xx-i2c.yaml b/Documentation/devicetree/bindings/i2c/nuvoton,npcm7xx-i2c.yaml index e3ef2d36f372ce9a0b90a6bed35e86207c0f847b..128444942aec396e27f141b61eee544dbaa09c6a 100644 --- a/Documentation/devicetree/bindings/i2c/nuvoton,npcm7xx-i2c.yaml +++ b/Documentation/devicetree/bindings/i2c/nuvoton,npcm7xx-i2c.yaml @@ -17,7 +17,7 @@ maintainers: properties: compatible: - const: nuvoton,npcm7xx-i2c + const: nuvoton,npcm750-i2c reg: maxItems: 1 diff --git a/Documentation/devicetree/bindings/i2c/renesas,i2c.txt b/Documentation/devicetree/bindings/i2c/renesas,i2c.txt index 96d869ac38390f0319d1659103926c9f489d616d..5762d2d1ab9c9658a9f36cfc36fc48042f00170d 100644 --- a/Documentation/devicetree/bindings/i2c/renesas,i2c.txt +++ b/Documentation/devicetree/bindings/i2c/renesas,i2c.txt @@ -26,6 +26,7 @@ Required properties: "renesas,i2c-r8a77980" if the device is a part of a R8A77980 SoC. "renesas,i2c-r8a77990" if the device is a part of a R8A77990 SoC. "renesas,i2c-r8a77995" if the device is a part of a R8A77995 SoC. + "renesas,i2c-r8a779a0" if the device is a part of a R8A779A0 SoC. "renesas,rcar-gen1-i2c" for a generic R-Car Gen1 compatible device. "renesas,rcar-gen2-i2c" for a generic R-Car Gen2 or RZ/G1 compatible device. diff --git a/Documentation/devicetree/bindings/i3c/i3c.txt b/Documentation/devicetree/bindings/i3c/i3c.txt deleted file mode 100644 index 4ffe059f0feca1a6bc1d26489cf51a619ee7e265..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/i3c/i3c.txt +++ /dev/null @@ -1,140 +0,0 @@ -Generic device tree bindings for I3C busses -=========================================== - -This document describes generic bindings that should be used to describe I3C -busses in a device tree. - -Required properties -------------------- - -- #address-cells - should be <3>. Read more about addresses below. -- #size-cells - should be <0>. -- compatible - name of the I3C master controller driving the I3C bus - -For other required properties e.g. to describe register sets, -clocks, etc. check the binding documentation of the specific driver. -The node describing an I3C bus should be named i3c-master. - -Optional properties -------------------- - -These properties may not be supported by all I3C master drivers. Each I3C -master bindings should specify which of them are supported. - -- i3c-scl-hz: frequency of the SCL signal used for I3C transfers. - When undefined the core sets it to 12.5MHz. - -- i2c-scl-hz: frequency of the SCL signal used for I2C transfers. - When undefined, the core looks at LVR (Legacy Virtual Register) - values of I2C devices described in the device tree to determine - the maximum I2C frequency. - -I2C devices -=========== - -Each I2C device connected to the bus should be described in a subnode. All -properties described in Documentation/devicetree/bindings/i2c/i2c.txt are -valid here, but several new properties have been added. - -New constraint on existing properties: --------------------------------------- -- reg: contains 3 cells - + first cell : still encoding the I2C address. 10 bit addressing is not - supported. Devices with 10 bit address can't be properly passed through - DEFSLVS command. - - + second cell: shall be 0 - - + third cell: shall encode the I3C LVR (Legacy Virtual Register) - bit[31:8]: unused/ignored - bit[7:5]: I2C device index. Possible values - * 0: I2C device has a 50 ns spike filter - * 1: I2C device does not have a 50 ns spike filter but supports high - frequency on SCL - * 2: I2C device does not have a 50 ns spike filter and is not tolerant - to high frequencies - * 3-7: reserved - - bit[4]: tell whether the device operates in FM (Fast Mode) or FM+ mode - * 0: FM+ mode - * 1: FM mode - - bit[3:0]: device type - * 0-15: reserved - -The I2C node unit-address should always match the first cell of the reg -property: @. - -I3C devices -=========== - -All I3C devices are supposed to support DAA (Dynamic Address Assignment), and -are thus discoverable. So, by default, I3C devices do not have to be described -in the device tree. -This being said, one might want to attach extra resources to these devices, -and those resources may have to be described in the device tree, which in turn -means we have to describe I3C devices. - -Another use case for describing an I3C device in the device tree is when this -I3C device has a static I2C address and we want to assign it a specific I3C -dynamic address before the DAA takes place (so that other devices on the bus -can't take this dynamic address). - -The I3C device should be names @,, -where device-type is describing the type of device connected on the bus -(gpio-controller, sensor, ...). - -Required properties -------------------- -- reg: contains 3 cells - + first cell : encodes the static I2C address. Should be 0 if the device does - not have one (0 is not a valid I2C address). - - + second and third cells: should encode the ProvisionalID. The second cell - contains the manufacturer ID left-shifted by 1. - The third cell contains ORing of the part ID - left-shifted by 16, the instance ID left-shifted - by 12 and the extra information. This encoding is - following the PID definition provided by the I3C - specification. - -Optional properties -------------------- -- assigned-address: dynamic address to be assigned to this device. This - property is only valid if the I3C device has a static - address (first cell of the reg property != 0). - - -Example: - - i3c-master@d040000 { - compatible = "cdns,i3c-master"; - clocks = <&coreclock>, <&i3csysclock>; - clock-names = "pclk", "sysclk"; - interrupts = <3 0>; - reg = <0x0d040000 0x1000>; - #address-cells = <3>; - #size-cells = <0>; - i2c-scl-hz = <100000>; - - /* I2C device. */ - nunchuk: nunchuk@52 { - compatible = "nintendo,nunchuk"; - reg = <0x52 0x0 0x10>; - }; - - /* I3C device with a static I2C address. */ - thermal_sensor: sensor@68,39200144004 { - reg = <0x68 0x392 0x144004>; - assigned-address = <0xa>; - }; - - /* - * I3C device without a static I2C address but requiring - * resources described in the DT. - */ - sensor@0,39200154004 { - reg = <0x0 0x392 0x154004>; - clocks = <&clock_provider 0>; - }; - }; diff --git a/Documentation/devicetree/bindings/i3c/i3c.yaml b/Documentation/devicetree/bindings/i3c/i3c.yaml new file mode 100644 index 0000000000000000000000000000000000000000..52042aa44d19bdeacc91b542a47f43bb7c5b0804 --- /dev/null +++ b/Documentation/devicetree/bindings/i3c/i3c.yaml @@ -0,0 +1,179 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/i3c/i3c.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: I3C bus binding + +maintainers: + - Alexandre Belloni + - Miquel Raynal + +description: | + I3C busses can be described with a node for the primary I3C controller device + and a set of child nodes for each I2C or I3C slave on the bus. Each of them + may, during the life of the bus, request mastership. + +properties: + $nodename: + pattern: "^i3c-master@[0-9a-f]+$" + + "#address-cells": + const: 3 + description: | + Each I2C device connected to the bus should be described in a subnode. + + All I3C devices are supposed to support DAA (Dynamic Address Assignment), + and are thus discoverable. So, by default, I3C devices do not have to be + described in the device tree. This being said, one might want to attach + extra resources to these devices, and those resources may have to be + described in the device tree, which in turn means we have to describe + I3C devices. + + Another use case for describing an I3C device in the device tree is when + this I3C device has a static I2C address and we want to assign it a + specific I3C dynamic address before the DAA takes place (so that other + devices on the bus can't take this dynamic address). + + "#size-cells": + const: 0 + + i3c-scl-hz: + description: | + Frequency of the SCL signal used for I3C transfers. When undefined, the + default value should be 12.5MHz. + + May not be supported by all controllers. + + i2c-scl-hz: + description: | + Frequency of the SCL signal used for I2C transfers. When undefined, the + default should be to look at LVR (Legacy Virtual Register) values of + I2C devices described in the device tree to determine the maximum I2C + frequency. + + May not be supported by all controllers. + +required: + - "#address-cells" + - "#size-cells" + +patternProperties: + "@[0-9a-f]+$": + type: object + description: | + I2C child, should be named: @ + + All properties described in Documentation/devicetree/bindings/i2c/i2c.txt + are valid here, except the reg property whose content is changed. + + properties: + compatible: + description: + Compatible of the I2C device. + + reg: + items: + - items: + - description: | + I2C address. 10 bit addressing is not supported. Devices with + 10-bit address can't be properly passed through DEFSLVS + command. + minimum: 0 + maximum: 0x7f + - const: 0 + - description: | + Shall encode the I3C LVR (Legacy Virtual Register): + bit[31:8]: unused/ignored + bit[7:5]: I2C device index. Possible values: + * 0: I2C device has a 50 ns spike filter + * 1: I2C device does not have a 50 ns spike filter but + supports high frequency on SCL + * 2: I2C device does not have a 50 ns spike filter and is + not tolerant to high frequencies + * 3-7: reserved + bit[4]: tell whether the device operates in FM (Fast Mode) + or FM+ mode: + * 0: FM+ mode + * 1: FM mode + bit[3:0]: device type + * 0-15: reserved + + required: + - compatible + - reg + + "@[0-9a-f]+,[0-9a-f]+$": + type: object + description: | + I3C child, should be named: @, + + properties: + reg: + items: + - items: + - description: | + Encodes the static I2C address. Should be 0 if the device does + not have one (0 is not a valid I2C address). + minimum: 0 + maximum: 0x7f + - description: | + First half of the Provisional ID (following the PID + definition provided by the I3C specification). + + Contains the manufacturer ID left-shifted by 1. + - description: | + Second half of the Provisional ID (following the PID + definition provided by the I3C specification). + + Contains the ORing of the part ID left-shifted by 16, + the instance ID left-shifted by 12 and extra information. + + assigned-address: + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 0x1 + maximum: 0xff + description: | + Dynamic address to be assigned to this device. This property is only + valid if the I3C device has a static address (first cell of the reg + property != 0). + + required: + - reg + +additionalProperties: true + +examples: + - | + i3c-master@d040000 { + compatible = "cdns,i3c-master"; + clocks = <&coreclock>, <&i3csysclock>; + clock-names = "pclk", "sysclk"; + interrupts = <3 0>; + reg = <0x0d040000 0x1000>; + #address-cells = <3>; + #size-cells = <0>; + i2c-scl-hz = <100000>; + + /* I2C device. */ + nunchuk: nunchuk@52 { + compatible = "nintendo,nunchuk"; + reg = <0x52 0x0 0x10>; + }; + + /* I3C device with a static I2C address. */ + thermal_sensor: sensor@68,39200144004 { + reg = <0x68 0x392 0x144004>; + assigned-address = <0xa>; + }; + + /* + * I3C device without a static I2C address but requiring + * resources described in the DT. + */ + sensor@0,39200154004 { + reg = <0x0 0x392 0x154004>; + clocks = <&clock_provider 0>; + }; + }; diff --git a/Documentation/devicetree/bindings/i3c/mipi-i3c-hci.yaml b/Documentation/devicetree/bindings/i3c/mipi-i3c-hci.yaml index 07a7b10163a3e394aedeb3b096fec76f6ffe9c4a..04da001fc6ec952786b09470cfeba3e06e80685c 100644 --- a/Documentation/devicetree/bindings/i3c/mipi-i3c-hci.yaml +++ b/Documentation/devicetree/bindings/i3c/mipi-i3c-hci.yaml @@ -9,6 +9,9 @@ title: MIPI I3C HCI Device Tree Bindings maintainers: - Nicolas Pitre +allOf: + - $ref: /schemas/i3c/i3c.yaml# + description: | MIPI I3C Host Controller Interface @@ -36,12 +39,14 @@ required: - reg - interrupts -additionalProperties: false +unevaluatedProperties: false examples: - | - i3c@a0000000 { + i3c-master@a0000000 { compatible = "mipi-i3c-hci"; reg = <0xa0000000 0x2000>; interrupts = <89>; + #address-cells = <3>; + #size-cells = <0>; }; diff --git a/Documentation/devicetree/bindings/i3c/silvaco,i3c-master.yaml b/Documentation/devicetree/bindings/i3c/silvaco,i3c-master.yaml new file mode 100644 index 0000000000000000000000000000000000000000..adb5165505aa3098498a39b1e91792d377d17aec --- /dev/null +++ b/Documentation/devicetree/bindings/i3c/silvaco,i3c-master.yaml @@ -0,0 +1,60 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/i3c/silvaco,i3c-master.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Silvaco I3C master + +maintainers: + - Conor Culhane + +allOf: + - $ref: "i3c.yaml#" + +properties: + compatible: + const: silvaco,i3c-master-v1 + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + + clocks: + items: + - description: system clock + - description: bus clock + - description: other (slower) events clock + + clock-names: + items: + - const: pclk + - const: fast_clk + - const: slow_clk + + resets: + maxItems: 1 + +required: + - compatible + - reg + - interrupts + - clock-names + - clocks + +additionalProperties: true + +examples: + - | + i3c-master@a0000000 { + compatible = "silvaco,i3c-master"; + clocks = <&zynqmp_clk 71>, <&fclk>, <&sclk>; + clock-names = "pclk", "fast_clk", "slow_clk"; + interrupt-parent = <&gic>; + interrupts = <0 89 4>; + reg = <0xa0000000 0x1000>; + #address-cells = <3>; + #size-cells = <0>; + }; diff --git a/Documentation/devicetree/bindings/iio/adc/adi,ad7768-1.yaml b/Documentation/devicetree/bindings/iio/adc/adi,ad7768-1.yaml index 924477dfb833dd5491147b56d9e799a13db09197..a85a28145ef6b8f3f673524a52a0aa2483d7239c 100644 --- a/Documentation/devicetree/bindings/iio/adc/adi,ad7768-1.yaml +++ b/Documentation/devicetree/bindings/iio/adc/adi,ad7768-1.yaml @@ -40,6 +40,7 @@ properties: ADC reference voltage supply adi,sync-in-gpios: + maxItems: 1 description: Enables synchronization of multiple devices that require simultaneous sampling. A pulse is always required if the configuration is changed @@ -76,6 +77,7 @@ patternProperties: properties: reg: + maxItems: 1 description: | The channel number. diff --git a/Documentation/devicetree/bindings/iio/adc/aspeed,ast2400-adc.yaml b/Documentation/devicetree/bindings/iio/adc/aspeed,ast2400-adc.yaml index 7f534a933e9246d9484fe794a67df74ce7c55a97..a726b6c2ab65a25024ae8f3153e089f6805384d3 100644 --- a/Documentation/devicetree/bindings/iio/adc/aspeed,ast2400-adc.yaml +++ b/Documentation/devicetree/bindings/iio/adc/aspeed,ast2400-adc.yaml @@ -23,6 +23,7 @@ properties: maxItems: 1 clocks: + maxItems: 1 description: Input clock used to derive the sample clock. Expected to be the SoC's APB clock. diff --git a/Documentation/devicetree/bindings/iio/adc/lltc,ltc2496.yaml b/Documentation/devicetree/bindings/iio/adc/lltc,ltc2496.yaml index 2716d4e953296adcf66507362947b3922b1aeea8..0bd2fc0356c86476eb12dbab07244ca57bd94e83 100644 --- a/Documentation/devicetree/bindings/iio/adc/lltc,ltc2496.yaml +++ b/Documentation/devicetree/bindings/iio/adc/lltc,ltc2496.yaml @@ -20,7 +20,7 @@ properties: description: Power supply for the reference voltage reg: - description: spi chipselect number according to the usual spi bindings + maxItems: 1 spi-max-frequency: description: maximal spi bus frequency supported diff --git a/Documentation/devicetree/bindings/iio/adc/qcom,spmi-vadc.yaml b/Documentation/devicetree/bindings/iio/adc/qcom,spmi-vadc.yaml index 95cc705b961bf849620d8d964082521da7053be5..74a4a9d95798c8d771d553ea4ead04c17a8c728c 100644 --- a/Documentation/devicetree/bindings/iio/adc/qcom,spmi-vadc.yaml +++ b/Documentation/devicetree/bindings/iio/adc/qcom,spmi-vadc.yaml @@ -68,6 +68,7 @@ patternProperties: properties: reg: + maxItems: 1 description: | ADC channel number. See include/dt-bindings/iio/qcom,spmi-vadc.h diff --git a/Documentation/devicetree/bindings/iio/adc/st,stm32-adc.yaml b/Documentation/devicetree/bindings/iio/adc/st,stm32-adc.yaml index 517e32976c30426a49e6e49a9f927c6547daecce..a58334c3bb76d685867bc1c45e65e01581b46d35 100644 --- a/Documentation/devicetree/bindings/iio/adc/st,stm32-adc.yaml +++ b/Documentation/devicetree/bindings/iio/adc/st,stm32-adc.yaml @@ -41,6 +41,8 @@ properties: maxItems: 2 clocks: + minItems: 1 + maxItems: 2 description: | Core can use up to two clocks, depending on part used: - "adc" clock: for the analog circuitry, common to all ADCs. diff --git a/Documentation/devicetree/bindings/iio/dac/adi,ad5686.yaml b/Documentation/devicetree/bindings/iio/dac/adi,ad5696.yaml similarity index 77% rename from Documentation/devicetree/bindings/iio/dac/adi,ad5686.yaml rename to Documentation/devicetree/bindings/iio/dac/adi,ad5696.yaml index 8065228e5df8fb8f9f5e42ab47acea8efb48215e..56b0cda0f30ab88cdec6064d9c31ab6c3bb6e5db 100644 --- a/Documentation/devicetree/bindings/iio/dac/adi,ad5686.yaml +++ b/Documentation/devicetree/bindings/iio/dac/adi,ad5696.yaml @@ -1,16 +1,16 @@ # SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) %YAML 1.2 --- -$id: http://devicetree.org/schemas/iio/dac/adi,ad5686.yaml# +$id: http://devicetree.org/schemas/iio/dac/adi,ad5696.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Analog Devices AD5686 and similar multi-channel DACs +title: Analog Devices AD5696 and similar multi-channel DACs maintainers: - Michael Auchter description: | - Binding for Analog Devices AD5686 and similar multi-channel DACs + Binding for Analog Devices AD5696 and similar multi-channel DACs properties: compatible: @@ -48,8 +48,8 @@ examples: #address-cells = <1>; #size-cells = <0>; - ad5686: dac@0 { - compatible = "adi,ad5686"; + ad5696: dac@0 { + compatible = "adi,ad5696"; reg = <0>; vcc-supply = <&dac_vref>; }; diff --git a/Documentation/devicetree/bindings/iio/health/ti,afe4404.yaml b/Documentation/devicetree/bindings/iio/health/ti,afe4404.yaml index 3b4d6c48b8bb0105995144ae5e7c4101ffbe52ff..c0e815d9999e38df1bf4cd5a5ccdc9beac3b1dcd 100644 --- a/Documentation/devicetree/bindings/iio/health/ti,afe4404.yaml +++ b/Documentation/devicetree/bindings/iio/health/ti,afe4404.yaml @@ -11,7 +11,7 @@ maintainers: properties: compatible: - const: ti,afe4403 + const: ti,afe4404 reg: maxItems: 1 diff --git a/Documentation/devicetree/bindings/iio/magnetometer/asahi-kasei,ak8975.yaml b/Documentation/devicetree/bindings/iio/magnetometer/asahi-kasei,ak8975.yaml index a25590a16ba7e95b7c2f438eb11ef87f55276c09..a0a1ffe017df85514869b5efb112795184b128c9 100644 --- a/Documentation/devicetree/bindings/iio/magnetometer/asahi-kasei,ak8975.yaml +++ b/Documentation/devicetree/bindings/iio/magnetometer/asahi-kasei,ak8975.yaml @@ -47,6 +47,7 @@ properties: description: an optional 3x3 mounting rotation matrix. reset-gpios: + maxItems: 1 description: | an optional pin needed for AK09911 to set the reset state. This should be usually active low diff --git a/Documentation/devicetree/bindings/iio/potentiometer/adi,ad5272.yaml b/Documentation/devicetree/bindings/iio/potentiometer/adi,ad5272.yaml index 1aee9f9be951e8c0d6f0874ef3ccfadf17c4a831..0ebb6725a1af93b202b2b1f8a56bd989fb278fca 100644 --- a/Documentation/devicetree/bindings/iio/potentiometer/adi,ad5272.yaml +++ b/Documentation/devicetree/bindings/iio/potentiometer/adi,ad5272.yaml @@ -25,6 +25,7 @@ properties: maxItems: 1 reset-gpios: + maxItems: 1 description: Active low signal to the AD5272 RESET input. diff --git a/Documentation/devicetree/bindings/input/goodix,gt7375p.yaml b/Documentation/devicetree/bindings/input/goodix,gt7375p.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fe1c5016f7f316601b9f54124f74eacec1a8a5f1 --- /dev/null +++ b/Documentation/devicetree/bindings/input/goodix,gt7375p.yaml @@ -0,0 +1,65 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/input/goodix,gt7375p.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Goodix GT7375P touchscreen + +maintainers: + - Douglas Anderson + +description: + Supports the Goodix GT7375P touchscreen. + This touchscreen uses the i2c-hid protocol but has some non-standard + power sequencing required. + +properties: + compatible: + items: + - const: goodix,gt7375p + + reg: + enum: + - 0x5d + - 0x14 + + interrupts: + maxItems: 1 + + reset-gpios: + true + + vdd-supply: + description: The 3.3V supply to the touchscreen. + +required: + - compatible + - reg + - interrupts + - reset-gpios + - vdd-supply + +additionalProperties: false + +examples: + - | + #include + #include + #include + + i2c { + #address-cells = <1>; + #size-cells = <0>; + + ap_ts: touchscreen@5d { + compatible = "goodix,gt7375p"; + reg = <0x5d>; + + interrupt-parent = <&tlmm>; + interrupts = <9 IRQ_TYPE_LEVEL_LOW>; + + reset-gpios = <&tlmm 8 GPIO_ACTIVE_LOW>; + vdd-supply = <&pp3300_ts>; + }; + }; diff --git a/Documentation/devicetree/bindings/input/google,cros-ec-keyb.yaml b/Documentation/devicetree/bindings/input/google,cros-ec-keyb.yaml index 8e50c14a9d778945cf7344b2efbc7ed25633f08a..5377b232fa10f85862bd280dd7e53dfc7947bb56 100644 --- a/Documentation/devicetree/bindings/input/google,cros-ec-keyb.yaml +++ b/Documentation/devicetree/bindings/input/google,cros-ec-keyb.yaml @@ -31,6 +31,17 @@ properties: if the EC does not have its own logic or hardware for this. type: boolean + function-row-physmap: + minItems: 1 + maxItems: 15 + description: | + An ordered u32 array describing the rows/columns (in the scan matrix) + of top row keys from physical left (KEY_F1) to right. Each entry + encodes the row/column as: + (((row) & 0xFF) << 24) | (((column) & 0xFF) << 16) + where the lower 16 bits are reserved. This property is specified only + when the keyboard has a custom design for the top row keys. + required: - compatible @@ -38,11 +49,24 @@ unevaluatedProperties: false examples: - | + #include cros-ec-keyb { compatible = "google,cros-ec-keyb"; keypad,num-rows = <8>; keypad,num-columns = <13>; google,needs-ghost-filter; + function-row-physmap = < + MATRIX_KEY(0x00, 0x02, 0) /* T1 */ + MATRIX_KEY(0x03, 0x02, 0) /* T2 */ + MATRIX_KEY(0x02, 0x02, 0) /* T3 */ + MATRIX_KEY(0x01, 0x02, 0) /* T4 */ + MATRIX_KEY(0x03, 0x04, 0) /* T5 */ + MATRIX_KEY(0x02, 0x04, 0) /* T6 */ + MATRIX_KEY(0x01, 0x04, 0) /* T7 */ + MATRIX_KEY(0x02, 0x09, 0) /* T8 */ + MATRIX_KEY(0x01, 0x09, 0) /* T9 */ + MATRIX_KEY(0x00, 0x04, 0) /* T10 */ + >; /* * Keymap entries take the form of 0xRRCCKKKK where * RR=Row CC=Column KKKK=Key Code diff --git a/Documentation/devicetree/bindings/input/touchscreen/elan,elants_i2c.yaml b/Documentation/devicetree/bindings/input/touchscreen/elan,elants_i2c.yaml index a792d6377b1daf08e4d38dcf4c84343ce5013350..a9b53c2e6f0abc8e8bc495981995105322e7bfaa 100644 --- a/Documentation/devicetree/bindings/input/touchscreen/elan,elants_i2c.yaml +++ b/Documentation/devicetree/bindings/input/touchscreen/elan,elants_i2c.yaml @@ -29,6 +29,7 @@ properties: description: touchscreen can be used as a wakeup source. reset-gpios: + maxItems: 1 description: reset gpio the chip is connected to. vcc33-supply: diff --git a/Documentation/devicetree/bindings/interconnect/qcom,qcs404.yaml b/Documentation/devicetree/bindings/interconnect/qcom,qcs404.yaml deleted file mode 100644 index 3fbb8785fbc938e94a4903fd38cf8e99ec9ed93c..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/interconnect/qcom,qcs404.yaml +++ /dev/null @@ -1,77 +0,0 @@ -# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) -%YAML 1.2 ---- -$id: http://devicetree.org/schemas/interconnect/qcom,qcs404.yaml# -$schema: http://devicetree.org/meta-schemas/core.yaml# - -title: Qualcomm QCS404 Network-On-Chip interconnect - -maintainers: - - Georgi Djakov - -description: | - The Qualcomm QCS404 interconnect providers support adjusting the - bandwidth requirements between the various NoC fabrics. - -properties: - reg: - maxItems: 1 - - compatible: - enum: - - qcom,qcs404-bimc - - qcom,qcs404-pcnoc - - qcom,qcs404-snoc - - '#interconnect-cells': - const: 1 - - clock-names: - items: - - const: bus - - const: bus_a - - clocks: - items: - - description: Bus Clock - - description: Bus A Clock - -required: - - compatible - - reg - - '#interconnect-cells' - - clock-names - - clocks - -additionalProperties: false - -examples: - - | - #include - - bimc: interconnect@400000 { - reg = <0x00400000 0x80000>; - compatible = "qcom,qcs404-bimc"; - #interconnect-cells = <1>; - clock-names = "bus", "bus_a"; - clocks = <&rpmcc RPM_SMD_BIMC_CLK>, - <&rpmcc RPM_SMD_BIMC_A_CLK>; - }; - - pnoc: interconnect@500000 { - reg = <0x00500000 0x15080>; - compatible = "qcom,qcs404-pcnoc"; - #interconnect-cells = <1>; - clock-names = "bus", "bus_a"; - clocks = <&rpmcc RPM_SMD_PNOC_CLK>, - <&rpmcc RPM_SMD_PNOC_A_CLK>; - }; - - snoc: interconnect@580000 { - reg = <0x00580000 0x23080>; - compatible = "qcom,qcs404-snoc"; - #interconnect-cells = <1>; - clock-names = "bus", "bus_a"; - clocks = <&rpmcc RPM_SMD_SNOC_CLK>, - <&rpmcc RPM_SMD_SNOC_A_CLK>; - }; diff --git a/Documentation/devicetree/bindings/interconnect/qcom,msm8916.yaml b/Documentation/devicetree/bindings/interconnect/qcom,rpm.yaml similarity index 77% rename from Documentation/devicetree/bindings/interconnect/qcom,msm8916.yaml rename to Documentation/devicetree/bindings/interconnect/qcom,rpm.yaml index e1009ae4e8f75eb28a9916aeabb62e028085e85a..983d71fb53993951e33a1f4169d40dd9af303ea9 100644 --- a/Documentation/devicetree/bindings/interconnect/qcom,msm8916.yaml +++ b/Documentation/devicetree/bindings/interconnect/qcom,rpm.yaml @@ -1,27 +1,35 @@ # SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) %YAML 1.2 --- -$id: http://devicetree.org/schemas/interconnect/qcom,msm8916.yaml# +$id: http://devicetree.org/schemas/interconnect/qcom,rpm.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Qualcomm MSM8916 Network-On-Chip interconnect +title: Qualcomm RPM Network-On-Chip Interconnect maintainers: - Georgi Djakov description: | - The Qualcomm MSM8916 interconnect providers support adjusting the - bandwidth requirements between the various NoC fabrics. + RPM interconnect providers support system bandwidth requirements through + RPM processor. The provider is able to communicate with the RPM through + the RPM shared memory device. properties: + reg: + maxItems: 1 + compatible: enum: - qcom,msm8916-bimc - qcom,msm8916-pcnoc - qcom,msm8916-snoc - - reg: - maxItems: 1 + - qcom,msm8939-bimc + - qcom,msm8939-pcnoc + - qcom,msm8939-snoc + - qcom,msm8939-snoc-mm + - qcom,qcs404-bimc + - qcom,qcs404-pcnoc + - qcom,qcs404-snoc '#interconnect-cells': const: 1 diff --git a/Documentation/devicetree/bindings/interconnect/qcom,rpmh.yaml b/Documentation/devicetree/bindings/interconnect/qcom,rpmh.yaml index 30c2a092d2d34febd58e664e6428f399b43a00a4..799e73cdb90b494af79c98a8dac61ecaf61b74fd 100644 --- a/Documentation/devicetree/bindings/interconnect/qcom,rpmh.yaml +++ b/Documentation/devicetree/bindings/interconnect/qcom,rpmh.yaml @@ -45,6 +45,10 @@ properties: - qcom,sdm845-mem-noc - qcom,sdm845-mmss-noc - qcom,sdm845-system-noc + - qcom,sdx55-ipa-virt + - qcom,sdx55-mc-virt + - qcom,sdx55-mem-noc + - qcom,sdx55-system-noc - qcom,sm8150-aggre1-noc - qcom,sm8150-aggre2-noc - qcom,sm8150-camnoc-noc @@ -69,7 +73,7 @@ properties: - qcom,sm8250-system-noc '#interconnect-cells': - const: 1 + enum: [ 1, 2 ] qcom,bcm-voters: $ref: /schemas/types.yaml#/definitions/phandle-array diff --git a/Documentation/devicetree/bindings/interrupt-controller/fsl,intmux.yaml b/Documentation/devicetree/bindings/interrupt-controller/fsl,intmux.yaml index 43c6effbb5bd8fb4e1590bd04a03e7522debfdbe..1d6e0f64a807a167755af494766fdf6c756cb6a9 100644 --- a/Documentation/devicetree/bindings/interrupt-controller/fsl,intmux.yaml +++ b/Documentation/devicetree/bindings/interrupt-controller/fsl,intmux.yaml @@ -31,7 +31,7 @@ properties: The 1st cell is hw interrupt number, the 2nd cell is channel index. clocks: - description: ipg clock. + maxItems: 1 clock-names: const: ipg diff --git a/Documentation/devicetree/bindings/interrupt-controller/sifive,plic-1.0.0.yaml b/Documentation/devicetree/bindings/interrupt-controller/sifive,plic-1.0.0.yaml index b9a61c9f753082921342ee11636e21083e5ade35..08d5a57ce00ff4465eaf11b6853d4b1b03324785 100644 --- a/Documentation/devicetree/bindings/interrupt-controller/sifive,plic-1.0.0.yaml +++ b/Documentation/devicetree/bindings/interrupt-controller/sifive,plic-1.0.0.yaml @@ -8,10 +8,11 @@ $schema: http://devicetree.org/meta-schemas/core.yaml# title: SiFive Platform-Level Interrupt Controller (PLIC) description: - SiFive SOCs include an implementation of the Platform-Level Interrupt Controller - (PLIC) high-level specification in the RISC-V Privileged Architecture - specification. The PLIC connects all external interrupts in the system to all - hart contexts in the system, via the external interrupt source in each hart. + SiFive SoCs and other RISC-V SoCs include an implementation of the + Platform-Level Interrupt Controller (PLIC) high-level specification in + the RISC-V Privileged Architecture specification. The PLIC connects all + external interrupts in the system to all hart contexts in the system, via + the external interrupt source in each hart. A hart context is a privilege mode in a hardware execution thread. For example, in an 4 core system with 2-way SMT, you have 8 harts and probably at least two @@ -42,7 +43,9 @@ maintainers: properties: compatible: items: - - const: sifive,fu540-c000-plic + - enum: + - sifive,fu540-c000-plic + - canaan,k210-plic - const: sifive,plic-1.0.0 reg: diff --git a/Documentation/devicetree/bindings/interrupt-controller/st,stm32-exti.yaml b/Documentation/devicetree/bindings/interrupt-controller/st,stm32-exti.yaml index 2a5b29567926cec646728eed08bad4ce59ce47ca..6d3e68eb2e8ba4af9d45013c4ea3ad00073a1f31 100644 --- a/Documentation/devicetree/bindings/interrupt-controller/st,stm32-exti.yaml +++ b/Documentation/devicetree/bindings/interrupt-controller/st,stm32-exti.yaml @@ -36,6 +36,8 @@ properties: Reference to a phandle of a hardware spinlock provider node. interrupts: + minItems: 1 + maxItems: 96 description: Interrupts references to primary interrupt controller diff --git a/Documentation/devicetree/bindings/interrupt-controller/ti,pruss-intc.yaml b/Documentation/devicetree/bindings/interrupt-controller/ti,pruss-intc.yaml index c2ce215501a589041ca8f198646b28f907b53406..9731dd4421a1578258da8db784b10f434453c223 100644 --- a/Documentation/devicetree/bindings/interrupt-controller/ti,pruss-intc.yaml +++ b/Documentation/devicetree/bindings/interrupt-controller/ti,pruss-intc.yaml @@ -33,6 +33,9 @@ description: | corresponding PRUSS node. The node should be named "interrupt-controller". properties: + $nodename: + pattern: "^interrupt-controller@[0-9a-f]+$" + compatible: enum: - ti,pruss-intc diff --git a/Documentation/devicetree/bindings/iommu/arm,smmu.yaml b/Documentation/devicetree/bindings/iommu/arm,smmu.yaml index 3b63f2ae24dbb4dd09fc9b15788f984bc59c0549..6ba161dea4d82450ec23f50f83fd86525497a0de 100644 --- a/Documentation/devicetree/bindings/iommu/arm,smmu.yaml +++ b/Documentation/devicetree/bindings/iommu/arm,smmu.yaml @@ -34,9 +34,11 @@ properties: items: - enum: - qcom,sc7180-smmu-500 + - qcom,sc8180x-smmu-500 - qcom,sdm845-smmu-500 - qcom,sm8150-smmu-500 - qcom,sm8250-smmu-500 + - qcom,sm8350-smmu-500 - const: arm,mmu-500 - description: Qcom Adreno GPUs implementing "arm,smmu-v2" items: diff --git a/Documentation/devicetree/bindings/iommu/mediatek,iommu.txt b/Documentation/devicetree/bindings/iommu/mediatek,iommu.txt deleted file mode 100644 index ac949f7fe3d475df0e04b6f39c000c7ed0e0fe22..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/iommu/mediatek,iommu.txt +++ /dev/null @@ -1,105 +0,0 @@ -* Mediatek IOMMU Architecture Implementation - - Some Mediatek SOCs contain a Multimedia Memory Management Unit (M4U), and -this M4U have two generations of HW architecture. Generation one uses flat -pagetable, and only supports 4K size page mapping. Generation two uses the -ARM Short-Descriptor translation table format for address translation. - - About the M4U Hardware Block Diagram, please check below: - - EMI (External Memory Interface) - | - m4u (Multimedia Memory Management Unit) - | - +--------+ - | | - gals0-rx gals1-rx (Global Async Local Sync rx) - | | - | | - gals0-tx gals1-tx (Global Async Local Sync tx) - | | Some SoCs may have GALS. - +--------+ - | - SMI Common(Smart Multimedia Interface Common) - | - +----------------+------- - | | - | gals-rx There may be GALS in some larbs. - | | - | | - | gals-tx - | | - SMI larb0 SMI larb1 ... SoCs have several SMI local arbiter(larb). - (display) (vdec) - | | - | | - +-----+-----+ +----+----+ - | | | | | | - | | |... | | | ... There are different ports in each larb. - | | | | | | -OVL0 RDMA0 WDMA0 MC PP VLD - - As above, The Multimedia HW will go through SMI and M4U while it -access EMI. SMI is a bridge between m4u and the Multimedia HW. It contain -smi local arbiter and smi common. It will control whether the Multimedia -HW should go though the m4u for translation or bypass it and talk -directly with EMI. And also SMI help control the power domain and clocks for -each local arbiter. - Normally we specify a local arbiter(larb) for each multimedia HW -like display, video decode, and camera. And there are different ports -in each larb. Take a example, There are many ports like MC, PP, VLD in the -video decode local arbiter, all these ports are according to the video HW. - In some SoCs, there may be a GALS(Global Async Local Sync) module between -smi-common and m4u, and additional GALS module between smi-larb and -smi-common. GALS can been seen as a "asynchronous fifo" which could help -synchronize for the modules in different clock frequency. - -Required properties: -- compatible : must be one of the following string: - "mediatek,mt2701-m4u" for mt2701 which uses generation one m4u HW. - "mediatek,mt2712-m4u" for mt2712 which uses generation two m4u HW. - "mediatek,mt6779-m4u" for mt6779 which uses generation two m4u HW. - "mediatek,mt7623-m4u", "mediatek,mt2701-m4u" for mt7623 which uses - generation one m4u HW. - "mediatek,mt8167-m4u" for mt8167 which uses generation two m4u HW. - "mediatek,mt8173-m4u" for mt8173 which uses generation two m4u HW. - "mediatek,mt8183-m4u" for mt8183 which uses generation two m4u HW. -- reg : m4u register base and size. -- interrupts : the interrupt of m4u. -- clocks : must contain one entry for each clock-names. -- clock-names : Only 1 optional clock: - - "bclk": the block clock of m4u. - Here is the list which require this "bclk": - - mt2701, mt2712, mt7623 and mt8173. - Note that m4u use the EMI clock which always has been enabled before kernel - if there is no this "bclk". -- mediatek,larbs : List of phandle to the local arbiters in the current Socs. - Refer to bindings/memory-controllers/mediatek,smi-larb.txt. It must sort - according to the local arbiter index, like larb0, larb1, larb2... -- iommu-cells : must be 1. This is the mtk_m4u_id according to the HW. - Specifies the mtk_m4u_id as defined in - dt-binding/memory/mt2701-larb-port.h for mt2701, mt7623 - dt-binding/memory/mt2712-larb-port.h for mt2712, - dt-binding/memory/mt6779-larb-port.h for mt6779, - dt-binding/memory/mt8167-larb-port.h for mt8167, - dt-binding/memory/mt8173-larb-port.h for mt8173, and - dt-binding/memory/mt8183-larb-port.h for mt8183. - -Example: - iommu: iommu@10205000 { - compatible = "mediatek,mt8173-m4u"; - reg = <0 0x10205000 0 0x1000>; - interrupts = ; - clocks = <&infracfg CLK_INFRA_M4U>; - clock-names = "bclk"; - mediatek,larbs = <&larb0 &larb1 &larb2 &larb3 &larb4 &larb5>; - #iommu-cells = <1>; - }; - -Example for a client device: - display { - compatible = "mediatek,mt8173-disp"; - iommus = <&iommu M4U_PORT_DISP_OVL0>, - <&iommu M4U_PORT_DISP_RDMA0>; - ... - }; diff --git a/Documentation/devicetree/bindings/iommu/mediatek,iommu.yaml b/Documentation/devicetree/bindings/iommu/mediatek,iommu.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0f26fe14c8e21761a8de9f37ae6377fa9e0eb96c --- /dev/null +++ b/Documentation/devicetree/bindings/iommu/mediatek,iommu.yaml @@ -0,0 +1,183 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/iommu/mediatek,iommu.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: MediaTek IOMMU Architecture Implementation + +maintainers: + - Yong Wu + +description: |+ + Some MediaTek SOCs contain a Multimedia Memory Management Unit (M4U), and + this M4U have two generations of HW architecture. Generation one uses flat + pagetable, and only supports 4K size page mapping. Generation two uses the + ARM Short-Descriptor translation table format for address translation. + + About the M4U Hardware Block Diagram, please check below: + + EMI (External Memory Interface) + | + m4u (Multimedia Memory Management Unit) + | + +--------+ + | | + gals0-rx gals1-rx (Global Async Local Sync rx) + | | + | | + gals0-tx gals1-tx (Global Async Local Sync tx) + | | Some SoCs may have GALS. + +--------+ + | + SMI Common(Smart Multimedia Interface Common) + | + +----------------+------- + | | + | gals-rx There may be GALS in some larbs. + | | + | | + | gals-tx + | | + SMI larb0 SMI larb1 ... SoCs have several SMI local arbiter(larb). + (display) (vdec) + | | + | | + +-----+-----+ +----+----+ + | | | | | | + | | |... | | | ... There are different ports in each larb. + | | | | | | + OVL0 RDMA0 WDMA0 MC PP VLD + + As above, The Multimedia HW will go through SMI and M4U while it + access EMI. SMI is a bridge between m4u and the Multimedia HW. It contain + smi local arbiter and smi common. It will control whether the Multimedia + HW should go though the m4u for translation or bypass it and talk + directly with EMI. And also SMI help control the power domain and clocks for + each local arbiter. + + Normally we specify a local arbiter(larb) for each multimedia HW + like display, video decode, and camera. And there are different ports + in each larb. Take a example, There are many ports like MC, PP, VLD in the + video decode local arbiter, all these ports are according to the video HW. + + In some SoCs, there may be a GALS(Global Async Local Sync) module between + smi-common and m4u, and additional GALS module between smi-larb and + smi-common. GALS can been seen as a "asynchronous fifo" which could help + synchronize for the modules in different clock frequency. + +properties: + compatible: + oneOf: + - enum: + - mediatek,mt2701-m4u # generation one + - mediatek,mt2712-m4u # generation two + - mediatek,mt6779-m4u # generation two + - mediatek,mt8167-m4u # generation two + - mediatek,mt8173-m4u # generation two + - mediatek,mt8183-m4u # generation two + - mediatek,mt8192-m4u # generation two + + - description: mt7623 generation one + items: + - const: mediatek,mt7623-m4u + - const: mediatek,mt2701-m4u + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + + clocks: + items: + - description: bclk is the block clock. + + clock-names: + items: + - const: bclk + + mediatek,larbs: + $ref: /schemas/types.yaml#/definitions/phandle-array + minItems: 1 + maxItems: 32 + description: | + List of phandle to the local arbiters in the current Socs. + Refer to bindings/memory-controllers/mediatek,smi-larb.yaml. It must sort + according to the local arbiter index, like larb0, larb1, larb2... + + '#iommu-cells': + const: 1 + description: | + This is the mtk_m4u_id according to the HW. Specifies the mtk_m4u_id as + defined in + dt-binding/memory/mt2701-larb-port.h for mt2701 and mt7623, + dt-binding/memory/mt2712-larb-port.h for mt2712, + dt-binding/memory/mt6779-larb-port.h for mt6779, + dt-binding/memory/mt8167-larb-port.h for mt8167, + dt-binding/memory/mt8173-larb-port.h for mt8173, + dt-binding/memory/mt8183-larb-port.h for mt8183, + dt-binding/memory/mt8192-larb-port.h for mt8192. + + power-domains: + maxItems: 1 + +required: + - compatible + - reg + - interrupts + - mediatek,larbs + - '#iommu-cells' + +allOf: + - if: + properties: + compatible: + contains: + enum: + - mediatek,mt2701-m4u + - mediatek,mt2712-m4u + - mediatek,mt8173-m4u + - mediatek,mt8192-m4u + + then: + required: + - clocks + + - if: + properties: + compatible: + enum: + - mediatek,mt8192-m4u + + then: + required: + - power-domains + +additionalProperties: false + +examples: + - | + #include + #include + + iommu: iommu@10205000 { + compatible = "mediatek,mt8173-m4u"; + reg = <0x10205000 0x1000>; + interrupts = ; + clocks = <&infracfg CLK_INFRA_M4U>; + clock-names = "bclk"; + mediatek,larbs = <&larb0 &larb1 &larb2 + &larb3 &larb4 &larb5>; + #iommu-cells = <1>; + }; + + - | + #include + + /* Example for a client device */ + display { + compatible = "mediatek,mt8173-disp"; + iommus = <&iommu M4U_PORT_DISP_OVL0>, + <&iommu M4U_PORT_DISP_RDMA0>; + }; diff --git a/Documentation/devicetree/bindings/iommu/renesas,ipmmu-vmsa.yaml b/Documentation/devicetree/bindings/iommu/renesas,ipmmu-vmsa.yaml index cde1afa8dfd639e94085b1cfa65d12f55513d08c..dda44976acc144232656ea1efe9fb8ed293a0e7c 100644 --- a/Documentation/devicetree/bindings/iommu/renesas,ipmmu-vmsa.yaml +++ b/Documentation/devicetree/bindings/iommu/renesas,ipmmu-vmsa.yaml @@ -76,7 +76,6 @@ required: - compatible - reg - '#iommu-cells' - - power-domains oneOf: - required: @@ -86,6 +85,17 @@ oneOf: additionalProperties: false +allOf: + - if: + properties: + compatible: + not: + contains: + const: renesas,ipmmu-vmsa + then: + required: + - power-domains + examples: - | #include @@ -93,7 +103,7 @@ examples: #include ipmmu_mx: iommu@fe951000 { - compatible = "renasas,ipmmu-r8a7791", "renasas,ipmmu-vmsa"; + compatible = "renesas,ipmmu-r8a7791", "renesas,ipmmu-vmsa"; reg = <0xfe951000 0x1000>; interrupts = , ; diff --git a/Documentation/devicetree/bindings/leds/leds-lgm.yaml b/Documentation/devicetree/bindings/leds/leds-lgm.yaml new file mode 100644 index 0000000000000000000000000000000000000000..32bbf146c01d3670ebc8b770542233626e000ba2 --- /dev/null +++ b/Documentation/devicetree/bindings/leds/leds-lgm.yaml @@ -0,0 +1,113 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/leds/leds-lgm.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Intel Lightning Mountain (LGM) SoC LED Serial Shift Output (SSO) Controller driver + +maintainers: + - Zhu, Yi Xin + - Amireddy Mallikarjuna reddy + +properties: + compatible: + const: intel,lgm-ssoled + + gpio-controller: true + + '#gpio-cells': + const: 2 + + ngpios: + minimum: 0 + maximum: 32 + description: + Number of GPIOs this controller provides. + + intel,sso-update-rate-hz: + description: + Blink frequency for SOUTs in Hz. + + led-controller: + type: object + description: + This sub-node must contain a sub-node for each leds. + + additionalProperties: false + + patternProperties: + "^led@[0-23]$": + type: object + + properties: + reg: + description: Index of the LED. + minimum: 0 + maximum: 2 + + intel,sso-hw-trigger: + type: boolean + description: This property indicates Hardware driven/control LED. + + intel,sso-hw-blink: + type: boolean + description: This property indicates Enable LED blink by Hardware. + + intel,sso-blink-rate-hz: + description: LED HW blink frequency. + + retain-state-suspended: + type: boolean + description: The suspend state of LED can be retained. + + retain-state-shutdown: + type: boolean + description: Retain the state of the LED on shutdown. + +required: + - compatible + - reg + - clocks + - clock-names + - "#gpio-cells" + - gpio-controller + +additionalProperties: false + +examples: + - | + #include + #include + + ssogpio: ssogpio@e0d40000 { + compatible = "intel,sso-led"; + reg = <0xE0D40000 0x2E4>; + gpio-controller; + #gpio-cells = <2>; + ngpios = <32>; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_ledc>; + clocks = <&cgu0 LGM_GCLK_LEDC0>, <&afeclk>; + clock-names = "sso", "fpid"; + intel,sso-update-rate-hz = <250000>; + + led-controller { + #address-cells = <1>; + #size-cells = <0>; + + led@0 { + reg = <0>; + function = "gphy"; + color = ; + led-gpio = <&ssogpio 0 0>; + }; + + led@23 { + reg = <23>; + function = LED_FUNCTION_POWER; + color = ; + led-gpio = <&ssogpio 23 0>; + }; + }; + }; diff --git a/Documentation/devicetree/bindings/leds/ti,tca6507.yaml b/Documentation/devicetree/bindings/leds/ti,tca6507.yaml index 94c307c987621c5bcf3182ae2384889f4dd0b516..32c60038789586926d5bf8d3e86734c51caf095c 100644 --- a/Documentation/devicetree/bindings/leds/ti,tca6507.yaml +++ b/Documentation/devicetree/bindings/leds/ti,tca6507.yaml @@ -69,6 +69,7 @@ patternProperties: if: patternProperties: "^gpio@[0-6]$": + type: object properties: compatible: contains: diff --git a/Documentation/devicetree/bindings/mailbox/omap-mailbox.txt b/Documentation/devicetree/bindings/mailbox/omap-mailbox.txt index 5fe80c1c19fcc3b8101f8d3a17067dcdf63951b5..12371f5c6cd9ebb618165e2291337eaf37a71157 100644 --- a/Documentation/devicetree/bindings/mailbox/omap-mailbox.txt +++ b/Documentation/devicetree/bindings/mailbox/omap-mailbox.txt @@ -28,6 +28,9 @@ SoCs has each of these instances form a cluster and combine multiple clusters into a single IP block present within the Main NavSS. The interrupt lines from all these clusters are multiplexed and routed to different processor subsystems over a limited number of common interrupt output lines of an Interrupt Router. +The AM64x SoCS also uses a single IP block comprising of multiple clusters, +but the number of clusters are smaller, and the interrupt output lines are +connected directly to various processors. Mailbox Device Node: ==================== @@ -42,6 +45,7 @@ Required properties: "ti,omap4-mailbox" for OMAP44xx, OMAP54xx, AM33xx, AM43xx and DRA7xx SoCs "ti,am654-mailbox" for K3 AM65x and J721E SoCs + "ti,am64-mailbox" for K3 AM64x SoCs - reg: Contains the mailbox register address range (base address and length) - interrupts: Contains the interrupt information for the mailbox diff --git a/Documentation/devicetree/bindings/mailbox/qcom,apcs-kpss-global.yaml b/Documentation/devicetree/bindings/mailbox/qcom,apcs-kpss-global.yaml index ffd09b664ff50ac8f0289d851ecd08f1a177d3ae..5dc1173d03fd5074f1c31fed74bde09dab7b6827 100644 --- a/Documentation/devicetree/bindings/mailbox/qcom,apcs-kpss-global.yaml +++ b/Documentation/devicetree/bindings/mailbox/qcom,apcs-kpss-global.yaml @@ -24,6 +24,7 @@ properties: - qcom,msm8998-apcs-hmss-global - qcom,qcs404-apcs-apps-global - qcom,sc7180-apss-shared + - qcom,sc8180x-apss-shared - qcom,sdm660-apcs-hmss-global - qcom,sdm845-apss-shared - qcom,sm8150-apss-shared @@ -33,9 +34,11 @@ properties: clocks: description: phandles to the parent clocks of the clock driver + minItems: 2 items: - description: primary pll parent of the clock driver - description: auxiliary parent + - description: reference clock '#mbox-cells': const: 1 @@ -44,9 +47,11 @@ properties: const: 0 clock-names: + minItems: 2 items: - const: pll - const: aux + - const: ref required: - compatible @@ -55,6 +60,35 @@ required: additionalProperties: false +allOf: + - if: + properties: + compatible: + enum: + - qcom,ipq6018-apcs-apps-global + - qcom,ipq8074-apcs-apps-global + - qcom,msm8916-apcs-kpss-global + - qcom,msm8994-apcs-kpss-global + - qcom,msm8996-apcs-hmss-global + - qcom,msm8998-apcs-hmss-global + - qcom,qcs404-apcs-apps-global + - qcom,sc7180-apss-shared + - qcom,sdm660-apcs-hmss-global + - qcom,sdm845-apss-shared + - qcom,sm8150-apss-shared + then: + properties: + clocks: + maxItems: 2 + - if: + properties: + compatible: + enum: + - qcom,sdx55-apcs-gcc + then: + properties: + clocks: + maxItems: 3 examples: # Example apcs with msm8996 diff --git a/Documentation/devicetree/bindings/media/allwinner,sun4i-a10-video-engine.yaml b/Documentation/devicetree/bindings/media/allwinner,sun4i-a10-video-engine.yaml index 2f7058f7760cd38e91c696f4bf350a9e21eecdef..c34303b87a5be81e5c6f48146936926641a3ad98 100644 --- a/Documentation/devicetree/bindings/media/allwinner,sun4i-a10-video-engine.yaml +++ b/Documentation/devicetree/bindings/media/allwinner,sun4i-a10-video-engine.yaml @@ -53,6 +53,7 @@ properties: maxItems: 1 memory-region: + maxItems: 1 description: CMA pool to use for buffers allocation instead of the default CMA pool. diff --git a/Documentation/devicetree/bindings/media/i2c/imx219.yaml b/Documentation/devicetree/bindings/media/i2c/imx219.yaml index 012c0565d8ae9f4b664c2344a43ee2829fa8f81a..5fc96944b448decafdfd6dc775e4e5926f3ad55f 100644 --- a/Documentation/devicetree/bindings/media/i2c/imx219.yaml +++ b/Documentation/devicetree/bindings/media/i2c/imx219.yaml @@ -40,6 +40,7 @@ properties: Digital core voltage supply, 1.2 volts reset-gpios: + maxItems: 1 description: |- Reference to the GPIO connected to the xclr pin, if any. Must be released (set high) after all supplies are applied. diff --git a/Documentation/devicetree/bindings/memory-controllers/exynos-srom.yaml b/Documentation/devicetree/bindings/memory-controllers/exynos-srom.yaml index 637e24f0f73b44984dd5c70f27fc56b0f1f59eef..c6e44f47ce7cfca978fac48f936f33c30913855f 100644 --- a/Documentation/devicetree/bindings/memory-controllers/exynos-srom.yaml +++ b/Documentation/devicetree/bindings/memory-controllers/exynos-srom.yaml @@ -28,6 +28,8 @@ properties: const: 1 ranges: + minItems: 1 + maxItems: 4 description: | Reflects the memory layout with four integer values per bank. Format: 0 diff --git a/Documentation/devicetree/bindings/mfd/bd9571mwv.txt b/Documentation/devicetree/bindings/mfd/bd9571mwv.txt index 8c4678650d1a3087423811b7e60e115d0d60e208..1d6413e96c376e4b40c6d020e4e430558ba6c5aa 100644 --- a/Documentation/devicetree/bindings/mfd/bd9571mwv.txt +++ b/Documentation/devicetree/bindings/mfd/bd9571mwv.txt @@ -1,7 +1,7 @@ -* ROHM BD9571MWV Power Management Integrated Circuit (PMIC) bindings +* ROHM BD9571MWV/BD9574MWF Power Management Integrated Circuit (PMIC) bindings Required properties: - - compatible : Should be "rohm,bd9571mwv". + - compatible : Should be "rohm,bd9571mwv" or "rohm,bd9574mwf". - reg : I2C slave address. - interrupts : The interrupt line the device is connected to. - interrupt-controller : Marks the device node as an interrupt controller. diff --git a/Documentation/devicetree/bindings/mfd/canaan,k210-sysctl.yaml b/Documentation/devicetree/bindings/mfd/canaan,k210-sysctl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c24ad45cabb506aa8a1acca4aeb4812a7d0d7854 --- /dev/null +++ b/Documentation/devicetree/bindings/mfd/canaan,k210-sysctl.yaml @@ -0,0 +1,109 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/mfd/canaan,k210-sysctl.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Canaan Kendryte K210 System Controller Device Tree Bindings + +maintainers: + - Damien Le Moal + +description: + Canaan Inc. Kendryte K210 SoC system controller which provides a + register map for controlling the clocks, reset signals and pin power + domains of the SoC. + +properties: + compatible: + items: + - const: canaan,k210-sysctl + - const: syscon + - const: simple-mfd + + clocks: + maxItems: 1 + description: + System controller Advanced Power Bus (APB) interface clock source. + + clock-names: + items: + - const: pclk + + reg: + maxItems: 1 + + clock-controller: + # Child node + type: object + $ref: "../clock/canaan,k210-clk.yaml" + description: + Clock controller for the SoC clocks. This child node definition + should follow the bindings specified in + Documentation/devicetree/bindings/clock/canaan,k210-clk.yaml. + + reset-controller: + # Child node + type: object + $ref: "../reset/canaan,k210-rst.yaml" + description: + Reset controller for the SoC. This child node definition + should follow the bindings specified in + Documentation/devicetree/bindings/reset/canaan,k210-rst.yaml. + + syscon-reboot: + # Child node + type: object + $ref: "../power/reset/syscon-reboot.yaml" + description: + Reboot method for the SoC. This child node definition + should follow the bindings specified in + Documentation/devicetree/bindings/power/reset/syscon-reboot.yaml. + +required: + - compatible + - clocks + - reg + - clock-controller + +additionalProperties: false + +examples: + - | + #include + #include + + clocks { + in0: oscllator { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <26000000>; + }; + }; + + sysctl: syscon@50440000 { + compatible = "canaan,k210-sysctl", + "syscon", "simple-mfd"; + reg = <0x50440000 0x100>; + clocks = <&sysclk K210_CLK_APB1>; + clock-names = "pclk"; + + sysclk: clock-controller { + #clock-cells = <1>; + compatible = "canaan,k210-clk"; + clocks = <&in0>; + }; + + sysrst: reset-controller { + compatible = "canaan,k210-rst"; + #reset-cells = <1>; + }; + + reboot: syscon-reboot { + compatible = "syscon-reboot"; + regmap = <&sysctl>; + offset = <48>; + mask = <1>; + value = <1>; + }; + }; diff --git a/Documentation/devicetree/bindings/mfd/ene-kb930.yaml b/Documentation/devicetree/bindings/mfd/ene-kb930.yaml new file mode 100644 index 0000000000000000000000000000000000000000..06ed9ec8f4bbce472cc0a7d52253f9e371ca031f --- /dev/null +++ b/Documentation/devicetree/bindings/mfd/ene-kb930.yaml @@ -0,0 +1,65 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/mfd/ene-kb930.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: ENE KB930 Embedded Controller bindings + +description: | + This binding describes the ENE KB930 Embedded Controller attached to an + I2C bus. + +maintainers: + - Dmitry Osipenko + +properties: + compatible: + items: + - enum: + - acer,a500-iconia-ec # Acer A500 Iconia tablet device + - const: ene,kb930 + reg: + maxItems: 1 + + monitored-battery: true + power-supplies: true + system-power-controller: true + +required: + - compatible + - reg + +additionalProperties: false + +examples: + - | + battery: battery-cell { + compatible = "simple-battery"; + charge-full-design-microamp-hours = <3260000>; + energy-full-design-microwatt-hours = <24000000>; + operating-range-celsius = <0 40>; + }; + + mains: ac-adapter { + compatible = "gpio-charger"; + charger-type = "mains"; + gpios = <&gpio 125 0>; + }; + + i2c { + #address-cells = <1>; + #size-cells = <0>; + + embedded-controller@58 { + compatible = "acer,a500-iconia-ec", "ene,kb930"; + reg = <0x58>; + + system-power-controller; + + monitored-battery = <&battery>; + power-supplies = <&mains>; + }; + }; + +... diff --git a/Documentation/devicetree/bindings/mfd/gateworks-gsc.yaml b/Documentation/devicetree/bindings/mfd/gateworks-gsc.yaml index d08e8fe7644684795735abbb51209db899cf07aa..5a1e8d21f7a09e24d1989733ee329eed96d0c20a 100644 --- a/Documentation/devicetree/bindings/mfd/gateworks-gsc.yaml +++ b/Documentation/devicetree/bindings/mfd/gateworks-gsc.yaml @@ -83,8 +83,9 @@ properties: 2 - scaled voltage based on an optional resistor divider and optional offset 3 - pre-scaled 16-bit voltage value + 4 - fan tach input to report RPM's $ref: /schemas/types.yaml#/definitions/uint32 - enum: [0, 1, 2, 3] + enum: [0, 1, 2, 3, 4] gw,voltage-divider-ohms: description: Values of resistors for divider on raw ADC input diff --git a/Documentation/devicetree/bindings/mfd/iqs62x.yaml b/Documentation/devicetree/bindings/mfd/iqs62x.yaml index 541b06d80e7337a8f1b5c195ecdb3061d6b7f4b5..044cd7542c2bcf5030b93de98b3314e6d63fce7f 100644 --- a/Documentation/devicetree/bindings/mfd/iqs62x.yaml +++ b/Documentation/devicetree/bindings/mfd/iqs62x.yaml @@ -93,7 +93,7 @@ examples: pwmleds { compatible = "pwm-leds"; - panel { + led-1 { pwms = <&iqs620a_pwm 0 1000000>; max-brightness = <255>; }; diff --git a/Documentation/devicetree/bindings/misc/eeprom-93xx46.txt b/Documentation/devicetree/bindings/misc/eeprom-93xx46.txt index a8ebb4621f7903ee920636ec1ff5fc124efb695f..7b636b7a8311f71b6d760e5ae5b05298dbfcc3ba 100644 --- a/Documentation/devicetree/bindings/misc/eeprom-93xx46.txt +++ b/Documentation/devicetree/bindings/misc/eeprom-93xx46.txt @@ -4,6 +4,7 @@ Required properties: - compatible : shall be one of: "atmel,at93c46d" "eeprom-93xx46" + "microchip,93lc46b" - data-size : number of data bits per word (either 8 or 16) Optional properties: diff --git a/Documentation/devicetree/bindings/misc/fsl,dpaa2-console.yaml b/Documentation/devicetree/bindings/misc/fsl,dpaa2-console.yaml index 271a3eafc054e62947ef75c8745385fa2bb69eb3..8cc951feb7df4bd785c0b282b07abf1c3f27b4bd 100644 --- a/Documentation/devicetree/bindings/misc/fsl,dpaa2-console.yaml +++ b/Documentation/devicetree/bindings/misc/fsl,dpaa2-console.yaml @@ -15,6 +15,7 @@ properties: const: "fsl,dpaa2-console" reg: + maxItems: 1 description: A standard property. Specifies the region where the MCFBA (MC firmware base address) register can be found. diff --git a/Documentation/devicetree/bindings/mmc/allwinner,sun4i-a10-mmc.yaml b/Documentation/devicetree/bindings/mmc/allwinner,sun4i-a10-mmc.yaml index e82c9a07b6fb020f773984b3b50754af2d6f2780..e75b3a8ba816ca0a12a00a7dafa5309f3d38534e 100644 --- a/Documentation/devicetree/bindings/mmc/allwinner,sun4i-a10-mmc.yaml +++ b/Documentation/devicetree/bindings/mmc/allwinner,sun4i-a10-mmc.yaml @@ -26,6 +26,8 @@ properties: - const: allwinner,sun9i-a80-mmc - const: allwinner,sun50i-a64-emmc - const: allwinner,sun50i-a64-mmc + - const: allwinner,sun50i-a100-emmc + - const: allwinner,sun50i-a100-mmc - items: - const: allwinner,sun8i-a83t-mmc - const: allwinner,sun7i-a20-mmc @@ -47,6 +49,12 @@ properties: - items: - const: allwinner,sun50i-h6-mmc - const: allwinner,sun50i-a64-mmc + - items: + - const: allwinner,sun50i-h616-emmc + - const: allwinner,sun50i-a100-emmc + - items: + - const: allwinner,sun50i-h616-mmc + - const: allwinner,sun50i-a100-mmc reg: maxItems: 1 diff --git a/Documentation/devicetree/bindings/mmc/arm,pl18x.yaml b/Documentation/devicetree/bindings/mmc/arm,pl18x.yaml new file mode 100644 index 0000000000000000000000000000000000000000..47595cb483becee2229152c73e52b4f8721d67cc --- /dev/null +++ b/Documentation/devicetree/bindings/mmc/arm,pl18x.yaml @@ -0,0 +1,223 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/mmc/arm,pl18x.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: ARM PrimeCell MultiMedia Card Interface (MMCI) PL180 and PL181 + +maintainers: + - Linus Walleij + - Ulf Hansson + +description: + The ARM PrimeCells MMCI PL180 and PL181 provides an interface for + reading and writing to MultiMedia and SD cards alike. Over the years + vendors have use the VHDL code from ARM to create derivative MMC/SD/SDIO + host controllers with very similar characteristics. + +allOf: + - $ref: /schemas/arm/primecell.yaml# + - $ref: mmc-controller.yaml# + +# We need a select here so we don't match all nodes with 'arm,primecell' +select: + properties: + compatible: + contains: + enum: + - arm,pl180 + - arm,pl181 + - arm,pl18x + required: + - compatible + +properties: + compatible: + oneOf: + - description: The first version of the block, simply called + PL180 and found in the ARM Integrator IM/PD1 logic module. + items: + - const: arm,pl180 + - const: arm,primecell + - description: The improved version of the block, found in the + ARM Versatile and later reference designs. Further revisions + exist but get detected at runtime by reading some magic numbers + in the PrimeCell ID registers. + items: + - const: arm,pl181 + - const: arm,primecell + - description: Wildcard entry that will let the operating system + inspect the PrimeCell ID registers to determine which hardware + variant of PL180 or PL181 this is. + items: + - const: arm,pl18x + - const: arm,primecell + + clocks: + description: One or two clocks, the "apb_pclk" and the "MCLK" + which is the core block clock. The names are not compulsory. + minItems: 1 + maxItems: 2 + + power-domains: true + + resets: + maxItems: 1 + + reg: + description: the MMIO memory window must be exactly 4KB (0x1000) and the + layout should provide the PrimeCell ID registers so that the device can + be discovered. On ST Micro variants, a second register window may be + defined if a delay block is present and used for tuning. + + interrupts: + description: The first interrupt is the command interrupt and corresponds + to the event at the end of a command. The second interrupt is the + PIO (polled I/O) interrupt and occurs when the FIFO needs to be + emptied as part of a bulk read from the card. Some variants have these + two interrupts wired into the same line (logic OR) and in that case + only one interrupt may be provided. + minItems: 1 + maxItems: 2 + + st,sig-dir-dat0: + $ref: /schemas/types.yaml#/definitions/flag + description: ST Micro-specific property, bus signal direction pins used for + DAT[0]. + + st,sig-dir-dat2: + $ref: /schemas/types.yaml#/definitions/flag + description: ST Micro-specific property, bus signal direction pins used for + DAT[2]. + + st,sig-dir-dat31: + $ref: /schemas/types.yaml#/definitions/flag + description: ST Micro-specific property, bus signal direction pins used for + DAT[3] and DAT[1]. + + st,sig-dir-dat74: + $ref: /schemas/types.yaml#/definitions/flag + description: ST Micro-specific property, bus signal direction pins used for + DAT[7] and DAT[4]. + + st,sig-dir-cmd: + $ref: /schemas/types.yaml#/definitions/flag + description: ST Micro-specific property, CMD signal direction used for + pin CMD. + + st,sig-pin-fbclk: + $ref: /schemas/types.yaml#/definitions/flag + description: ST Micro-specific property, feedback clock FBCLK signal pin + in use. + + st,sig-dir: + $ref: /schemas/types.yaml#/definitions/flag + description: ST Micro-specific property, signal direction polarity used for + pins CMD, DAT[0], DAT[1], DAT[2] and DAT[3]. + + st,neg-edge: + $ref: /schemas/types.yaml#/definitions/flag + description: ST Micro-specific property, data and command phase relation, + generated on the sd clock falling edge. + + st,use-ckin: + $ref: /schemas/types.yaml#/definitions/flag + description: ST Micro-specific property, use CKIN pin from an external + driver to sample the receive data (for example with a voltage switch + transceiver). + + st,cmd-gpios: + maxItems: 1 + description: + The GPIO matching the CMD pin. + + st,ck-gpios: + maxItems: 1 + description: + The GPIO matching the CK pin. + + st,ckin-gpios: + maxItems: 1 + description: + The GPIO matching the CKIN pin. + +dependencies: + st,cmd-gpios: [ "st,use-ckin" ] + st,ck-gpios: [ "st,use-ckin" ] + st,ckin-gpios: [ "st,use-ckin" ] + +unevaluatedProperties: false + +required: + - compatible + - reg + - interrupts + +examples: + - | + #include + #include + + mmc@5000 { + compatible = "arm,pl180", "arm,primecell"; + reg = <0x5000 0x1000>; + interrupts-extended = <&vic 22 &sic 1>; + clocks = <&xtal24mhz>, <&pclk>; + clock-names = "mclk", "apb_pclk"; + }; + + mmc@80126000 { + compatible = "arm,pl18x", "arm,primecell"; + reg = <0x80126000 0x1000>; + interrupts = <0 60 IRQ_TYPE_LEVEL_HIGH>; + dmas = <&dma 29 0 0x2>, <&dma 29 0 0x0>; + dma-names = "rx", "tx"; + clocks = <&prcc_kclk 1 5>, <&prcc_pclk 1 5>; + clock-names = "sdi", "apb_pclk"; + max-frequency = <100000000>; + bus-width = <4>; + cap-sd-highspeed; + cap-mmc-highspeed; + cd-gpios = <&gpio2 31 0x4>; + st,sig-dir-dat0; + st,sig-dir-dat2; + st,sig-dir-cmd; + st,sig-pin-fbclk; + vmmc-supply = <&ab8500_ldo_aux3_reg>; + vqmmc-supply = <&vmmci>; + }; + + mmc@101f6000 { + compatible = "arm,pl18x", "arm,primecell"; + reg = <0x101f6000 0x1000>; + clocks = <&sdiclk>, <&pclksdi>; + clock-names = "mclk", "apb_pclk"; + interrupt-parent = <&vica>; + interrupts = <22>; + max-frequency = <400000>; + bus-width = <4>; + cap-mmc-highspeed; + cap-sd-highspeed; + full-pwr-cycle; + st,sig-dir-dat0; + st,sig-dir-dat2; + st,sig-dir-dat31; + st,sig-dir-cmd; + st,sig-pin-fbclk; + vmmc-supply = <&vmmc_regulator>; + }; + + mmc@52007000 { + compatible = "arm,pl18x", "arm,primecell"; + arm,primecell-periphid = <0x10153180>; + reg = <0x52007000 0x1000>; + interrupts = <49>; + interrupt-names = "cmd_irq"; + clocks = <&rcc 0>; + clock-names = "apb_pclk"; + resets = <&rcc 1>; + cap-sd-highspeed; + cap-mmc-highspeed; + max-frequency = <120000000>; + }; diff --git a/Documentation/devicetree/bindings/mmc/marvell,xenon-sdhci.txt b/Documentation/devicetree/bindings/mmc/marvell,xenon-sdhci.txt index ed1456f5c94dda21ecaf2eebd86bcd7934d11376..c51a62d751dcfdd70f21702d49ceaae080d67e7f 100644 --- a/Documentation/devicetree/bindings/mmc/marvell,xenon-sdhci.txt +++ b/Documentation/devicetree/bindings/mmc/marvell,xenon-sdhci.txt @@ -12,6 +12,7 @@ Required Properties: - "marvell,armada-3700-sdhci": For controllers on Armada-3700 SoC. Must provide a second register area and marvell,pad-type. - "marvell,armada-ap806-sdhci": For controllers on Armada AP806. + - "marvell,armada-ap807-sdhci": For controllers on Armada AP807. - "marvell,armada-cp110-sdhci": For controllers on Armada CP110. - clocks: diff --git a/Documentation/devicetree/bindings/mmc/mmc-controller.yaml b/Documentation/devicetree/bindings/mmc/mmc-controller.yaml index e674bba52ee97a1202754aa4b93151e56a0034a4..e141330c111478df32fa8c61da76414f0b5074b1 100644 --- a/Documentation/devicetree/bindings/mmc/mmc-controller.yaml +++ b/Documentation/devicetree/bindings/mmc/mmc-controller.yaml @@ -40,6 +40,7 @@ properties: There is no card detection available; polling must be used. cd-gpios: + maxItems: 1 description: The card detection will be done using the GPIO provided. @@ -104,6 +105,7 @@ properties: line. Not used in combination with eMMC or SDIO. wp-gpios: + maxItems: 1 description: GPIO to use for the write-protect detection. diff --git a/Documentation/devicetree/bindings/mmc/mmci.txt b/Documentation/devicetree/bindings/mmc/mmci.txt deleted file mode 100644 index 4ec921e4bf344f7b2594baff99cb8447cc63c0d4..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/mmc/mmci.txt +++ /dev/null @@ -1,74 +0,0 @@ -* ARM PrimeCell MultiMedia Card Interface (MMCI) PL180/1 - -The ARM PrimeCell MMCI PL180 and PL181 provides an interface for -reading and writing to MultiMedia and SD cards alike. - -This file documents differences between the core properties described -by mmc.txt and the properties used by the mmci driver. Using "st" as -the prefix for a property, indicates support by the ST Micro variant. - -Required properties: -- compatible : contains "arm,pl18x", "arm,primecell". -- vmmc-supply : phandle to the regulator device tree node, mentioned - as the VCC/VDD supply in the eMMC/SD specs. - -Optional properties: -- arm,primecell-periphid : contains the PrimeCell Peripheral ID, it overrides - the ID provided by the HW -- resets : phandle to internal reset line. - Should be defined for sdmmc variant. -- vqmmc-supply : phandle to the regulator device tree node, mentioned - as the VCCQ/VDD_IO supply in the eMMC/SD specs. -specific for ux500 variant: -- st,sig-dir-dat0 : bus signal direction pin used for DAT[0]. -- st,sig-dir-dat2 : bus signal direction pin used for DAT[2]. -- st,sig-dir-dat31 : bus signal direction pin used for DAT[3] and DAT[1]. -- st,sig-dir-dat74 : bus signal direction pin used for DAT[4] to DAT[7]. -- st,sig-dir-cmd : cmd signal direction pin used for CMD. -- st,sig-pin-fbclk : feedback clock signal pin used. - -specific for sdmmc variant: -- reg : a second base register may be defined if a delay - block is present and used for tuning. -- st,sig-dir : signal direction polarity used for cmd, dat0 dat123. -- st,neg-edge : data & command phase relation, generated on - sd clock falling edge. -- st,use-ckin : use ckin pin from an external driver to sample - the receive data (example: with voltage - switch transceiver). - -Deprecated properties: -- mmc-cap-mmc-highspeed : indicates whether MMC is high speed capable. -- mmc-cap-sd-highspeed : indicates whether SD is high speed capable. - -Example: - -sdi0_per1@80126000 { - compatible = "arm,pl18x", "arm,primecell"; - reg = <0x80126000 0x1000>; - interrupts = <0 60 IRQ_TYPE_LEVEL_HIGH>; - - dmas = <&dma 29 0 0x2>, /* Logical - DevToMem */ - <&dma 29 0 0x0>; /* Logical - MemToDev */ - dma-names = "rx", "tx"; - - clocks = <&prcc_kclk 1 5>, <&prcc_pclk 1 5>; - clock-names = "sdi", "apb_pclk"; - - max-frequency = <100000000>; - bus-width = <4>; - cap-sd-highspeed; - cap-mmc-highspeed; - cd-gpios = <&gpio2 31 0x4>; // 95 - st,sig-dir-dat0; - st,sig-dir-dat2; - st,sig-dir-cmd; - st,sig-pin-fbclk; - - vmmc-supply = <&ab8500_ldo_aux3_reg>; - vqmmc-supply = <&vmmci>; - - pinctrl-names = "default", "sleep"; - pinctrl-0 = <&sdi0_default_mode>; - pinctrl-1 = <&sdi0_sleep_mode>; -}; diff --git a/Documentation/devicetree/bindings/mmc/renesas,sdhi.yaml b/Documentation/devicetree/bindings/mmc/renesas,sdhi.yaml index 6bbf29b5c2392acd373c5aa50abf29dbc7527251..1118b6fa93c97aae014df3c013c81bbff5456740 100644 --- a/Documentation/devicetree/bindings/mmc/renesas,sdhi.yaml +++ b/Documentation/devicetree/bindings/mmc/renesas,sdhi.yaml @@ -59,6 +59,7 @@ properties: - renesas,sdhi-r8a77980 # R-Car V3H - renesas,sdhi-r8a77990 # R-Car E3 - renesas,sdhi-r8a77995 # R-Car D3 + - renesas,sdhi-r8a779a0 # R-Car V3U - const: renesas,rcar-gen3-sdhi # R-Car Gen3 or RZ/G2 reg: @@ -123,7 +124,7 @@ required: if: properties: compatible: - items: + contains: enum: - renesas,sdhi-r7s72100 - renesas,sdhi-r7s9210 diff --git a/Documentation/devicetree/bindings/mmc/sdhci-am654.yaml b/Documentation/devicetree/bindings/mmc/sdhci-am654.yaml index 1ae945434c53c8ed38b0e094c84043202dbf3c5a..3a79e39253d249bde331aa85fc74214dc3794880 100644 --- a/Documentation/devicetree/bindings/mmc/sdhci-am654.yaml +++ b/Documentation/devicetree/bindings/mmc/sdhci-am654.yaml @@ -15,12 +15,19 @@ allOf: properties: compatible: - enum: - - ti,am654-sdhci-5.1 - - ti,j721e-sdhci-8bit - - ti,j721e-sdhci-4bit - - ti,j7200-sdhci-8bit - - ti,j721e-sdhci-4bit + oneOf: + - const: ti,am654-sdhci-5.1 + - const: ti,j721e-sdhci-8bit + - const: ti,j721e-sdhci-4bit + - const: ti,j721e-sdhci-4bit + - const: ti,am64-sdhci-8bit + - const: ti,am64-sdhci-4bit + - items: + - const: ti,j7200-sdhci-8bit + - const: ti,j721e-sdhci-8bit + - items: + - const: ti,j7200-sdhci-4bit + - const: ti,j721e-sdhci-4bit reg: maxItems: 2 diff --git a/Documentation/devicetree/bindings/mmc/sdhci-msm.txt b/Documentation/devicetree/bindings/mmc/sdhci-msm.txt index 3b602fd6180bf61c560e7c34bc03311722a7f64b..4c7fa6a4ed15cdf9065949707a6a4bb90e1c2bf8 100644 --- a/Documentation/devicetree/bindings/mmc/sdhci-msm.txt +++ b/Documentation/devicetree/bindings/mmc/sdhci-msm.txt @@ -17,10 +17,11 @@ Required properties: "qcom,msm8916-sdhci", "qcom,sdhci-msm-v4" "qcom,msm8992-sdhci", "qcom,sdhci-msm-v4" "qcom,msm8996-sdhci", "qcom,sdhci-msm-v4" - "qcom,sm8250-sdhci", "qcom,sdhci-msm-v5" - "qcom,sdm845-sdhci", "qcom,sdhci-msm-v5" "qcom,qcs404-sdhci", "qcom,sdhci-msm-v5" "qcom,sc7180-sdhci", "qcom,sdhci-msm-v5"; + "qcom,sdm845-sdhci", "qcom,sdhci-msm-v5" + "qcom,sdx55-sdhci", "qcom,sdhci-msm-v5"; + "qcom,sm8250-sdhci", "qcom,sdhci-msm-v5" NOTE that some old device tree files may be floating around that only have the string "qcom,sdhci-msm-v4" without the SoC compatible string but doing that should be considered a deprecated practice. @@ -30,10 +31,12 @@ Required properties: - SD Core register map (required for controllers earlier than msm-v5) - CQE register map (Optional, CQE support is present on SDHC instance meant for eMMC and version v4.2 and above) + - Inline Crypto Engine register map (optional) - reg-names: When CQE register map is supplied, below reg-names are required - "hc" for Host controller register map - "core" for SD core register map - "cqhci" for CQE register map + - "ice" for Inline Crypto Engine register map (optional) - interrupts: Should contain an interrupt-specifiers for the interrupts: - Host controller interrupt (required) - pinctrl-names: Should contain only one value - "default". @@ -46,6 +49,7 @@ Required properties: "xo" - TCXO clock (optional) "cal" - reference clock for RCLK delay calibration (optional) "sleep" - sleep clock for RCLK delay calibration (optional) + "ice" - clock for Inline Crypto Engine (optional) - qcom,ddr-config: Certain chipsets and platforms require particular settings for the DDR_CONFIG register. Use this field to specify the register diff --git a/Documentation/devicetree/bindings/mmc/sdhci-sirf.txt b/Documentation/devicetree/bindings/mmc/sdhci-sirf.txt deleted file mode 100644 index dd6ed464bcb86432ec1fc6028cadb9080761ff5b..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/mmc/sdhci-sirf.txt +++ /dev/null @@ -1,18 +0,0 @@ -* SiRFprimII/marco/atlas6 SDHCI Controller - -This file documents differences between the core properties in mmc.txt -and the properties used by the sdhci-sirf driver. - -Required properties: -- compatible: sirf,prima2-sdhc - -Optional properties: -- cd-gpios: card detect gpio, with zero flags. - -Example: - - sd0: sdhci@56000000 { - compatible = "sirf,prima2-sdhc"; - reg = <0xcd000000 0x100000>; - cd-gpios = <&gpio 6 0>; - }; diff --git a/Documentation/devicetree/bindings/mmc/zx-dw-mshc.txt b/Documentation/devicetree/bindings/mmc/zx-dw-mshc.txt deleted file mode 100644 index 0f59bd5361f530c4c6b343a2b5a39396637940f5..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/mmc/zx-dw-mshc.txt +++ /dev/null @@ -1,31 +0,0 @@ -* ZTE specific extensions to the Synopsys Designware Mobile Storage - Host Controller - -The Synopsys designware mobile storage host controller is used to interface -a SoC with storage medium such as eMMC or SD/MMC cards. This file documents -differences between the core Synopsys dw mshc controller properties described -by synopsys-dw-mshc.txt and the properties used by the ZTE specific -extensions to the Synopsys Designware Mobile Storage Host Controller. - -Required Properties: - -* compatible: should be - - "zte,zx296718-dw-mshc": for ZX SoCs - -Example: - - mmc1: mmc@1110000 { - compatible = "zte,zx296718-dw-mshc"; - reg = <0x01110000 0x1000>; - interrupts = ; - fifo-depth = <32>; - data-addr = <0x200>; - fifo-watermark-aligned; - bus-width = <4>; - clock-frequency = <50000000>; - clocks = <&topcrm SD0_AHB>, <&topcrm SD0_WCLK>; - clock-names = "biu", "ciu"; - max-frequency = <50000000>; - cap-sdio-irq; - cap-sd-highspeed; - }; diff --git a/Documentation/devicetree/bindings/mtd/jedec,spi-nor.txt b/Documentation/devicetree/bindings/mtd/jedec,spi-nor.txt deleted file mode 100644 index f03be904d3c228d8e46cc117ea0d5893f68e6e9b..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/mtd/jedec,spi-nor.txt +++ /dev/null @@ -1,91 +0,0 @@ -* SPI NOR flash: ST M25Pxx (and similar) serial flash chips - -Required properties: -- #address-cells, #size-cells : Must be present if the device has sub-nodes - representing partitions. -- compatible : May include a device-specific string consisting of the - manufacturer and name of the chip. A list of supported chip - names follows. - Must also include "jedec,spi-nor" for any SPI NOR flash that can - be identified by the JEDEC READ ID opcode (0x9F). - - Supported chip names: - at25df321a - at25df641 - at26df081a - mr25h128 - mr25h256 - mr25h10 - mr25h40 - mx25l4005a - mx25l1606e - mx25l6405d - mx25l12805d - mx25l25635e - n25q064 - n25q128a11 - n25q128a13 - n25q512a - s25fl256s1 - s25fl512s - s25sl12801 - s25fl008k - s25fl064k - sst25vf040b - m25p40 - m25p80 - m25p16 - m25p32 - m25p64 - m25p128 - w25x80 - w25x32 - w25q32 - w25q64 - w25q32dw - w25q80bl - w25q128 - w25q256 - - The following chip names have been used historically to - designate quirky versions of flash chips that do not support the - JEDEC READ ID opcode (0x9F): - m25p05-nonjedec - m25p10-nonjedec - m25p20-nonjedec - m25p40-nonjedec - m25p80-nonjedec - m25p16-nonjedec - m25p32-nonjedec - m25p64-nonjedec - m25p128-nonjedec - -- reg : Chip-Select number -- spi-max-frequency : Maximum frequency of the SPI bus the chip can operate at - -Optional properties: -- m25p,fast-read : Use the "fast read" opcode to read data from the chip instead - of the usual "read" opcode. This opcode is not supported by - all chips and support for it can not be detected at runtime. - Refer to your chips' datasheet to check if this is supported - by your chip. -- broken-flash-reset : Some flash devices utilize stateful addressing modes - (e.g., for 32-bit addressing) which need to be managed - carefully by a system. Because these sorts of flash don't - have a standardized software reset command, and because some - systems don't toggle the flash RESET# pin upon system reset - (if the pin even exists at all), there are systems which - cannot reboot properly if the flash is left in the "wrong" - state. This boolean flag can be used on such systems, to - denote the absence of a reliable reset mechanism. - -Example: - - flash: m25p80@0 { - #address-cells = <1>; - #size-cells = <1>; - compatible = "spansion,m25p80", "jedec,spi-nor"; - reg = <0>; - spi-max-frequency = <40000000>; - m25p,fast-read; - }; diff --git a/Documentation/devicetree/bindings/mtd/jedec,spi-nor.yaml b/Documentation/devicetree/bindings/mtd/jedec,spi-nor.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5e7e5349f9a17f4d2747178700b59025e48f37c6 --- /dev/null +++ b/Documentation/devicetree/bindings/mtd/jedec,spi-nor.yaml @@ -0,0 +1,102 @@ +# SPDX-License-Identifier: GPL-2.0-only +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/mtd/jedec,spi-nor.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: SPI NOR flash ST M25Pxx (and similar) serial flash chips + +maintainers: + - Rob Herring + +properties: + compatible: + oneOf: + - items: + - pattern: "^((((micron|spansion|st),)?\ + (m25p(40|80|16|32|64|128)|\ + n25q(32b|064|128a11|128a13|256a|512a|164k)))|\ + atmel,at25df(321a|641|081a)|\ + everspin,mr25h(10|40|128|256)|\ + (mxicy|macronix),mx25l(4005a|1606e|6405d|8005|12805d|25635e)|\ + (mxicy|macronix),mx25u(4033|4035)|\ + (spansion,)?s25fl(128s|256s1|512s|008k|064k|164k)|\ + (sst|microchip),sst25vf(016b|032b|040b)|\ + (sst,)?sst26wf016b|\ + (sst,)?sst25wf(040b|080)|\ + winbond,w25x(80|32)|\ + (winbond,)?w25q(16|32(w|dw)?|64(dw)?|80bl|128(fw)?|256))$" + - const: jedec,spi-nor + - items: + - enum: + - issi,is25lp016d + - micron,mt25qu02g + - mxicy,mx25r1635f + - mxicy,mx25u6435f + - mxicy,mx25v8035f + - spansion,s25sl12801 + - spansion,s25fs512s + - const: jedec,spi-nor + - const: jedec,spi-nor + description: + Must also include "jedec,spi-nor" for any SPI NOR flash that can be + identified by the JEDEC READ ID opcode (0x9F). + + reg: + maxItems: 1 + + spi-max-frequency: true + spi-rx-bus-width: true + spi-tx-bus-width: true + + m25p,fast-read: + type: boolean + description: + Use the "fast read" opcode to read data from the chip instead of the usual + "read" opcode. This opcode is not supported by all chips and support for + it can not be detected at runtime. Refer to your chips' datasheet to check + if this is supported by your chip. + + broken-flash-reset: + type: boolean + description: + Some flash devices utilize stateful addressing modes (e.g., for 32-bit + addressing) which need to be managed carefully by a system. Because these + sorts of flash don't have a standardized software reset command, and + because some systems don't toggle the flash RESET# pin upon system reset + (if the pin even exists at all), there are systems which cannot reboot + properly if the flash is left in the "wrong" state. This boolean flag can + be used on such systems, to denote the absence of a reliable reset + mechanism. + + label: true + + partitions: + type: object + + '#address-cells': true + '#size-cells': true + +patternProperties: + # Note: use 'partitions' node for new users + '^partition@': + type: object + +additionalProperties: false + +examples: + - | + spi { + #address-cells = <1>; + #size-cells = <0>; + + flash@0 { + #address-cells = <1>; + #size-cells = <1>; + compatible = "spansion,m25p80", "jedec,spi-nor"; + reg = <0>; + spi-max-frequency = <40000000>; + m25p,fast-read; + }; + }; +... diff --git a/Documentation/devicetree/bindings/net/can/rcar_canfd.txt b/Documentation/devicetree/bindings/net/can/rcar_canfd.txt index 22cf2a889b2c7d6bdefc1a61853c2e7ca9a2de74..248c4ed97a0a078e75477f6da5024e3d736caeba 100644 --- a/Documentation/devicetree/bindings/net/can/rcar_canfd.txt +++ b/Documentation/devicetree/bindings/net/can/rcar_canfd.txt @@ -97,7 +97,7 @@ E.g. below enables Channel 0 alone in the board using External clock as fCAN clock. &canfd { - pinctrl-0 = <&canfd0_pins &can_clk_pins>; + pinctrl-0 = <&canfd0_pins>, <&can_clk_pins>; pinctrl-names = "default"; status = "okay"; diff --git a/Documentation/devicetree/bindings/net/ethernet-controller.yaml b/Documentation/devicetree/bindings/net/ethernet-controller.yaml index f599c1d9c961763f9c3df2e14b22ac8259f1ced0..4b7d1e5d003c7f282aa1e61852ed4b7bcce4af1d 100644 --- a/Documentation/devicetree/bindings/net/ethernet-controller.yaml +++ b/Documentation/devicetree/bindings/net/ethernet-controller.yaml @@ -206,6 +206,11 @@ properties: Indicates that full-duplex is used. When absent, half duplex is assumed. + pause: + $ref: /schemas/types.yaml#definitions/flag + description: + Indicates that pause should be enabled. + asym-pause: $ref: /schemas/types.yaml#/definitions/flag description: diff --git a/Documentation/devicetree/bindings/net/ti,k3-am654-cpsw-nuss.yaml b/Documentation/devicetree/bindings/net/ti,k3-am654-cpsw-nuss.yaml index 3fae9a5f0c6a4f264ce3692fac8be52767bc090c..783b9e32cf6633a4ee6d7f8786b55f22a279a5b1 100644 --- a/Documentation/devicetree/bindings/net/ti,k3-am654-cpsw-nuss.yaml +++ b/Documentation/devicetree/bindings/net/ti,k3-am654-cpsw-nuss.yaml @@ -72,6 +72,7 @@ properties: dma-coherent: true clocks: + maxItems: 1 description: CPSWxG NUSS functional clock clock-names: diff --git a/Documentation/devicetree/bindings/net/ti,k3-am654-cpts.yaml b/Documentation/devicetree/bindings/net/ti,k3-am654-cpts.yaml index ce43a1c58a57152bd0e2e6531ae276fc2204a633..4317eba503ca079b121b5608ca83178207107310 100644 --- a/Documentation/devicetree/bindings/net/ti,k3-am654-cpts.yaml +++ b/Documentation/devicetree/bindings/net/ti,k3-am654-cpts.yaml @@ -59,6 +59,7 @@ properties: - const: cpts clocks: + maxItems: 1 description: CPTS reference clock clock-names: diff --git a/Documentation/devicetree/bindings/nvmem/rmem.yaml b/Documentation/devicetree/bindings/nvmem/rmem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1d85a0a30846e8c43d8faabf37e34d7c3b5e9eac --- /dev/null +++ b/Documentation/devicetree/bindings/nvmem/rmem.yaml @@ -0,0 +1,49 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/nvmem/rmem.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Reserved Memory Based nvmem Device + +maintainers: + - Nicolas Saenz Julienne + +allOf: + - $ref: "nvmem.yaml#" + +properties: + compatible: + items: + - enum: + - raspberrypi,bootloader-config + - const: nvmem-rmem + + no-map: + $ref: /schemas/types.yaml#/definitions/flag + description: + Avoid creating a virtual mapping of the region as part of the OS' + standard mapping of system memory. + +required: + - compatible + - no-map + +unevaluatedProperties: false + +examples: + - | + reserved-memory { + #address-cells = <1>; + #size-cells = <1>; + + blconfig: nvram@10000000 { + compatible = "raspberrypi,bootloader-config", "nvmem-rmem"; + #address-cells = <1>; + #size-cells = <1>; + reg = <0x10000000 0x1000>; + no-map; + }; + }; + +... diff --git a/Documentation/devicetree/bindings/pci/brcm,stb-pcie.yaml b/Documentation/devicetree/bindings/pci/brcm,stb-pcie.yaml index 807694b4f41f68b37aab27cb205299e7aa005631..f90557f6deb843b66d7144837f416acddfb9d9c1 100644 --- a/Documentation/devicetree/bindings/pci/brcm,stb-pcie.yaml +++ b/Documentation/devicetree/bindings/pci/brcm,stb-pcie.yaml @@ -14,6 +14,7 @@ properties: items: - enum: - brcm,bcm2711-pcie # The Raspberry Pi 4 + - brcm,bcm4908-pcie - brcm,bcm7211-pcie # Broadcom STB version of RPi4 - brcm,bcm7278-pcie # Broadcom 7278 Arm - brcm,bcm7216-pcie # Broadcom 7216 Arm @@ -63,15 +64,6 @@ properties: aspm-no-l0s: true - resets: - description: for "brcm,bcm7216-pcie", must be a valid reset - phandle pointing to the RESCAL reset controller provider node. - $ref: "/schemas/types.yaml#/definitions/phandle" - - reset-names: - items: - - const: rescal - brcm,scb-sizes: description: u64 giving the 64bit PCIe memory viewport size of a memory controller. There may be up to @@ -98,12 +90,39 @@ required: allOf: - $ref: /schemas/pci/pci-bus.yaml# + - if: + properties: + compatible: + contains: + const: brcm,bcm4908-pcie + then: + properties: + resets: + items: + - description: reset controller handling the PERST# signal + + reset-names: + items: + - const: perst + + required: + - resets + - reset-names - if: properties: compatible: contains: const: brcm,bcm7216-pcie then: + properties: + resets: + items: + - description: phandle pointing to the RESCAL reset controller + + reset-names: + items: + - const: rescal + required: - resets - reset-names diff --git a/Documentation/devicetree/bindings/pci/layerscape-pci.txt b/Documentation/devicetree/bindings/pci/layerscape-pci.txt index daa99f7d4c3fb4637f5b6d9dbb4c901543af1dec..6d898dd4a8e2a1fed25983a058f073d9753593dd 100644 --- a/Documentation/devicetree/bindings/pci/layerscape-pci.txt +++ b/Documentation/devicetree/bindings/pci/layerscape-pci.txt @@ -26,6 +26,7 @@ Required properties: "fsl,ls1046a-pcie-ep", "fsl,ls-pcie-ep" "fsl,ls1088a-pcie-ep", "fsl,ls-pcie-ep" "fsl,ls2088a-pcie-ep", "fsl,ls-pcie-ep" + "fsl,lx2160ar2-pcie-ep", "fsl,ls-pcie-ep" - reg: base addresses and lengths of the PCIe controller register blocks. - interrupts: A list of interrupt outputs of the controller. Must contain an entry for each entry in the interrupt-names property. diff --git a/Documentation/devicetree/bindings/pci/microchip,pcie-host.yaml b/Documentation/devicetree/bindings/pci/microchip,pcie-host.yaml new file mode 100644 index 0000000000000000000000000000000000000000..04251d71f56bd46dcf0800368695f9b880d55941 --- /dev/null +++ b/Documentation/devicetree/bindings/pci/microchip,pcie-host.yaml @@ -0,0 +1,92 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/pci/microchip,pcie-host.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Microchip PCIe Root Port Bridge Controller Device Tree Bindings + +maintainers: + - Daire McNamara + +allOf: + - $ref: /schemas/pci/pci-bus.yaml# + +properties: + compatible: + const: microchip,pcie-host-1.0 # PolarFire + + reg: + maxItems: 2 + + reg-names: + items: + - const: cfg + - const: apb + + interrupts: + minItems: 1 + maxItems: 2 + items: + - description: PCIe host controller + - description: builtin MSI controller + + interrupt-names: + minItems: 1 + maxItems: 2 + items: + - const: pcie + - const: msi + + ranges: + maxItems: 1 + + msi-controller: + description: Identifies the node as an MSI controller. + + msi-parent: + description: MSI controller the device is capable of using. + +required: + - reg + - reg-names + - "#interrupt-cells" + - interrupts + - interrupt-map-mask + - interrupt-map + - msi-controller + +unevaluatedProperties: false + +examples: + - | + soc { + #address-cells = <2>; + #size-cells = <2>; + pcie0: pcie@2030000000 { + compatible = "microchip,pcie-host-1.0"; + reg = <0x0 0x70000000 0x0 0x08000000>, + <0x0 0x43000000 0x0 0x00010000>; + reg-names = "cfg", "apb"; + device_type = "pci"; + #address-cells = <3>; + #size-cells = <2>; + #interrupt-cells = <1>; + interrupts = <119>; + interrupt-map-mask = <0x0 0x0 0x0 0x7>; + interrupt-map = <0 0 0 1 &pcie_intc0 0>, + <0 0 0 2 &pcie_intc0 1>, + <0 0 0 3 &pcie_intc0 2>, + <0 0 0 4 &pcie_intc0 3>; + interrupt-parent = <&plic0>; + msi-parent = <&pcie0>; + msi-controller; + bus-range = <0x00 0x7f>; + ranges = <0x03000000 0x0 0x78000000 0x0 0x78000000 0x0 0x04000000>; + pcie_intc0: interrupt-controller { + #address-cells = <0>; + #interrupt-cells = <1>; + interrupt-controller; + }; + }; + }; diff --git a/Documentation/devicetree/bindings/pci/qcom,pcie.txt b/Documentation/devicetree/bindings/pci/qcom,pcie.txt index 3b55310390a0a74113ae7194cf5393e24eb168c5..0da458a051b6c5d4073dd7d8a33446c71fdce938 100644 --- a/Documentation/devicetree/bindings/pci/qcom,pcie.txt +++ b/Documentation/devicetree/bindings/pci/qcom,pcie.txt @@ -132,8 +132,20 @@ - "master_bus" AXI Master clock - "slave_bus" AXI Slave clock --clock-names: - Usage: required for sdm845 and sm8250 +- clock-names: + Usage: required for sdm845 + Value type: + Definition: Should contain the following entries + - "aux" Auxiliary clock + - "cfg" Configuration clock + - "bus_master" Master AXI clock + - "bus_slave" Slave AXI clock + - "slave_q2a" Slave Q2A clock + - "tbu" PCIe TBU clock + - "pipe" PIPE clock + +- clock-names: + Usage: required for sm8250 Value type: Definition: Should contain the following entries - "aux" Auxiliary clock @@ -142,6 +154,7 @@ - "bus_slave" Slave AXI clock - "slave_q2a" Slave Q2A clock - "tbu" PCIe TBU clock + - "ddrss_sf_tbu" PCIe SF TBU clock - "pipe" PIPE clock - resets: diff --git a/Documentation/devicetree/bindings/phy/allwinner,sun4i-a10-usb-phy.yaml b/Documentation/devicetree/bindings/phy/allwinner,sun4i-a10-usb-phy.yaml index 94ac23687b7eba99d846fdadc57acdd6c57cac6d..77606c899fe2bf622b2cf791db2f6ebc3ed81f26 100644 --- a/Documentation/devicetree/bindings/phy/allwinner,sun4i-a10-usb-phy.yaml +++ b/Documentation/devicetree/bindings/phy/allwinner,sun4i-a10-usb-phy.yaml @@ -51,9 +51,11 @@ properties: - const: usb2_reset usb0_id_det-gpios: + maxItems: 1 description: GPIO to the USB OTG ID pin usb0_vbus_det-gpios: + maxItems: 1 description: GPIO to the USB OTG VBUS detect pin usb0_vbus_power-supply: diff --git a/Documentation/devicetree/bindings/phy/allwinner,sun50i-a64-usb-phy.yaml b/Documentation/devicetree/bindings/phy/allwinner,sun50i-a64-usb-phy.yaml index fd6e126fcf18f2812140bb6d9d6b0bbe2fd521a7..078af52b16eddb37c5bf7ba0a0c4051bd9e1307a 100644 --- a/Documentation/devicetree/bindings/phy/allwinner,sun50i-a64-usb-phy.yaml +++ b/Documentation/devicetree/bindings/phy/allwinner,sun50i-a64-usb-phy.yaml @@ -50,9 +50,11 @@ properties: - const: usb1_reset usb0_id_det-gpios: + maxItems: 1 description: GPIO to the USB OTG ID pin usb0_vbus_det-gpios: + maxItems: 1 description: GPIO to the USB OTG VBUS detect pin usb0_vbus_power-supply: diff --git a/Documentation/devicetree/bindings/phy/allwinner,sun50i-h6-usb-phy.yaml b/Documentation/devicetree/bindings/phy/allwinner,sun50i-h6-usb-phy.yaml index 7670411002c95a21d5005de60ed6c2a24deca10b..e632140722a27c5a0a7f90319f4ccfd40693af64 100644 --- a/Documentation/devicetree/bindings/phy/allwinner,sun50i-h6-usb-phy.yaml +++ b/Documentation/devicetree/bindings/phy/allwinner,sun50i-h6-usb-phy.yaml @@ -50,9 +50,11 @@ properties: - const: usb3_reset usb0_id_det-gpios: + maxItems: 1 description: GPIO to the USB OTG ID pin usb0_vbus_det-gpios: + maxItems: 1 description: GPIO to the USB OTG VBUS detect pin usb0_vbus_power-supply: diff --git a/Documentation/devicetree/bindings/phy/allwinner,sun5i-a13-usb-phy.yaml b/Documentation/devicetree/bindings/phy/allwinner,sun5i-a13-usb-phy.yaml index 9b319381d1ad2e80f3358c64af2652f745af0007..5bad9b06e2e71a6f428e8ed86d5b35cd5a1ef696 100644 --- a/Documentation/devicetree/bindings/phy/allwinner,sun5i-a13-usb-phy.yaml +++ b/Documentation/devicetree/bindings/phy/allwinner,sun5i-a13-usb-phy.yaml @@ -45,9 +45,11 @@ properties: - const: usb1_reset usb0_id_det-gpios: + maxItems: 1 description: GPIO to the USB OTG ID pin usb0_vbus_det-gpios: + maxItems: 1 description: GPIO to the USB OTG VBUS detect pin usb0_vbus_power-supply: diff --git a/Documentation/devicetree/bindings/phy/allwinner,sun6i-a31-usb-phy.yaml b/Documentation/devicetree/bindings/phy/allwinner,sun6i-a31-usb-phy.yaml index b0ed01bbf3db95fa9682d73cbd80a9c4d8f3e779..922b4665e00d09be4eb84db7580f323db74ce83b 100644 --- a/Documentation/devicetree/bindings/phy/allwinner,sun6i-a31-usb-phy.yaml +++ b/Documentation/devicetree/bindings/phy/allwinner,sun6i-a31-usb-phy.yaml @@ -54,9 +54,11 @@ properties: - const: usb2_reset usb0_id_det-gpios: + maxItems: 1 description: GPIO to the USB OTG ID pin usb0_vbus_det-gpios: + maxItems: 1 description: GPIO to the USB OTG VBUS detect pin usb0_vbus_power-supply: diff --git a/Documentation/devicetree/bindings/phy/allwinner,sun8i-a23-usb-phy.yaml b/Documentation/devicetree/bindings/phy/allwinner,sun8i-a23-usb-phy.yaml index b0674406f8aaf3f03daf44a80a1aa14eb117cb53..a94019efc2f3a7c0357eedb05531782c1bd258c7 100644 --- a/Documentation/devicetree/bindings/phy/allwinner,sun8i-a23-usb-phy.yaml +++ b/Documentation/devicetree/bindings/phy/allwinner,sun8i-a23-usb-phy.yaml @@ -50,9 +50,11 @@ properties: - const: usb1_reset usb0_id_det-gpios: + maxItems: 1 description: GPIO to the USB OTG ID pin usb0_vbus_det-gpios: + maxItems: 1 description: GPIO to the USB OTG VBUS detect pin usb0_vbus_power-supply: diff --git a/Documentation/devicetree/bindings/phy/allwinner,sun8i-a83t-usb-phy.yaml b/Documentation/devicetree/bindings/phy/allwinner,sun8i-a83t-usb-phy.yaml index 48dc9c834a9bacc8e7295275854e8985f00b98e6..33f3ddc0492d7855155e1570789ff298219cc348 100644 --- a/Documentation/devicetree/bindings/phy/allwinner,sun8i-a83t-usb-phy.yaml +++ b/Documentation/devicetree/bindings/phy/allwinner,sun8i-a83t-usb-phy.yaml @@ -56,9 +56,11 @@ properties: - const: usb2_reset usb0_id_det-gpios: + maxItems: 1 description: GPIO to the USB OTG ID pin usb0_vbus_det-gpios: + maxItems: 1 description: GPIO to the USB OTG VBUS detect pin usb0_vbus_power-supply: diff --git a/Documentation/devicetree/bindings/phy/allwinner,sun8i-h3-usb-phy.yaml b/Documentation/devicetree/bindings/phy/allwinner,sun8i-h3-usb-phy.yaml index 60c344585276719312b689d21052f4fbeb887430..f8043106080392073aca98d15aa96fa1bcb546f3 100644 --- a/Documentation/devicetree/bindings/phy/allwinner,sun8i-h3-usb-phy.yaml +++ b/Documentation/devicetree/bindings/phy/allwinner,sun8i-h3-usb-phy.yaml @@ -62,9 +62,11 @@ properties: - const: usb3_reset usb0_id_det-gpios: + maxItems: 1 description: GPIO to the USB OTG ID pin usb0_vbus_det-gpios: + maxItems: 1 description: GPIO to the USB OTG VBUS detect pin usb0_vbus_power-supply: diff --git a/Documentation/devicetree/bindings/phy/allwinner,sun8i-r40-usb-phy.yaml b/Documentation/devicetree/bindings/phy/allwinner,sun8i-r40-usb-phy.yaml index a2bb36790fbde45e7d07ca588a2f9b0894083167..d947e50a49d26ac2970963707eb8649b11b60a48 100644 --- a/Documentation/devicetree/bindings/phy/allwinner,sun8i-r40-usb-phy.yaml +++ b/Documentation/devicetree/bindings/phy/allwinner,sun8i-r40-usb-phy.yaml @@ -56,9 +56,11 @@ properties: - const: usb2_reset usb0_id_det-gpios: + maxItems: 1 description: GPIO to the USB OTG ID pin usb0_vbus_det-gpios: + maxItems: 1 description: GPIO to the USB OTG VBUS detect pin usb0_vbus_power-supply: diff --git a/Documentation/devicetree/bindings/phy/allwinner,sun8i-v3s-usb-phy.yaml b/Documentation/devicetree/bindings/phy/allwinner,sun8i-v3s-usb-phy.yaml index eadfd0c9493c7d53fe0f7465dbc4c08b59667679..a2836c296cc47033c55e2ef7f48e776a44c4ef04 100644 --- a/Documentation/devicetree/bindings/phy/allwinner,sun8i-v3s-usb-phy.yaml +++ b/Documentation/devicetree/bindings/phy/allwinner,sun8i-v3s-usb-phy.yaml @@ -42,9 +42,11 @@ properties: const: usb0_reset usb0_id_det-gpios: + maxItems: 1 description: GPIO to the USB OTG ID pin usb0_vbus_det-gpios: + maxItems: 1 description: GPIO to the USB OTG VBUS detect pin usb0_vbus_power-supply: diff --git a/Documentation/devicetree/bindings/phy/allwinner,sun9i-a80-usb-phy.yaml b/Documentation/devicetree/bindings/phy/allwinner,sun9i-a80-usb-phy.yaml index ded7d6f0a119370a88ac8deae8c97e9d7aca91bf..2eb493fa64fd62eff7b713cdc089c1bb8752fde8 100644 --- a/Documentation/devicetree/bindings/phy/allwinner,sun9i-a80-usb-phy.yaml +++ b/Documentation/devicetree/bindings/phy/allwinner,sun9i-a80-usb-phy.yaml @@ -22,7 +22,8 @@ properties: clocks: anyOf: - - description: Main PHY Clock + - maxItems: 1 + description: Main PHY Clock - items: - description: Main PHY clock @@ -39,20 +40,16 @@ properties: - const: hsic_480M resets: - anyOf: + minItems: 1 + items: - description: Normal USB PHY reset - - - items: - - description: Normal USB PHY reset - - description: HSIC Reset + - description: HSIC Reset reset-names: - oneOf: + minItems: 1 + items: - const: phy - - - items: - - const: phy - - const: hsic + - const: hsic phy_type: const: hsic diff --git a/Documentation/devicetree/bindings/phy/brcm,brcmstb-usb-phy.txt b/Documentation/devicetree/bindings/phy/brcm,brcmstb-usb-phy.txt deleted file mode 100644 index 698aacbdcfc4ebdeefc819feb03aea41692ebb47..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/phy/brcm,brcmstb-usb-phy.txt +++ /dev/null @@ -1,86 +0,0 @@ -Broadcom STB USB PHY - -Required properties: -- compatible: should be one of - "brcm,brcmstb-usb-phy" - "brcm,bcm7216-usb-phy" - "brcm,bcm7211-usb-phy" - -- reg and reg-names properties requirements are specific to the - compatible string. - "brcm,brcmstb-usb-phy": - - reg: 1 or 2 offset and length pairs. One for the base CTRL registers - and an optional pair for systems with USB 3.x support - - reg-names: not specified - "brcm,bcm7216-usb-phy": - - reg: 3 offset and length pairs for CTRL, XHCI_EC and XHCI_GBL - registers - - reg-names: "ctrl", "xhci_ec", "xhci_gbl" - "brcm,bcm7211-usb-phy": - - reg: 5 offset and length pairs for CTRL, XHCI_EC, XHCI_GBL, - USB_PHY and USB_MDIO registers and an optional pair - for the BDC registers - - reg-names: "ctrl", "xhci_ec", "xhci_gbl", "usb_phy", "usb_mdio", "bdc_ec" - -- #phy-cells: Shall be 1 as it expects one argument for setting - the type of the PHY. Possible values are: - - PHY_TYPE_USB2 for USB1.1/2.0 PHY - - PHY_TYPE_USB3 for USB3.x PHY - -Optional Properties: -- clocks : clock phandles. -- clock-names: String, clock name. -- interrupts: wakeup interrupt -- interrupt-names: "wakeup" -- brcm,ipp: Boolean, Invert Port Power. - Possible values are: 0 (Don't invert), 1 (Invert) -- brcm,ioc: Boolean, Invert Over Current detection. - Possible values are: 0 (Don't invert), 1 (Invert) -- dr_mode: String, PHY Device mode. - Possible values are: "host", "peripheral ", "drd" or "typec-pd" - If this property is not defined, the phy will default to "host" mode. -- brcm,syscon-piarbctl: phandle to syscon for handling config registers -NOTE: one or both of the following two properties must be set -- brcm,has-xhci: Boolean indicating the phy has an XHCI phy. -- brcm,has-eohci: Boolean indicating the phy has an EHCI/OHCI phy. - - -Example: - -usbphy_0: usb-phy@f0470200 { - reg = <0xf0470200 0xb8>, - <0xf0471940 0x6c0>; - compatible = "brcm,brcmstb-usb-phy"; - #phy-cells = <1>; - dr_mode = "host" - brcm,ioc = <1>; - brcm,ipp = <1>; - brcm,has-xhci; - brcm,has-eohci; - clocks = <&usb20>, <&usb30>; - clock-names = "sw_usb", "sw_usb3"; -}; - -usb-phy@29f0200 { - reg = <0x29f0200 0x200>, - <0x29c0880 0x30>, - <0x29cc100 0x534>, - <0x2808000 0x24>, - <0x2980080 0x8>; - reg-names = "ctrl", - "xhci_ec", - "xhci_gbl", - "usb_phy", - "usb_mdio"; - brcm,ioc = <0x0>; - brcm,ipp = <0x0>; - compatible = "brcm,bcm7211-usb-phy"; - interrupts = <0x30>; - interrupt-parent = <&vpu_intr1_nosec_intc>; - interrupt-names = "wake"; - #phy-cells = <0x1>; - brcm,has-xhci; - syscon-piarbctl = <&syscon_piarbctl>; - clocks = <&scmi_clk 256>; - clock-names = "sw_usb"; -}; diff --git a/Documentation/devicetree/bindings/phy/brcm,brcmstb-usb-phy.yaml b/Documentation/devicetree/bindings/phy/brcm,brcmstb-usb-phy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0497368d1fcafb1f0a903de8da1f38466347ea31 --- /dev/null +++ b/Documentation/devicetree/bindings/phy/brcm,brcmstb-usb-phy.yaml @@ -0,0 +1,196 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/phy/brcm,brcmstb-usb-phy.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Broadcom STB USB PHY + +description: Broadcom's PHY that handles EHCI/OHCI and/or XHCI + +maintainers: + - Al Cooper + - RafaÅ‚ MiÅ‚ecki + +properties: + compatible: + enum: + - brcm,bcm4908-usb-phy + - brcm,bcm7211-usb-phy + - brcm,bcm7216-usb-phy + - brcm,brcmstb-usb-phy + + reg: + minItems: 1 + maxItems: 6 + items: + - description: the base CTRL register + - description: XHCI EC register + - description: XHCI GBL register + - description: USB PHY register + - description: USB MDIO register + - description: BDC register + + reg-names: + minItems: 1 + maxItems: 6 + items: + - const: ctrl + - const: xhci_ec + - const: xhci_gbl + - const: usb_phy + - const: usb_mdio + - const: bdc_ec + + clocks: + minItems: 1 + maxItems: 2 + + clock-names: + minItems: 1 + maxItems: 2 + items: + - const: sw_usb + - const: sw_usb3 + + interrupts: + description: wakeup interrupt + + interrupt-names: + const: wake + + brcm,ipp: + $ref: /schemas/types.yaml#/definitions/uint32 + description: Invert Port Power + minimum: 0 + maximum: 1 + + brcm,ioc: + $ref: /schemas/types.yaml#/definitions/uint32 + description: Invert Over Current detection + minimum: 0 + maximum: 1 + + dr_mode: + description: PHY Device mode. If this property is not defined, the PHY will + default to "host" mode. + enum: + - host + - peripheral + - drd + - typec-pd + + brcm,syscon-piarbctl: + description: phandle to syscon for handling config registers + $ref: /schemas/types.yaml#/definitions/phandle + + brcm,has-xhci: + description: Indicates the PHY has an XHCI PHY. + type: boolean + + brcm,has-eohci: + description: Indicates the PHY has an EHCI/OHCI PHY. + type: boolean + + "#phy-cells": + description: | + Cell allows setting the type of the PHY. Possible values are: + - PHY_TYPE_USB2 for USB1.1/2.0 PHY + - PHY_TYPE_USB3 for USB3.x PHY + const: 1 + +required: + - reg + - "#phy-cells" + +anyOf: + - required: + - brcm,has-xhci + - required: + - brcm,has-eohci + +allOf: + - if: + properties: + compatible: + contains: + enum: + - const: brcm,bcm4908-usb-phy + - const: brcm,brcmstb-usb-phy + then: + properties: + reg: + minItems: 1 + maxItems: 2 + - if: + properties: + compatible: + contains: + const: brcm,bcm7211-usb-phy + then: + properties: + reg: + minItems: 5 + maxItems: 6 + reg-names: + minItems: 5 + maxItems: 6 + - if: + properties: + compatible: + contains: + const: brcm,bcm7216-usb-phy + then: + properties: + reg: + minItems: 3 + maxItems: 3 + reg-names: + minItems: 3 + maxItems: 3 + +additionalProperties: false + +examples: + - | + #include + + usb-phy@f0470200 { + compatible = "brcm,brcmstb-usb-phy"; + reg = <0xf0470200 0xb8>, + <0xf0471940 0x6c0>; + #phy-cells = <1>; + dr_mode = "host"; + brcm,ioc = <1>; + brcm,ipp = <1>; + brcm,has-xhci; + brcm,has-eohci; + clocks = <&usb20>, <&usb30>; + clock-names = "sw_usb", "sw_usb3"; + }; + - | + #include + + usb-phy@29f0200 { + compatible = "brcm,bcm7211-usb-phy"; + reg = <0x29f0200 0x200>, + <0x29c0880 0x30>, + <0x29cc100 0x534>, + <0x2808000 0x24>, + <0x2980080 0x8>; + reg-names = "ctrl", + "xhci_ec", + "xhci_gbl", + "usb_phy", + "usb_mdio"; + brcm,ioc = <0x0>; + brcm,ipp = <0x0>; + interrupts = <0x30>; + interrupt-parent = <&vpu_intr1_nosec_intc>; + interrupt-names = "wake"; + #phy-cells = <0x1>; + brcm,has-xhci; + brcm,syscon-piarbctl = <&syscon_piarbctl>; + clocks = <&scmi_clk 256>; + clock-names = "sw_usb"; + }; diff --git a/Documentation/devicetree/bindings/phy/brcm,sata-phy.yaml b/Documentation/devicetree/bindings/phy/brcm,sata-phy.yaml index 58c3ef8004ad83d0b3d6f70b61730681344ccfe1..04edda504ab6a985ec3643d509567a666420355a 100644 --- a/Documentation/devicetree/bindings/phy/brcm,sata-phy.yaml +++ b/Documentation/devicetree/bindings/phy/brcm,sata-phy.yaml @@ -99,8 +99,7 @@ patternProperties: if: properties: compatible: - items: - const: brcm,iproc-ns2-sata-phy + const: brcm,iproc-ns2-sata-phy then: properties: reg: diff --git a/Documentation/devicetree/bindings/phy/mediatek,dsi-phy.yaml b/Documentation/devicetree/bindings/phy/mediatek,dsi-phy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..71d4acea1f669a7d01a6cde25e67ea1ed4ea2093 --- /dev/null +++ b/Documentation/devicetree/bindings/phy/mediatek,dsi-phy.yaml @@ -0,0 +1,85 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +# Copyright (c) 2020 MediaTek +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/phy/mediatek,dsi-phy.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: MediaTek MIPI Display Serial Interface (DSI) PHY binding + +maintainers: + - Chun-Kuang Hu + - Philipp Zabel + - Chunfeng Yun + +description: The MIPI DSI PHY supports up to 4-lane output. + +properties: + $nodename: + pattern: "^dsi-phy@[0-9a-f]+$" + + compatible: + enum: + - mediatek,mt2701-mipi-tx + - mediatek,mt7623-mipi-tx + - mediatek,mt8173-mipi-tx + - mediatek,mt8183-mipi-tx + + reg: + maxItems: 1 + + clocks: + items: + - description: PLL reference clock + + clock-output-names: + maxItems: 1 + + "#phy-cells": + const: 0 + + "#clock-cells": + const: 0 + + nvmem-cells: + maxItems: 1 + description: A phandle to the calibration data provided by a nvmem device, + if unspecified, default values shall be used. + + nvmem-cell-names: + items: + - const: calibration-data + + drive-strength-microamp: + description: adjust driving current + multipleOf: 200 + minimum: 2000 + maximum: 6000 + default: 4600 + +required: + - compatible + - reg + - clocks + - clock-output-names + - "#phy-cells" + - "#clock-cells" + +additionalProperties: false + +examples: + - | + #include + dsi-phy@10215000 { + compatible = "mediatek,mt8173-mipi-tx"; + reg = <0x10215000 0x1000>; + clocks = <&clk26m>; + clock-output-names = "mipi_tx0_pll"; + drive-strength-microamp = <4000>; + nvmem-cells= <&mipi_tx_calibration>; + nvmem-cell-names = "calibration-data"; + #clock-cells = <0>; + #phy-cells = <0>; + }; + +... diff --git a/Documentation/devicetree/bindings/phy/mediatek,hdmi-phy.yaml b/Documentation/devicetree/bindings/phy/mediatek,hdmi-phy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4752517a144604cadcbd761d861f9e8d07645f45 --- /dev/null +++ b/Documentation/devicetree/bindings/phy/mediatek,hdmi-phy.yaml @@ -0,0 +1,92 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +# Copyright (c) 2020 MediaTek +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/phy/mediatek,hdmi-phy.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: MediaTek High Definition Multimedia Interface (HDMI) PHY binding + +maintainers: + - Chun-Kuang Hu + - Philipp Zabel + - Chunfeng Yun + +description: | + The HDMI PHY serializes the HDMI encoder's three channel 10-bit parallel + output and drives the HDMI pads. + +properties: + $nodename: + pattern: "^hdmi-phy@[0-9a-f]+$" + + compatible: + enum: + - mediatek,mt2701-hdmi-phy + - mediatek,mt7623-hdmi-phy + - mediatek,mt8173-hdmi-phy + + reg: + maxItems: 1 + + clocks: + items: + - description: PLL reference clock + + clock-names: + items: + - const: pll_ref + + clock-output-names: + items: + - const: hdmitx_dig_cts + + "#phy-cells": + const: 0 + + "#clock-cells": + const: 0 + + mediatek,ibias: + description: + TX DRV bias current for < 1.65Gbps + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 0 + maximum: 63 + default: 0xa + + mediatek,ibias_up: + description: + TX DRV bias current for >= 1.65Gbps + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 0 + maximum: 63 + default: 0x1c + +required: + - compatible + - reg + - clocks + - clock-names + - clock-output-names + - "#phy-cells" + - "#clock-cells" + +additionalProperties: false + +examples: + - | + #include + hdmi_phy: hdmi-phy@10209100 { + compatible = "mediatek,mt8173-hdmi-phy"; + reg = <0x10209100 0x24>; + clocks = <&apmixedsys CLK_APMIXED_HDMI_REF>; + clock-names = "pll_ref"; + clock-output-names = "hdmitx_dig_cts"; + mediatek,ibias = <0xa>; + mediatek,ibias_up = <0x1c>; + #clock-cells = <0>; + #phy-cells = <0>; + }; + +... diff --git a/Documentation/devicetree/bindings/phy/mediatek,tphy.yaml b/Documentation/devicetree/bindings/phy/mediatek,tphy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..602e6ff45785e8f2e6797155263e0a1d5f8a30ec --- /dev/null +++ b/Documentation/devicetree/bindings/phy/mediatek,tphy.yaml @@ -0,0 +1,260 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +# Copyright (c) 2020 MediaTek +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/phy/mediatek,tphy.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: MediaTek T-PHY Controller Device Tree Bindings + +maintainers: + - Chunfeng Yun + +description: | + The T-PHY controller supports physical layer functionality for a number of + controllers on MediaTek SoCs, includes USB2.0, USB3.0, PCIe and SATA. + + Layout differences of banks between T-PHY V1 (mt8173/mt2701) and + T-PHY V2 (mt2712) when works on USB mode: + ----------------------------------- + Version 1: + port offset bank + shared 0x0000 SPLLC + 0x0100 FMREG + u2 port0 0x0800 U2PHY_COM + u3 port0 0x0900 U3PHYD + 0x0a00 U3PHYD_BANK2 + 0x0b00 U3PHYA + 0x0c00 U3PHYA_DA + u2 port1 0x1000 U2PHY_COM + u3 port1 0x1100 U3PHYD + 0x1200 U3PHYD_BANK2 + 0x1300 U3PHYA + 0x1400 U3PHYA_DA + u2 port2 0x1800 U2PHY_COM + ... + + Version 2: + port offset bank + u2 port0 0x0000 MISC + 0x0100 FMREG + 0x0300 U2PHY_COM + u3 port0 0x0700 SPLLC + 0x0800 CHIP + 0x0900 U3PHYD + 0x0a00 U3PHYD_BANK2 + 0x0b00 U3PHYA + 0x0c00 U3PHYA_DA + u2 port1 0x1000 MISC + 0x1100 FMREG + 0x1300 U2PHY_COM + u3 port1 0x1700 SPLLC + 0x1800 CHIP + 0x1900 U3PHYD + 0x1a00 U3PHYD_BANK2 + 0x1b00 U3PHYA + 0x1c00 U3PHYA_DA + u2 port2 0x2000 MISC + ... + + SPLLC shared by u3 ports and FMREG shared by u2 ports on V1 are put back + into each port; a new bank MISC for u2 ports and CHIP for u3 ports are + added on V2. + +properties: + $nodename: + pattern: "^t-phy@[0-9a-f]+$" + + compatible: + oneOf: + - items: + - enum: + - mediatek,mt2701-tphy + - mediatek,mt7623-tphy + - mediatek,mt7622-tphy + - mediatek,mt8516-tphy + - const: mediatek,generic-tphy-v1 + - items: + - enum: + - mediatek,mt2712-tphy + - mediatek,mt7629-tphy + - mediatek,mt8183-tphy + - const: mediatek,generic-tphy-v2 + - const: mediatek,mt2701-u3phy + deprecated: true + - const: mediatek,mt2712-u3phy + deprecated: true + - const: mediatek,mt8173-u3phy + + reg: + description: + Register shared by multiple ports, exclude port's private register. + It is needed for T-PHY V1, such as mt2701 and mt8173, but not for + T-PHY V2, such as mt2712. + maxItems: 1 + + "#address-cells": + enum: [1, 2] + + "#size-cells": + enum: [1, 2] + + # Used with non-empty value if optional 'reg' is not provided. + # The format of the value is an arbitrary number of triplets of + # (child-bus-address, parent-bus-address, length). + ranges: true + + mediatek,src-ref-clk-mhz: + description: + Frequency of reference clock for slew rate calibrate + default: 26 + + mediatek,src-coef: + description: + Coefficient for slew rate calibrate, depends on SoC process + $ref: /schemas/types.yaml#/definitions/uint32 + default: 28 + +# Required child node: +patternProperties: + "^usb-phy@[0-9a-f]+$": + type: object + description: + A sub-node is required for each port the controller provides. + Address range information including the usual 'reg' property + is used inside these nodes to describe the controller's topology. + + properties: + reg: + maxItems: 1 + + clocks: + minItems: 1 + maxItems: 2 + items: + - description: Reference clock, (HS is 48Mhz, SS/P is 24~27Mhz) + - description: Reference clock of analog phy + description: + Uses both clocks if the clock of analog and digital phys are + separated, otherwise uses "ref" clock only if needed. + + clock-names: + minItems: 1 + maxItems: 2 + items: + - const: ref + - const: da_ref + + "#phy-cells": + const: 1 + description: | + The cells contain the following arguments. + + - description: The PHY type + enum: + - PHY_TYPE_USB2 + - PHY_TYPE_USB3 + - PHY_TYPE_PCIE + - PHY_TYPE_SATA + + # The following optional vendor properties are only for debug or HQA test + mediatek,eye-src: + description: + The value of slew rate calibrate (U2 phy) + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 1 + maximum: 7 + + mediatek,eye-vrt: + description: + The selection of VRT reference voltage (U2 phy) + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 1 + maximum: 7 + + mediatek,eye-term: + description: + The selection of HS_TX TERM reference voltage (U2 phy) + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 1 + maximum: 7 + + mediatek,intr: + description: + The selection of internal resistor (U2 phy) + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 1 + maximum: 31 + + mediatek,discth: + description: + The selection of disconnect threshold (U2 phy) + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 1 + maximum: 15 + + mediatek,bc12: + description: + Specify the flag to enable BC1.2 if support it + type: boolean + + required: + - reg + - "#phy-cells" + + additionalProperties: false + +required: + - compatible + - "#address-cells" + - "#size-cells" + - ranges + +additionalProperties: false + +examples: + - | + #include + #include + #include + #include + usb@11271000 { + compatible = "mediatek,mt8173-mtu3", "mediatek,mtu3"; + reg = <0x11271000 0x3000>, <0x11280700 0x0100>; + reg-names = "mac", "ippc"; + phys = <&u2port0 PHY_TYPE_USB2>, + <&u3port0 PHY_TYPE_USB3>, + <&u2port1 PHY_TYPE_USB2>; + interrupts = ; + clocks = <&topckgen CLK_TOP_USB30_SEL>; + clock-names = "sys_ck"; + }; + + t-phy@11290000 { + compatible = "mediatek,mt8173-u3phy"; + reg = <0x11290000 0x800>; + #address-cells = <1>; + #size-cells = <1>; + ranges; + + u2port0: usb-phy@11290800 { + reg = <0x11290800 0x100>; + clocks = <&apmixedsys CLK_APMIXED_REF2USB_TX>, <&clk48m>; + clock-names = "ref", "da_ref"; + #phy-cells = <1>; + }; + + u3port0: usb-phy@11290900 { + reg = <0x11290900 0x700>; + clocks = <&clk26m>; + clock-names = "ref"; + #phy-cells = <1>; + }; + + u2port1: usb-phy@11291000 { + reg = <0x11291000 0x100>; + #phy-cells = <1>; + }; + }; + +... diff --git a/Documentation/devicetree/bindings/phy/mediatek,ufs-phy.yaml b/Documentation/devicetree/bindings/phy/mediatek,ufs-phy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3a9be82e7f13f7775d7b6dc233d1aa6dbd982332 --- /dev/null +++ b/Documentation/devicetree/bindings/phy/mediatek,ufs-phy.yaml @@ -0,0 +1,64 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +# Copyright (c) 2020 MediaTek +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/phy/mediatek,ufs-phy.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: MediaTek Universal Flash Storage (UFS) M-PHY binding + +maintainers: + - Stanley Chu + - Chunfeng Yun + +description: | + UFS M-PHY nodes are defined to describe on-chip UFS M-PHY hardware macro. + Each UFS M-PHY node should have its own node. + To bind UFS M-PHY with UFS host controller, the controller node should + contain a phandle reference to UFS M-PHY node. + +properties: + $nodename: + pattern: "^ufs-phy@[0-9a-f]+$" + + compatible: + const: mediatek,mt8183-ufsphy + + reg: + maxItems: 1 + + clocks: + items: + - description: Unipro core control clock. + - description: M-PHY core control clock. + + clock-names: + items: + - const: unipro + - const: mp + + "#phy-cells": + const: 0 + +required: + - compatible + - reg + - "#phy-cells" + - clocks + - clock-names + +additionalProperties: false + +examples: + - | + #include + ufsphy: ufs-phy@11fa0000 { + compatible = "mediatek,mt8183-ufsphy"; + reg = <0x11fa0000 0xc000>; + clocks = <&infracfg CLK_INFRA_UNIPRO_SCK>, + <&infracfg CLK_INFRA_UFS_MP_SAP_BCLK>; + clock-names = "unipro", "mp"; + #phy-cells = <0>; + }; + +... diff --git a/Documentation/devicetree/bindings/phy/mediatek,xsphy.yaml b/Documentation/devicetree/bindings/phy/mediatek,xsphy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..598fd2b95c298811ff0fe19342b60504277b1f81 --- /dev/null +++ b/Documentation/devicetree/bindings/phy/mediatek,xsphy.yaml @@ -0,0 +1,199 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +# Copyright (c) 2020 MediaTek +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/phy/mediatek,xsphy.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: MediaTek XS-PHY Controller Device Tree Bindings + +maintainers: + - Chunfeng Yun + +description: | + The XS-PHY controller supports physical layer functionality for USB3.1 + GEN2 controller on MediaTek SoCs. + + Banks layout of xsphy + ---------------------------------- + port offset bank + u2 port0 0x0000 MISC + 0x0100 FMREG + 0x0300 U2PHY_COM + u2 port1 0x1000 MISC + 0x1100 FMREG + 0x1300 U2PHY_COM + u2 port2 0x2000 MISC + ... + u31 common 0x3000 DIG_GLB + 0x3100 PHYA_GLB + u31 port0 0x3400 DIG_LN_TOP + 0x3500 DIG_LN_TX0 + 0x3600 DIG_LN_RX0 + 0x3700 DIG_LN_DAIF + 0x3800 PHYA_LN + u31 port1 0x3a00 DIG_LN_TOP + 0x3b00 DIG_LN_TX0 + 0x3c00 DIG_LN_RX0 + 0x3d00 DIG_LN_DAIF + 0x3e00 PHYA_LN + ... + DIG_GLB & PHYA_GLB are shared by U31 ports. + +properties: + $nodename: + pattern: "^xs-phy@[0-9a-f]+$" + + compatible: + items: + - enum: + - mediatek,mt3611-xsphy + - mediatek,mt3612-xsphy + - const: mediatek,xsphy + + reg: + description: + Register shared by multiple U3 ports, exclude port's private register, + if only U2 ports provided, shouldn't use the property. + maxItems: 1 + + "#address-cells": + enum: [1, 2] + + "#size-cells": + enum: [1, 2] + + ranges: true + + mediatek,src-ref-clk-mhz: + description: + Frequency of reference clock for slew rate calibrate + default: 26 + + mediatek,src-coef: + description: + Coefficient for slew rate calibrate, depends on SoC process + $ref: /schemas/types.yaml#/definitions/uint32 + default: 17 + +# Required child node: +patternProperties: + "^usb-phy@[0-9a-f]+$": + type: object + description: + A sub-node is required for each port the controller provides. + Address range information including the usual 'reg' property + is used inside these nodes to describe the controller's topology. + + properties: + reg: + maxItems: 1 + + clocks: + items: + - description: Reference clock, (HS is 48Mhz, SS/P is 24~27Mhz) + + clock-names: + items: + - const: ref + + "#phy-cells": + const: 1 + description: | + The cells contain the following arguments. + + - description: The PHY type + enum: + - PHY_TYPE_USB2 + - PHY_TYPE_USB3 + + # The following optional vendor properties are only for debug or HQA test + mediatek,eye-src: + description: + The value of slew rate calibrate (U2 phy) + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 1 + maximum: 7 + + mediatek,eye-vrt: + description: + The selection of VRT reference voltage (U2 phy) + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 1 + maximum: 7 + + mediatek,eye-term: + description: + The selection of HS_TX TERM reference voltage (U2 phy) + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 1 + maximum: 7 + + mediatek,efuse-intr: + description: + The selection of Internal Resistor (U2/U3 phy) + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 1 + maximum: 63 + + mediatek,efuse-tx-imp: + description: + The selection of TX Impedance (U3 phy) + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 1 + maximum: 31 + + mediatek,efuse-rx-imp: + description: + The selection of RX Impedance (U3 phy) + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 1 + maximum: 31 + + required: + - reg + - clocks + - clock-names + - "#phy-cells" + + additionalProperties: false + +required: + - compatible + - "#address-cells" + - "#size-cells" + - ranges + +additionalProperties: false + +examples: + - | + #include + + u3phy: xs-phy@11c40000 { + compatible = "mediatek,mt3611-xsphy", "mediatek,xsphy"; + reg = <0x11c43000 0x0200>; + mediatek,src-ref-clk-mhz = <26>; + mediatek,src-coef = <17>; + #address-cells = <1>; + #size-cells = <1>; + ranges; + + u2port0: usb-phy@11c40000 { + reg = <0x11c40000 0x0400>; + clocks = <&clk48m>; + clock-names = "ref"; + mediatek,eye-src = <4>; + #phy-cells = <1>; + }; + + u3port0: usb-phy@11c43000 { + reg = <0x11c43400 0x0500>; + clocks = <&clk26m>; + clock-names = "ref"; + mediatek,efuse-intr = <28>; + #phy-cells = <1>; + }; + }; + +... diff --git a/Documentation/devicetree/bindings/phy/phy-mtk-tphy.txt b/Documentation/devicetree/bindings/phy/phy-mtk-tphy.txt deleted file mode 100644 index dd75b676b71d05c2e8d840954d038902a66ac72a..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/phy/phy-mtk-tphy.txt +++ /dev/null @@ -1,162 +0,0 @@ -MediaTek T-PHY binding --------------------------- - -T-phy controller supports physical layer functionality for a number of -controllers on MediaTek SoCs, such as, USB2.0, USB3.0, PCIe, and SATA. - -Required properties (controller (parent) node): - - compatible : should be one of - "mediatek,generic-tphy-v1" - "mediatek,generic-tphy-v2" - "mediatek,mt2701-u3phy" (deprecated) - "mediatek,mt2712-u3phy" (deprecated) - "mediatek,mt8173-u3phy"; - make use of "mediatek,generic-tphy-v1" on mt2701 instead and - "mediatek,generic-tphy-v2" on mt2712 instead. - -- #address-cells: the number of cells used to represent physical - base addresses. -- #size-cells: the number of cells used to represent the size of an address. -- ranges: the address mapping relationship to the parent, defined with - - empty value: if optional 'reg' is used. - - non-empty value: if optional 'reg' is not used. should set - the child's base address to 0, the physical address - within parent's address space, and the length of - the address map. - -Required nodes : a sub-node is required for each port the controller - provides. Address range information including the usual - 'reg' property is used inside these nodes to describe - the controller's topology. - -Optional properties (controller (parent) node): - - reg : offset and length of register shared by multiple ports, - exclude port's private register. It is needed on mt2701 - and mt8173, but not on mt2712. - - mediatek,src-ref-clk-mhz : frequency of reference clock for slew rate - calibrate - - mediatek,src-coef : coefficient for slew rate calibrate, depends on - SoC process - -Required properties (port (child) node): -- reg : address and length of the register set for the port. -- #phy-cells : should be 1 (See second example) - cell after port phandle is phy type from: - - PHY_TYPE_USB2 - - PHY_TYPE_USB3 - - PHY_TYPE_PCIE - - PHY_TYPE_SATA - -Optional properties (PHY_TYPE_USB2 port (child) node): -- clocks : a list of phandle + clock-specifier pairs, one for each - entry in clock-names -- clock-names : may contain - "ref": 48M reference clock for HighSpeed (digital) phy; and 26M - reference clock for SuperSpeed (digital) phy, sometimes is - 24M, 25M or 27M, depended on platform. - "da_ref": the reference clock of analog phy, used if the clocks - of analog and digital phys are separated, otherwise uses - "ref" clock only if needed. - -- mediatek,eye-src : u32, the value of slew rate calibrate -- mediatek,eye-vrt : u32, the selection of VRT reference voltage -- mediatek,eye-term : u32, the selection of HS_TX TERM reference voltage -- mediatek,bc12 : bool, enable BC12 of u2phy if support it -- mediatek,discth : u32, the selection of disconnect threshold -- mediatek,intr : u32, the selection of internal R (resistance) - -Example: - -u3phy: usb-phy@11290000 { - compatible = "mediatek,mt8173-u3phy"; - reg = <0 0x11290000 0 0x800>; - #address-cells = <2>; - #size-cells = <2>; - ranges; - - u2port0: usb-phy@11290800 { - reg = <0 0x11290800 0 0x100>; - clocks = <&apmixedsys CLK_APMIXED_REF2USB_TX>; - clock-names = "ref"; - #phy-cells = <1>; - }; - - u3port0: usb-phy@11290900 { - reg = <0 0x11290800 0 0x700>; - clocks = <&clk26m>; - clock-names = "ref"; - #phy-cells = <1>; - }; - - u2port1: usb-phy@11291000 { - reg = <0 0x11291000 0 0x100>; - clocks = <&apmixedsys CLK_APMIXED_REF2USB_TX>; - clock-names = "ref"; - #phy-cells = <1>; - }; -}; - -Specifying phy control of devices ---------------------------------- - -Device nodes should specify the configuration required in their "phys" -property, containing a phandle to the phy port node and a device type; -phy-names for each port are optional. - -Example: - -#include - -usb30: usb@11270000 { - ... - phys = <&u2port0 PHY_TYPE_USB2>, <&u3port0 PHY_TYPE_USB3>; - phy-names = "usb2-0", "usb3-0"; - ... -}; - - -Layout differences of banks between mt8173/mt2701 and mt2712 -------------------------------------------------------------- -mt8173 and mt2701: -port offset bank -shared 0x0000 SPLLC - 0x0100 FMREG -u2 port0 0x0800 U2PHY_COM -u3 port0 0x0900 U3PHYD - 0x0a00 U3PHYD_BANK2 - 0x0b00 U3PHYA - 0x0c00 U3PHYA_DA -u2 port1 0x1000 U2PHY_COM -u3 port1 0x1100 U3PHYD - 0x1200 U3PHYD_BANK2 - 0x1300 U3PHYA - 0x1400 U3PHYA_DA -u2 port2 0x1800 U2PHY_COM - ... - -mt2712: -port offset bank -u2 port0 0x0000 MISC - 0x0100 FMREG - 0x0300 U2PHY_COM -u3 port0 0x0700 SPLLC - 0x0800 CHIP - 0x0900 U3PHYD - 0x0a00 U3PHYD_BANK2 - 0x0b00 U3PHYA - 0x0c00 U3PHYA_DA -u2 port1 0x1000 MISC - 0x1100 FMREG - 0x1300 U2PHY_COM -u3 port1 0x1700 SPLLC - 0x1800 CHIP - 0x1900 U3PHYD - 0x1a00 U3PHYD_BANK2 - 0x1b00 U3PHYA - 0x1c00 U3PHYA_DA -u2 port2 0x2000 MISC - ... - - SPLLC shared by u3 ports and FMREG shared by u2 ports on -mt8173/mt2701 are put back into each port; a new bank MISC for -u2 ports and CHIP for u3 ports are added on mt2712. diff --git a/Documentation/devicetree/bindings/phy/phy-mtk-ufs.txt b/Documentation/devicetree/bindings/phy/phy-mtk-ufs.txt deleted file mode 100644 index 5789029a1d42a970ea57f09eff79238c28fa3217..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/phy/phy-mtk-ufs.txt +++ /dev/null @@ -1,38 +0,0 @@ -MediaTek Universal Flash Storage (UFS) M-PHY binding --------------------------------------------------------- - -UFS M-PHY nodes are defined to describe on-chip UFS M-PHY hardware macro. -Each UFS M-PHY node should have its own node. - -To bind UFS M-PHY with UFS host controller, the controller node should -contain a phandle reference to UFS M-PHY node. - -Required properties for UFS M-PHY nodes: -- compatible : Compatible list, contains the following controller: - "mediatek,mt8183-ufsphy" for ufs phy - persent on MT81xx chipsets. -- reg : Address and length of the UFS M-PHY register set. -- #phy-cells : This property shall be set to 0. -- clocks : List of phandle and clock specifier pairs. -- clock-names : List of clock input name strings sorted in the same - order as the clocks property. Following clocks are - mandatory. - "unipro": Unipro core control clock. - "mp": M-PHY core control clock. - -Example: - - ufsphy: phy@11fa0000 { - compatible = "mediatek,mt8183-ufsphy"; - reg = <0 0x11fa0000 0 0xc000>; - #phy-cells = <0>; - - clocks = <&infracfg_ao INFRACFG_AO_UNIPRO_SCK_CG>, - <&infracfg_ao INFRACFG_AO_UFS_MP_SAP_BCLK_CG>; - clock-names = "unipro", "mp"; - }; - - ufshci@11270000 { - ... - phys = <&ufsphy>; - }; diff --git a/Documentation/devicetree/bindings/phy/phy-mtk-xsphy.txt b/Documentation/devicetree/bindings/phy/phy-mtk-xsphy.txt deleted file mode 100644 index e7caefa0b9c2687475e1458098c3ebb6dbbf66bf..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/phy/phy-mtk-xsphy.txt +++ /dev/null @@ -1,109 +0,0 @@ -MediaTek XS-PHY binding --------------------------- - -The XS-PHY controller supports physical layer functionality for USB3.1 -GEN2 controller on MediaTek SoCs. - -Required properties (controller (parent) node): - - compatible : should be "mediatek,-xsphy", "mediatek,xsphy", - soc-model is the name of SoC, such as mt3611 etc; - when using "mediatek,xsphy" compatible string, you need SoC specific - ones in addition, one of: - - "mediatek,mt3611-xsphy" - - - #address-cells, #size-cells : should use the same values as the root node - - ranges: must be present - -Optional properties (controller (parent) node): - - reg : offset and length of register shared by multiple U3 ports, - exclude port's private register, if only U2 ports provided, - shouldn't use the property. - - mediatek,src-ref-clk-mhz : u32, frequency of reference clock for slew rate - calibrate - - mediatek,src-coef : u32, coefficient for slew rate calibrate, depends on - SoC process - -Required nodes : a sub-node is required for each port the controller - provides. Address range information including the usual - 'reg' property is used inside these nodes to describe - the controller's topology. - -Required properties (port (child) node): -- reg : address and length of the register set for the port. -- clocks : a list of phandle + clock-specifier pairs, one for each - entry in clock-names -- clock-names : must contain - "ref": 48M reference clock for HighSpeed analog phy; and 26M - reference clock for SuperSpeedPlus analog phy, sometimes is - 24M, 25M or 27M, depended on platform. -- #phy-cells : should be 1 - cell after port phandle is phy type from: - - PHY_TYPE_USB2 - - PHY_TYPE_USB3 - -The following optional properties are only for debug or HQA test -Optional properties (PHY_TYPE_USB2 port (child) node): -- mediatek,eye-src : u32, the value of slew rate calibrate -- mediatek,eye-vrt : u32, the selection of VRT reference voltage -- mediatek,eye-term : u32, the selection of HS_TX TERM reference voltage -- mediatek,efuse-intr : u32, the selection of Internal Resistor - -Optional properties (PHY_TYPE_USB3 port (child) node): -- mediatek,efuse-intr : u32, the selection of Internal Resistor -- mediatek,efuse-tx-imp : u32, the selection of TX Impedance -- mediatek,efuse-rx-imp : u32, the selection of RX Impedance - -Banks layout of xsphy -------------------------------------------------------------- -port offset bank -u2 port0 0x0000 MISC - 0x0100 FMREG - 0x0300 U2PHY_COM -u2 port1 0x1000 MISC - 0x1100 FMREG - 0x1300 U2PHY_COM -u2 port2 0x2000 MISC - ... -u31 common 0x3000 DIG_GLB - 0x3100 PHYA_GLB -u31 port0 0x3400 DIG_LN_TOP - 0x3500 DIG_LN_TX0 - 0x3600 DIG_LN_RX0 - 0x3700 DIG_LN_DAIF - 0x3800 PHYA_LN -u31 port1 0x3a00 DIG_LN_TOP - 0x3b00 DIG_LN_TX0 - 0x3c00 DIG_LN_RX0 - 0x3d00 DIG_LN_DAIF - 0x3e00 PHYA_LN - ... - -DIG_GLB & PHYA_GLB are shared by U31 ports. - -Example: - -u3phy: usb-phy@11c40000 { - compatible = "mediatek,mt3611-xsphy", "mediatek,xsphy"; - reg = <0 0x11c43000 0 0x0200>; - mediatek,src-ref-clk-mhz = <26>; - mediatek,src-coef = <17>; - #address-cells = <2>; - #size-cells = <2>; - ranges; - - u2port0: usb-phy@11c40000 { - reg = <0 0x11c40000 0 0x0400>; - clocks = <&clk48m>; - clock-names = "ref"; - mediatek,eye-src = <4>; - #phy-cells = <1>; - }; - - u3port0: usb-phy@11c43000 { - reg = <0 0x11c43400 0 0x0500>; - clocks = <&clk26m>; - clock-names = "ref"; - mediatek,efuse-intr = <28>; - #phy-cells = <1>; - }; -}; diff --git a/Documentation/devicetree/bindings/phy/phy-stm32-usbphyc.yaml b/Documentation/devicetree/bindings/phy/phy-stm32-usbphyc.yaml index 0ba61979b970d700f66887f19306432c8c797f69..46df6786727a6b40343ddb454469a5c51246b4fb 100644 --- a/Documentation/devicetree/bindings/phy/phy-stm32-usbphyc.yaml +++ b/Documentation/devicetree/bindings/phy/phy-stm32-usbphyc.yaml @@ -45,6 +45,12 @@ properties: "#size-cells": const: 0 + vdda1v1-supply: + description: regulator providing 1V1 power supply to the PLL block + + vdda1v8-supply: + description: regulator providing 1V8 power supply to the PLL block + #Required child nodes: patternProperties: @@ -61,12 +67,6 @@ patternProperties: phy-supply: description: regulator providing 3V3 power supply to the PHY. - vdda1v1-supply: - description: regulator providing 1V1 power supply to the PLL block - - vdda1v8-supply: - description: regulator providing 1V8 power supply to the PLL block - "#phy-cells": enum: [ 0x0, 0x1 ] @@ -90,8 +90,6 @@ patternProperties: required: - reg - phy-supply - - vdda1v1-supply - - vdda1v8-supply - "#phy-cells" additionalProperties: false @@ -102,6 +100,8 @@ required: - clocks - "#address-cells" - "#size-cells" + - vdda1v1-supply + - vdda1v8-supply - usb-phy@0 - usb-phy@1 @@ -116,22 +116,20 @@ examples: reg = <0x5a006000 0x1000>; clocks = <&rcc USBPHY_K>; resets = <&rcc USBPHY_R>; + vdda1v1-supply = <®11>; + vdda1v8-supply = <®18>; #address-cells = <1>; #size-cells = <0>; usbphyc_port0: usb-phy@0 { reg = <0>; phy-supply = <&vdd_usb>; - vdda1v1-supply = <®11>; - vdda1v8-supply = <®18>; #phy-cells = <0>; }; usbphyc_port1: usb-phy@1 { reg = <1>; phy-supply = <&vdd_usb>; - vdda1v1-supply = <®11>; - vdda1v8-supply = <®18>; #phy-cells = <1>; }; }; diff --git a/Documentation/devicetree/bindings/phy/qcom,qmp-phy.yaml b/Documentation/devicetree/bindings/phy/qcom,qmp-phy.yaml index ec05db374645da14030afa99e97ae562b6c98021..626447fee09252d179e40b5192edc757159245c3 100644 --- a/Documentation/devicetree/bindings/phy/qcom,qmp-phy.yaml +++ b/Documentation/devicetree/bindings/phy/qcom,qmp-phy.yaml @@ -25,19 +25,32 @@ properties: - qcom,msm8998-qmp-pcie-phy - qcom,msm8998-qmp-ufs-phy - qcom,msm8998-qmp-usb3-phy + - qcom,sc8180x-qmp-ufs-phy + - qcom,sc8180x-qmp-usb3-phy - qcom,sdm845-qhp-pcie-phy - qcom,sdm845-qmp-pcie-phy - qcom,sdm845-qmp-ufs-phy - qcom,sdm845-qmp-usb3-uni-phy - qcom,sm8150-qmp-ufs-phy + - qcom,sm8150-qmp-usb3-phy + - qcom,sm8150-qmp-usb3-uni-phy - qcom,sm8250-qmp-ufs-phy - qcom,sm8250-qmp-gen3x1-pcie-phy - qcom,sm8250-qmp-gen3x2-pcie-phy - qcom,sm8250-qmp-modem-pcie-phy + - qcom,sm8250-qmp-usb3-phy + - qcom,sm8250-qmp-usb3-uni-phy + - qcom,sm8350-qmp-ufs-phy + - qcom,sm8350-qmp-usb3-phy + - qcom,sm8350-qmp-usb3-uni-phy + - qcom,sdx55-qmp-usb3-uni-phy reg: + minItems: 1 + maxItems: 2 items: - description: Address and length of PHY's common serdes block. + - description: Address and length of PHY's DP_COM control block. "#clock-cells": enum: [ 1, 2 ] @@ -131,6 +144,32 @@ allOf: items: - const: phy - const: common + - if: + properties: + compatible: + contains: + enum: + - qcom,sdx55-qmp-usb3-uni-phy + then: + properties: + clocks: + items: + - description: Phy aux clock. + - description: Phy config clock. + - description: 19.2 MHz ref clk. + clock-names: + items: + - const: aux + - const: cfg_ahb + - const: ref + resets: + items: + - description: reset of phy block. + - description: phy common block reset. + reset-names: + items: + - const: phy + - const: common - if: properties: compatible: @@ -285,6 +324,64 @@ allOf: reset-names: items: - const: phy + - if: + properties: + compatible: + contains: + enum: + - qcom,sm8150-qmp-usb3-phy + - qcom,sm8150-qmp-usb3-uni-phy + - qcom,sm8250-qmp-usb3-uni-phy + - qcom,sm8350-qmp-usb3-uni-phy + then: + properties: + clocks: + items: + - description: Phy aux clock. + - description: 19.2 MHz ref clk source. + - description: 19.2 MHz ref clk. + - description: Phy common block aux clock. + clock-names: + items: + - const: aux + - const: ref_clk_src + - const: ref + - const: com_aux + resets: + items: + - description: reset of phy block. + - description: phy common block reset. + reset-names: + items: + - const: phy + - const: common + - if: + properties: + compatible: + contains: + enum: + - qcom,sm8250-qmp-usb3-phy + - qcom,sm8350-qmp-usb3-phy + then: + properties: + clocks: + items: + - description: Phy aux clock. + - description: 19.2 MHz ref clk. + - description: Phy common block aux clock. + clock-names: + items: + - const: aux + - const: ref_clk_src + - const: com_aux + resets: + items: + - description: reset of phy block. + - description: phy common block reset. + reset-names: + items: + - const: phy + - const: common examples: - | diff --git a/Documentation/devicetree/bindings/phy/qcom,qusb2-phy.yaml b/Documentation/devicetree/bindings/phy/qcom,qusb2-phy.yaml index d457fb6a47794b292dac4a6b1a3aac62fa3ce903..9f9cf07b7d45529b9bffc315c175cf0b7356a25d 100644 --- a/Documentation/devicetree/bindings/phy/qcom,qusb2-phy.yaml +++ b/Documentation/devicetree/bindings/phy/qcom,qusb2-phy.yaml @@ -21,6 +21,8 @@ properties: - qcom,ipq8074-qusb2-phy - qcom,msm8996-qusb2-phy - qcom,msm8998-qusb2-phy + - qcom,sdm660-qusb2-phy + - qcom,ipq6018-qusb2-phy - items: - enum: - qcom,sc7180-qusb2-phy diff --git a/Documentation/devicetree/bindings/phy/qcom,usb-hs-28nm.yaml b/Documentation/devicetree/bindings/phy/qcom,usb-hs-28nm.yaml index ca6a0836b53c4168e29830b1a5591cfc9453ed70..abcc4373f39e1898244abeca9f7308f4de357d51 100644 --- a/Documentation/devicetree/bindings/phy/qcom,usb-hs-28nm.yaml +++ b/Documentation/devicetree/bindings/phy/qcom,usb-hs-28nm.yaml @@ -16,6 +16,7 @@ properties: compatible: enum: - qcom,usb-hs-28nm-femtophy + - qcom,usb-hs-28nm-mdm9607 reg: maxItems: 1 diff --git a/Documentation/devicetree/bindings/phy/qcom,usb-snps-femto-v2.yaml b/Documentation/devicetree/bindings/phy/qcom,usb-snps-femto-v2.yaml index 4949a2851532492a1d50764128286acd6458c0c9..ee77c645832667d70ed150d634a66215dce48cb1 100644 --- a/Documentation/devicetree/bindings/phy/qcom,usb-snps-femto-v2.yaml +++ b/Documentation/devicetree/bindings/phy/qcom,usb-snps-femto-v2.yaml @@ -17,6 +17,8 @@ properties: enum: - qcom,usb-snps-hs-7nm-phy - qcom,sm8150-usb-hs-phy + - qcom,sm8250-usb-hs-phy + - qcom,sm8350-usb-hs-phy - qcom,usb-snps-femto-v2-phy reg: diff --git a/Documentation/devicetree/bindings/phy/renesas,usb2-phy.yaml b/Documentation/devicetree/bindings/phy/renesas,usb2-phy.yaml index 829e8c7e467ae75ef3f024822e4ec69651305b90..0f358d5b84ef74d4ba58e3fbcf63c389129e5e97 100644 --- a/Documentation/devicetree/bindings/phy/renesas,usb2-phy.yaml +++ b/Documentation/devicetree/bindings/phy/renesas,usb2-phy.yaml @@ -81,9 +81,8 @@ properties: if: properties: compatible: - items: - enum: - - renesas,usb2-phy-r7s9210 + contains: + const: renesas,usb2-phy-r7s9210 then: required: - clock-names diff --git a/Documentation/devicetree/bindings/phy/rockchip-emmc-phy.txt b/Documentation/devicetree/bindings/phy/rockchip-emmc-phy.txt index 00aa2d349e55ccbbaec80fd2281fe78f9781b44b..57d28c0d5696a62b58c8bcf24e3f2ef8681c84f7 100644 --- a/Documentation/devicetree/bindings/phy/rockchip-emmc-phy.txt +++ b/Documentation/devicetree/bindings/phy/rockchip-emmc-phy.txt @@ -16,11 +16,11 @@ Optional properties: - drive-impedance-ohm: Specifies the drive impedance in Ohm. Possible values are 33, 40, 50, 66 and 100. If not set, the default value of 50 will be applied. - - enable-strobe-pulldown: Enable internal pull-down for the strobe line. - If not set, pull-down is not used. - - output-tapdelay-select: Specifies the phyctrl_otapdlysec register. - If not set, the register defaults to 0x4. - Maximum value 0xf. + - rockchip,enable-strobe-pulldown: Enable internal pull-down for the strobe + line. If not set, pull-down is not used. + - rockchip,output-tapdelay-select: Specifies the phyctrl_otapdlysec register. + If not set, the register defaults to 0x4. + Maximum value 0xf. Example: diff --git a/Documentation/devicetree/bindings/phy/socionext,uniphier-ahci-phy.yaml b/Documentation/devicetree/bindings/phy/socionext,uniphier-ahci-phy.yaml index 34756347a14ef59e5242f022a41e35fde3f77097..745c525ce6b903f1f8753341954f13fa01dcb36a 100644 --- a/Documentation/devicetree/bindings/phy/socionext,uniphier-ahci-phy.yaml +++ b/Documentation/devicetree/bindings/phy/socionext,uniphier-ahci-phy.yaml @@ -20,7 +20,7 @@ properties: - socionext,uniphier-pxs3-ahci-phy reg: - description: PHY register region (offset and length) + maxItems: 1 "#phy-cells": const: 0 diff --git a/Documentation/devicetree/bindings/phy/socionext,uniphier-pcie-phy.yaml b/Documentation/devicetree/bindings/phy/socionext,uniphier-pcie-phy.yaml index a06831fd64b9b3f7698dc2eb8bb8b70b07913668..3e05668990414191348d7d1b18923c1cba5ab6eb 100644 --- a/Documentation/devicetree/bindings/phy/socionext,uniphier-pcie-phy.yaml +++ b/Documentation/devicetree/bindings/phy/socionext,uniphier-pcie-phy.yaml @@ -21,7 +21,7 @@ properties: - socionext,uniphier-pxs3-pcie-phy reg: - description: PHY register region (offset and length) + maxItems: 1 "#phy-cells": const: 0 diff --git a/Documentation/devicetree/bindings/phy/socionext,uniphier-usb3hs-phy.yaml b/Documentation/devicetree/bindings/phy/socionext,uniphier-usb3hs-phy.yaml index 6fa5caab148760917a0f98ac8feac5a5cf1b4ba8..a681cbc3b4eff658b1b30efc9a8caabde4e07214 100644 --- a/Documentation/devicetree/bindings/phy/socionext,uniphier-usb3hs-phy.yaml +++ b/Documentation/devicetree/bindings/phy/socionext,uniphier-usb3hs-phy.yaml @@ -24,7 +24,7 @@ properties: - socionext,uniphier-pxs3-usb3-hsphy reg: - description: PHY register region (offset and length) + maxItems: 1 "#phy-cells": const: 0 diff --git a/Documentation/devicetree/bindings/phy/socionext,uniphier-usb3ss-phy.yaml b/Documentation/devicetree/bindings/phy/socionext,uniphier-usb3ss-phy.yaml index 9d46715ed03615a7b71893e9d1428dbdf7630a3e..41c0dd68ee258084fb32adb503a7cd5beaa17781 100644 --- a/Documentation/devicetree/bindings/phy/socionext,uniphier-usb3ss-phy.yaml +++ b/Documentation/devicetree/bindings/phy/socionext,uniphier-usb3ss-phy.yaml @@ -25,7 +25,7 @@ properties: - socionext,uniphier-pxs3-usb3-ssphy reg: - description: PHY register region (offset and length) + maxItems: 1 "#phy-cells": const: 0 diff --git a/Documentation/devicetree/bindings/phy/ti,phy-gmii-sel.yaml b/Documentation/devicetree/bindings/phy/ti,phy-gmii-sel.yaml index bcec422d77342083b5126fc677ac1b518ba3a82b..ff8a6d9eb15314897a76b9b81ebe472ab7f62189 100644 --- a/Documentation/devicetree/bindings/phy/ti,phy-gmii-sel.yaml +++ b/Documentation/devicetree/bindings/phy/ti,phy-gmii-sel.yaml @@ -55,7 +55,7 @@ properties: - ti,am654-phy-gmii-sel reg: - description: Address and length of the register set for the device + maxItems: 1 '#phy-cells': true diff --git a/Documentation/devicetree/bindings/phy/ti,phy-j721e-wiz.yaml b/Documentation/devicetree/bindings/phy/ti,phy-j721e-wiz.yaml index c33e9bc795217846fefca8d6bb329e575b1a5411..bbbd85501ada84565dd648da6057917f8b9192e8 100644 --- a/Documentation/devicetree/bindings/phy/ti,phy-j721e-wiz.yaml +++ b/Documentation/devicetree/bindings/phy/ti,phy-j721e-wiz.yaml @@ -151,7 +151,7 @@ patternProperties: WIZ node should have '1' subnode for the SERDES. It could be either Sierra SERDES or Torrent SERDES. Sierra SERDES should follow the bindings specified in - Documentation/devicetree/bindings/phy/phy-cadence-sierra.txt + Documentation/devicetree/bindings/phy/phy-cadence-sierra.yaml Torrent SERDES should follow the bindings specified in Documentation/devicetree/bindings/phy/phy-cadence-torrent.yaml diff --git a/Documentation/devicetree/bindings/pinctrl/allwinner,sun4i-a10-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/allwinner,sun4i-a10-pinctrl.yaml index 5240487dfe50e0279298a815a2ec4ff7359e7812..cce63c3cc46309c853a955678f2dd96723da7238 100644 --- a/Documentation/devicetree/bindings/pinctrl/allwinner,sun4i-a10-pinctrl.yaml +++ b/Documentation/devicetree/bindings/pinctrl/allwinner,sun4i-a10-pinctrl.yaml @@ -53,6 +53,8 @@ properties: - allwinner,sun50i-h5-pinctrl - allwinner,sun50i-h6-pinctrl - allwinner,sun50i-h6-r-pinctrl + - allwinner,sun50i-h616-pinctrl + - allwinner,sun50i-h616-r-pinctrl - allwinner,suniv-f1c100s-pinctrl - nextthing,gr8-pinctrl @@ -61,7 +63,7 @@ properties: interrupts: minItems: 1 - maxItems: 7 + maxItems: 8 description: One interrupt per external interrupt bank supported on the controller, sorted by bank number ascending order. @@ -91,7 +93,7 @@ properties: bank found in the controller $ref: /schemas/types.yaml#/definitions/uint32-array minItems: 1 - maxItems: 5 + maxItems: 8 patternProperties: # It's pretty scary, but the basic idea is that: @@ -145,6 +147,17 @@ allOf: # boards are defining it at the moment so it would generate a lot of # warnings. + - if: + properties: + compatible: + enum: + - allwinner,sun50i-h616-pinctrl + + then: + properties: + interrupts: + minItems: 8 + - if: properties: compatible: diff --git a/Documentation/devicetree/bindings/pinctrl/aspeed,ast2400-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/aspeed,ast2400-pinctrl.yaml index 54631dc1adb045b79d56e97b266632472f83971b..100bb6dea3ec98042212889700e45eb7613e65e0 100644 --- a/Documentation/devicetree/bindings/pinctrl/aspeed,ast2400-pinctrl.yaml +++ b/Documentation/devicetree/bindings/pinctrl/aspeed,ast2400-pinctrl.yaml @@ -23,8 +23,7 @@ properties: compatible: const: aspeed,ast2400-pinctrl reg: - description: | - A hint for the memory regions associated with the pin-controller + maxItems: 2 patternProperties: '^.*$': @@ -63,7 +62,7 @@ examples: reg = <0x1e6e2000 0x1a8>; pinctrl: pinctrl { - compatible = "aspeed,g4-pinctrl"; + compatible = "aspeed,ast2400-pinctrl"; pinctrl_i2c3_default: i2c3_default { function = "I2C3"; diff --git a/Documentation/devicetree/bindings/pinctrl/aspeed,ast2500-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/aspeed,ast2500-pinctrl.yaml index a90c0fe0495f34213ab1b1bb82a510d637a1b24f..904697bc9415d98782bbd5b08cd0a53155c662df 100644 --- a/Documentation/devicetree/bindings/pinctrl/aspeed,ast2500-pinctrl.yaml +++ b/Documentation/devicetree/bindings/pinctrl/aspeed,ast2500-pinctrl.yaml @@ -24,8 +24,8 @@ properties: compatible: const: aspeed,ast2500-pinctrl reg: - description: | - A hint for the memory regions associated with the pin-controller + maxItems: 2 + aspeed,external-nodes: minItems: 2 maxItems: 2 @@ -81,7 +81,7 @@ examples: reg = <0x1e6e2000 0x1a8>; pinctrl: pinctrl { - compatible = "aspeed,g5-pinctrl"; + compatible = "aspeed,ast2500-pinctrl"; aspeed,external-nodes = <&gfx>, <&lhc>; pinctrl_i2c3_default: i2c3_default { diff --git a/Documentation/devicetree/bindings/pinctrl/aspeed,ast2600-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/aspeed,ast2600-pinctrl.yaml index c78ab7e2eee708b0a95ad6d264f880fd19d2aae6..ad91c0bc54da669c8e6808fee2cf887706c40d13 100644 --- a/Documentation/devicetree/bindings/pinctrl/aspeed,ast2600-pinctrl.yaml +++ b/Documentation/devicetree/bindings/pinctrl/aspeed,ast2600-pinctrl.yaml @@ -95,7 +95,7 @@ examples: reg = <0x1e6e2000 0xf6c>; pinctrl: pinctrl { - compatible = "aspeed,g6-pinctrl"; + compatible = "aspeed,ast2600-pinctrl"; pinctrl_pwm10g1_default: pwm10g1_default { function = "PWM10"; diff --git a/Documentation/devicetree/bindings/pinctrl/atmel,at91-pio4-pinctrl.txt b/Documentation/devicetree/bindings/pinctrl/atmel,at91-pio4-pinctrl.txt index 265015bc0603d0fcad7779579316c653ed33dab1..e2b861ce16d81725175eca5a491fdd3a7fbd4888 100644 --- a/Documentation/devicetree/bindings/pinctrl/atmel,at91-pio4-pinctrl.txt +++ b/Documentation/devicetree/bindings/pinctrl/atmel,at91-pio4-pinctrl.txt @@ -35,9 +35,11 @@ ioset settings. Use the macros from boot/dts/-pinfunc.h file to get the right representation of the pin. Optional properties: -- GENERIC_PINCONFIG: generic pinconfig options to use, bias-disable, -bias-pull-down, bias-pull-up, drive-open-drain, input-schmitt-enable, -input-debounce, output-low, output-high. +- GENERIC_PINCONFIG: generic pinconfig options to use: + - bias-disable, bias-pull-down, bias-pull-up, drive-open-drain, + input-schmitt-enable, input-debounce, output-low, output-high. + - for microchip,sama7g5-pinctrl only: + - slew-rate: 0 - disabled, 1 - enabled (default) - atmel,drive-strength: 0 or 1 for low drive, 2 for medium drive and 3 for high drive. The default value is low drive. diff --git a/Documentation/devicetree/bindings/pinctrl/brcm,ns2-pinmux.txt b/Documentation/devicetree/bindings/pinctrl/brcm,ns2-pinmux.txt index e295dda4bbbab958ea2e229e1899c99bc4394540..40e0a9a19525b40dc6f80a40f5a5416ed45f3b71 100644 --- a/Documentation/devicetree/bindings/pinctrl/brcm,ns2-pinmux.txt +++ b/Documentation/devicetree/bindings/pinctrl/brcm,ns2-pinmux.txt @@ -39,7 +39,7 @@ For example: <0x660009b0 0x40>; pinctrl-names = "default"; - pinctrl-0 = <&nand_sel &uart3_rx &sdio0_d4>; + pinctrl-0 = <&nand_sel>, <&uart3_rx>, <&sdio0_d4>; /* Select nand function */ nand_sel: nand_sel { diff --git a/Documentation/devicetree/bindings/pinctrl/brcm,nsp-pinmux.txt b/Documentation/devicetree/bindings/pinctrl/brcm,nsp-pinmux.txt index 603564e5fe6f0e54b2dfd042be9806a8f4687a98..dede11e4ef78087ff48b6373bb1f6b503d4b292e 100644 --- a/Documentation/devicetree/bindings/pinctrl/brcm,nsp-pinmux.txt +++ b/Documentation/devicetree/bindings/pinctrl/brcm,nsp-pinmux.txt @@ -30,7 +30,7 @@ For example: <0x1803f408 0x04>; pinctrl-names = "default"; - pinctrl-0 = <&pwm &gpio_b &nand_sel>; + pinctrl-0 = <&pwm>, <&gpio_b>, <&nand_sel>; pwm: pwm { function = "pwm"; diff --git a/Documentation/devicetree/bindings/pinctrl/canaan,k210-fpioa.yaml b/Documentation/devicetree/bindings/pinctrl/canaan,k210-fpioa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..46fbc73ab26b5146ef9708df1864e78b0ba569eb --- /dev/null +++ b/Documentation/devicetree/bindings/pinctrl/canaan,k210-fpioa.yaml @@ -0,0 +1,171 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/pinctrl/canaan,k210-fpioa.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Canaan Kendryte K210 FPIOA Device Tree Bindings + +maintainers: + - Damien Le Moal + +description: + The Canaan Kendryte K210 SoC Fully Programmable IO Array (FPIOA) + controller allows assiging any of 256 possible functions to any of + 48 IO pins of the SoC. Pin function configuration is performed on + a per-pin basis. + +properties: + compatible: + const: canaan,k210-fpioa + + reg: + maxItems: 1 + description: + Address and length of the register set for the FPIOA controller. + + clocks: + items: + - description: Controller reference clock source + - description: APB interface clock source + + clock-names: + items: + - const: ref + - const: pclk + + resets: + maxItems: 1 + + canaan,k210-sysctl-power: + $ref: /schemas/types.yaml#/definitions/phandle-array + description: | + phandle of the K210 system controller node and offset of its + power domain control register. + +patternProperties: + '-pinmux$': + type: object + $ref: /schemas/pinctrl/pinmux-node.yaml + description: + FPIOA client devices use sub-nodes to define the desired pin + configuration. Client device sub-nodes use the pinux property + below. + + properties: + pinmux: + description: + List of IO pins alternate functions. The values for each IO + pin is a combination of an IO pin number (0 to 47) with the + desired function for the IO pin. Functions are defined as + macros in include/dt-bindings/pinctrl/k210-fpioa.h. + The K210_FPIOA(IO pin, function) macro is provided to + facilitate the combination of IO pin numbers and functions. + + required: + - pinmux + + additionalProperties: false + + '-pins$': + type: object + $ref: /schemas/pinctrl/pincfg-node.yaml + description: + FPIOA client devices use sub-nodes to define the desired + configuration of pins. Client device sub-nodes use the + properties below. + + properties: + pins: + description: + List of IO pins affected by the properties specified in this + subnode. IO pins are identified using the pin names "IO_xx". + Pin configuration nodes can also define the power domain to + be used for the SoC pin groups A0 (IO pins 0-5), + A1 (IO pins 6-11), A2 (IO pins 12-17), B0 (IO pins 18-23), + B1 (IO pins 24-29), B2 (IO pins 30-35), B3 (IO pins 30-35), + C0 (IO pins 36-41) and C1 (IO pins 42-47) using the + power-source property. + items: + anyOf: + - pattern: "^(IO_([0-9]*))|(A[0-2])|(B[3-5])|(C[6-7])$" + - enum: [ IO_0, IO_1, IO_2, IO_3, IO_4, IO_5, IO_6, IO_7, + IO_8, IO_9, IO_10, IO_11, IO_12, IO_13, IO_14, + IO_15, IO_16, IO_17, IO_18, IO_19, IO_20, IO_21, + IO_22, IO_23, IO_24, IO_25, IO_26, IO_27, IO_28, + IO_29, IO_30, IO_31, IO_32, IO_33, IO_34, IO_35, + IO_36, IO_37, IO_38, IO_39, IO_40, IO_41, IO_42, + IO_43, IO_44, IO_45, IO_46, IO_47, + A0, A1, A2, B3, B4, B5, C6, C7 ] + bias-disable: true + + bias-pull-down: true + + bias-pull-up: true + + drive-strength: true + + drive-strength-microamp: true + + input-enable: true + + input-disable: true + + input-schmitt-enable: true + + input-schmitt-disable: true + + input-polarity-invert: + description: + Enable or disable pin input polarity inversion. + + output-enable: true + + output-disable: true + + output-high: true + + output-low: true + + output-polarity-invert: + description: + Enable or disable pin output polarity inversion. + + slew-rate: true + + power-source: true + + additionalProperties: false + +required: + - compatible + - reg + - clocks + - canaan,k210-sysctl-power + +additionalProperties: false + +examples: + - | + #include + #include + #include + + fpioa: pinmux@502B0000 { + compatible = "canaan,k210-fpioa"; + reg = <0x502B0000 0x100>; + clocks = <&sysclk K210_CLK_FPIOA>, + <&sysclk K210_CLK_APB0>; + clock-names = "ref", "pclk"; + resets = <&sysrst K210_RST_FPIOA>; + canaan,k210-sysctl-power = <&sysctl 108>; + pinctrl-0 = <&jtag_pinctrl>; + pinctrl-names = "default"; + + jtag_pinctrl: jtag-pinmux { + pinmux = , + , + , + ; + }; + }; diff --git a/Documentation/devicetree/bindings/pinctrl/fsl,imx7d-pinctrl.txt b/Documentation/devicetree/bindings/pinctrl/fsl,imx7d-pinctrl.txt index 8ac1d0851a0f8f0a343455bbfd3cdbb77cf04d45..bfab5ca49fd1211a78922f3984c65b170b72f985 100644 --- a/Documentation/devicetree/bindings/pinctrl/fsl,imx7d-pinctrl.txt +++ b/Documentation/devicetree/bindings/pinctrl/fsl,imx7d-pinctrl.txt @@ -60,7 +60,7 @@ iomuxc-lpsr controller and SDA pad from iomuxc controller as: i2c1: i2c@30a20000 { pinctrl-names = "default"; - pinctrl-0 = <&pinctrl_i2c1_1 &pinctrl_i2c1_2>; + pinctrl-0 = <&pinctrl_i2c1_1>, <&pinctrl_i2c1_2>; }; iomuxc-lpsr@302c0000 { diff --git a/Documentation/devicetree/bindings/pinctrl/microchip,sparx5-sgpio.yaml b/Documentation/devicetree/bindings/pinctrl/microchip,sparx5-sgpio.yaml index df0c83cb1c6e46495243aba50f618b24a48369b9..4fe35e650909ce8c606d60d7006781240bbc2f95 100644 --- a/Documentation/devicetree/bindings/pinctrl/microchip,sparx5-sgpio.yaml +++ b/Documentation/devicetree/bindings/pinctrl/microchip,sparx5-sgpio.yaml @@ -99,8 +99,8 @@ patternProperties: '#interrupt-cells': description: - Specifies the pin (port and bit) and flags, as defined in - defined in include/dt-bindings/interrupt-controller/irq.h + Specifies the pin (port and bit) and flags, as defined in + defined in include/dt-bindings/interrupt-controller/irq.h const: 3 ngpios: diff --git a/Documentation/devicetree/bindings/pinctrl/pinctrl-atlas7.txt b/Documentation/devicetree/bindings/pinctrl/pinctrl-atlas7.txt deleted file mode 100644 index fbdd1a716a1e11860ecb98c12ce25d22e2d88921..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/pinctrl/pinctrl-atlas7.txt +++ /dev/null @@ -1,109 +0,0 @@ -CSR SiRFatlas7 pinmux controller - -Required properties: -- compatible : "sirf,atlas7-ioc" -- reg : Address range of the pinctrl registers - -For example, pinctrl might have properties like the following: - pinctrl: ioc@18880000 { - compatible = "sirf,atlas7-ioc"; - reg = <0x18880000 0x1000>; - - a_ac97_pmx: ac97@0 { - ac97 { - groups = "audio_ac97_grp"; - function = "audio_ac97"; - }; - }; - - ... - - sd2_pmx: sd2@0 { - sd2 { - groups = "sd2_grp0"; - function = "sd2"; - }; - }; - - ... - - - sample0_cfg: sample0@0 { - sample0 { - pins = "ldd_0", "ldd_1"; - bias-pull-up; - }; - }; - - sample1_cfg: sample1@0 { - sample1 { - pins = "ldd_2", "ldd_3"; - input-schmitt-enable; - }; - }; - - sample2_cfg: sample2@0 { - sample2 { - groups = "uart4_nopause_grp"; - bias-pull-down; - }; - }; - - sample3_cfg: sample3@0 { - sample3 { - pins = "ldd_4", "ldd_5"; - drive-strength = <2>; - }; - }; - }; - -Please refer to pinctrl-bindings.txt in this directory for details of the common -pinctrl bindings used by client devices. - -SiRFatlas7's pinmux nodes act as a container for an arbitrary number of subnodes. -Each of these subnodes represents some desired configuration for a group of pins. - -Required subnode-properties: -- groups : An array of strings. Each string contains the name of a group. -- function: A string containing the name of the function to mux to the - group. - - Valid values for group and function names can be found from looking at the - group and function arrays in driver files: - drivers/pinctrl/pinctrl-sirf.c - -For example, pinctrl might have subnodes like the following: - sd0_pmx: sd0@0 { - sd0 { - groups = "sd0_grp"; - function = "sd0"; - }; - }; - - sd1_pmx0: sd1@0 { - sd1 { - groups = "sd1_grp0"; - function = "sd1_m0"; - }; - }; - - sd1_pmx1: sd1@1 { - sd1 { - groups = "sd1_grp1"; - function = "sd1_m1"; - }; - }; - -For a specific board, if it wants to use sd1, -it can add the following to its board-specific .dts file. -sd1: sd@12340000 { - pinctrl-names = "default"; - pinctrl-0 = <&sd1_pmx0>; -} - -or - -sd1: sd@12340000 { - pinctrl-names = "default"; - pinctrl-0 = <&sd1_pmx1>; -} diff --git a/Documentation/devicetree/bindings/pinctrl/pinctrl-bindings.txt b/Documentation/devicetree/bindings/pinctrl/pinctrl-bindings.txt index 4613bb17ace3f6e15feeb77408dcc366cb22374b..9dae60acf950641d04832a134a92aecbd3b83a43 100644 --- a/Documentation/devicetree/bindings/pinctrl/pinctrl-bindings.txt +++ b/Documentation/devicetree/bindings/pinctrl/pinctrl-bindings.txt @@ -77,13 +77,13 @@ For example: device { pinctrl-names = "active", "idle"; pinctrl-0 = <&state_0_node_a>; - pinctrl-1 = <&state_1_node_a &state_1_node_b>; + pinctrl-1 = <&state_1_node_a>, <&state_1_node_b>; }; /* For the same device if using state IDs */ device { pinctrl-0 = <&state_0_node_a>; - pinctrl-1 = <&state_1_node_a &state_1_node_b>; + pinctrl-1 = <&state_1_node_a>, <&state_1_node_b>; }; /* diff --git a/Documentation/devicetree/bindings/pinctrl/pinctrl-mcp23s08.txt b/Documentation/devicetree/bindings/pinctrl/pinctrl-mcp23s08.txt index 8b94aa8f5971ceb7733511f7aea0af942e02cad1..6ec3c8d79f496b52f86ddeeb223464997d9eb269 100644 --- a/Documentation/devicetree/bindings/pinctrl/pinctrl-mcp23s08.txt +++ b/Documentation/devicetree/bindings/pinctrl/pinctrl-mcp23s08.txt @@ -134,7 +134,7 @@ gpio21: gpio@21 { #interrupt-cells = <0x2>; microchip,irq-mirror; pinctrl-names = "default"; - pinctrl-0 = <&i2cgpio0irq &gpio21pullups>; + pinctrl-0 = <&i2cgpio0irq>, <&gpio21pullups>; gpio21pullups: pinmux { pins = "gpio0", "gpio1", "gpio2", "gpio3", diff --git a/Documentation/devicetree/bindings/pinctrl/pinctrl-mt65xx.txt b/Documentation/devicetree/bindings/pinctrl/pinctrl-mt65xx.txt index 931a18cd1e238e6c2f20cf060d76bc6c503695a7..360e59c9301a93a8614c461eef70ce3650a29c8e 100644 --- a/Documentation/devicetree/bindings/pinctrl/pinctrl-mt65xx.txt +++ b/Documentation/devicetree/bindings/pinctrl/pinctrl-mt65xx.txt @@ -91,7 +91,7 @@ Examples: pinctrl@1c20800 { compatible = "mediatek,mt8135-pinctrl"; reg = <0 0x1000B000 0 0x1000>; - mediatek,pctl-regmap = <&syscfg_pctl_a &syscfg_pctl_b>; + mediatek,pctl-regmap = <&syscfg_pctl_a>, <&syscfg_pctl_b>; pins-are-numbered; gpio-controller; #gpio-cells = <2>; diff --git a/Documentation/devicetree/bindings/pinctrl/pinctrl-single.txt b/Documentation/devicetree/bindings/pinctrl/pinctrl-single.txt index f903eb4471f87133c39f73e2ef98a90cfef02b9f..bfd222b0549569dfb6287c76cd5bc177f4b7c720 100644 --- a/Documentation/devicetree/bindings/pinctrl/pinctrl-single.txt +++ b/Documentation/devicetree/bindings/pinctrl/pinctrl-single.txt @@ -8,7 +8,7 @@ Required properties: - reg : offset and length of the register set for the mux registers - #pinctrl-cells : number of cells in addition to the index, set to 1 - for pinctrl-single,pins and 2 for pinctrl-single,bits + or 2 for pinctrl-single,pins and set to 2 for pinctrl-single,bits - pinctrl-single,register-width : pinmux register access width in bits @@ -80,7 +80,7 @@ Optional properties: property. /* pin base, nr pins & gpio function */ - pinctrl-single,gpio-range = <&range 0 3 0 &range 3 9 1>; + pinctrl-single,gpio-range = <&range 0 3 0>, <&range 3 9 1>; - interrupt-controller : standard interrupt controller binding if using interrupts for wake-up events for example. In this case pinctrl-single @@ -185,10 +185,10 @@ pmx_gpio: pinmux@d401e000 { pinctrl-single,function-mask = <7>; /* sparse GPIO range could be supported */ - pinctrl-single,gpio-range = <&range 0 3 0 &range 3 9 1 - &range 12 1 0 &range 13 29 1 - &range 43 1 0 &range 44 49 1 - &range 94 1 1 &range 96 2 1>; + pinctrl-single,gpio-range = <&range 0 3 0>, <&range 3 9 1>, + <&range 12 1 0>, <&range 13 29 1>, + <&range 43 1 0>, <&range 44 49 1>, + <&range 94 1 1>, <&range 96 2 1>; range: gpio-range { #pinctrl-single,gpio-range-cells = <3>; diff --git a/Documentation/devicetree/bindings/pinctrl/pinctrl-zx.txt b/Documentation/devicetree/bindings/pinctrl/pinctrl-zx.txt deleted file mode 100644 index 39170f372599b2a95f5ce23db7a33967387f8ea7..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/pinctrl/pinctrl-zx.txt +++ /dev/null @@ -1,84 +0,0 @@ -* ZTE ZX Pin Controller - -The pin controller on ZTE ZX platforms is kinda of hybrid. It consists of -a main controller and an auxiliary one. For example, on ZX296718 SoC, the -main controller is TOP_PMM and the auxiliary one is AON_IOCFG. Both -controllers work together to control pin multiplexing and configuration in -the way illustrated as below. - - - GMII_RXD3 ---+ - | - DVI1_HS ---+----------------------------- GMII_RXD3 (TOP pin) - | - BGPIO16 ---+ ^ - | pinconf - ^ | - | pinmux | - | | - - TOP_PMM (main) AON_IOCFG (aux) - - | | | - | pinmux | | - | pinmux v | - v | pinconf - KEY_ROW2 ---+ v - PORT1_LCD_TE ---+ | - | AGPIO10 ---+------ KEY_ROW2 (AON pin) - I2S0_DOUT3 ---+ | - |-----------------------+ - PWM_OUT3 ---+ - | - VGA_VS1 ---+ - - -For most of pins like GMII_RXD3 in the figure, the pinmux function is -controlled by TOP_PMM block only, and this type of pins are meant by term -'TOP pins'. For pins like KEY_ROW2, the pinmux is controlled by both -TOP_PMM and AON_IOCFG blocks, as the available multiplexing functions for -the pin spread in both controllers. This type of pins are called 'AON pins'. -Though pinmux implementation is quite different, pinconf is same for both -types of pins. Both are controlled by auxiliary controller, i.e. AON_IOCFG -on ZX296718. - -Required properties: -- compatible: should be "zte,zx296718-pmm". -- reg: the register physical address and length. -- zte,auxiliary-controller: phandle to the auxiliary pin controller which - implements pinmux for AON pins and pinconf for all pins. - -The following pin configuration are supported. Please refer to -pinctrl-bindings.txt in this directory for more details of the common -pinctrl bindings used by client devices. - -- bias-pull-up -- bias-pull-down -- drive-strength -- input-enable -- slew-rate - -Examples: - -iocfg: pin-controller@119000 { - compatible = "zte,zx296718-iocfg"; - reg = <0x119000 0x1000>; -}; - -pmm: pin-controller@1462000 { - compatible = "zte,zx296718-pmm"; - reg = <0x1462000 0x1000>; - zte,auxiliary-controller = <&iocfg>; -}; - -&pmm { - vga_pins: vga { - pins = "KEY_COL1", "KEY_COL2", "KEY_ROW1", "KEY_ROW2"; - function = "VGA"; - }; -}; - -&vga { - pinctrl-names = "default"; - pinctrl-0 = <&vga_pins>; -}; diff --git a/Documentation/devicetree/bindings/pinctrl/qcom,pmic-mpp.txt b/Documentation/devicetree/bindings/pinctrl/qcom,pmic-mpp.txt index 448d36a85730f55316d9ff2b4951d627d21580a8..0ba07bc96c55f054eec11f732a3efc757c866240 100644 --- a/Documentation/devicetree/bindings/pinctrl/qcom,pmic-mpp.txt +++ b/Documentation/devicetree/bindings/pinctrl/qcom,pmic-mpp.txt @@ -8,6 +8,7 @@ of PMIC's from Qualcomm. Value type: Definition: Should contain one of: "qcom,pm8018-mpp", + "qcom,pm8019-mpp", "qcom,pm8038-mpp", "qcom,pm8058-mpp", "qcom,pm8821-mpp", diff --git a/Documentation/devicetree/bindings/pinctrl/qcom,sc8180x-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/qcom,sc8180x-pinctrl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a82dab8983957b72ded86569b5dd88bd8ceb418e --- /dev/null +++ b/Documentation/devicetree/bindings/pinctrl/qcom,sc8180x-pinctrl.yaml @@ -0,0 +1,152 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/pinctrl/qcom,sc8180x-pinctrl.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Qualcomm Technologies, Inc. SC8180X TLMM block + +maintainers: + - Bjorn Andersson + +description: | + This binding describes the Top Level Mode Multiplexer block found in the + SC8180X platform. + +allOf: + - $ref: /schemas/pinctrl/qcom,tlmm-common.yaml# + +properties: + compatible: + const: qcom,sc8180x-tlmm + + reg: + maxItems: 3 + + reg-names: + items: + - const: "west" + - const: "east" + - const: "south" + + interrupts: true + interrupt-controller: true + '#interrupt-cells': true + gpio-controller: true + gpio-reserved-ranges: true + '#gpio-cells': true + gpio-ranges: true + wakeup-parent: true + +required: + - compatible + - reg + - reg-names + +additionalProperties: false + +patternProperties: + '-state$': + oneOf: + - $ref: "#/$defs/qcom-sc8180x-tlmm-state" + - patternProperties: + ".*": + $ref: "#/$defs/qcom-sc8180x-tlmm-state" + +'$defs': + qcom-sc8180x-tlmm-state: + type: object + description: + Pinctrl node's client devices use subnodes for desired pin configuration. + Client device subnodes use below standard properties. + $ref: "qcom,tlmm-common.yaml#/$defs/qcom-tlmm-state" + + properties: + pins: + description: + List of gpio pins affected by the properties specified in this + subnode. + items: + oneOf: + - pattern: "^gpio([0-9]|[1-9][0-9]|1[0-8][0-9])$" + - enum: [ sdc2_clk, sdc2_cmd, sdc2_data, ufs_reset ] + minItems: 1 + maxItems: 16 + + function: + description: + Specify the alternative function to be configured for the specified + pins. + + enum: [ adsp_ext, agera_pll, aoss_cti, atest_char, atest_tsens, + atest_tsens2, atest_usb0, atest_usb1, atest_usb2, atest_usb3, + atest_usb4, audio_ref, btfm_slimbus, cam_mclk, cci_async, + cci_i2c, cci_timer0, cci_timer1, cci_timer2, cci_timer3, + cci_timer4, cci_timer5, cci_timer6, cci_timer7, cci_timer8, + cci_timer9, cri_trng, dbg_out, ddr_bist, ddr_pxi, debug_hot, + dp_hot, edp_hot, edp_lcd, emac_phy, emac_pps, gcc_gp1, gcc_gp2, + gcc_gp3, gcc_gp4, gcc_gp5, gpio, gps, grfc, hs1_mi2s, hs2_mi2s, + hs3_mi2s, jitter_bist, lpass_slimbus, m_voc, mdp_vsync, + mdp_vsync0, mdp_vsync1, mdp_vsync2, mdp_vsync3, mdp_vsync4, + mdp_vsync5, mss_lte, nav_pps, pa_indicator, pci_e0, pci_e1, + pci_e2, pci_e3, phase_flag, pll_bist, pll_bypassnl, pll_reset, + pri_mi2s, pri_mi2s_ws, prng_rosc, qdss_cti, qdss_gpio, qlink, + qspi0, qspi0_clk, qspi0_cs, qspi1, qspi1_clk, qspi1_cs, + qua_mi2s, qup0, qup1, qup2, qup3, qup4, qup5, qup6, qup7, qup8, + qup9, qup10, qup11, qup12, qup13, qup14, qup15, qup16, qup17, + qup18, qup19, qup_l4, qup_l5, qup_l6, rgmii, sd_write, sdc4, + sdc4_clk, sdc4_cmd, sec_mi2s, sp_cmu, spkr_i2s, ter_mi2s, tgu, + tsense_pwm1, tsense_pwm2, tsif1, tsif2, uim1, uim2, uim_batt, + usb0_phy, usb1_phy, usb2phy_ac, vfr_1, vsense_trigger, + wlan1_adc, wlan2_adc, wmss_reset ] + + bias-disable: true + bias-pull-down: true + bias-pull-up: true + drive-strength: true + input-enable: true + output-high: true + output-low: true + + required: + - pins + - function + + additionalProperties: false + +examples: + - | + #include + pinctrl@3100000 { + compatible = "qcom,sc8180x-tlmm"; + reg = <0x03100000 0x300000>, + <0x03500000 0x700000>, + <0x03d00000 0x300000>; + reg-names = "west", "east", "south"; + interrupts = ; + gpio-controller; + #gpio-cells = <2>; + interrupt-controller; + #interrupt-cells = <2>; + gpio-ranges = <&tlmm 0 0 190>; + + gpio-wo-subnode-state { + pins = "gpio1"; + function = "gpio"; + }; + + uart-w-subnodes-state { + rx { + pins = "gpio4"; + function = "qup6"; + bias-pull-up; + }; + + tx { + pins = "gpio5"; + function = "qup6"; + bias-disable; + }; + }; + }; +... diff --git a/Documentation/devicetree/bindings/pinctrl/qcom,sm8350-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/qcom,sm8350-pinctrl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4f2667ea2805ebb9dfa573b02c1340bee0cdb593 --- /dev/null +++ b/Documentation/devicetree/bindings/pinctrl/qcom,sm8350-pinctrl.yaml @@ -0,0 +1,145 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/pinctrl/qcom,sm8350-pinctrl.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Qualcomm Technologies, Inc. SM8350 TLMM block + +maintainers: + - Vinod Koul + +description: | + This binding describes the Top Level Mode Multiplexer (TLMM) block found + in the SM8350 platform. + +allOf: + - $ref: /schemas/pinctrl/qcom,tlmm-common.yaml# + +properties: + compatible: + const: qcom,sm8350-tlmm + + reg: + maxItems: 1 + + interrupts: true + interrupt-controller: true + '#interrupt-cells': true + gpio-controller: true + gpio-reserved-ranges: true + '#gpio-cells': true + gpio-ranges: true + wakeup-parent: true + +required: + - compatible + - reg + +additionalProperties: false + +patternProperties: + '-state$': + oneOf: + - $ref: "#/$defs/qcom-sm8350-tlmm-state" + - patternProperties: + ".*": + $ref: "#/$defs/qcom-sm8350-tlmm-state" + +$defs: + qcom-sm8350-tlmm-state: + type: object + description: + Pinctrl node's client devices use subnodes for desired pin configuration. + Client device subnodes use below standard properties. + $ref: "qcom,tlmm-common.yaml#/$defs/qcom-tlmm-state" + + properties: + pins: + description: + List of gpio pins affected by the properties specified in this + subnode. + items: + oneOf: + - pattern: "^gpio([0-9]|[1-9][0-9]|1[0-9][0-9]|20[0-3])$" + - enum: [ sdc1_clk, sdc1_cmd, sdc1_data, sdc2_clk, sdc2_cmd, sdc2_data ] + minItems: 1 + maxItems: 36 + + function: + description: + Specify the alternative function to be configured for the specified + pins. + + enum: [ atest_char, atest_usb, audio_ref, cam_mclk, cci_async, + cci_i2c, cci_timer, cmu_rng, coex_uart1, coex_uart2, cri_trng, + cri_trng0, cri_trng1, dbg_out, ddr_bist, ddr_pxi0, ddr_pxi1, + ddr_pxi2, ddr_pxi3, dp_hot, dp_lcd, gcc_gp1, gcc_gp2, gcc_gp3, + gpio, ibi_i3c, jitter_bist, lpass_slimbus, mdp_vsync, mdp_vsync0, + mdp_vsync1, mdp_vsync2, mdp_vsync3, mi2s0_data0, mi2s0_data1, + mi2s0_sck, mi2s0_ws, mi2s1_data0, mi2s1_data1, mi2s1_sck, + mi2s1_ws, mi2s2_data0, mi2s2_data1, mi2s2_sck, mi2s2_ws, + mss_grfc0, mss_grfc1, mss_grfc10, mss_grfc11, mss_grfc12, + mss_grfc2, mss_grfc3, mss_grfc4, mss_grfc5, mss_grfc6, + mss_grfc7, mss_grfc8, mss_grfc9, nav_gpio, pa_indicator, + pcie0_clkreqn, pcie1_clkreqn, phase_flag, pll_bist, pll_clk, + pri_mi2s, prng_rosc, qdss_cti, qdss_gpio, qlink0_enable, + qlink0_request, qlink0_wmss, qlink1_enable, qlink1_request, + qlink1_wmss, qlink2_enable, qlink2_request, qlink2_wmss, qspi0, + qspi1, qspi2, qspi3, qspi_clk, qspi_cs, qup0, qup1, qup10, + qup11, qup12, qup13, qup14, qup15, qup16, qup17, qup18, qup19, + qup2, qup3, qup4, qup5, qup6, qup7, qup8, qup9, qup_l4, qup_l5, + qup_l6, sd_write, sdc40, sdc41, sdc42, sdc43, sdc4_clk, + sdc4_cmd, sec_mi2s, tb_trig, tgu_ch0, tgu_ch1, tgu_ch2, + tgu_ch3, tsense_pwm1, tsense_pwm2, uim0_clk, uim0_data, + uim0_present, uim0_reset, uim1_clk, uim1_data, uim1_present, + uim1_reset, usb2phy_ac, usb_phy, vfr_0, vfr_1, vsense_trigger ] + + + bias-disable: true + bias-pull-down: true + bias-pull-up: true + drive-strength: true + input-enable: true + output-high: true + output-low: true + + required: + - pins + - function + + additionalProperties: false + +examples: + - | + #include + pinctrl@f100000 { + compatible = "qcom,sm8350-tlmm"; + reg = <0x0f100000 0x300000>; + interrupts = ; + gpio-controller; + #gpio-cells = <2>; + interrupt-controller; + #interrupt-cells = <2>; + gpio-ranges = <&tlmm 0 0 203>; + + gpio-wo-subnode-state { + pins = "gpio1"; + function = "gpio"; + }; + + uart-w-subnodes-state { + rx { + pins = "gpio18"; + function = "qup3"; + bias-pull-up; + }; + + tx { + pins = "gpio19"; + function = "qup3"; + bias-disable; + }; + }; + }; +... diff --git a/Documentation/devicetree/bindings/pinctrl/qcom,tlmm-common.yaml b/Documentation/devicetree/bindings/pinctrl/qcom,tlmm-common.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3b37cf102d41e521e9a9b71920c7b850f7a62385 --- /dev/null +++ b/Documentation/devicetree/bindings/pinctrl/qcom,tlmm-common.yaml @@ -0,0 +1,85 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/pinctrl/qcom,tlmm-common.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Qualcomm Technologies, Inc. Top Level Mode Multiplexer (TLMM) definitions + +maintainers: + - Bjorn Andersson + +description: + This defines the common properties used to describe all Qualcomm Top Level + Mode Multiplexer bindings and pinconf/pinmux states for these. + +properties: + interrupts: + description: + Specifies the TLMM summary IRQ + maxItems: 1 + + interrupt-controller: true + + '#interrupt-cells': + description: + Specifies the PIN numbers and Flags, as defined in defined in + include/dt-bindings/interrupt-controller/irq.h + const: 2 + + gpio-controller: true + + '#gpio-cells': + description: + Specifying the pin number and flags, as defined in + include/dt-bindings/gpio/gpio.h + const: 2 + + gpio-ranges: + maxItems: 1 + + wakeup-parent: + description: + Specifying the interrupt-controller used to wake up the system when the + TLMM block has been powered down. + maxItems: 1 + + gpio-reserved-ranges: + description: + Pins can be reserved for trusted applications and thereby unaccessible + from the OS. This property can be used to mark the pins which resources + should not be accessed by the OS. Please see the ../gpio/gpio.txt for more + information. + +required: + - interrupts + - interrupt-controller + - '#interrupt-cells' + - gpio-controller + - '#gpio-cells' + - gpio-ranges + +additionalProperties: true + +$defs: + qcom-tlmm-state: + allOf: + - $ref: pincfg-node.yaml# + - $ref: pinmux-node.yaml# + + properties: + drive-strength: + enum: [2, 4, 6, 8, 10, 12, 14, 16] + default: 2 + description: + Selects the drive strength for the specified pins, in mA. + + bias-pull-down: true + bias-pull-up: true + bias-disable: true + input-enable: true + output-high: true + output-low: true + + additionalProperties: true +... diff --git a/Documentation/devicetree/bindings/pinctrl/ralink,rt2880-pinmux.yaml b/Documentation/devicetree/bindings/pinctrl/ralink,rt2880-pinmux.yaml index 7dea3e26d99e1f11b38ebdd3a84e058a77318d16..b32f2676cab57e0f7d4c6161d6685992ba70e9b9 100644 --- a/Documentation/devicetree/bindings/pinctrl/ralink,rt2880-pinmux.yaml +++ b/Documentation/devicetree/bindings/pinctrl/ralink,rt2880-pinmux.yaml @@ -15,39 +15,38 @@ description: properties: compatible: - enum: - - ralink,rt2880-pinmux + const: ralink,rt2880-pinmux - pinctrl-0: - description: - A phandle to the node containing the subnodes containing default - configurations. This is for pinctrl hogs. +patternProperties: + '-pins$': + type: object + patternProperties: + '^(.*-)?pinmux$': + type: object + description: node for pinctrl. + $ref: pinmux-node.yaml# + + properties: + groups: + description: Name of the pin group to use for the functions. + enum: [i2c, spi, uart1, uart2, uart3, rgmii1, rgmii2, mdio, + pcie, sdhci] + function: + description: The mux function to select + enum: [gpio, i2c, spi, uart1, uart2, uart3, rgmii1, rgmii2, + mdio, nand1, nand2, sdhci] + + required: + - groups + - function + + additionalProperties: false - pinctrl-names: - description: - A pinctrl state named "default" can be defined. - const: default + additionalProperties: false required: - compatible -patternProperties: - '[a-z0-9_-]+': - if: - type: object - description: node for pinctrl. - $ref: "pinmux-node.yaml" - then: - properties: - groups: - description: Name of the pin group to use for the functions. - enum: [i2c, spi, uart1, uart2, uart3, rgmii1, rgmii2, mdio, - pcie, sdhci] - function: - description: The mux function to select - enum: [gpio, i2c, spi, uart1, uart2, uart3, rgmii1, rgmii2, - mdio, nand1, nand2, sdhci] - additionalProperties: false examples: @@ -55,14 +54,9 @@ examples: - | pinctrl { compatible = "ralink,rt2880-pinmux"; - pinctrl-names = "default"; - pinctrl-0 = <&state_default>; - - state_default: pinctrl0 { - }; - i2c_pins: i2c0 { - i2c0 { + i2c_pins: i2c0-pins { + pinmux { groups = "i2c"; function = "i2c"; }; diff --git a/Documentation/devicetree/bindings/pinctrl/renesas,pfc.yaml b/Documentation/devicetree/bindings/pinctrl/renesas,pfc.yaml index 5b5b1b9d2ec7422c89059970f6fa82896b384f60..ac4e068aa03f51686b281912486a341d7a72e8d1 100644 --- a/Documentation/devicetree/bindings/pinctrl/renesas,pfc.yaml +++ b/Documentation/devicetree/bindings/pinctrl/renesas,pfc.yaml @@ -43,11 +43,12 @@ properties: - renesas,pfc-r8a77980 # R-Car V3H - renesas,pfc-r8a77990 # R-Car E3 - renesas,pfc-r8a77995 # R-Car D3 + - renesas,pfc-r8a779a0 # R-Car V3U - renesas,pfc-sh73a0 # SH-Mobile AG5 reg: minItems: 1 - maxItems: 2 + maxItems: 10 gpio-controller: true @@ -76,11 +77,10 @@ required: if: properties: compatible: - items: - enum: - - renesas,pfc-r8a73a4 - - renesas,pfc-r8a7740 - - renesas,pfc-sh73a0 + enum: + - renesas,pfc-r8a73a4 + - renesas,pfc-r8a7740 + - renesas,pfc-sh73a0 then: required: - interrupts-extended diff --git a/Documentation/devicetree/bindings/pinctrl/samsung-pinctrl.txt b/Documentation/devicetree/bindings/pinctrl/samsung-pinctrl.txt index 7734ab6fec44947118c0ea99e6e47e8859ee4bd5..38a1416fd2cd8e1ab4c66340de3f8fd213570d6e 100644 --- a/Documentation/devicetree/bindings/pinctrl/samsung-pinctrl.txt +++ b/Documentation/devicetree/bindings/pinctrl/samsung-pinctrl.txt @@ -336,7 +336,7 @@ Example 3: A uart client node that supports 'default' and 'flow-control' states. interrupts = <0 52 0>; pinctrl-names = "default", "flow-control; pinctrl-0 = <&uart0_data>; - pinctrl-1 = <&uart0_data &uart0_fctl>; + pinctrl-1 = <&uart0_data>, <&uart0_fctl>; }; Example 4: Set up the default pin state for uart controller. diff --git a/Documentation/devicetree/bindings/power/renesas,apmu.yaml b/Documentation/devicetree/bindings/power/renesas,apmu.yaml index 60a23b3beb40c3c87e30b815854319244e2d2246..391897d897f241c96f46d797881ec57b562a09dd 100644 --- a/Documentation/devicetree/bindings/power/renesas,apmu.yaml +++ b/Documentation/devicetree/bindings/power/renesas,apmu.yaml @@ -52,5 +52,5 @@ examples: apmu@e6152000 { compatible = "renesas,r8a7791-apmu", "renesas,apmu"; reg = <0xe6152000 0x188>; - cpus = <&cpu0 &cpu1>; + cpus = <&cpu0>, <&cpu1>; }; diff --git a/Documentation/devicetree/bindings/power/supply/bq256xx.yaml b/Documentation/devicetree/bindings/power/supply/bq256xx.yaml new file mode 100644 index 0000000000000000000000000000000000000000..18b54783e11a23ed1f06e6bc639de816fb5c9d0a --- /dev/null +++ b/Documentation/devicetree/bindings/power/supply/bq256xx.yaml @@ -0,0 +1,110 @@ +# SPDX-License-Identifier: (GPL-2.0-only or BSD-2-Clause) +# Copyright (C) 2020 Texas Instruments Incorporated +%YAML 1.2 +--- +$id: "http://devicetree.org/schemas/power/supply/bq256xx.yaml#" +$schema: "http://devicetree.org/meta-schemas/core.yaml#" + +title: TI bq256xx Switch Mode Buck Charger + +maintainers: + - Ricardo Rivera-Matos + +description: | + The bq256xx devices are a family of highly-integrated battery charge + management and system power management ICs for single cell Li-ion and Li- + polymer batteries. + + Datasheets: + - https://www.ti.com/lit/ds/symlink/bq25600.pdf + - https://www.ti.com/lit/ds/symlink/bq25601.pdf + - https://www.ti.com/lit/ds/symlink/bq25600d.pdf + - https://www.ti.com/lit/ds/symlink/bq25601d.pdf + - https://www.ti.com/lit/ds/symlink/bq25611d.pdf + - https://www.ti.com/lit/ds/symlink/bq25618.pdf + - https://www.ti.com/lit/ds/symlink/bq25619.pdf + +properties: + compatible: + enum: + - ti,bq25600 + - ti,bq25601 + - ti,bq25600d + - ti,bq25601d + - ti,bq25611d + - ti,bq25618 + - ti,bq25619 + + reg: + maxItems: 1 + + ti,watchdog-timeout-ms: + $ref: /schemas/types.yaml#/definitions/uint32 + default: 0 + description: | + Watchdog timer in ms. 0 (default) disables the watchdog + minimum: 0 + maximum: 160000 + enum: [ 0, 40000, 80000, 160000] + + input-voltage-limit-microvolt: + description: | + Minimum input voltage limit in µV with a 100000 µV step + minimum: 3900000 + maximum: 5400000 + + input-current-limit-microamp: + description: | + Maximum input current limit in µA with a 100000 µA step + minimum: 100000 + maximum: 3200000 + + monitored-battery: + $ref: /schemas/types.yaml#/definitions/phandle + description: phandle to the battery node being monitored + + interrupts: + maxItems: 1 + description: | + Interrupt sends an active low, 256 μs pulse to host to report the charger + device status and faults. + +required: + - compatible + - reg + - monitored-battery + +additionalProperties: false + +examples: + - | + bat: battery { + compatible = "simple-battery"; + constant-charge-current-max-microamp = <2040000>; + constant-charge-voltage-max-microvolt = <4352000>; + precharge-current-microamp = <180000>; + charge-term-current-microamp = <180000>; + }; + #include + #include + i2c { + + clock-frequency = <400000>; + + #address-cells = <1>; + #size-cells = <0>; + + charger@6b { + compatible = "ti,bq25601"; + reg = <0x6b>; + monitored-battery = <&bat>; + + interrupt-parent = <&gpio1>; + interrupts = <16 IRQ_TYPE_EDGE_FALLING>; + ti,watchdog-timeout-ms = <40000>; + + input-voltage-limit-microvolt = <4500000>; + input-current-limit-microamp = <2400000>; + }; + }; +... diff --git a/Documentation/devicetree/bindings/power/supply/bq25980.yaml b/Documentation/devicetree/bindings/power/supply/bq25980.yaml index f6b3dd4093caaf483a47e495c3dfa5bb907b6005..06eca6667f67899d0cfe8b10634957b3a1220ed5 100644 --- a/Documentation/devicetree/bindings/power/supply/bq25980.yaml +++ b/Documentation/devicetree/bindings/power/supply/bq25980.yaml @@ -70,6 +70,7 @@ properties: description: Enables bypass mode at boot time interrupts: + maxItems: 1 description: | Indicates that the device state has changed. diff --git a/Documentation/devicetree/bindings/power/supply/ltc4162-l.yaml b/Documentation/devicetree/bindings/power/supply/ltc4162-l.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1f88c9e013f47085ddf1650c08c0d19ba4643373 --- /dev/null +++ b/Documentation/devicetree/bindings/power/supply/ltc4162-l.yaml @@ -0,0 +1,69 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +# Copyright (C) 2020 Topic Embedded Products +%YAML 1.2 +--- +$id: "http://devicetree.org/schemas/power/supply/ltc4162-l.yaml#" +$schema: "http://devicetree.org/meta-schemas/core.yaml#" + +title: Linear Technology (Analog Devices) LTC4162-L Charger + +maintainers: + - Mike Looijmans + +description: | + The LTC ® 4162-L is an advanced monolithic synchronous step-down switching + battery charger and PowerPath (TM) manager that seamlessly manages power + distribution between input sources such as wall adapters, backplanes, solar + panels, etc., and a rechargeable Lithium-Ion/Polymer battery. + + Specifications about the charger can be found at: + https://www.analog.com/en/products/ltc4162-s.html + +properties: + compatible: + enum: + - lltc,ltc4162-l + + reg: + maxItems: 1 + description: I2C address of the charger. + + lltc,rsnsb-micro-ohms: + $ref: /schemas/types.yaml#/definitions/uint32 + description: Battery sense resistor in microohm. + minimum: 1000 + + lltc,rsnsi-micro-ohms: + $ref: /schemas/types.yaml#/definitions/uint32 + description: Input current sense resistor in microohm. + minimum: 1000 + + lltc,cell-count: + $ref: /schemas/types.yaml#/definitions/uint32 + description: | + Number of battery cells. If not provided, will be obtained from the chip + once the external power is applied. Omit this when the number of cells + is somewhat dynamic. Without it, several measurements will return 0 until + the charger is connected to an external supply. + +required: + - compatible + - reg + - lltc,rsnsb-micro-ohms + - lltc,rsnsi-micro-ohms + +additionalProperties: false + +examples: + - | + i2c0 { + #address-cells = <1>; + #size-cells = <0>; + charger: battery-charger@68 { + compatible = "lltc,ltc4162-l"; + reg = <0x68>; + lltc,rsnsb-micro-ohms = <10000>; + lltc,rsnsi-micro-ohms = <16000>; + lltc,cell-count = <2>; + }; + }; diff --git a/Documentation/devicetree/bindings/ptp/ptp-idtcm.yaml b/Documentation/devicetree/bindings/ptp/ptp-idtcm.yaml index 239b49fad805c0610ee5af6775c55b27965818fd..658cec67743e1220888db2e6b3ae2e63651145d6 100644 --- a/Documentation/devicetree/bindings/ptp/ptp-idtcm.yaml +++ b/Documentation/devicetree/bindings/ptp/ptp-idtcm.yaml @@ -59,9 +59,7 @@ additionalProperties: false examples: - | - i2c@1 { - compatible = "abc,acme-1234"; - reg = <0x01 0x400>; + i2c { #address-cells = <1>; #size-cells = <0>; phc@5b { diff --git a/Documentation/devicetree/bindings/pwm/pwm-sifive.yaml b/Documentation/devicetree/bindings/pwm/pwm-sifive.yaml index 5ac25275d8bffc804a984ef5f44871822e60e009..84e66913d042ef425d35592782e7dc598afeb752 100644 --- a/Documentation/devicetree/bindings/pwm/pwm-sifive.yaml +++ b/Documentation/devicetree/bindings/pwm/pwm-sifive.yaml @@ -25,12 +25,15 @@ description: properties: compatible: items: - - const: sifive,fu540-c000-pwm + - enum: + - sifive,fu540-c000-pwm + - sifive,fu740-c000-pwm - const: sifive,pwm0 description: Should be "sifive,-pwm" and "sifive,pwm". Supported - compatible strings are "sifive,fu540-c000-pwm" for the SiFive PWM v0 - as integrated onto the SiFive FU540 chip, and "sifive,pwm0" for the + compatible strings are "sifive,fu540-c000-pwm" and + "sifive,fu740-c000-pwm" for the SiFive PWM v0 as integrated onto the + SiFive FU540 and FU740 chip respectively, and "sifive,pwm0" for the SiFive PWM v0 IP block with no chip integration tweaks. Please refer to sifive-blocks-ip-versioning.txt for details. diff --git a/Documentation/devicetree/bindings/pwm/pwm-zx.txt b/Documentation/devicetree/bindings/pwm/pwm-zx.txt deleted file mode 100644 index 3c8fe7aa8269f538038d05b1c39f34ca2ffbaf05..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/pwm/pwm-zx.txt +++ /dev/null @@ -1,22 +0,0 @@ -ZTE ZX PWM controller - -Required properties: - - compatible: Should be "zte,zx296718-pwm". - - reg: Physical base address and length of the controller's registers. - - clocks : The phandle and specifier referencing the controller's clocks. - - clock-names: "pclk" for PCLK, "wclk" for WCLK to the PWM controller. The - PCLK is for register access, while WCLK is the reference clock for - calculating period and duty cycles. - - #pwm-cells: Should be 3. See pwm.yaml in this directory for a description of - the cells format. - -Example: - - pwm: pwm@1439000 { - compatible = "zte,zx296718-pwm"; - reg = <0x1439000 0x1000>; - clocks = <&lsp1crm LSP1_PWM_PCLK>, - <&lsp1crm LSP1_PWM_WCLK>; - clock-names = "pclk", "wclk"; - #pwm-cells = <3>; - }; diff --git a/Documentation/devicetree/bindings/regulator/max8997-regulator.txt b/Documentation/devicetree/bindings/regulator/max8997-regulator.txt index 6fe825b8ac1b2335cefc50ab23b6677552087998..b53c5e2b335fc14dc7f8e894754d994e426a037c 100644 --- a/Documentation/devicetree/bindings/regulator/max8997-regulator.txt +++ b/Documentation/devicetree/bindings/regulator/max8997-regulator.txt @@ -35,6 +35,7 @@ Optional properties: - interrupts: Interrupt specifiers for two interrupt sources. - First interrupt specifier is for 'irq1' interrupt. - Second interrupt specifier is for 'alert' interrupt. +- charger-supply: regulator node for charging current. - max8997,pmic-buck1-uses-gpio-dvs: 'buck1' can be controlled by gpio dvs. - max8997,pmic-buck2-uses-gpio-dvs: 'buck2' can be controlled by gpio dvs. - max8997,pmic-buck5-uses-gpio-dvs: 'buck5' can be controlled by gpio dvs. diff --git a/Documentation/devicetree/bindings/regulator/mcp16502-regulator.txt b/Documentation/devicetree/bindings/regulator/mcp16502-regulator.txt index d86584ed4d9303354a113d43fea870df3cfdc3fc..451cc4e86b01898f85e4d218d6aa8b0bb4be9d96 100644 --- a/Documentation/devicetree/bindings/regulator/mcp16502-regulator.txt +++ b/Documentation/devicetree/bindings/regulator/mcp16502-regulator.txt @@ -4,7 +4,8 @@ Required properties: - compatible: "microchip,mcp16502" - reg: I2C slave address - lpm-gpios: GPIO for LPM pin. Note that this GPIO *must* remain high during - suspend-to-ram, keeping the PMIC into HIBERNATE mode. + suspend-to-ram, keeping the PMIC into HIBERNATE mode; this + property is optional; - regulators: A node that houses a sub-node for each regulator within the device. Each sub-node is identified using the node's name. The content of each sub-node is defined by the diff --git a/Documentation/devicetree/bindings/regulator/mt6315-regulator.yaml b/Documentation/devicetree/bindings/regulator/mt6315-regulator.yaml new file mode 100644 index 0000000000000000000000000000000000000000..61dd5af80db6755945694298624f03c74e8709de --- /dev/null +++ b/Documentation/devicetree/bindings/regulator/mt6315-regulator.yaml @@ -0,0 +1,69 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/regulator/mt6315-regulator.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Mediatek MT6315 Regulator + +maintainers: + - Hsin-Hsiung Wang + +description: | + The MT6315 is a power management IC (PMIC) configurable with SPMI. + that contains 4 BUCKs output which can combine with each other + by different efuse settings. + +properties: + compatible: + const: mediatek,mt6315-regulator + + reg: + maxItems: 1 + + regulators: + type: object + description: List of regulators and its properties + + patternProperties: + "^vbuck[1-4]$": + type: object + $ref: "regulator.yaml#" + + properties: + regulator-name: + pattern: "^vbuck[1-4]$" + + additionalProperties: false + +required: + - compatible + - reg + - regulators + +additionalProperties: false + +examples: + - | + pmic@6 { + compatible = "mediatek,mt6315-regulator"; + reg = <0x6 0>; + + regulators { + vbuck1 { + regulator-compatible = "vbuck1"; + regulator-min-microvolt = <300000>; + regulator-max-microvolt = <1193750>; + regulator-enable-ramp-delay = <256>; + regulator-allowed-modes = <0 1 2 4>; + }; + + vbuck3 { + regulator-compatible = "vbuck3"; + regulator-min-microvolt = <300000>; + regulator-max-microvolt = <1193750>; + regulator-enable-ramp-delay = <256>; + regulator-allowed-modes = <0 1 2 4>; + }; + }; + }; diff --git a/Documentation/devicetree/bindings/regulator/nxp,pca9450-regulator.yaml b/Documentation/devicetree/bindings/regulator/nxp,pca9450-regulator.yaml index c2b0a8b6da1ec50ee5a63d302ef1f5ae2372b1a9..f70f2e758a002fd3f6750401e32b0df778713c4e 100644 --- a/Documentation/devicetree/bindings/regulator/nxp,pca9450-regulator.yaml +++ b/Documentation/devicetree/bindings/regulator/nxp,pca9450-regulator.yaml @@ -87,6 +87,11 @@ properties: additionalProperties: false + sd-vsel-gpios: + description: GPIO that is used to switch LDO5 between being configured by + LDO5CTRL_L or LDO5CTRL_H register. Use this if the SD_VSEL signal is + connected to a host GPIO. + required: - compatible - reg diff --git a/Documentation/devicetree/bindings/regulator/nxp,pf8x00-regulator.yaml b/Documentation/devicetree/bindings/regulator/nxp,pf8x00-regulator.yaml index 956156fe52a3eb6040a44d8a4d393cf25b857484..8761437ed8adc3b033d3441bac9ed204a58446ae 100644 --- a/Documentation/devicetree/bindings/regulator/nxp,pf8x00-regulator.yaml +++ b/Documentation/devicetree/bindings/regulator/nxp,pf8x00-regulator.yaml @@ -62,8 +62,11 @@ properties: $ref: "/schemas/types.yaml#/definitions/uint32" minimum: 2100 maximum: 4500 + deprecated: true description: BUCK regulators current limit in mA. + This property is deprecated, please use + "regulator-max-microamp" instead. Listed current limits in mA are, 2100 (default) @@ -73,21 +76,11 @@ properties: nxp,phase-shift: $ref: "/schemas/types.yaml#/definitions/uint32" - minimum: 45 - maximum: 0 + default: 0 + enum: [ 0, 45, 90, 135, 180, 225, 270, 315 ] description: BUCK regulators phase shift control in degrees. - Listed phase shift control values in degrees are, - 45 - 90 - 135 - 180 - 225 - 270 - 315 - 0 (default) - unevaluatedProperties: false "^vsnvs$": diff --git a/Documentation/devicetree/bindings/regulator/qcom,rpmh-regulator.txt b/Documentation/devicetree/bindings/regulator/qcom,rpmh-regulator.txt index 7d462b899473ad726d184c4512d80804b077d180..ce1e04354006db449946b5b73254487d3a6ef174 100644 --- a/Documentation/devicetree/bindings/regulator/qcom,rpmh-regulator.txt +++ b/Documentation/devicetree/bindings/regulator/qcom,rpmh-regulator.txt @@ -50,6 +50,8 @@ First Level Nodes - PMIC "qcom,pm8350-rpmh-regulators" "qcom,pm8350c-rpmh-regulators" "qcom,pm8998-rpmh-regulators" + "qcom,pmc8180-rpmh-regulators" + "qcom,pmc8180c-rpmh-regulators" "qcom,pmi8998-rpmh-regulators" "qcom,pm6150-rpmh-regulators" "qcom,pm6150l-rpmh-regulators" diff --git a/Documentation/devicetree/bindings/regulator/qcom-labibb-regulator.yaml b/Documentation/devicetree/bindings/regulator/qcom-labibb-regulator.yaml index 53853ec20fe2412d56600922f2e8c97d786b1ca4..cf784bd1f5e50888c45b83dc02652988b84f0196 100644 --- a/Documentation/devicetree/bindings/regulator/qcom-labibb-regulator.yaml +++ b/Documentation/devicetree/bindings/regulator/qcom-labibb-regulator.yaml @@ -22,11 +22,17 @@ properties: type: object properties: + qcom,soft-start-us: + $ref: /schemas/types.yaml#/definitions/uint32 + description: Regulator soft start time in microseconds. + enum: [200, 400, 600, 800] + default: 200 interrupts: - maxItems: 1 + minItems: 1 + maxItems: 2 description: - Short-circuit interrupt for lab. + Short-circuit and over-current interrupts for lab. required: - interrupts @@ -35,11 +41,17 @@ properties: type: object properties: + qcom,discharge-resistor-kohms: + $ref: /schemas/types.yaml#/definitions/uint32 + description: Discharge resistor value in KiloOhms. + enum: [300, 64, 32, 16] + default: 300 interrupts: - maxItems: 1 + minItems: 1 + maxItems: 2 description: - Short-circuit interrupt for lab. + Short-circuit and over-current interrupts for ibb. required: - interrupts @@ -57,13 +69,15 @@ examples: compatible = "qcom,pmi8998-lab-ibb"; lab { - interrupts = <0x3 0x0 IRQ_TYPE_EDGE_RISING>; - interrupt-names = "sc-err"; + interrupts = <0x3 0xde 0x1 IRQ_TYPE_EDGE_RISING>, + <0x3 0xde 0x0 IRQ_TYPE_LEVEL_LOW>; + interrupt-names = "sc-err", "ocp"; }; ibb { - interrupts = <0x3 0x2 IRQ_TYPE_EDGE_RISING>; - interrupt-names = "sc-err"; + interrupts = <0x3 0xdc 0x2 IRQ_TYPE_EDGE_RISING>, + <0x3 0xdc 0x0 IRQ_TYPE_LEVEL_LOW>; + interrupt-names = "sc-err", "ocp"; }; }; diff --git a/Documentation/devicetree/bindings/regulator/richtek,rt4831-regulator.yaml b/Documentation/devicetree/bindings/regulator/richtek,rt4831-regulator.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d9c23333e15752a80e4f91fc45cf5a3cfa8883b0 --- /dev/null +++ b/Documentation/devicetree/bindings/regulator/richtek,rt4831-regulator.yaml @@ -0,0 +1,35 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/regulator/richtek,rt4831-regulator.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Richtek RT4831 Display Bias Voltage Regulator + +maintainers: + - ChiYuan Huang + +description: | + RT4831 is a multifunctional device that can provide power to the LCD display + and LCD backlight. + + For Display Bias Voltage DSVP and DSVN, the output range is about 4V to 6.5V. + It is sufficient to meet the current LCD power requirement. + + DSVLCM is a boost regulator in IC internal as DSVP and DSVN input power. + Its voltage should be configured above 0.15V to 0.2V gap larger than the + voltage needed for DSVP and DSVN. Too much voltage gap could improve the + voltage drop from the heavy loading scenario. But it also make the power + efficiency worse. It's a trade-off. + + Datasheet is available at + https://www.richtek.com/assets/product_file/RT4831A/DS4831A-05.pdf + +patternProperties: + "^DSV(LCM|P|N)$": + type: object + $ref: regulator.yaml# + description: + Properties for single Display Bias Voltage regulator. + +additionalProperties: false diff --git a/Documentation/devicetree/bindings/remoteproc/ingenic,vpu.yaml b/Documentation/devicetree/bindings/remoteproc/ingenic,vpu.yaml index c019f9fbe916b5b71664a2dbef77777315ee87ae..d0aa91bbf5e59c783710d569f0dd0af77fa72283 100644 --- a/Documentation/devicetree/bindings/remoteproc/ingenic,vpu.yaml +++ b/Documentation/devicetree/bindings/remoteproc/ingenic,vpu.yaml @@ -44,7 +44,7 @@ properties: - const: vpu interrupts: - description: VPU hardware interrupt + maxItems: 1 required: - compatible diff --git a/Documentation/devicetree/bindings/remoteproc/mtk,scp.txt b/Documentation/devicetree/bindings/remoteproc/mtk,scp.txt index 3ba668bab14b22929bfd092d276754f167bcd766..3f5f78764b6024c473f01d0ee60e4b5e54821991 100644 --- a/Documentation/devicetree/bindings/remoteproc/mtk,scp.txt +++ b/Documentation/devicetree/bindings/remoteproc/mtk,scp.txt @@ -6,10 +6,10 @@ Mediatek SoCs. Required properties: - compatible Should be "mediatek,mt8183-scp" -- reg Should contain the address ranges for the two memory - regions, SRAM and CFG. -- reg-names Contains the corresponding names for the two memory - regions. These should be named "sram" & "cfg". +- reg Should contain the address ranges for memory regions: + SRAM, CFG, and L1TCM. +- reg-names Contains the corresponding names for the memory regions: + "sram", "cfg", and "l1tcm". - clocks Clock for co-processor (See: ../clock/clock-bindings.txt) - clock-names Contains the corresponding name for the clock. This should be named "main". diff --git a/Documentation/devicetree/bindings/remoteproc/qcom,adsp.txt b/Documentation/devicetree/bindings/remoteproc/qcom,adsp.txt index 54737024da20886e52507852fed25f8be01d8598..1c330a8941f970c690971e4d535f0bedcb622a84 100644 --- a/Documentation/devicetree/bindings/remoteproc/qcom,adsp.txt +++ b/Documentation/devicetree/bindings/remoteproc/qcom,adsp.txt @@ -25,6 +25,10 @@ on the Qualcomm ADSP Hexagon core. "qcom,sm8250-adsp-pas" "qcom,sm8250-cdsp-pas" "qcom,sm8250-slpi-pas" + "qcom,sm8350-adsp-pas" + "qcom,sm8350-cdsp-pas" + "qcom,sm8350-slpi-pas" + "qcom,sm8350-mpss-pas" - interrupts-extended: Usage: required @@ -51,10 +55,14 @@ on the Qualcomm ADSP Hexagon core. qcom,sm8250-adsp-pas: qcom,sm8250-cdsp-pas: qcom,sm8250-slpi-pas: + qcom,sm8350-adsp-pas: + qcom,sm8350-cdsp-pas: + qcom,sm8350-slpi-pas: must be "wdog", "fatal", "ready", "handover", "stop-ack" qcom,qcs404-wcss-pas: qcom,sc7180-mpss-pas: qcom,sm8150-mpss-pas: + qcom,sm8350-mpss-pas: must be "wdog", "fatal", "ready", "handover", "stop-ack", "shutdown-ack" @@ -114,13 +122,17 @@ on the Qualcomm ADSP Hexagon core. qcom,sm8150-adsp-pas: qcom,sm8150-cdsp-pas: qcom,sm8250-cdsp-pas: + qcom,sm8350-cdsp-pas: must be "cx", "load_state" qcom,sc7180-mpss-pas: qcom,sm8150-mpss-pas: + qcom,sm8350-mpss-pas: must be "cx", "load_state", "mss" qcom,sm8250-adsp-pas: + qcom,sm8350-adsp-pas: qcom,sm8150-slpi-pas: qcom,sm8250-slpi-pas: + qcom,sm8350-slpi-pas: must be "lcx", "lmx", "load_state" - memory-region: diff --git a/Documentation/devicetree/bindings/remoteproc/qcom,wcnss-pil.txt b/Documentation/devicetree/bindings/remoteproc/qcom,wcnss-pil.txt index cc0b7fc1c29b6d4b3ece3e5e26aff431ac5ce23b..da09c0d79ac009f3b634e37a343800e0319b6a6d 100644 --- a/Documentation/devicetree/bindings/remoteproc/qcom,wcnss-pil.txt +++ b/Documentation/devicetree/bindings/remoteproc/qcom,wcnss-pil.txt @@ -80,6 +80,7 @@ and its resource dependencies. It is described by the following properties: Definition: must be one of: "qcom,wcn3620", "qcom,wcn3660", + "qcom,wcn3660b", "qcom,wcn3680" - clocks: diff --git a/Documentation/devicetree/bindings/remoteproc/ti,omap-remoteproc.yaml b/Documentation/devicetree/bindings/remoteproc/ti,omap-remoteproc.yaml index 084960a8f17ad2ce9d58a7c2761e2c3b7c0887fb..1a1159097a2a1e6731311b62ca1743cfe866dd42 100644 --- a/Documentation/devicetree/bindings/remoteproc/ti,omap-remoteproc.yaml +++ b/Documentation/devicetree/bindings/remoteproc/ti,omap-remoteproc.yaml @@ -70,10 +70,13 @@ properties: the firmware image. clocks: + maxItems: 1 description: | Main functional clock for the remote processor resets: + minItems: 1 + maxItems: 2 description: | Reset handles for the remote processor diff --git a/Documentation/devicetree/bindings/reset/canaan,k210-rst.yaml b/Documentation/devicetree/bindings/reset/canaan,k210-rst.yaml new file mode 100644 index 0000000000000000000000000000000000000000..53e4ede9c0bde2230ea2bf395defe9a0848cdb0b --- /dev/null +++ b/Documentation/devicetree/bindings/reset/canaan,k210-rst.yaml @@ -0,0 +1,40 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/reset/canaan,k210-rst.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Canaan Kendryte K210 Reset Controller Device Tree Bindings + +maintainers: + - Damien Le Moal + +description: | + Canaan Kendryte K210 reset controller driver which supports the SoC + system controller supplied reset registers for the various peripherals + of the SoC. The K210 reset controller node must be defined as a child + node of the K210 system controller node. + + See also: + - dt-bindings/reset/k210-rst.h + +properties: + compatible: + const: canaan,k210-rst + + '#reset-cells': + const: 1 + +required: + - '#reset-cells' + - compatible + +additionalProperties: false + +examples: + - | + #include + sysrst: reset-controller { + compatible = "canaan,k210-rst"; + #reset-cells = <1>; + }; diff --git a/Documentation/devicetree/bindings/riscv/canaan.yaml b/Documentation/devicetree/bindings/riscv/canaan.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f8f3f286bd55924d8e1ce5b8f5f38bf4a461a56a --- /dev/null +++ b/Documentation/devicetree/bindings/riscv/canaan.yaml @@ -0,0 +1,47 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/riscv/canaan.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Canaan SoC-based boards + +maintainers: + - Damien Le Moal + +description: + Canaan Kendryte K210 SoC-based boards + +properties: + $nodename: + const: '/' + compatible: + oneOf: + - items: + - const: sipeed,maix-bit + - const: sipeed,maix-bitm + - const: canaan,kendryte-k210 + + - items: + - const: sipeed,maix-go + - const: canaan,kendryte-k210 + + - items: + - const: sipeed,maix-dock-m1 + - const: sipeed,maix-dock-m1w + - const: canaan,kendryte-k210 + + - items: + - const: sipeed,maixduino + - const: canaan,kendryte-k210 + + - items: + - const: canaan,kendryte-kd233 + - const: canaan,kendryte-k210 + + - items: + - const: canaan,kendryte-k210 + +additionalProperties: true + +... diff --git a/Documentation/devicetree/bindings/riscv/cpus.yaml b/Documentation/devicetree/bindings/riscv/cpus.yaml index c6925e0b16e46bb546d04fc3dd53900126ff310d..e534f6a7cfa1f2e28da5c5e191bb3c21d0716785 100644 --- a/Documentation/devicetree/bindings/riscv/cpus.yaml +++ b/Documentation/devicetree/bindings/riscv/cpus.yaml @@ -28,11 +28,18 @@ properties: - items: - enum: - sifive,rocket0 + - sifive,bullet0 - sifive,e5 + - sifive,e7 - sifive,e51 + - sifive,e71 - sifive,u54-mc + - sifive,u74-mc - sifive,u54 + - sifive,u74 - sifive,u5 + - sifive,u7 + - canaan,k210 - const: riscv - const: riscv # Simulator only description: @@ -50,6 +57,7 @@ properties: - riscv,sv32 - riscv,sv39 - riscv,sv48 + - riscv,none riscv,isa: description: diff --git a/Documentation/devicetree/bindings/riscv/sifive-l2-cache.yaml b/Documentation/devicetree/bindings/riscv/sifive-l2-cache.yaml index efc0198eeb74d8fdcdedd73ff511557bb4d1932b..23b227614366748ae69accd1bf1cadff54e292cd 100644 --- a/Documentation/devicetree/bindings/riscv/sifive-l2-cache.yaml +++ b/Documentation/devicetree/bindings/riscv/sifive-l2-cache.yaml @@ -27,6 +27,7 @@ select: items: - enum: - sifive,fu540-c000-ccache + - sifive,fu740-c000-ccache required: - compatible @@ -34,7 +35,9 @@ select: properties: compatible: items: - - const: sifive,fu540-c000-ccache + - enum: + - sifive,fu540-c000-ccache + - sifive,fu740-c000-ccache - const: cache cache-block-size: @@ -52,10 +55,13 @@ properties: cache-unified: true interrupts: - description: | - Must contain entries for DirError, DataError and DataFail signals. minItems: 3 - maxItems: 3 + maxItems: 4 + items: + - description: DirError interrupt + - description: DataError interrupt + - description: DataFail interrupt + - description: DirFail interrupt reg: maxItems: 1 @@ -63,10 +69,31 @@ properties: next-level-cache: true memory-region: + maxItems: 1 description: | The reference to the reserved-memory for the L2 Loosely Integrated Memory region. The reserved memory node should be defined as per the bindings in reserved-memory.txt. +if: + properties: + compatible: + contains: + const: sifive,fu540-c000-ccache + +then: + properties: + interrupts: + description: | + Must contain entries for DirError, DataError and DataFail signals. + maxItems: 3 + +else: + properties: + interrupts: + description: | + Must contain entries for DirError, DataError, DataFail, DirFail signals. + minItems: 4 + additionalProperties: false required: diff --git a/Documentation/devicetree/bindings/riscv/sifive.yaml b/Documentation/devicetree/bindings/riscv/sifive.yaml index 3a8647d1da4c75890f5bda47ff201a407aac0782..ee0a239af4c29667453cba37dcdbdc051c466666 100644 --- a/Documentation/devicetree/bindings/riscv/sifive.yaml +++ b/Documentation/devicetree/bindings/riscv/sifive.yaml @@ -17,11 +17,18 @@ properties: $nodename: const: '/' compatible: - items: - - enum: - - sifive,hifive-unleashed-a00 - - const: sifive,fu540-c000 - - const: sifive,fu540 + oneOf: + - items: + - enum: + - sifive,hifive-unleashed-a00 + - const: sifive,fu540-c000 + - const: sifive,fu540 + + - items: + - enum: + - sifive,hifive-unmatched-a00 + - const: sifive,fu740-c000 + - const: sifive,fu740 additionalProperties: true diff --git a/Documentation/devicetree/bindings/rtc/atmel,at91rm9200-rtc.yaml b/Documentation/devicetree/bindings/rtc/atmel,at91rm9200-rtc.yaml index 02bbfe726c62f69ed3993eeebbde2ffe9225a657..994de43d17fadef9fcf4a8a4195cdff4f77e8e78 100644 --- a/Documentation/devicetree/bindings/rtc/atmel,at91rm9200-rtc.yaml +++ b/Documentation/devicetree/bindings/rtc/atmel,at91rm9200-rtc.yaml @@ -20,6 +20,7 @@ properties: - atmel,sama5d4-rtc - atmel,sama5d2-rtc - microchip,sam9x60-rtc + - microchip,sama7g5-rtc reg: maxItems: 1 diff --git a/Documentation/devicetree/bindings/rtc/nxp,pcf2127.yaml b/Documentation/devicetree/bindings/rtc/nxp,pcf2127.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cde7b1675ead4c50038d54e9465338f41f2f3682 --- /dev/null +++ b/Documentation/devicetree/bindings/rtc/nxp,pcf2127.yaml @@ -0,0 +1,51 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/rtc/nxp,pcf2127.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: NXP PCF2127 Real Time Clock + +allOf: + - $ref: "rtc.yaml#" + +maintainers: + - Alexandre Belloni + +properties: + compatible: + const: nxp,pcf2127 + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + + start-year: true + + reset-source: true + +required: + - compatible + - reg + +additionalProperties: false + +examples: + - | + #include + i2c { + #address-cells = <1>; + #size-cells = <0>; + + rtc@51 { + compatible = "nxp,pcf2127"; + reg = <0x51>; + pinctrl-0 = <&rtc_nint_pins>; + interrupts-extended = <&gpio1 16 IRQ_TYPE_LEVEL_HIGH>; + reset-source; + }; + }; + +... diff --git a/Documentation/devicetree/bindings/rtc/pcf8563.txt b/Documentation/devicetree/bindings/rtc/pcf8563.txt index 6076fe76dbfa3fa416cd387c384673abae16e3c6..0a900f7c8977821b7ba7abc490cca33fabddc65a 100644 --- a/Documentation/devicetree/bindings/rtc/pcf8563.txt +++ b/Documentation/devicetree/bindings/rtc/pcf8563.txt @@ -5,7 +5,8 @@ Philips PCF8563/Epson RTC8564 Real Time Clock Required properties: - compatible: Should contain "nxp,pcf8563", "epson,rtc8564" or - "microcrystal,rv8564" + "microcrystal,rv8564" or + "nxp,pca8565" - reg: I2C address for chip. Optional property: diff --git a/Documentation/devicetree/bindings/rtc/sirf,prima2-sysrtc.txt b/Documentation/devicetree/bindings/rtc/sirf,prima2-sysrtc.txt deleted file mode 100644 index 58885b55da21360187dd836c53690d33a8de9066..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/rtc/sirf,prima2-sysrtc.txt +++ /dev/null @@ -1,13 +0,0 @@ -SiRFSoC Real Time Clock - -Required properties: -- compatible: must be "sirf,prima2-sysrtc" -- reg: address range of rtc register set. -- interrupts: rtc alarm interrupts. - -Example: - rtc@2000 { - compatible = "sirf,prima2-sysrtc"; - reg = <0x2000 0x1000>; - interrupts = <52 53 54>; - }; diff --git a/Documentation/devicetree/bindings/rtc/stericsson,coh901331.txt b/Documentation/devicetree/bindings/rtc/stericsson,coh901331.txt deleted file mode 100644 index e615a897b20e3a519e03410edff93d53cb09f4b6..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/rtc/stericsson,coh901331.txt +++ /dev/null @@ -1,16 +0,0 @@ -ST-Ericsson COH 901 331 Real Time Clock - -Required properties: -- compatible: must be "stericsson,coh901331" -- reg: address range of rtc register set. -- interrupts: rtc alarm interrupt. -- clocks: phandle to the rtc clock source - -Example: - rtc: rtc@c0017000 { - compatible = "stericsson,coh901331"; - reg = <0xc0017000 0x1000>; - interrupt-parent = <&vicb>; - interrupts = <10>; - clocks = <&rtc_clk>; - }; diff --git a/Documentation/devicetree/bindings/rtc/trivial-rtc.yaml b/Documentation/devicetree/bindings/rtc/trivial-rtc.yaml index c7d14de214c4c2c6396772f460e5621a80476b49..7548d871487128ac70c13ef7c70a1761a61dde16 100644 --- a/Documentation/devicetree/bindings/rtc/trivial-rtc.yaml +++ b/Documentation/devicetree/bindings/rtc/trivial-rtc.yaml @@ -48,12 +48,8 @@ properties: - microcrystal,rv3029 # Real Time Clock - microcrystal,rv8523 - # Real-time clock - - nxp,pcf2127 - # Real-time clock - - nxp,pcf2129 - # Real-time clock - nxp,pca2129 + - nxp,pcf2129 # Real-time Clock Module - pericom,pt7c4338 # I2C bus SERIAL INTERFACE REAL-TIME CLOCK IC diff --git a/Documentation/devicetree/bindings/serial/renesas,hscif.yaml b/Documentation/devicetree/bindings/serial/renesas,hscif.yaml index 512a84942f78b2cc48e9e1a08ce6d73d4053af6e..ee9804cd49bbca8b177d35b9815d9b5caa3a450f 100644 --- a/Documentation/devicetree/bindings/serial/renesas,hscif.yaml +++ b/Documentation/devicetree/bindings/serial/renesas,hscif.yaml @@ -82,6 +82,8 @@ properties: maxItems: 1 dmas: + minItems: 2 + maxItems: 4 description: Must contain a list of pairs of references to DMA specifiers, one for transmission, and one for reception. diff --git a/Documentation/devicetree/bindings/serial/renesas,scif.yaml b/Documentation/devicetree/bindings/serial/renesas,scif.yaml index 672158906c33aca17e570293a0212610b899eea6..22d76829f7ae18fd865f8105497deefbe919896a 100644 --- a/Documentation/devicetree/bindings/serial/renesas,scif.yaml +++ b/Documentation/devicetree/bindings/serial/renesas,scif.yaml @@ -120,6 +120,8 @@ properties: maxItems: 1 dmas: + minItems: 2 + maxItems: 4 description: Must contain a list of pairs of references to DMA specifiers, one for transmission, and one for reception. diff --git a/Documentation/devicetree/bindings/serial/renesas,scifa.yaml b/Documentation/devicetree/bindings/serial/renesas,scifa.yaml index dbffb953483529cf4811edc1dbeb60c118450819..3c67d3202e1bfe56d946bd3108059064b727c5e6 100644 --- a/Documentation/devicetree/bindings/serial/renesas,scifa.yaml +++ b/Documentation/devicetree/bindings/serial/renesas,scifa.yaml @@ -55,6 +55,8 @@ properties: maxItems: 1 dmas: + minItems: 2 + maxItems: 4 description: Must contain a list of pairs of references to DMA specifiers, one for transmission, and one for reception. diff --git a/Documentation/devicetree/bindings/serial/renesas,scifb.yaml b/Documentation/devicetree/bindings/serial/renesas,scifb.yaml index 147f8a37e02a80dc7ef1d492601b51927d845128..d5571c7a442484d29ffc1463e45d58ad31afe74b 100644 --- a/Documentation/devicetree/bindings/serial/renesas,scifb.yaml +++ b/Documentation/devicetree/bindings/serial/renesas,scifb.yaml @@ -55,6 +55,8 @@ properties: maxItems: 1 dmas: + minItems: 2 + maxItems: 4 description: Must contain a list of pairs of references to DMA specifiers, one for transmission, and one for reception. diff --git a/Documentation/devicetree/bindings/serial/sifive-serial.yaml b/Documentation/devicetree/bindings/serial/sifive-serial.yaml index 3ac5c7ff27586428ff0f52f26f8fb1a27196c9f5..5fa94dacbba97b4d62b3e43696106a8d1e0dbdee 100644 --- a/Documentation/devicetree/bindings/serial/sifive-serial.yaml +++ b/Documentation/devicetree/bindings/serial/sifive-serial.yaml @@ -20,6 +20,7 @@ properties: - enum: - sifive,fu540-c000-uart - sifive,fu740-c000-uart + - canaan,k210-uarths - const: sifive,uart0 description: diff --git a/Documentation/devicetree/bindings/sound/allwinner,sun4i-a10-codec.yaml b/Documentation/devicetree/bindings/sound/allwinner,sun4i-a10-codec.yaml index dd47fef9854d76bd9ff573d9dbcfbebd5b03c84e..559aff13ae232c785f12fec2c3f84de75fa2a9fd 100644 --- a/Documentation/devicetree/bindings/sound/allwinner,sun4i-a10-codec.yaml +++ b/Documentation/devicetree/bindings/sound/allwinner,sun4i-a10-codec.yaml @@ -88,6 +88,7 @@ properties: description: Phandle to the codec analog controls in the PRCM allwinner,pa-gpios: + maxItems: 1 description: GPIO to enable the external amplifier required: diff --git a/Documentation/devicetree/bindings/sound/google,sc7180-trogdor.yaml b/Documentation/devicetree/bindings/sound/google,sc7180-trogdor.yaml index 5095b780e2c77f1883907562485bb7077664a9c2..837e3faa63a9161b371b6c268eefc7e08dbf8e27 100644 --- a/Documentation/devicetree/bindings/sound/google,sc7180-trogdor.yaml +++ b/Documentation/devicetree/bindings/sound/google,sc7180-trogdor.yaml @@ -55,6 +55,7 @@ patternProperties: maxItems: 1 reg: + maxItems: 1 description: dai link address. cpu: diff --git a/Documentation/devicetree/bindings/sound/samsung,aries-wm8994.yaml b/Documentation/devicetree/bindings/sound/samsung,aries-wm8994.yaml index 1c6947294825ecc2bbfbc09a2e194fbc9c021996..5fff586dc802983214a44010e9fc30ceaa3ff67f 100644 --- a/Documentation/devicetree/bindings/sound/samsung,aries-wm8994.yaml +++ b/Documentation/devicetree/bindings/sound/samsung,aries-wm8994.yaml @@ -62,12 +62,15 @@ properties: description: Supply for the micbias on the headset mic earpath-sel-gpios: + maxItems: 1 description: GPIO for switching between tv-out and mic paths headset-detect-gpios: + maxItems: 1 description: GPIO for detection of headset insertion headset-key-gpios: + maxItems: 1 description: GPIO for detection of headset key press io-channels: diff --git a/Documentation/devicetree/bindings/sound/samsung,midas-audio.yaml b/Documentation/devicetree/bindings/sound/samsung,midas-audio.yaml index 578928e67e5c4fd4214125b6890383ac7d0c63c6..095775c598fa1d5b8110e747e9b34f8482e880c3 100644 --- a/Documentation/devicetree/bindings/sound/samsung,midas-audio.yaml +++ b/Documentation/devicetree/bindings/sound/samsung,midas-audio.yaml @@ -53,9 +53,11 @@ properties: description: Supply for the micbias on the Sub microphone fm-sel-gpios: + maxItems: 1 description: GPIO pin for FM selection lineout-sel-gpios: + maxItems: 1 description: GPIO pin for line out selection required: diff --git a/Documentation/devicetree/bindings/sound/tas2562.yaml b/Documentation/devicetree/bindings/sound/tas2562.yaml index 27f7132ba2ef069575a49a2a771eec56b44aa850..acd4bbe6973153b5a76c542fb1ae6463e7a8d653 100644 --- a/Documentation/devicetree/bindings/sound/tas2562.yaml +++ b/Documentation/devicetree/bindings/sound/tas2562.yaml @@ -36,10 +36,12 @@ properties: I2C address of the device can be one of these 0x4c, 0x4d, 0x4e or 0x4f shut-down-gpios: + maxItems: 1 description: GPIO used to control the state of the device. deprecated: true shutdown-gpios: + maxItems: 1 description: GPIO used to control the state of the device. interrupts: diff --git a/Documentation/devicetree/bindings/sound/tas2770.yaml b/Documentation/devicetree/bindings/sound/tas2770.yaml index 07e7f9951d2ed0fa0d15001e3e4aa9421d94bff8..027bebf4e8cf568101dd7dde614f0d17d07f80a2 100644 --- a/Documentation/devicetree/bindings/sound/tas2770.yaml +++ b/Documentation/devicetree/bindings/sound/tas2770.yaml @@ -27,9 +27,11 @@ properties: I2C address of the device can be between 0x41 to 0x48. reset-gpio: + maxItems: 1 description: GPIO used to reset the device. shutdown-gpios: + maxItems: 1 description: GPIO used to control the state of the device. interrupts: diff --git a/Documentation/devicetree/bindings/sound/tlv320adcx140.yaml b/Documentation/devicetree/bindings/sound/tlv320adcx140.yaml index df18be9d7b15f289b0e1ab2d7a0a372d182cd127..54d64785aad2432ae6721a2f6df39bc4bee89c08 100644 --- a/Documentation/devicetree/bindings/sound/tlv320adcx140.yaml +++ b/Documentation/devicetree/bindings/sound/tlv320adcx140.yaml @@ -35,6 +35,7 @@ properties: I2C addresss of the device can be one of these 0x4c, 0x4d, 0x4e or 0x4f reset-gpios: + maxItems: 1 description: | GPIO used for hardware reset. diff --git a/Documentation/devicetree/bindings/spi/allwinner,sun6i-a31-spi.yaml b/Documentation/devicetree/bindings/spi/allwinner,sun6i-a31-spi.yaml index 7866a655d81c77c9fd2a8bea8c1609a63b8ba560..908248260afa3efa0c58686dd32f545929e8c03b 100644 --- a/Documentation/devicetree/bindings/spi/allwinner,sun6i-a31-spi.yaml +++ b/Documentation/devicetree/bindings/spi/allwinner,sun6i-a31-spi.yaml @@ -25,6 +25,7 @@ properties: - enum: - allwinner,sun8i-r40-spi - allwinner,sun50i-h6-spi + - allwinner,sun50i-h616-spi - const: allwinner,sun8i-h3-spi reg: diff --git a/Documentation/devicetree/bindings/mtd/cadence-quadspi.txt b/Documentation/devicetree/bindings/spi/cadence-quadspi.txt similarity index 97% rename from Documentation/devicetree/bindings/mtd/cadence-quadspi.txt rename to Documentation/devicetree/bindings/spi/cadence-quadspi.txt index 945be7d5b23608906807da1b314efc0b86382207..8ace832a2d80ced376f28ed96f9a310b19209bbb 100644 --- a/Documentation/devicetree/bindings/mtd/cadence-quadspi.txt +++ b/Documentation/devicetree/bindings/spi/cadence-quadspi.txt @@ -5,6 +5,7 @@ Required properties: Generic default - "cdns,qspi-nor". For TI 66AK2G SoC - "ti,k2g-qspi", "cdns,qspi-nor". For TI AM654 SoC - "ti,am654-ospi", "cdns,qspi-nor". + For Intel LGM SoC - "intel,lgm-qspi", "cdns,qspi-nor". - reg : Contains two entries, each of which is a tuple consisting of a physical address and length. The first entry is the address and length of the controller register set. The second entry is the diff --git a/Documentation/devicetree/bindings/spi/nvidia,tegra210-quad.yaml b/Documentation/devicetree/bindings/spi/nvidia,tegra210-quad.yaml new file mode 100644 index 0000000000000000000000000000000000000000..35a8045b2c70db82a1b4ec8961362818dda44ae0 --- /dev/null +++ b/Documentation/devicetree/bindings/spi/nvidia,tegra210-quad.yaml @@ -0,0 +1,117 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/spi/nvidia,tegra210-quad.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Tegra Quad SPI Controller + +maintainers: + - Thierry Reding + - Jonathan Hunter + +allOf: + - $ref: "spi-controller.yaml#" + +properties: + compatible: + enum: + - nvidia,tegra210-qspi + - nvidia,tegra186-qspi + - nvidia,tegra194-qspi + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + + clock-names: + items: + - const: qspi + - const: qspi_out + + clocks: + maxItems: 2 + + resets: + maxItems: 1 + + dmas: + maxItems: 2 + + dma-names: + items: + - const: rx + - const: tx + +patternProperties: + "@[0-9a-f]+": + type: object + + properties: + spi-rx-bus-width: + enum: [1, 2, 4] + + spi-tx-bus-width: + enum: [1, 2, 4] + + nvidia,tx-clk-tap-delay: + description: + Delays the clock going out to device with this tap value. + Tap value varies based on platform design trace lengths from Tegra + QSPI to corresponding slave device. + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 0 + maximum: 31 + + nvidia,rx-clk-tap-delay: + description: + Delays the clock coming in from the device with this tap value. + Tap value varies based on platform design trace lengths from Tegra + QSPI to corresponding slave device. + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 0 + maximum: 255 + + required: + - reg + +required: + - compatible + - reg + - interrupts + - clock-names + - clocks + - resets + +unevaluatedProperties: false + +examples: + - | + #include + #include + #include + spi@70410000 { + compatible = "nvidia,tegra210-qspi"; + reg = <0x70410000 0x1000>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + clocks = <&tegra_car TEGRA210_CLK_QSPI>, + <&tegra_car TEGRA210_CLK_QSPI_PM>; + clock-names = "qspi", "qspi_out"; + resets = <&tegra_car 211>; + dmas = <&apbdma 5>, <&apbdma 5>; + dma-names = "rx", "tx"; + + flash@0 { + compatible = "spi-nor"; + reg = <0>; + spi-max-frequency = <104000000>; + spi-tx-bus-width = <2>; + spi-rx-bus-width = <2>; + nvidia,tx-clk-tap-delay = <0>; + nvidia,rx-clk-tap-delay = <0>; + }; + }; diff --git a/Documentation/devicetree/bindings/spi/realtek,rtl-spi.yaml b/Documentation/devicetree/bindings/spi/realtek,rtl-spi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..30a62a211984f6d0264d98aeb3440d7cbb68d5a1 --- /dev/null +++ b/Documentation/devicetree/bindings/spi/realtek,rtl-spi.yaml @@ -0,0 +1,41 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/spi/realtek,rtl-spi.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Realtek RTL838x/RTL839x SPI controller + +maintainers: + - Bert Vermeulen + - Birger Koblitz + +allOf: + - $ref: "spi-controller.yaml#" + +properties: + compatible: + oneOf: + - const: realtek,rtl8380-spi + - const: realtek,rtl8382-spi + - const: realtek,rtl8391-spi + - const: realtek,rtl8392-spi + - const: realtek,rtl8393-spi + + reg: + maxItems: 1 + +required: + - compatible + - reg + +unevaluatedProperties: false + +examples: + - | + spi: spi@1200 { + compatible = "realtek,rtl8382-spi"; + reg = <0x1200 0x100>; + #address-cells = <1>; + #size-cells = <0>; + }; diff --git a/Documentation/devicetree/bindings/spi/renesas,rspi.yaml b/Documentation/devicetree/bindings/spi/renesas,rspi.yaml index 10e83cb17e8dd771c48410f1a5ec4ea09c514495..8397f60d80a2e527337392d4227e2325e5bb2266 100644 --- a/Documentation/devicetree/bindings/spi/renesas,rspi.yaml +++ b/Documentation/devicetree/bindings/spi/renesas,rspi.yaml @@ -68,6 +68,8 @@ properties: maxItems: 1 dmas: + minItems: 2 + maxItems: 4 description: Must contain a list of pairs of references to DMA specifiers, one for transmission, and one for reception. diff --git a/Documentation/devicetree/bindings/spi/renesas,sh-msiof.yaml b/Documentation/devicetree/bindings/spi/renesas,sh-msiof.yaml index 44c7ddb4b1098e159bc755e2efaed65bb85f6bd6..b104899205f6d2da17fd86325cb0fec4312e22d8 100644 --- a/Documentation/devicetree/bindings/spi/renesas,sh-msiof.yaml +++ b/Documentation/devicetree/bindings/spi/renesas,sh-msiof.yaml @@ -47,6 +47,7 @@ properties: - renesas,msiof-r8a77980 # R-Car V3H - renesas,msiof-r8a77990 # R-Car E3 - renesas,msiof-r8a77995 # R-Car D3 + - renesas,msiof-r8a779a0 # R-Car V3U - const: renesas,rcar-gen3-msiof # generic R-Car Gen3 and RZ/G2 # compatible device - items: diff --git a/Documentation/devicetree/bindings/spi/spi-controller.yaml b/Documentation/devicetree/bindings/spi/spi-controller.yaml index 5f505810104ddc1da4812f821ec1bd107e297b66..06786f1b43d26822c8ff8f9c6d551b64381d36aa 100644 --- a/Documentation/devicetree/bindings/spi/spi-controller.yaml +++ b/Documentation/devicetree/bindings/spi/spi-controller.yaml @@ -152,8 +152,9 @@ patternProperties: spi-rx-bus-width: description: Bus width to the SPI bus used for read transfers. + If 0 is provided, then no RX will be possible on this device. $ref: /schemas/types.yaml#/definitions/uint32 - enum: [1, 2, 4, 8] + enum: [0, 1, 2, 4, 8] default: 1 spi-rx-delay-us: @@ -163,8 +164,9 @@ patternProperties: spi-tx-bus-width: description: Bus width to the SPI bus used for write transfers. + If 0 is provided, then no TX will be possible on this device. $ref: /schemas/types.yaml#/definitions/uint32 - enum: [1, 2, 4, 8] + enum: [0, 1, 2, 4, 8] default: 1 spi-tx-delay-us: diff --git a/Documentation/devicetree/bindings/spi/spi-sirf.txt b/Documentation/devicetree/bindings/spi/spi-sirf.txt deleted file mode 100644 index ddd78ff68fae5b741b5125e7b19696b3c847fd8a..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/spi/spi-sirf.txt +++ /dev/null @@ -1,42 +0,0 @@ -* CSR SiRFprimaII Serial Peripheral Interface - -Required properties: -- compatible : Should be "sirf,prima2-spi", "sirf,prima2-usp" - or "sirf,atlas7-usp" -- reg : Offset and length of the register set for the device -- interrupts : Should contain SPI interrupt -- resets: phandle to the reset controller asserting this device in - reset - See ../reset/reset.txt for details. -- dmas : Must contain an entry for each entry in clock-names. - See ../dma/dma.txt for details. -- dma-names : Must include the following entries: - - rx - - tx -- clocks : Must contain an entry for each entry in clock-names. - See ../clocks/clock-bindings.txt for details. - -- #address-cells: Number of cells required to define a chip select - address on the SPI bus. Should be set to 1. -- #size-cells: Should be zero. - -Optional properties: -- spi-max-frequency: Specifies maximum SPI clock frequency, - Units - Hz. Definition as per - Documentation/devicetree/bindings/spi/spi-bus.txt -- cs-gpios: should specify GPIOs used for chipselects. - -Example: - -spi0: spi@b00d0000 { - compatible = "sirf,prima2-spi"; - reg = <0xb00d0000 0x10000>; - interrupts = <15>; - dmas = <&dmac1 9>, - <&dmac1 4>; - dma-names = "rx", "tx"; - #address-cells = <1>; - #size-cells = <0>; - clocks = <&clks 19>; - resets = <&rstc 26>; -}; diff --git a/Documentation/devicetree/bindings/sram/sram.yaml b/Documentation/devicetree/bindings/sram/sram.yaml index 6d65771e979c8ab10ecda9f56acc8da69259c55f..c1a5afa73cfe0280af4531ffab19c3e377c596d8 100644 --- a/Documentation/devicetree/bindings/sram/sram.yaml +++ b/Documentation/devicetree/bindings/sram/sram.yaml @@ -35,6 +35,7 @@ properties: maxItems: 1 clocks: + maxItems: 1 description: A list of phandle and clock specifier pair that controls the single SRAM clock. @@ -46,6 +47,7 @@ properties: const: 1 ranges: + maxItems: 1 description: Should translate from local addresses within the sram to bus addresses. diff --git a/Documentation/devicetree/bindings/thermal/allwinner,sun8i-a83t-ths.yaml b/Documentation/devicetree/bindings/thermal/allwinner,sun8i-a83t-ths.yaml index 31edd051295a71b895a0fdc00425e4b8b3130dc3..bf97d1fb33e71fa5cd02c34981c6bced62c7355a 100644 --- a/Documentation/devicetree/bindings/thermal/allwinner,sun8i-a83t-ths.yaml +++ b/Documentation/devicetree/bindings/thermal/allwinner,sun8i-a83t-ths.yaml @@ -103,12 +103,12 @@ allOf: compatible: contains: enum: - - const: allwinner,sun8i-h3-ths - - const: allwinner,sun8i-r40-ths - - const: allwinner,sun50i-a64-ths - - const: allwinner,sun50i-a100-ths - - const: allwinner,sun50i-h5-ths - - const: allwinner,sun50i-h6-ths + - allwinner,sun8i-h3-ths + - allwinner,sun8i-r40-ths + - allwinner,sun50i-a64-ths + - allwinner,sun50i-a100-ths + - allwinner,sun50i-h5-ths + - allwinner,sun50i-h6-ths then: required: diff --git a/Documentation/devicetree/bindings/thermal/qcom-spmi-adc-tm5.yaml b/Documentation/devicetree/bindings/thermal/qcom-spmi-adc-tm5.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7cd364430573dc5f63ac2f70a610b90a191748ef --- /dev/null +++ b/Documentation/devicetree/bindings/thermal/qcom-spmi-adc-tm5.yaml @@ -0,0 +1,153 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/thermal/qcom-spmi-adc-tm5.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Qualcomm's SPMI PMIC ADC Thermal Monitoring +maintainers: + - Dmitry Baryshkov + +properties: + compatible: + const: qcom,spmi-adc-tm5 + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + + "#thermal-sensor-cells": + const: 1 + description: + Number of cells required to uniquely identify the thermal sensors. Since + we have multiple sensors this is set to 1 + + "#address-cells": + const: 1 + + "#size-cells": + const: 0 + + qcom,avg-samples: + $ref: /schemas/types.yaml#/definitions/uint32 + description: Number of samples to be used for measurement. + enum: + - 1 + - 2 + - 4 + - 8 + - 16 + default: 1 + + qcom,decimation: + $ref: /schemas/types.yaml#/definitions/uint32 + description: This parameter is used to decrease ADC sampling rate. + Quicker measurements can be made by reducing decimation ratio. + enum: + - 250 + - 420 + - 840 + default: 840 + +patternProperties: + "^([-a-z0-9]*)@[0-7]$": + type: object + description: + Represent one thermal sensor. + + properties: + reg: + $ref: /schemas/types.yaml#/definitions/uint32 + description: Specify the sensor channel. There are 8 channels in PMIC5's ADC TM + minimum: 0 + maximum: 7 + + io-channels: + description: + From common IIO binding. Used to pipe PMIC ADC channel to thermal monitor + + qcom,ratiometric: + $ref: /schemas/types.yaml#/definitions/flag + description: + Channel calibration type. + If this property is specified VADC will use the VDD reference + (1.875V) and GND for channel calibration. If property is not found, + channel will be calibrated with 0V and 1.25V reference channels, + also known as absolute calibration. + + qcom,hw-settle-time-us: + $ref: /schemas/types.yaml#/definitions/uint32 + description: Time between AMUX getting configured and the ADC starting conversion. + enum: [15, 100, 200, 300, 400, 500, 600, 700, 1000, 2000, 4000, 8000, 16000, 32000, 64000, 128000] + + qcom,pre-scaling: + $ref: /schemas/types.yaml#/definitions/uint32-array + description: Used for scaling the channel input signal before the + signal is fed to VADC. The configuration for this node is to know the + pre-determined ratio and use it for post scaling. It is a pair of + integers, denoting the numerator and denominator of the fraction by + which input signal is multiplied. For example, <1 3> indicates the + signal is scaled down to 1/3 of its value before ADC measurement. If + property is not found default value depending on chip will be used. + items: + - const: 1 + - enum: [ 1, 3, 4, 6, 20, 8, 10 ] + + required: + - reg + - io-channels + + additionalProperties: + false + +required: + - compatible + - reg + - interrupts + - "#address-cells" + - "#size-cells" + - "#thermal-sensor-cells" + +additionalProperties: false + +examples: + - | + #include + #include + spmi_bus { + #address-cells = <1>; + #size-cells = <0>; + pm8150b_adc: adc@3100 { + reg = <0x3100>; + compatible = "qcom,spmi-adc5"; + #address-cells = <1>; + #size-cells = <0>; + #io-channel-cells = <1>; + + /* Other propreties are omitted */ + conn-therm@4f { + reg = ; + qcom,ratiometric; + qcom,hw-settle-time = <200>; + }; + }; + + pm8150b_adc_tm: adc-tm@3500 { + compatible = "qcom,spmi-adc-tm5"; + reg = <0x3500>; + interrupts = <0x2 0x35 0x0 IRQ_TYPE_EDGE_RISING>; + #thermal-sensor-cells = <1>; + #address-cells = <1>; + #size-cells = <0>; + + conn-therm@0 { + reg = <0>; + io-channels = <&pm8150b_adc ADC5_AMUX_THM3_100K_PU>; + qcom,ratiometric; + qcom,hw-settle-time-us = <200>; + }; + }; + }; +... diff --git a/Documentation/devicetree/bindings/thermal/tango-thermal.txt b/Documentation/devicetree/bindings/thermal/tango-thermal.txt deleted file mode 100644 index 2c918d742867a72c8e549c9f13728aa3c16bd975..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/thermal/tango-thermal.txt +++ /dev/null @@ -1,17 +0,0 @@ -* Tango Thermal - -The SMP8758 SoC includes 3 instances of this temperature sensor -(in the CPU, video decoder, and PCIe controller). - -Required properties: -- #thermal-sensor-cells: Should be 0 (see Documentation/devicetree/bindings/thermal/thermal-sensor.yaml) -- compatible: "sigma,smp8758-thermal" -- reg: Address range of the thermal registers - -Example: - - cpu_temp: thermal@920100 { - #thermal-sensor-cells = <0>; - compatible = "sigma,smp8758-thermal"; - reg = <0x920100 12>; - }; diff --git a/Documentation/devicetree/bindings/thermal/zx2967-thermal.txt b/Documentation/devicetree/bindings/thermal/zx2967-thermal.txt deleted file mode 100644 index 3dc1c6bf0478fa6d598654f7def790597c159e1a..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/thermal/zx2967-thermal.txt +++ /dev/null @@ -1,116 +0,0 @@ -* ZTE zx2967 family Thermal - -Required Properties: -- compatible: should be one of the following. - * zte,zx296718-thermal -- reg: physical base address of the controller and length of memory mapped - region. -- clocks : Pairs of phandle and specifier referencing the controller's clocks. -- clock-names: "topcrm" for the topcrm clock. - "apb" for the apb clock. -- #thermal-sensor-cells: must be 0. - -Please note: slope coefficient defined in thermal-zones section need to be -multiplied by 1000. - -Example for tempsensor: - - tempsensor: tempsensor@148a000 { - compatible = "zte,zx296718-thermal"; - reg = <0x0148a000 0x20>; - clocks = <&topcrm TEMPSENSOR_GATE>, <&audiocrm AUDIO_TS_PCLK>; - clock-names = "topcrm", "apb"; - #thermal-sensor-cells = <0>; - }; - -Example for cooling device: - - cooling_dev: cooling_dev { - cluster0_cooling_dev: cluster0-cooling-dev { - #cooling-cells = <2>; - cpumask = <0xf>; - capacitance = <1500>; - }; - - cluster1_cooling_dev: cluster1-cooling-dev { - #cooling-cells = <2>; - cpumask = <0x30>; - capacitance = <2000>; - }; - }; - -Example for thermal zones: - - thermal-zones { - zx296718_thermal: zx296718_thermal { - polling-delay-passive = <500>; - polling-delay = <1000>; - sustainable-power = <6500>; - - thermal-sensors = <&tempsensor 0>; - /* - * slope need to be multiplied by 1000. - */ - coefficients = <1951 (-922)>; - - trips { - trip0: switch_on_temperature { - temperature = <90000>; - hysteresis = <2000>; - type = "passive"; - }; - - trip1: desired_temperature { - temperature = <100000>; - hysteresis = <2000>; - type = "passive"; - }; - - crit: critical_temperature { - temperature = <110000>; - hysteresis = <2000>; - type = "critical"; - }; - }; - - cooling-maps { - map0 { - trip = <&trip0>; - cooling-device = <&gpu 2 5>; - }; - - map1 { - trip = <&trip0>; - cooling-device = <&cluster0_cooling_dev 1 2>; - }; - - map2 { - trip = <&trip1>; - cooling-device = <&cluster0_cooling_dev 1 2>; - }; - - map3 { - trip = <&crit>; - cooling-device = <&cluster0_cooling_dev 1 2>; - }; - - map4 { - trip = <&trip0>; - cooling-device = <&cluster1_cooling_dev 1 2>; - contribution = <9000>; - }; - - map5 { - trip = <&trip1>; - cooling-device = <&cluster1_cooling_dev 1 2>; - contribution = <4096>; - }; - - map6 { - trip = <&crit>; - cooling-device = <&cluster1_cooling_dev 1 2>; - contribution = <4096>; - }; - }; - }; - }; diff --git a/Documentation/devicetree/bindings/timer/allwinner,sun4i-a10-timer.yaml b/Documentation/devicetree/bindings/timer/allwinner,sun4i-a10-timer.yaml index d918cee100aced4e31c4a0dddf5ec176ade12460..1c7cf32e7ac2a91da11a1061822112b17eca790a 100644 --- a/Documentation/devicetree/bindings/timer/allwinner,sun4i-a10-timer.yaml +++ b/Documentation/devicetree/bindings/timer/allwinner,sun4i-a10-timer.yaml @@ -22,6 +22,8 @@ properties: maxItems: 1 interrupts: + minItems: 2 + maxItems: 6 description: List of timers interrupts diff --git a/Documentation/devicetree/bindings/timer/allwinner,sun5i-a13-hstimer.yaml b/Documentation/devicetree/bindings/timer/allwinner,sun5i-a13-hstimer.yaml index 40fc4bcb31457db7374f4266839894469cd2366f..b6a6d03a08b21ecbea7fcff9b213ed73ee92aead 100644 --- a/Documentation/devicetree/bindings/timer/allwinner,sun5i-a13-hstimer.yaml +++ b/Documentation/devicetree/bindings/timer/allwinner,sun5i-a13-hstimer.yaml @@ -46,8 +46,7 @@ required: if: properties: compatible: - items: - const: allwinner,sun5i-a13-hstimer + const: allwinner,sun5i-a13-hstimer then: properties: diff --git a/Documentation/devicetree/bindings/timer/intel,ixp4xx-timer.yaml b/Documentation/devicetree/bindings/timer/intel,ixp4xx-timer.yaml index 1a721d8af67a5450021a5a9ba310ee0684654bff..a8de99b0c0f924718e33ae49e2f7e8f9d775f57c 100644 --- a/Documentation/devicetree/bindings/timer/intel,ixp4xx-timer.yaml +++ b/Documentation/devicetree/bindings/timer/intel,ixp4xx-timer.yaml @@ -18,7 +18,7 @@ properties: - const: intel,ixp4xx-timer reg: - description: Should contain registers location and length + maxItems: 1 interrupts: minItems: 1 diff --git a/Documentation/devicetree/bindings/timer/sifive,clint.yaml b/Documentation/devicetree/bindings/timer/sifive,clint.yaml index 2a0e9cd9fbcf0ffded1fb1d2809c9a6094c7dfaa..a35952f487426988641dcde96981904a761e05ab 100644 --- a/Documentation/devicetree/bindings/timer/sifive,clint.yaml +++ b/Documentation/devicetree/bindings/timer/sifive,clint.yaml @@ -23,15 +23,19 @@ description: properties: compatible: items: - - const: sifive,fu540-c000-clint + - enum: + - sifive,fu540-c000-clint + - canaan,k210-clint - const: sifive,clint0 description: - Should be "sifive,-clint" and "sifive,clint". + Should be ",-clint" and "sifive,clint". Supported compatible strings are - "sifive,fu540-c000-clint" for the SiFive CLINT v0 as integrated - onto the SiFive FU540 chip, and "sifive,clint0" for the SiFive - CLINT v0 IP block with no chip integration tweaks. + onto the SiFive FU540 chip, "canaan,k210-clint" for the SiFive + CLINT v0 as integrated onto the Canaan Kendryte K210 chip, and + "sifive,clint0" for the SiFive CLINT v0 IP block with no chip + integration tweaks. Please refer to sifive-blocks-ip-versioning.txt for details reg: diff --git a/Documentation/devicetree/bindings/timer/snps,dw-apb-timer.yaml b/Documentation/devicetree/bindings/timer/snps,dw-apb-timer.yaml index d65faf289a833cb6479a612badeed35eff3b3f18..d33c9205a9093c1da564d4d508f9ea7261486c52 100644 --- a/Documentation/devicetree/bindings/timer/snps,dw-apb-timer.yaml +++ b/Documentation/devicetree/bindings/timer/snps,dw-apb-timer.yaml @@ -24,6 +24,9 @@ properties: interrupts: maxItems: 1 + resets: + maxItems: 1 + clocks: minItems: 1 items: diff --git a/Documentation/devicetree/bindings/usb/allwinner,sun4i-a10-musb.yaml b/Documentation/devicetree/bindings/usb/allwinner,sun4i-a10-musb.yaml index d9207bf9d8946e4ee9c799f1adc92ec5906d0359..0f520f17735ee63defae79979b75670aec9d9261 100644 --- a/Documentation/devicetree/bindings/usb/allwinner,sun4i-a10-musb.yaml +++ b/Documentation/devicetree/bindings/usb/allwinner,sun4i-a10-musb.yaml @@ -39,7 +39,7 @@ properties: maxItems: 1 phys: - description: PHY specifier for the OTG PHY + maxItems: 1 phy-names: const: usb diff --git a/Documentation/devicetree/bindings/usb/brcm,usb-pinmap.yaml b/Documentation/devicetree/bindings/usb/brcm,usb-pinmap.yaml index ffa148b9eaa8213951395b89148dd68a57c5b40d..d4618d15ecc1f791d1353d2d6b7c9724e7400a9f 100644 --- a/Documentation/devicetree/bindings/usb/brcm,usb-pinmap.yaml +++ b/Documentation/devicetree/bindings/usb/brcm,usb-pinmap.yaml @@ -22,6 +22,8 @@ properties: description: Interrupt for signals mirrored to out-gpios. in-gpios: + minItems: 1 + maxItems: 2 description: Array of one or two GPIO pins used for input signals. brcm,in-functions: @@ -33,6 +35,7 @@ properties: description: Array of enable and mask pairs, one per gpio in-gpios. out-gpios: + maxItems: 1 description: Array of one GPIO pin used for output signals. brcm,out-functions: diff --git a/Documentation/devicetree/bindings/usb/generic-ehci.yaml b/Documentation/devicetree/bindings/usb/generic-ehci.yaml index 6816de7dfc0029999f86aee14931d30503e443e0..cf83f2d9afacbcbc64f9bbc9df5de4607f39de94 100644 --- a/Documentation/devicetree/bindings/usb/generic-ehci.yaml +++ b/Documentation/devicetree/bindings/usb/generic-ehci.yaml @@ -128,7 +128,7 @@ properties: Phandle of a companion. phys: - description: PHY specifier for the USB PHY + maxItems: 1 phy-names: const: usb diff --git a/Documentation/devicetree/bindings/usb/generic-ohci.yaml b/Documentation/devicetree/bindings/usb/generic-ohci.yaml index 53df281f618c0db0168eadae94dc15f0a587499a..0f5f6ea702d06063d5b9de802ca2580d410f2680 100644 --- a/Documentation/devicetree/bindings/usb/generic-ohci.yaml +++ b/Documentation/devicetree/bindings/usb/generic-ohci.yaml @@ -101,7 +101,7 @@ properties: Overrides the detected port count phys: - description: PHY specifier for the USB PHY + maxItems: 1 phy-names: const: usb diff --git a/Documentation/devicetree/bindings/usb/ingenic,musb.yaml b/Documentation/devicetree/bindings/usb/ingenic,musb.yaml index 678396eeeb783b26e79e11560592141d357404b3..f506225a4d577d24a42d066a87a5ffa70940828f 100644 --- a/Documentation/devicetree/bindings/usb/ingenic,musb.yaml +++ b/Documentation/devicetree/bindings/usb/ingenic,musb.yaml @@ -40,7 +40,7 @@ properties: - const: mc phys: - description: PHY specifier for the USB PHY + maxItems: 1 usb-role-switch: type: boolean diff --git a/Documentation/devicetree/bindings/usb/renesas,usb3-peri.yaml b/Documentation/devicetree/bindings/usb/renesas,usb3-peri.yaml index 929a3f413b44eba0c9e7db9d9edb2d5de0d2afe6..9fcf54b10b0744469e851be035bc780425d36058 100644 --- a/Documentation/devicetree/bindings/usb/renesas,usb3-peri.yaml +++ b/Documentation/devicetree/bindings/usb/renesas,usb3-peri.yaml @@ -54,18 +54,19 @@ properties: description: phandle of a companion. ports: + $ref: /schemas/graph.yaml#/properties/ports description: | any connector to the data bus of this controller should be modelled using the OF graph bindings specified, if the "usb-role-switch" property is used. - type: object + properties: port@0: - type: object + $ref: /schemas/graph.yaml#/properties/port description: High Speed (HS) data bus. port@1: - type: object + $ref: /schemas/graph.yaml#/properties/port description: Super Speed (SS) data bus. required: diff --git a/Documentation/devicetree/bindings/usb/renesas,usbhs.yaml b/Documentation/devicetree/bindings/usb/renesas,usbhs.yaml index 54c361d4a7af0805da86d6c944e7770be0c3c1a3..e67223d90bb7cb04e5787931a809bd5fd70a5235 100644 --- a/Documentation/devicetree/bindings/usb/renesas,usbhs.yaml +++ b/Documentation/devicetree/bindings/usb/renesas,usbhs.yaml @@ -68,6 +68,7 @@ properties: Integer to use BUSWAIT register. renesas,enable-gpio: + maxItems: 1 description: | gpio specifier to check GPIO determining if USB function should be enabled. diff --git a/Documentation/devicetree/bindings/usb/ti,hd3ss3220.yaml b/Documentation/devicetree/bindings/usb/ti,hd3ss3220.yaml index 52ceb07294a35877e916499b39c8b125fa3d8992..b86bf6bc9cd67431e4267cb445478d5e25d7d6da 100644 --- a/Documentation/devicetree/bindings/usb/ti,hd3ss3220.yaml +++ b/Documentation/devicetree/bindings/usb/ti,hd3ss3220.yaml @@ -26,17 +26,17 @@ properties: maxItems: 1 ports: + $ref: /schemas/graph.yaml#/properties/ports description: OF graph bindings (specified in bindings/graph.txt) that model SS data bus to the SS capable connector. - type: object + properties: port@0: - type: object + $ref: /schemas/graph.yaml#/properties/port description: Super Speed (SS) MUX inputs connected to SS capable connector. - $ref: /connector/usb-connector.yaml#/properties/ports/properties/port@1 port@1: - type: object + $ref: /schemas/graph.yaml#/properties/port description: Output of 2:1 MUX connected to Super Speed (SS) data bus. required: diff --git a/Documentation/devicetree/bindings/usb/ti,j721e-usb.yaml b/Documentation/devicetree/bindings/usb/ti,j721e-usb.yaml index c80a835719196b6ec1ca1eef6d1bbaa61e704872..7ec87a783c5cfaac5c0444e4bb8a84bca0ac7407 100644 --- a/Documentation/devicetree/bindings/usb/ti,j721e-usb.yaml +++ b/Documentation/devicetree/bindings/usb/ti,j721e-usb.yaml @@ -19,7 +19,7 @@ properties: - const: ti,am64-usb reg: - description: module registers + maxItems: 1 ranges: true @@ -28,6 +28,7 @@ properties: PM domain provider node and an args specifier containing the USB device id value. See, Documentation/devicetree/bindings/soc/ti/sci-pm-domain.txt + maxItems: 1 clocks: description: Clock phandles to usb2_refclk and lpm_clk diff --git a/Documentation/devicetree/bindings/usb/ti,keystone-dwc3.yaml b/Documentation/devicetree/bindings/usb/ti,keystone-dwc3.yaml index ca7fbe3ed22e3f43b2ea3da1e8e7116919bbf149..9a068d3bc73b65a2f39c7a905ac052d537af5cae 100644 --- a/Documentation/devicetree/bindings/usb/ti,keystone-dwc3.yaml +++ b/Documentation/devicetree/bindings/usb/ti,keystone-dwc3.yaml @@ -43,12 +43,14 @@ properties: maxItems: 2 power-domains: + maxItems: 1 description: Should contain a phandle to a PM domain provider node and an args specifier containing the USB device id value. This property is as per the binding, Documentation/devicetree/bindings/soc/ti/sci-pm-domain.txt phys: + maxItems: 1 description: PHY specifier for the USB3.0 PHY. Some SoCs need the USB3.0 PHY to be turned on before the controller. diff --git a/Documentation/devicetree/bindings/vendor-prefixes.yaml b/Documentation/devicetree/bindings/vendor-prefixes.yaml index b4e026f2d6c52d892bd3f1ad3ec556ce53630607..f6064d84a424de4db2bc3bf6dc149569d8b35f17 100644 --- a/Documentation/devicetree/bindings/vendor-prefixes.yaml +++ b/Documentation/devicetree/bindings/vendor-prefixes.yaml @@ -1085,6 +1085,8 @@ patternProperties: description: Shenzhen Sunchip Technology Co., Ltd "^SUNW,.*": description: Sun Microsystems, Inc + "^silvaco,.*": + description: Silvaco, Inc. "^swir,.*": description: Sierra Wireless "^syna,.*": @@ -1272,6 +1274,8 @@ patternProperties: description: YSH & ATIL "^yones-toptech,.*": description: Yones Toptech Co., Ltd. + "^ys,.*": + description: Shenzhen Yashi Changhua Intelligent Technology Co., Ltd. "^ysoft,.*": description: Y Soft Corporation a.s. "^zealz,.*": diff --git a/Documentation/devicetree/bindings/watchdog/allwinner,sun4i-a10-wdt.yaml b/Documentation/devicetree/bindings/watchdog/allwinner,sun4i-a10-wdt.yaml index 5ac607de8be41e572ab094b00c7c8e1b260e3c05..9aa3c313c49f2ef12d44c295c50acee4d1f21e95 100644 --- a/Documentation/devicetree/bindings/watchdog/allwinner,sun4i-a10-wdt.yaml +++ b/Documentation/devicetree/bindings/watchdog/allwinner,sun4i-a10-wdt.yaml @@ -19,13 +19,11 @@ properties: - const: allwinner,sun4i-a10-wdt - const: allwinner,sun6i-a31-wdt - items: - - const: allwinner,sun50i-a64-wdt - - const: allwinner,sun6i-a31-wdt - - items: - - const: allwinner,sun50i-a100-wdt - - const: allwinner,sun6i-a31-wdt - - items: - - const: allwinner,sun50i-h6-wdt + - enum: + - allwinner,sun50i-a64-wdt + - allwinner,sun50i-a100-wdt + - allwinner,sun50i-h6-wdt + - allwinner,sun50i-h616-wdt - const: allwinner,sun6i-a31-wdt - items: - const: allwinner,suniv-f1c100s-wdt diff --git a/Documentation/devicetree/bindings/watchdog/intel,keembay-wdt.yaml b/Documentation/devicetree/bindings/watchdog/intel,keembay-wdt.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1437ff8a122f2be0ceccc8c4c9c8fa59b38a2f8b --- /dev/null +++ b/Documentation/devicetree/bindings/watchdog/intel,keembay-wdt.yaml @@ -0,0 +1,57 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/watchdog/intel,keembay-wdt.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Intel Keem Bay SoC non-secure Watchdog Timer + +maintainers: + - Wan Ahmad Zainie + +properties: + compatible: + enum: + - intel,keembay-wdt + + reg: + maxItems: 1 + + clocks: + maxItems: 1 + + interrupts: + items: + - description: interrupt specifier for threshold interrupt line + - description: interrupt specifier for timeout interrupt line + + interrupt-names: + items: + - const: threshold + - const: timeout + +required: + - compatible + - reg + - interrupts + - interrupt-names + - clocks + +additionalProperties: false + +examples: + - | + #include + #include + #define KEEM_BAY_A53_TIM + + watchdog: watchdog@2033009c { + compatible = "intel,keembay-wdt"; + reg = <0x2033009c 0x10>; + interrupts = , + ; + interrupt-names = "threshold", "timeout"; + clocks = <&scmi_clk KEEM_BAY_A53_TIM>; + }; + +... diff --git a/Documentation/devicetree/bindings/watchdog/mtk-wdt.txt b/Documentation/devicetree/bindings/watchdog/mtk-wdt.txt index 4dd36bd3f1adc1b5af29c2f236246e36de756160..e36ba60de829f366fba68862fe61c7a3cfeacd47 100644 --- a/Documentation/devicetree/bindings/watchdog/mtk-wdt.txt +++ b/Documentation/devicetree/bindings/watchdog/mtk-wdt.txt @@ -4,14 +4,15 @@ Required properties: - compatible should contain: "mediatek,mt2701-wdt", "mediatek,mt6589-wdt": for MT2701 - "mediatek,mt2712-wdt", "mediatek,mt6589-wdt": for MT2712 + "mediatek,mt2712-wdt": for MT2712 "mediatek,mt6589-wdt": for MT6589 "mediatek,mt6797-wdt", "mediatek,mt6589-wdt": for MT6797 "mediatek,mt7622-wdt", "mediatek,mt6589-wdt": for MT7622 "mediatek,mt7623-wdt", "mediatek,mt6589-wdt": for MT7623 "mediatek,mt7629-wdt", "mediatek,mt6589-wdt": for MT7629 - "mediatek,mt8183-wdt", "mediatek,mt6589-wdt": for MT8183 + "mediatek,mt8183-wdt": for MT8183 "mediatek,mt8516-wdt", "mediatek,mt6589-wdt": for MT8516 + "mediatek,mt8192-wdt": for MT8192 - reg : Specifies base physical address and size of the registers. diff --git a/Documentation/devicetree/bindings/watchdog/renesas,wdt.yaml b/Documentation/devicetree/bindings/watchdog/renesas,wdt.yaml index 6933005b52bd928eefe25ccf9df3cc58f7907c65..ab66d3f0c4765567a73dca00e6995e6ccca5ef8c 100644 --- a/Documentation/devicetree/bindings/watchdog/renesas,wdt.yaml +++ b/Documentation/devicetree/bindings/watchdog/renesas,wdt.yaml @@ -50,6 +50,7 @@ properties: - renesas,r8a77980-wdt # R-Car V3H - renesas,r8a77990-wdt # R-Car E3 - renesas,r8a77995-wdt # R-Car D3 + - renesas,r8a779a0-wdt # R-Car V3U - const: renesas,rcar-gen3-wdt # R-Car Gen3 and RZ/G2 reg: diff --git a/Documentation/devicetree/bindings/watchdog/sigma,smp8642-wdt.txt b/Documentation/devicetree/bindings/watchdog/sigma,smp8642-wdt.txt deleted file mode 100644 index 5b7ec2c707d8bb83432a0726d4117459234090c3..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/watchdog/sigma,smp8642-wdt.txt +++ /dev/null @@ -1,18 +0,0 @@ -Sigma Designs SMP86xx/SMP87xx watchdog - -Required properties: -- compatible: Should be "sigma,smp8642-wdt" -- reg: Specifies the physical address region -- clocks: Should be a phandle to the clock - -Optional properties: -- timeout-sec: watchdog timeout in seconds - -Example: - -watchdog@1fd00 { - compatible = "sigma,smp8642-wdt"; - reg = <0x1fd00 8>; - clocks = <&xtal_in_clk>; - timeout-sec = <30>; -}; diff --git a/Documentation/devicetree/bindings/watchdog/sirfsoc_wdt.txt b/Documentation/devicetree/bindings/watchdog/sirfsoc_wdt.txt deleted file mode 100644 index 0dce5e3100b499e550278052a20b7f7d75c2c5ec..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/watchdog/sirfsoc_wdt.txt +++ /dev/null @@ -1,18 +0,0 @@ -SiRFSoC Timer and Watchdog Timer(WDT) Controller - -Required properties: -- compatible: "sirf,prima2-tick" -- reg: Address range of tick timer/WDT register set -- interrupts: interrupt number to the cpu - -Optional properties: -- timeout-sec : Contains the watchdog timeout in seconds - -Example: - -timer@b0020000 { - compatible = "sirf,prima2-tick"; - reg = <0xb0020000 0x1000>; - interrupts = <0>; - timeout-sec = <30>; -}; diff --git a/Documentation/devicetree/bindings/watchdog/snps,dw-wdt.yaml b/Documentation/devicetree/bindings/watchdog/snps,dw-wdt.yaml index f7ee9229c29f969339e1ea1ba0feaa1bb34d5594..b58596b1831de33af1d604d570337b0ce250902f 100644 --- a/Documentation/devicetree/bindings/watchdog/snps,dw-wdt.yaml +++ b/Documentation/devicetree/bindings/watchdog/snps,dw-wdt.yaml @@ -18,10 +18,16 @@ properties: - const: snps,dw-wdt - items: - enum: + - rockchip,px30-wdt - rockchip,rk3066-wdt - rockchip,rk3188-wdt + - rockchip,rk3228-wdt - rockchip,rk3288-wdt + - rockchip,rk3308-wdt + - rockchip,rk3328-wdt - rockchip,rk3368-wdt + - rockchip,rk3399-wdt + - rockchip,rv1108-wdt - const: snps,dw-wdt reg: diff --git a/Documentation/devicetree/bindings/watchdog/stericsson-coh901327.txt b/Documentation/devicetree/bindings/watchdog/stericsson-coh901327.txt deleted file mode 100644 index 8ffb88e39e7652bbb533a1cf70155cb568c3d67d..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/watchdog/stericsson-coh901327.txt +++ /dev/null @@ -1,19 +0,0 @@ -ST-Ericsson COH 901 327 Watchdog timer - -Required properties: -- compatible: must be "stericsson,coh901327". -- reg: physical base address of the controller and length of memory mapped - region. -- interrupts: the interrupt used for the watchdog timeout warning. - -Optional properties: -- timeout-sec: contains the watchdog timeout in seconds. - -Example: - -watchdog: watchdog@c0012000 { - compatible = "stericsson,coh901327"; - reg = <0xc0012000 0x1000>; - interrupts = <3>; - timeout-sec = <60>; -}; diff --git a/Documentation/devicetree/bindings/watchdog/ti,rti-wdt.yaml b/Documentation/devicetree/bindings/watchdog/ti,rti-wdt.yaml index c1348db593746a89083d1efd6b43f669702fefcc..054584d7543a53955d0c1556a040aac6386c3c11 100644 --- a/Documentation/devicetree/bindings/watchdog/ti,rti-wdt.yaml +++ b/Documentation/devicetree/bindings/watchdog/ti,rti-wdt.yaml @@ -57,8 +57,8 @@ examples: */ #include - watchdog0: rti@2200000 { - compatible = "ti,rti-wdt"; + watchdog@2200000 { + compatible = "ti,j7-rti-wdt"; reg = <0x2200000 0x100>; clocks = <&k3_clks 252 1>; power-domains = <&k3_pds 252 TI_SCI_PD_EXCLUSIVE>; diff --git a/Documentation/devicetree/bindings/watchdog/zte,zx2967-wdt.txt b/Documentation/devicetree/bindings/watchdog/zte,zx2967-wdt.txt deleted file mode 100644 index 06ce67766756e972e7532ebc8173a674e54776bc..0000000000000000000000000000000000000000 --- a/Documentation/devicetree/bindings/watchdog/zte,zx2967-wdt.txt +++ /dev/null @@ -1,32 +0,0 @@ -ZTE zx2967 Watchdog timer - -Required properties: - -- compatible : should be one of the following. - * zte,zx296718-wdt -- reg : Specifies base physical address and size of the registers. -- clocks : Pairs of phandle and specifier referencing the controller's clocks. -- resets : Reference to the reset controller controlling the watchdog - controller. - -Optional properties: - -- timeout-sec : Contains the watchdog timeout in seconds. -- zte,wdt-reset-sysctrl : Directs how to reset system by the watchdog. - if we don't want to restart system when watchdog been triggered, - it's not required, vice versa. - It should include following fields. - * phandle of aon-sysctrl. - * offset of register that be written, should be 0xb0. - * configure value that be written to aon-sysctrl. - * bit mask, corresponding bits will be affected. - -Example: - -wdt: watchdog@1465000 { - compatible = "zte,zx296718-wdt"; - reg = <0x1465000 0x1000>; - clocks = <&topcrm WDT_WCLK>; - resets = <&toprst 35>; - zte,wdt-reset-sysctrl = <&aon_sysctrl 0xb0 1 0x115>; -}; diff --git a/Documentation/devicetree/usage-model.rst b/Documentation/devicetree/usage-model.rst index e1b42dc63f0108d1eb997b36267e3406f7256026..1eb83496ca1ea89e5be546609087f1c8019c0c16 100644 --- a/Documentation/devicetree/usage-model.rst +++ b/Documentation/devicetree/usage-model.rst @@ -12,7 +12,7 @@ This article describes how Linux uses the device tree. An overview of the device tree data format can be found on the device tree usage page at devicetree.org\ [1]_. -.. [1] https://elinux.org/Device_Tree_Usage +.. [1] https://www.devicetree.org/specifications/ The "Open Firmware Device Tree", or simply Device Tree (DT), is a data structure and language for describing hardware. More specifically, it diff --git a/Documentation/doc-guide/sphinx.rst b/Documentation/doc-guide/sphinx.rst index 36ac2166ad675ee8e8881871700d04525d77576a..ec3e71f56009f9c66edb9b6e969640de17b9d657 100644 --- a/Documentation/doc-guide/sphinx.rst +++ b/Documentation/doc-guide/sphinx.rst @@ -340,16 +340,26 @@ Rendered as: Cross-referencing ----------------- -Cross-referencing from one documentation page to another can be done by passing -the path to the file starting from the Documentation folder. -For example, to cross-reference to this page (the .rst extension is optional):: - - See Documentation/doc-guide/sphinx.rst. - -If you want to use a relative path, you need to use Sphinx's ``doc`` directive. -For example, referencing this page from the same directory would be done as:: - - See :doc:`sphinx`. +Cross-referencing from one documentation page to another can be done simply by +writing the path to the document file, no special syntax required. The path can +be either absolute or relative. For absolute paths, start it with +"Documentation/". For example, to cross-reference to this page, all the +following are valid options, depending on the current document's directory (note +that the ``.rst`` extension is required):: + + See Documentation/doc-guide/sphinx.rst. This always works. + Take a look at sphinx.rst, which is at this same directory. + Read ../sphinx.rst, which is one directory above. + +If you want the link to have a different rendered text other than the document's +title, you need to use Sphinx's ``doc`` role. For example:: + + See :doc:`my custom link text for document sphinx `. + +For most use cases, the former is preferred, as it is cleaner and more suited +for people reading the source files. If you come across a ``:doc:`` usage that +isn't adding any value, please feel free to convert it to just the document +path. For information on cross-referencing to kernel-doc functions or types, see Documentation/doc-guide/kernel-doc.rst. diff --git a/Documentation/driver-api/cxl/index.rst b/Documentation/driver-api/cxl/index.rst new file mode 100644 index 0000000000000000000000000000000000000000..036e49553542b332d49921305fdd2f17c491e5a1 --- /dev/null +++ b/Documentation/driver-api/cxl/index.rst @@ -0,0 +1,12 @@ +.. SPDX-License-Identifier: GPL-2.0 + +==================== +Compute Express Link +==================== + +.. toctree:: + :maxdepth: 1 + + memory-devices + +.. only:: subproject and html diff --git a/Documentation/driver-api/cxl/memory-devices.rst b/Documentation/driver-api/cxl/memory-devices.rst new file mode 100644 index 0000000000000000000000000000000000000000..1bad466f916797077a70714e82d80dbffec82891 --- /dev/null +++ b/Documentation/driver-api/cxl/memory-devices.rst @@ -0,0 +1,46 @@ +.. SPDX-License-Identifier: GPL-2.0 +.. include:: + +=================================== +Compute Express Link Memory Devices +=================================== + +A Compute Express Link Memory Device is a CXL component that implements the +CXL.mem protocol. It contains some amount of volatile memory, persistent memory, +or both. It is enumerated as a PCI device for configuration and passing +messages over an MMIO mailbox. Its contribution to the System Physical +Address space is handled via HDM (Host Managed Device Memory) decoders +that optionally define a device's contribution to an interleaved address +range across multiple devices underneath a host-bridge or interleaved +across host-bridges. + +Driver Infrastructure +===================== + +This section covers the driver infrastructure for a CXL memory device. + +CXL Memory Device +----------------- + +.. kernel-doc:: drivers/cxl/mem.c + :doc: cxl mem + +.. kernel-doc:: drivers/cxl/mem.c + :internal: + +CXL Bus +------- +.. kernel-doc:: drivers/cxl/bus.c + :doc: cxl bus + +External Interfaces +=================== + +CXL IOCTL Interface +------------------- + +.. kernel-doc:: include/uapi/linux/cxl_mem.h + :doc: UAPI + +.. kernel-doc:: include/uapi/linux/cxl_mem.h + :internal: diff --git a/Documentation/driver-api/gpio/consumer.rst b/Documentation/driver-api/gpio/consumer.rst index 173e4c7b037d0db81727ec2bac5009e738b0aea7..22271c342d923f6a6d101f8e9c7dbb19c6187767 100644 --- a/Documentation/driver-api/gpio/consumer.rst +++ b/Documentation/driver-api/gpio/consumer.rst @@ -361,12 +361,13 @@ corresponding chip driver. In that case a significantly improved performance can be expected. If simultaneous access is not possible the GPIOs will be accessed sequentially. -The functions take three arguments: +The functions take four arguments: + * array_size - the number of array elements * desc_array - an array of GPIO descriptors * array_info - optional information obtained from gpiod_get_array() * value_bitmap - a bitmap to store the GPIOs' values (get) or - a bitmap of values to assign to the GPIOs (set) + a bitmap of values to assign to the GPIOs (set) The descriptor array can be obtained using the gpiod_get_array() function or one of its variants. If the group of descriptors returned by that function diff --git a/Documentation/driver-api/gpio/driver.rst b/Documentation/driver-api/gpio/driver.rst index 0fb57e298b41e88b38c62c342fabef3053a4911c..d6b0d779859b3499ae7f3c90f31df382bbcab6f9 100644 --- a/Documentation/driver-api/gpio/driver.rst +++ b/Documentation/driver-api/gpio/driver.rst @@ -640,8 +640,8 @@ compliance: level and edge IRQs * [1] http://www.spinics.net/lists/linux-omap/msg120425.html -* [2] https://lkml.org/lkml/2015/9/25/494 -* [3] https://lkml.org/lkml/2015/9/25/495 +* [2] https://lore.kernel.org/r/1443209283-20781-2-git-send-email-grygorii.strashko@ti.com +* [3] https://lore.kernel.org/r/1443209283-20781-3-git-send-email-grygorii.strashko@ti.com Requesting self-owned GPIO pins diff --git a/Documentation/driver-api/gpio/intro.rst b/Documentation/driver-api/gpio/intro.rst index 74591489d0b55f677d0371b7f21b2345be58b92f..94dd7185e76eb5df0051f0cd04bf52586f9e2b33 100644 --- a/Documentation/driver-api/gpio/intro.rst +++ b/Documentation/driver-api/gpio/intro.rst @@ -106,11 +106,11 @@ don't. When you need open drain signaling but your hardware doesn't directly support it, there's a common idiom you can use to emulate it with any GPIO pin that can be used as either an input or an output: - LOW: gpiod_direction_output(gpio, 0) ... this drives the signal and overrides - the pullup. + **LOW**: ``gpiod_direction_output(gpio, 0)`` ... this drives the signal and + overrides the pullup. - HIGH: gpiod_direction_input(gpio) ... this turns off the output, so the pullup - (or some other device) controls the signal. + **HIGH**: ``gpiod_direction_input(gpio)`` ... this turns off the output, so + the pullup (or some other device) controls the signal. The same logic can be applied to emulate open source signaling, by driving the high signal and configuring the GPIO as input for low. This open drain/open diff --git a/Documentation/driver-api/index.rst b/Documentation/driver-api/index.rst index 2456d0a97ed812dc0418d350fa3cfc106a0883e9..b0ab367896ab04008cce89232d8f53e3a2658246 100644 --- a/Documentation/driver-api/index.rst +++ b/Documentation/driver-api/index.rst @@ -35,6 +35,7 @@ available subsections can be seen below. usb/index firewire pci/index + cxl/index spi i2c ipmb @@ -93,12 +94,12 @@ available subsections can be seen below. pps ptp phy/index - pti_intel_mid pwm pldmfw/index rfkill serial/index sm501 + surface_aggregator/index switchtec sync_file vfio-mediated-device diff --git a/Documentation/driver-api/men-chameleon-bus.rst b/Documentation/driver-api/men-chameleon-bus.rst index 1b1f048aa7483d6cd5c105e219c2e9f1d1e78878..6f0b9ee4759503b15aecbd8c75cc41a917d162a7 100644 --- a/Documentation/driver-api/men-chameleon-bus.rst +++ b/Documentation/driver-api/men-chameleon-bus.rst @@ -18,6 +18,7 @@ MEN Chameleon Bus 4.1 The driver structure 4.2 Probing and attaching 4.3 Initializing the driver + 4.4 Using DMA Introduction @@ -173,3 +174,14 @@ module at the MCB core:: The module_mcb_driver() macro can be used to reduce the above code:: module_mcb_driver(foo_driver); + +Using DMA +--------- + +To make use of the kernel's DMA-API's function, you will need to use the +carrier device's 'struct device'. Fortunately 'struct mcb_device' embeds a +pointer (->dma_dev) to the carrier's device for DMA purposes:: + + ret = dma_set_mask_and_coherent(&mdev->dma_dev, DMA_BIT_MASK(dma_bits)); + if (rc) + /* Handle errors */ diff --git a/Documentation/driver-api/pti_intel_mid.rst b/Documentation/driver-api/pti_intel_mid.rst deleted file mode 100644 index bacc2a4ee89fd686fd36d0342f5d9ac7d6c6e4fe..0000000000000000000000000000000000000000 --- a/Documentation/driver-api/pti_intel_mid.rst +++ /dev/null @@ -1,108 +0,0 @@ -.. SPDX-License-Identifier: GPL-2.0 - -============= -Intel MID PTI -============= - -The Intel MID PTI project is HW implemented in Intel Atom -system-on-a-chip designs based on the Parallel Trace -Interface for MIPI P1149.7 cJTAG standard. The kernel solution -for this platform involves the following files:: - - ./include/linux/pti.h - ./drivers/.../n_tracesink.h - ./drivers/.../n_tracerouter.c - ./drivers/.../n_tracesink.c - ./drivers/.../pti.c - -pti.c is the driver that enables various debugging features -popular on platforms from certain mobile manufacturers. -n_tracerouter.c and n_tracesink.c allow extra system information to -be collected and routed to the pti driver, such as trace -debugging data from a modem. Although n_tracerouter -and n_tracesink are a part of the complete PTI solution, -these two line disciplines can work separately from -pti.c and route any data stream from one /dev/tty node -to another /dev/tty node via kernel-space. This provides -a stable, reliable connection that will not break unless -the user-space application shuts down (plus avoids -kernel->user->kernel context switch overheads of routing -data). - -An example debugging usage for this driver system: - - * Hook /dev/ttyPTI0 to syslogd. Opening this port will also start - a console device to further capture debugging messages to PTI. - * Hook /dev/ttyPTI1 to modem debugging data to write to PTI HW. - This is where n_tracerouter and n_tracesink are used. - * Hook /dev/pti to a user-level debugging application for writing - to PTI HW. - * `Use mipi_` Kernel Driver API in other device drivers for - debugging to PTI by first requesting a PTI write address via - mipi_request_masterchannel(1). - -Below is example pseudo-code on how a 'privileged' application -can hook up n_tracerouter and n_tracesink to any tty on -a system. 'Privileged' means the application has enough -privileges to successfully manipulate the ldisc drivers -but is not just blindly executing as 'root'. Keep in mind -the use of ioctl(,TIOCSETD,) is not specific to the n_tracerouter -and n_tracesink line discpline drivers but is a generic -operation for a program to use a line discpline driver -on a tty port other than the default n_tty: - -.. code-block:: c - - /////////// To hook up n_tracerouter and n_tracesink ///////// - - // Note that n_tracerouter depends on n_tracesink. - #include - #define ONE_TTY "/dev/ttyOne" - #define TWO_TTY "/dev/ttyTwo" - - // needed global to hand onto ldisc connection - static int g_fd_source = -1; - static int g_fd_sink = -1; - - // these two vars used to grab LDISC values from loaded ldisc drivers - // in OS. Look at /proc/tty/ldiscs to get the right numbers from - // the ldiscs loaded in the system. - int source_ldisc_num, sink_ldisc_num = -1; - int retval; - - g_fd_source = open(ONE_TTY, O_RDWR); // must be R/W - g_fd_sink = open(TWO_TTY, O_RDWR); // must be R/W - - if (g_fd_source <= 0) || (g_fd_sink <= 0) { - // doubt you'll want to use these exact error lines of code - printf("Error on open(). errno: %d\n",errno); - return errno; - } - - retval = ioctl(g_fd_sink, TIOCSETD, &sink_ldisc_num); - if (retval < 0) { - printf("Error on ioctl(). errno: %d\n", errno); - return errno; - } - - retval = ioctl(g_fd_source, TIOCSETD, &source_ldisc_num); - if (retval < 0) { - printf("Error on ioctl(). errno: %d\n", errno); - return errno; - } - - /////////// To disconnect n_tracerouter and n_tracesink //////// - - // First make sure data through the ldiscs has stopped. - - // Second, disconnect ldiscs. This provides a - // little cleaner shutdown on tty stack. - sink_ldisc_num = 0; - source_ldisc_num = 0; - ioctl(g_fd_uart, TIOCSETD, &sink_ldisc_num); - ioctl(g_fd_gadget, TIOCSETD, &source_ldisc_num); - - // Three, program closes connection, and cleanup: - close(g_fd_uart); - close(g_fd_gadget); - g_fd_uart = g_fd_gadget = NULL; diff --git a/Documentation/driver-api/surface_aggregator/client-api.rst b/Documentation/driver-api/surface_aggregator/client-api.rst new file mode 100644 index 0000000000000000000000000000000000000000..8e0b000d0e6429a62896ca466529e6f6984060b3 --- /dev/null +++ b/Documentation/driver-api/surface_aggregator/client-api.rst @@ -0,0 +1,38 @@ +.. SPDX-License-Identifier: GPL-2.0+ + +=============================== +Client Driver API Documentation +=============================== + +.. contents:: + :depth: 2 + + +Serial Hub Communication +======================== + +.. kernel-doc:: include/linux/surface_aggregator/serial_hub.h + +.. kernel-doc:: drivers/platform/surface/aggregator/ssh_packet_layer.c + :export: + + +Controller and Core Interface +============================= + +.. kernel-doc:: include/linux/surface_aggregator/controller.h + +.. kernel-doc:: drivers/platform/surface/aggregator/controller.c + :export: + +.. kernel-doc:: drivers/platform/surface/aggregator/core.c + :export: + + +Client Bus and Client Device API +================================ + +.. kernel-doc:: include/linux/surface_aggregator/device.h + +.. kernel-doc:: drivers/platform/surface/aggregator/bus.c + :export: diff --git a/Documentation/driver-api/surface_aggregator/client.rst b/Documentation/driver-api/surface_aggregator/client.rst new file mode 100644 index 0000000000000000000000000000000000000000..26d13085a11754aaed8d31d20574d10b796fabe3 --- /dev/null +++ b/Documentation/driver-api/surface_aggregator/client.rst @@ -0,0 +1,393 @@ +.. SPDX-License-Identifier: GPL-2.0+ + +.. |ssam_controller| replace:: :c:type:`struct ssam_controller ` +.. |ssam_device| replace:: :c:type:`struct ssam_device ` +.. |ssam_device_driver| replace:: :c:type:`struct ssam_device_driver ` +.. |ssam_client_bind| replace:: :c:func:`ssam_client_bind` +.. |ssam_client_link| replace:: :c:func:`ssam_client_link` +.. |ssam_get_controller| replace:: :c:func:`ssam_get_controller` +.. |ssam_controller_get| replace:: :c:func:`ssam_controller_get` +.. |ssam_controller_put| replace:: :c:func:`ssam_controller_put` +.. |ssam_device_alloc| replace:: :c:func:`ssam_device_alloc` +.. |ssam_device_add| replace:: :c:func:`ssam_device_add` +.. |ssam_device_remove| replace:: :c:func:`ssam_device_remove` +.. |ssam_device_driver_register| replace:: :c:func:`ssam_device_driver_register` +.. |ssam_device_driver_unregister| replace:: :c:func:`ssam_device_driver_unregister` +.. |module_ssam_device_driver| replace:: :c:func:`module_ssam_device_driver` +.. |SSAM_DEVICE| replace:: :c:func:`SSAM_DEVICE` +.. |ssam_notifier_register| replace:: :c:func:`ssam_notifier_register` +.. |ssam_notifier_unregister| replace:: :c:func:`ssam_notifier_unregister` +.. |ssam_request_sync| replace:: :c:func:`ssam_request_sync` +.. |ssam_event_mask| replace:: :c:type:`enum ssam_event_mask ` + + +====================== +Writing Client Drivers +====================== + +For the API documentation, refer to: + +.. toctree:: + :maxdepth: 2 + + client-api + + +Overview +======== + +Client drivers can be set up in two main ways, depending on how the +corresponding device is made available to the system. We specifically +differentiate between devices that are presented to the system via one of +the conventional ways, e.g. as platform devices via ACPI, and devices that +are non-discoverable and instead need to be explicitly provided by some +other mechanism, as discussed further below. + + +Non-SSAM Client Drivers +======================= + +All communication with the SAM EC is handled via the |ssam_controller| +representing that EC to the kernel. Drivers targeting a non-SSAM device (and +thus not being a |ssam_device_driver|) need to explicitly establish a +connection/relation to that controller. This can be done via the +|ssam_client_bind| function. Said function returns a reference to the SSAM +controller, but, more importantly, also establishes a device link between +client device and controller (this can also be done separate via +|ssam_client_link|). It is important to do this, as it, first, guarantees +that the returned controller is valid for use in the client driver for as +long as this driver is bound to its device, i.e. that the driver gets +unbound before the controller ever becomes invalid, and, second, as it +ensures correct suspend/resume ordering. This setup should be done in the +driver's probe function, and may be used to defer probing in case the SSAM +subsystem is not ready yet, for example: + +.. code-block:: c + + static int client_driver_probe(struct platform_device *pdev) + { + struct ssam_controller *ctrl; + + ctrl = ssam_client_bind(&pdev->dev); + if (IS_ERR(ctrl)) + return PTR_ERR(ctrl) == -ENODEV ? -EPROBE_DEFER : PTR_ERR(ctrl); + + // ... + + return 0; + } + +The controller may be separately obtained via |ssam_get_controller| and its +lifetime be guaranteed via |ssam_controller_get| and |ssam_controller_put|. +Note that none of these functions, however, guarantee that the controller +will not be shut down or suspended. These functions essentially only operate +on the reference, i.e. only guarantee a bare minimum of accessibility +without any guarantees at all on practical operability. + + +Adding SSAM Devices +=================== + +If a device does not already exist/is not already provided via conventional +means, it should be provided as |ssam_device| via the SSAM client device +hub. New devices can be added to this hub by entering their UID into the +corresponding registry. SSAM devices can also be manually allocated via +|ssam_device_alloc|, subsequently to which they have to be added via +|ssam_device_add| and eventually removed via |ssam_device_remove|. By +default, the parent of the device is set to the controller device provided +for allocation, however this may be changed before the device is added. Note +that, when changing the parent device, care must be taken to ensure that the +controller lifetime and suspend/resume ordering guarantees, in the default +setup provided through the parent-child relation, are preserved. If +necessary, by use of |ssam_client_link| as is done for non-SSAM client +drivers and described in more detail above. + +A client device must always be removed by the party which added the +respective device before the controller shuts down. Such removal can be +guaranteed by linking the driver providing the SSAM device to the controller +via |ssam_client_link|, causing it to unbind before the controller driver +unbinds. Client devices registered with the controller as parent are +automatically removed when the controller shuts down, but this should not be +relied upon, especially as this does not extend to client devices with a +different parent. + + +SSAM Client Drivers +=================== + +SSAM client device drivers are, in essence, no different than other device +driver types. They are represented via |ssam_device_driver| and bind to a +|ssam_device| via its UID (:c:type:`struct ssam_device.uid `) +member and the match table +(:c:type:`struct ssam_device_driver.match_table `), +which should be set when declaring the driver struct instance. Refer to the +|SSAM_DEVICE| macro documentation for more details on how to define members +of the driver's match table. + +The UID for SSAM client devices consists of a ``domain``, a ``category``, +a ``target``, an ``instance``, and a ``function``. The ``domain`` is used +differentiate between physical SAM devices +(:c:type:`SSAM_DOMAIN_SERIALHUB `), i.e. devices that can +be accessed via the Surface Serial Hub, and virtual ones +(:c:type:`SSAM_DOMAIN_VIRTUAL `), such as client-device +hubs, that have no real representation on the SAM EC and are solely used on +the kernel/driver-side. For physical devices, ``category`` represents the +target category, ``target`` the target ID, and ``instance`` the instance ID +used to access the physical SAM device. In addition, ``function`` references +a specific device functionality, but has no meaning to the SAM EC. The +(default) name of a client device is generated based on its UID. + +A driver instance can be registered via |ssam_device_driver_register| and +unregistered via |ssam_device_driver_unregister|. For convenience, the +|module_ssam_device_driver| macro may be used to define module init- and +exit-functions registering the driver. + +The controller associated with a SSAM client device can be found in its +:c:type:`struct ssam_device.ctrl ` member. This reference is +guaranteed to be valid for at least as long as the client driver is bound, +but should also be valid for as long as the client device exists. Note, +however, that access outside of the bound client driver must ensure that the +controller device is not suspended while making any requests or +(un-)registering event notifiers (and thus should generally be avoided). This +is guaranteed when the controller is accessed from inside the bound client +driver. + + +Making Synchronous Requests +=========================== + +Synchronous requests are (currently) the main form of host-initiated +communication with the EC. There are a couple of ways to define and execute +such requests, however, most of them boil down to something similar as shown +in the example below. This example defines a write-read request, meaning +that the caller provides an argument to the SAM EC and receives a response. +The caller needs to know the (maximum) length of the response payload and +provide a buffer for it. + +Care must be taken to ensure that any command payload data passed to the SAM +EC is provided in little-endian format and, similarly, any response payload +data received from it is converted from little-endian to host endianness. + +.. code-block:: c + + int perform_request(struct ssam_controller *ctrl, u32 arg, u32 *ret) + { + struct ssam_request rqst; + struct ssam_response resp; + int status; + + /* Convert request argument to little-endian. */ + __le32 arg_le = cpu_to_le32(arg); + __le32 ret_le = cpu_to_le32(0); + + /* + * Initialize request specification. Replace this with your values. + * The rqst.payload field may be NULL if rqst.length is zero, + * indicating that the request does not have any argument. + * + * Note: The request parameters used here are not valid, i.e. + * they do not correspond to an actual SAM/EC request. + */ + rqst.target_category = SSAM_SSH_TC_SAM; + rqst.target_id = 0x01; + rqst.command_id = 0x02; + rqst.instance_id = 0x03; + rqst.flags = SSAM_REQUEST_HAS_RESPONSE; + rqst.length = sizeof(arg_le); + rqst.payload = (u8 *)&arg_le; + + /* Initialize request response. */ + resp.capacity = sizeof(ret_le); + resp.length = 0; + resp.pointer = (u8 *)&ret_le; + + /* + * Perform actual request. The response pointer may be null in case + * the request does not have any response. This must be consistent + * with the SSAM_REQUEST_HAS_RESPONSE flag set in the specification + * above. + */ + status = ssam_request_sync(ctrl, &rqst, &resp); + + /* + * Alternatively use + * + * ssam_request_sync_onstack(ctrl, &rqst, &resp, sizeof(arg_le)); + * + * to perform the request, allocating the message buffer directly + * on the stack as opposed to allocation via kzalloc(). + */ + + /* + * Convert request response back to native format. Note that in the + * error case, this value is not touched by the SSAM core, i.e. + * 'ret_le' will be zero as specified in its initialization. + */ + *ret = le32_to_cpu(ret_le); + + return status; + } + +Note that |ssam_request_sync| in its essence is a wrapper over lower-level +request primitives, which may also be used to perform requests. Refer to its +implementation and documentation for more details. + +An arguably more user-friendly way of defining such functions is by using +one of the generator macros, for example via: + +.. code-block:: c + + SSAM_DEFINE_SYNC_REQUEST_W(__ssam_tmp_perf_mode_set, __le32, { + .target_category = SSAM_SSH_TC_TMP, + .target_id = 0x01, + .command_id = 0x03, + .instance_id = 0x00, + }); + +This example defines a function + +.. code-block:: c + + int __ssam_tmp_perf_mode_set(struct ssam_controller *ctrl, const __le32 *arg); + +executing the specified request, with the controller passed in when calling +said function. In this example, the argument is provided via the ``arg`` +pointer. Note that the generated function allocates the message buffer on +the stack. Thus, if the argument provided via the request is large, these +kinds of macros should be avoided. Also note that, in contrast to the +previous non-macro example, this function does not do any endianness +conversion, which has to be handled by the caller. Apart from those +differences the function generated by the macro is similar to the one +provided in the non-macro example above. + +The full list of such function-generating macros is + +- :c:func:`SSAM_DEFINE_SYNC_REQUEST_N` for requests without return value and + without argument. +- :c:func:`SSAM_DEFINE_SYNC_REQUEST_R` for requests with return value but no + argument. +- :c:func:`SSAM_DEFINE_SYNC_REQUEST_W` for requests without return value but + with argument. + +Refer to their respective documentation for more details. For each one of +these macros, a special variant is provided, which targets request types +applicable to multiple instances of the same device type: + +- :c:func:`SSAM_DEFINE_SYNC_REQUEST_MD_N` +- :c:func:`SSAM_DEFINE_SYNC_REQUEST_MD_R` +- :c:func:`SSAM_DEFINE_SYNC_REQUEST_MD_W` + +The difference of those macros to the previously mentioned versions is, that +the device target and instance IDs are not fixed for the generated function, +but instead have to be provided by the caller of said function. + +Additionally, variants for direct use with client devices, i.e. +|ssam_device|, are also provided. These can, for example, be used as +follows: + +.. code-block:: c + + SSAM_DEFINE_SYNC_REQUEST_CL_R(ssam_bat_get_sta, __le32, { + .target_category = SSAM_SSH_TC_BAT, + .command_id = 0x01, + }); + +This invocation of the macro defines a function + +.. code-block:: c + + int ssam_bat_get_sta(struct ssam_device *sdev, __le32 *ret); + +executing the specified request, using the device IDs and controller given +in the client device. The full list of such macros for client devices is: + +- :c:func:`SSAM_DEFINE_SYNC_REQUEST_CL_N` +- :c:func:`SSAM_DEFINE_SYNC_REQUEST_CL_R` +- :c:func:`SSAM_DEFINE_SYNC_REQUEST_CL_W` + + +Handling Events +=============== + +To receive events from the SAM EC, an event notifier must be registered for +the desired event via |ssam_notifier_register|. The notifier must be +unregistered via |ssam_notifier_unregister| once it is not required any +more. + +Event notifiers are registered by providing (at minimum) a callback to call +in case an event has been received, the registry specifying how the event +should be enabled, an event ID specifying for which target category and, +optionally and depending on the registry used, for which instance ID events +should be enabled, and finally, flags describing how the EC will send these +events. If the specific registry does not enable events by instance ID, the +instance ID must be set to zero. Additionally, a priority for the respective +notifier may be specified, which determines its order in relation to any +other notifier registered for the same target category. + +By default, event notifiers will receive all events for the specific target +category, regardless of the instance ID specified when registering the +notifier. The core may be instructed to only call a notifier if the target +ID or instance ID (or both) of the event match the ones implied by the +notifier IDs (in case of target ID, the target ID of the registry), by +providing an event mask (see |ssam_event_mask|). + +In general, the target ID of the registry is also the target ID of the +enabled event (with the notable exception being keyboard input events on the +Surface Laptop 1 and 2, which are enabled via a registry with target ID 1, +but provide events with target ID 2). + +A full example for registering an event notifier and handling received +events is provided below: + +.. code-block:: c + + u32 notifier_callback(struct ssam_event_notifier *nf, + const struct ssam_event *event) + { + int status = ... + + /* Handle the event here ... */ + + /* Convert return value and indicate that we handled the event. */ + return ssam_notifier_from_errno(status) | SSAM_NOTIF_HANDLED; + } + + int setup_notifier(struct ssam_device *sdev, + struct ssam_event_notifier *nf) + { + /* Set priority wrt. other handlers of same target category. */ + nf->base.priority = 1; + + /* Set event/notifier callback. */ + nf->base.fn = notifier_callback; + + /* Specify event registry, i.e. how events get enabled/disabled. */ + nf->event.reg = SSAM_EVENT_REGISTRY_KIP; + + /* Specify which event to enable/disable */ + nf->event.id.target_category = sdev->uid.category; + nf->event.id.instance = sdev->uid.instance; + + /* + * Specify for which events the notifier callback gets executed. + * This essentially tells the core if it can skip notifiers that + * don't have target or instance IDs matching those of the event. + */ + nf->event.mask = SSAM_EVENT_MASK_STRICT; + + /* Specify event flags. */ + nf->event.flags = SSAM_EVENT_SEQUENCED; + + return ssam_notifier_register(sdev->ctrl, nf); + } + +Multiple event notifiers can be registered for the same event. The event +handler core takes care of enabling and disabling events when notifiers are +registered and unregistered, by keeping track of how many notifiers for a +specific event (combination of registry, event target category, and event +instance ID) are currently registered. This means that a specific event will +be enabled when the first notifier for it is being registered and disabled +when the last notifier for it is being unregistered. Note that the event +flags are therefore only used on the first registered notifier, however, one +should take care that notifiers for a specific event are always registered +with the same flag and it is considered a bug to do otherwise. diff --git a/Documentation/driver-api/surface_aggregator/clients/cdev.rst b/Documentation/driver-api/surface_aggregator/clients/cdev.rst new file mode 100644 index 0000000000000000000000000000000000000000..248c1372d879113c7f924846d92fa6f01143db97 --- /dev/null +++ b/Documentation/driver-api/surface_aggregator/clients/cdev.rst @@ -0,0 +1,87 @@ +.. SPDX-License-Identifier: GPL-2.0+ + +.. |u8| replace:: :c:type:`u8 ` +.. |u16| replace:: :c:type:`u16 ` +.. |ssam_cdev_request| replace:: :c:type:`struct ssam_cdev_request ` +.. |ssam_cdev_request_flags| replace:: :c:type:`enum ssam_cdev_request_flags ` + +============================== +User-Space EC Interface (cdev) +============================== + +The ``surface_aggregator_cdev`` module provides a misc-device for the SSAM +controller to allow for a (more or less) direct connection from user-space to +the SAM EC. It is intended to be used for development and debugging, and +therefore should not be used or relied upon in any other way. Note that this +module is not loaded automatically, but instead must be loaded manually. + +The provided interface is accessible through the ``/dev/surface/aggregator`` +device-file. All functionality of this interface is provided via IOCTLs. +These IOCTLs and their respective input/output parameter structs are defined in +``include/uapi/linux/surface_aggregator/cdev.h``. + +A small python library and scripts for accessing this interface can be found +at https://github.com/linux-surface/surface-aggregator-module/tree/master/scripts/ssam. + + +Controller IOCTLs +================= + +The following IOCTLs are provided: + +.. flat-table:: Controller IOCTLs + :widths: 1 1 1 1 4 + :header-rows: 1 + + * - Type + - Number + - Direction + - Name + - Description + + * - ``0xA5`` + - ``1`` + - ``WR`` + - ``REQUEST`` + - Perform synchronous SAM request. + + +``REQUEST`` +----------- + +Defined as ``_IOWR(0xA5, 1, struct ssam_cdev_request)``. + +Executes a synchronous SAM request. The request specification is passed in +as argument of type |ssam_cdev_request|, which is then written to/modified +by the IOCTL to return status and result of the request. + +Request payload data must be allocated separately and is passed in via the +``payload.data`` and ``payload.length`` members. If a response is required, +the response buffer must be allocated by the caller and passed in via the +``response.data`` member. The ``response.length`` member must be set to the +capacity of this buffer, or if no response is required, zero. Upon +completion of the request, the call will write the response to the response +buffer (if its capacity allows it) and overwrite the length field with the +actual size of the response, in bytes. + +Additionally, if the request has a response, this must be indicated via the +request flags, as is done with in-kernel requests. Request flags can be set +via the ``flags`` member and the values correspond to the values found in +|ssam_cdev_request_flags|. + +Finally, the status of the request itself is returned in the ``status`` +member (a negative errno value indicating failure). Note that failure +indication of the IOCTL is separated from failure indication of the request: +The IOCTL returns a negative status code if anything failed during setup of +the request (``-EFAULT``) or if the provided argument or any of its fields +are invalid (``-EINVAL``). In this case, the status value of the request +argument may be set, providing more detail on what went wrong (e.g. +``-ENOMEM`` for out-of-memory), but this value may also be zero. The IOCTL +will return with a zero status code in case the request has been set up, +submitted, and completed (i.e. handed back to user-space) successfully from +inside the IOCTL, but the request ``status`` member may still be negative in +case the actual execution of the request failed after it has been submitted. + +A full definition of the argument struct is provided below: + +.. kernel-doc:: include/uapi/linux/surface_aggregator/cdev.h diff --git a/Documentation/driver-api/surface_aggregator/clients/index.rst b/Documentation/driver-api/surface_aggregator/clients/index.rst new file mode 100644 index 0000000000000000000000000000000000000000..3ccabce2327150c67bf83258be46ddf231e7c63e --- /dev/null +++ b/Documentation/driver-api/surface_aggregator/clients/index.rst @@ -0,0 +1,21 @@ +.. SPDX-License-Identifier: GPL-2.0+ + +=========================== +Client Driver Documentation +=========================== + +This is the documentation for client drivers themselves. Refer to +:doc:`../client` for documentation on how to write client drivers. + +.. toctree:: + :maxdepth: 1 + + cdev + san + +.. only:: subproject and html + + Indices + ======= + + * :ref:`genindex` diff --git a/Documentation/driver-api/surface_aggregator/clients/san.rst b/Documentation/driver-api/surface_aggregator/clients/san.rst new file mode 100644 index 0000000000000000000000000000000000000000..38c2580e7758029173fb4c8fe8d6f64d20b0d4bc --- /dev/null +++ b/Documentation/driver-api/surface_aggregator/clients/san.rst @@ -0,0 +1,44 @@ +.. SPDX-License-Identifier: GPL-2.0+ + +.. |san_client_link| replace:: :c:func:`san_client_link` +.. |san_dgpu_notifier_register| replace:: :c:func:`san_dgpu_notifier_register` +.. |san_dgpu_notifier_unregister| replace:: :c:func:`san_dgpu_notifier_unregister` + +=================== +Surface ACPI Notify +=================== + +The Surface ACPI Notify (SAN) device provides the bridge between ACPI and +SAM controller. Specifically, ACPI code can execute requests and handle +battery and thermal events via this interface. In addition to this, events +relating to the discrete GPU (dGPU) of the Surface Book 2 can be sent from +ACPI code (note: the Surface Book 3 uses a different method for this). The +only currently known event sent via this interface is a dGPU power-on +notification. While this driver handles the former part internally, it only +relays the dGPU events to any other driver interested via its public API and +does not handle them. + +The public interface of this driver is split into two parts: Client +registration and notifier-block registration. + +A client to the SAN interface can be linked as consumer to the SAN device +via |san_client_link|. This can be used to ensure that the a client +receiving dGPU events does not miss any events due to the SAN interface not +being set up as this forces the client driver to unbind once the SAN driver +is unbound. + +Notifier-blocks can be registered by any device for as long as the module is +loaded, regardless of being linked as client or not. Registration is done +with |san_dgpu_notifier_register|. If the notifier is not needed any more, it +should be unregistered via |san_dgpu_notifier_unregister|. + +Consult the API documentation below for more details. + + +API Documentation +================= + +.. kernel-doc:: include/linux/surface_acpi_notify.h + +.. kernel-doc:: drivers/platform/surface/surface_acpi_notify.c + :export: diff --git a/Documentation/driver-api/surface_aggregator/index.rst b/Documentation/driver-api/surface_aggregator/index.rst new file mode 100644 index 0000000000000000000000000000000000000000..6f3e1094904d827edcc59057f8a7993bef5153ff --- /dev/null +++ b/Documentation/driver-api/surface_aggregator/index.rst @@ -0,0 +1,21 @@ +.. SPDX-License-Identifier: GPL-2.0+ + +======================================= +Surface System Aggregator Module (SSAM) +======================================= + +.. toctree:: + :maxdepth: 2 + + overview + client + clients/index + ssh + internal + +.. only:: subproject and html + + Indices + ======= + + * :ref:`genindex` diff --git a/Documentation/driver-api/surface_aggregator/internal-api.rst b/Documentation/driver-api/surface_aggregator/internal-api.rst new file mode 100644 index 0000000000000000000000000000000000000000..639a67b5a392aca19b901c9f842747b1fc23fe20 --- /dev/null +++ b/Documentation/driver-api/surface_aggregator/internal-api.rst @@ -0,0 +1,67 @@ +.. SPDX-License-Identifier: GPL-2.0+ + +========================== +Internal API Documentation +========================== + +.. contents:: + :depth: 2 + + +Packet Transport Layer +====================== + +.. kernel-doc:: drivers/platform/surface/aggregator/ssh_parser.h + :internal: + +.. kernel-doc:: drivers/platform/surface/aggregator/ssh_parser.c + :internal: + +.. kernel-doc:: drivers/platform/surface/aggregator/ssh_msgb.h + :internal: + +.. kernel-doc:: drivers/platform/surface/aggregator/ssh_packet_layer.h + :internal: + +.. kernel-doc:: drivers/platform/surface/aggregator/ssh_packet_layer.c + :internal: + + +Request Transport Layer +======================= + +.. kernel-doc:: drivers/platform/surface/aggregator/ssh_request_layer.h + :internal: + +.. kernel-doc:: drivers/platform/surface/aggregator/ssh_request_layer.c + :internal: + + +Controller +========== + +.. kernel-doc:: drivers/platform/surface/aggregator/controller.h + :internal: + +.. kernel-doc:: drivers/platform/surface/aggregator/controller.c + :internal: + + +Client Device Bus +================= + +.. kernel-doc:: drivers/platform/surface/aggregator/bus.c + :internal: + + +Core +==== + +.. kernel-doc:: drivers/platform/surface/aggregator/core.c + :internal: + + +Trace Helpers +============= + +.. kernel-doc:: drivers/platform/surface/aggregator/trace.h diff --git a/Documentation/driver-api/surface_aggregator/internal.rst b/Documentation/driver-api/surface_aggregator/internal.rst new file mode 100644 index 0000000000000000000000000000000000000000..72704734982abf5df594005bf6995a9ea440fac5 --- /dev/null +++ b/Documentation/driver-api/surface_aggregator/internal.rst @@ -0,0 +1,577 @@ +.. SPDX-License-Identifier: GPL-2.0+ + +.. |ssh_ptl| replace:: :c:type:`struct ssh_ptl ` +.. |ssh_ptl_submit| replace:: :c:func:`ssh_ptl_submit` +.. |ssh_ptl_cancel| replace:: :c:func:`ssh_ptl_cancel` +.. |ssh_ptl_shutdown| replace:: :c:func:`ssh_ptl_shutdown` +.. |ssh_ptl_rx_rcvbuf| replace:: :c:func:`ssh_ptl_rx_rcvbuf` +.. |ssh_rtl| replace:: :c:type:`struct ssh_rtl ` +.. |ssh_rtl_submit| replace:: :c:func:`ssh_rtl_submit` +.. |ssh_rtl_cancel| replace:: :c:func:`ssh_rtl_cancel` +.. |ssh_rtl_shutdown| replace:: :c:func:`ssh_rtl_shutdown` +.. |ssh_packet| replace:: :c:type:`struct ssh_packet ` +.. |ssh_packet_get| replace:: :c:func:`ssh_packet_get` +.. |ssh_packet_put| replace:: :c:func:`ssh_packet_put` +.. |ssh_packet_ops| replace:: :c:type:`struct ssh_packet_ops ` +.. |ssh_packet_base_priority| replace:: :c:type:`enum ssh_packet_base_priority ` +.. |ssh_packet_flags| replace:: :c:type:`enum ssh_packet_flags ` +.. |SSH_PACKET_PRIORITY| replace:: :c:func:`SSH_PACKET_PRIORITY` +.. |ssh_frame| replace:: :c:type:`struct ssh_frame ` +.. |ssh_command| replace:: :c:type:`struct ssh_command ` +.. |ssh_request| replace:: :c:type:`struct ssh_request ` +.. |ssh_request_get| replace:: :c:func:`ssh_request_get` +.. |ssh_request_put| replace:: :c:func:`ssh_request_put` +.. |ssh_request_ops| replace:: :c:type:`struct ssh_request_ops ` +.. |ssh_request_init| replace:: :c:func:`ssh_request_init` +.. |ssh_request_flags| replace:: :c:type:`enum ssh_request_flags ` +.. |ssam_controller| replace:: :c:type:`struct ssam_controller ` +.. |ssam_device| replace:: :c:type:`struct ssam_device ` +.. |ssam_device_driver| replace:: :c:type:`struct ssam_device_driver ` +.. |ssam_client_bind| replace:: :c:func:`ssam_client_bind` +.. |ssam_client_link| replace:: :c:func:`ssam_client_link` +.. |ssam_request_sync| replace:: :c:type:`struct ssam_request_sync ` +.. |ssam_event_registry| replace:: :c:type:`struct ssam_event_registry ` +.. |ssam_event_id| replace:: :c:type:`struct ssam_event_id ` +.. |ssam_nf| replace:: :c:type:`struct ssam_nf ` +.. |ssam_nf_refcount_inc| replace:: :c:func:`ssam_nf_refcount_inc` +.. |ssam_nf_refcount_dec| replace:: :c:func:`ssam_nf_refcount_dec` +.. |ssam_notifier_register| replace:: :c:func:`ssam_notifier_register` +.. |ssam_notifier_unregister| replace:: :c:func:`ssam_notifier_unregister` +.. |ssam_cplt| replace:: :c:type:`struct ssam_cplt ` +.. |ssam_event_queue| replace:: :c:type:`struct ssam_event_queue ` +.. |ssam_request_sync_submit| replace:: :c:func:`ssam_request_sync_submit` + +===================== +Core Driver Internals +===================== + +Architectural overview of the Surface System Aggregator Module (SSAM) core +and Surface Serial Hub (SSH) driver. For the API documentation, refer to: + +.. toctree:: + :maxdepth: 2 + + internal-api + + +Overview +======== + +The SSAM core implementation is structured in layers, somewhat following the +SSH protocol structure: + +Lower-level packet transport is implemented in the *packet transport layer +(PTL)*, directly building on top of the serial device (serdev) +infrastructure of the kernel. As the name indicates, this layer deals with +the packet transport logic and handles things like packet validation, packet +acknowledgment (ACKing), packet (retransmission) timeouts, and relaying +packet payloads to higher-level layers. + +Above this sits the *request transport layer (RTL)*. This layer is centered +around command-type packet payloads, i.e. requests (sent from host to EC), +responses of the EC to those requests, and events (sent from EC to host). +It, specifically, distinguishes events from request responses, matches +responses to their corresponding requests, and implements request timeouts. + +The *controller* layer is building on top of this and essentially decides +how request responses and, especially, events are dealt with. It provides an +event notifier system, handles event activation/deactivation, provides a +workqueue for event and asynchronous request completion, and also manages +the message counters required for building command messages (``SEQ``, +``RQID``). This layer basically provides a fundamental interface to the SAM +EC for use in other kernel drivers. + +While the controller layer already provides an interface for other kernel +drivers, the client *bus* extends this interface to provide support for +native SSAM devices, i.e. devices that are not defined in ACPI and not +implemented as platform devices, via |ssam_device| and |ssam_device_driver| +simplify management of client devices and client drivers. + +Refer to :doc:`client` for documentation regarding the client device/driver +API and interface options for other kernel drivers. It is recommended to +familiarize oneself with that chapter and the :doc:`ssh` before continuing +with the architectural overview below. + + +Packet Transport Layer +====================== + +The packet transport layer is represented via |ssh_ptl| and is structured +around the following key concepts: + +Packets +------- + +Packets are the fundamental transmission unit of the SSH protocol. They are +managed by the packet transport layer, which is essentially the lowest layer +of the driver and is built upon by other components of the SSAM core. +Packets to be transmitted by the SSAM core are represented via |ssh_packet| +(in contrast, packets received by the core do not have any specific +structure and are managed entirely via the raw |ssh_frame|). + +This structure contains the required fields to manage the packet inside the +transport layer, as well as a reference to the buffer containing the data to +be transmitted (i.e. the message wrapped in |ssh_frame|). Most notably, it +contains an internal reference count, which is used for managing its +lifetime (accessible via |ssh_packet_get| and |ssh_packet_put|). When this +counter reaches zero, the ``release()`` callback provided to the packet via +its |ssh_packet_ops| reference is executed, which may then deallocate the +packet or its enclosing structure (e.g. |ssh_request|). + +In addition to the ``release`` callback, the |ssh_packet_ops| reference also +provides a ``complete()`` callback, which is run once the packet has been +completed and provides the status of this completion, i.e. zero on success +or a negative errno value in case of an error. Once the packet has been +submitted to the packet transport layer, the ``complete()`` callback is +always guaranteed to be executed before the ``release()`` callback, i.e. the +packet will always be completed, either successfully, with an error, or due +to cancellation, before it will be released. + +The state of a packet is managed via its ``state`` flags +(|ssh_packet_flags|), which also contains the packet type. In particular, +the following bits are noteworthy: + +* ``SSH_PACKET_SF_LOCKED_BIT``: This bit is set when completion, either + through error or success, is imminent. It indicates that no further + references of the packet should be taken and any existing references + should be dropped as soon as possible. The process setting this bit is + responsible for removing any references to this packet from the packet + queue and pending set. + +* ``SSH_PACKET_SF_COMPLETED_BIT``: This bit is set by the process running the + ``complete()`` callback and is used to ensure that this callback only runs + once. + +* ``SSH_PACKET_SF_QUEUED_BIT``: This bit is set when the packet is queued on + the packet queue and cleared when it is dequeued. + +* ``SSH_PACKET_SF_PENDING_BIT``: This bit is set when the packet is added to + the pending set and cleared when it is removed from it. + +Packet Queue +------------ + +The packet queue is the first of the two fundamental collections in the +packet transport layer. It is a priority queue, with priority of the +respective packets based on the packet type (major) and number of tries +(minor). See |SSH_PACKET_PRIORITY| for more details on the priority value. + +All packets to be transmitted by the transport layer must be submitted to +this queue via |ssh_ptl_submit|. Note that this includes control packets +sent by the transport layer itself. Internally, data packets can be +re-submitted to this queue due to timeouts or NAK packets sent by the EC. + +Pending Set +----------- + +The pending set is the second of the two fundamental collections in the +packet transport layer. It stores references to packets that have already +been transmitted, but wait for acknowledgment (e.g. the corresponding ACK +packet) by the EC. + +Note that a packet may both be pending and queued if it has been +re-submitted due to a packet acknowledgment timeout or NAK. On such a +re-submission, packets are not removed from the pending set. + +Transmitter Thread +------------------ + +The transmitter thread is responsible for most of the actual work regarding +packet transmission. In each iteration, it (waits for and) checks if the +next packet on the queue (if any) can be transmitted and, if so, removes it +from the queue and increments its counter for the number of transmission +attempts, i.e. tries. If the packet is sequenced, i.e. requires an ACK by +the EC, the packet is added to the pending set. Next, the packet's data is +submitted to the serdev subsystem. In case of an error or timeout during +this submission, the packet is completed by the transmitter thread with the +status value of the callback set accordingly. In case the packet is +unsequenced, i.e. does not require an ACK by the EC, the packet is completed +with success on the transmitter thread. + +Transmission of sequenced packets is limited by the number of concurrently +pending packets, i.e. a limit on how many packets may be waiting for an ACK +from the EC in parallel. This limit is currently set to one (see :doc:`ssh` +for the reasoning behind this). Control packets (i.e. ACK and NAK) can +always be transmitted. + +Receiver Thread +--------------- + +Any data received from the EC is put into a FIFO buffer for further +processing. This processing happens on the receiver thread. The receiver +thread parses and validates the received message into its |ssh_frame| and +corresponding payload. It prepares and submits the necessary ACK (and on +validation error or invalid data NAK) packets for the received messages. + +This thread also handles further processing, such as matching ACK messages +to the corresponding pending packet (via sequence ID) and completing it, as +well as initiating re-submission of all currently pending packets on +receival of a NAK message (re-submission in case of a NAK is similar to +re-submission due to timeout, see below for more details on that). Note that +the successful completion of a sequenced packet will always run on the +receiver thread (whereas any failure-indicating completion will run on the +process where the failure occurred). + +Any payload data is forwarded via a callback to the next upper layer, i.e. +the request transport layer. + +Timeout Reaper +-------------- + +The packet acknowledgment timeout is a per-packet timeout for sequenced +packets, started when the respective packet begins (re-)transmission (i.e. +this timeout is armed once per transmission attempt on the transmitter +thread). It is used to trigger re-submission or, when the number of tries +has been exceeded, cancellation of the packet in question. + +This timeout is handled via a dedicated reaper task, which is essentially a +work item (re-)scheduled to run when the next packet is set to time out. The +work item then checks the set of pending packets for any packets that have +exceeded the timeout and, if there are any remaining packets, re-schedules +itself to the next appropriate point in time. + +If a timeout has been detected by the reaper, the packet will either be +re-submitted if it still has some remaining tries left, or completed with +``-ETIMEDOUT`` as status if not. Note that re-submission, in this case and +triggered by receival of a NAK, means that the packet is added to the queue +with a now incremented number of tries, yielding a higher priority. The +timeout for the packet will be disabled until the next transmission attempt +and the packet remains on the pending set. + +Note that due to transmission and packet acknowledgment timeouts, the packet +transport layer is always guaranteed to make progress, if only through +timing out packets, and will never fully block. + +Concurrency and Locking +----------------------- + +There are two main locks in the packet transport layer: One guarding access +to the packet queue and one guarding access to the pending set. These +collections may only be accessed and modified under the respective lock. If +access to both collections is needed, the pending lock must be acquired +before the queue lock to avoid deadlocks. + +In addition to guarding the collections, after initial packet submission +certain packet fields may only be accessed under one of the locks. +Specifically, the packet priority must only be accessed while holding the +queue lock and the packet timestamp must only be accessed while holding the +pending lock. + +Other parts of the packet transport layer are guarded independently. State +flags are managed by atomic bit operations and, if necessary, memory +barriers. Modifications to the timeout reaper work item and expiration date +are guarded by their own lock. + +The reference of the packet to the packet transport layer (``ptl``) is +somewhat special. It is either set when the upper layer request is submitted +or, if there is none, when the packet is first submitted. After it is set, +it will not change its value. Functions that may run concurrently with +submission, i.e. cancellation, can not rely on the ``ptl`` reference to be +set. Access to it in these functions is guarded by ``READ_ONCE()``, whereas +setting ``ptl`` is equally guarded with ``WRITE_ONCE()`` for symmetry. + +Some packet fields may be read outside of the respective locks guarding +them, specifically priority and state for tracing. In those cases, proper +access is ensured by employing ``WRITE_ONCE()`` and ``READ_ONCE()``. Such +read-only access is only allowed when stale values are not critical. + +With respect to the interface for higher layers, packet submission +(|ssh_ptl_submit|), packet cancellation (|ssh_ptl_cancel|), data receival +(|ssh_ptl_rx_rcvbuf|), and layer shutdown (|ssh_ptl_shutdown|) may always be +executed concurrently with respect to each other. Note that packet +submission may not run concurrently with itself for the same packet. +Equally, shutdown and data receival may also not run concurrently with +themselves (but may run concurrently with each other). + + +Request Transport Layer +======================= + +The request transport layer is represented via |ssh_rtl| and builds on top +of the packet transport layer. It deals with requests, i.e. SSH packets sent +by the host containing a |ssh_command| as frame payload. This layer +separates responses to requests from events, which are also sent by the EC +via a |ssh_command| payload. While responses are handled in this layer, +events are relayed to the next upper layer, i.e. the controller layer, via +the corresponding callback. The request transport layer is structured around +the following key concepts: + +Request +------- + +Requests are packets with a command-type payload, sent from host to EC to +query data from or trigger an action on it (or both simultaneously). They +are represented by |ssh_request|, wrapping the underlying |ssh_packet| +storing its message data (i.e. SSH frame with command payload). Note that +all top-level representations, e.g. |ssam_request_sync| are built upon this +struct. + +As |ssh_request| extends |ssh_packet|, its lifetime is also managed by the +reference counter inside the packet struct (which can be accessed via +|ssh_request_get| and |ssh_request_put|). Once the counter reaches zero, the +``release()`` callback of the |ssh_request_ops| reference of the request is +called. + +Requests can have an optional response that is equally sent via a SSH +message with command-type payload (from EC to host). The party constructing +the request must know if a response is expected and mark this in the request +flags provided to |ssh_request_init|, so that the request transport layer +can wait for this response. + +Similar to |ssh_packet|, |ssh_request| also has a ``complete()`` callback +provided via its request ops reference and is guaranteed to be completed +before it is released once it has been submitted to the request transport +layer via |ssh_rtl_submit|. For a request without a response, successful +completion will occur once the underlying packet has been successfully +transmitted by the packet transport layer (i.e. from within the packet +completion callback). For a request with response, successful completion +will occur once the response has been received and matched to the request +via its request ID (which happens on the packet layer's data-received +callback running on the receiver thread). If the request is completed with +an error, the status value will be set to the corresponding (negative) errno +value. + +The state of a request is again managed via its ``state`` flags +(|ssh_request_flags|), which also encode the request type. In particular, +the following bits are noteworthy: + +* ``SSH_REQUEST_SF_LOCKED_BIT``: This bit is set when completion, either + through error or success, is imminent. It indicates that no further + references of the request should be taken and any existing references + should be dropped as soon as possible. The process setting this bit is + responsible for removing any references to this request from the request + queue and pending set. + +* ``SSH_REQUEST_SF_COMPLETED_BIT``: This bit is set by the process running the + ``complete()`` callback and is used to ensure that this callback only runs + once. + +* ``SSH_REQUEST_SF_QUEUED_BIT``: This bit is set when the request is queued on + the request queue and cleared when it is dequeued. + +* ``SSH_REQUEST_SF_PENDING_BIT``: This bit is set when the request is added to + the pending set and cleared when it is removed from it. + +Request Queue +------------- + +The request queue is the first of the two fundamental collections in the +request transport layer. In contrast to the packet queue of the packet +transport layer, it is not a priority queue and the simple first come first +serve principle applies. + +All requests to be transmitted by the request transport layer must be +submitted to this queue via |ssh_rtl_submit|. Once submitted, requests may +not be re-submitted, and will not be re-submitted automatically on timeout. +Instead, the request is completed with a timeout error. If desired, the +caller can create and submit a new request for another try, but it must not +submit the same request again. + +Pending Set +----------- + +The pending set is the second of the two fundamental collections in the +request transport layer. This collection stores references to all pending +requests, i.e. requests awaiting a response from the EC (similar to what the +pending set of the packet transport layer does for packets). + +Transmitter Task +---------------- + +The transmitter task is scheduled when a new request is available for +transmission. It checks if the next request on the request queue can be +transmitted and, if so, submits its underlying packet to the packet +transport layer. This check ensures that only a limited number of +requests can be pending, i.e. waiting for a response, at the same time. If +the request requires a response, the request is added to the pending set +before its packet is submitted. + +Packet Completion Callback +-------------------------- + +The packet completion callback is executed once the underlying packet of a +request has been completed. In case of an error completion, the +corresponding request is completed with the error value provided in this +callback. + +On successful packet completion, further processing depends on the request. +If the request expects a response, it is marked as transmitted and the +request timeout is started. If the request does not expect a response, it is +completed with success. + +Data-Received Callback +---------------------- + +The data received callback notifies the request transport layer of data +being received by the underlying packet transport layer via a data-type +frame. In general, this is expected to be a command-type payload. + +If the request ID of the command is one of the request IDs reserved for +events (one to ``SSH_NUM_EVENTS``, inclusively), it is forwarded to the +event callback registered in the request transport layer. If the request ID +indicates a response to a request, the respective request is looked up in +the pending set and, if found and marked as transmitted, completed with +success. + +Timeout Reaper +-------------- + +The request-response-timeout is a per-request timeout for requests expecting +a response. It is used to ensure that a request does not wait indefinitely +on a response from the EC and is started after the underlying packet has +been successfully completed. + +This timeout is, similar to the packet acknowledgment timeout on the packet +transport layer, handled via a dedicated reaper task. This task is +essentially a work-item (re-)scheduled to run when the next request is set +to time out. The work item then scans the set of pending requests for any +requests that have timed out and completes them with ``-ETIMEDOUT`` as +status. Requests will not be re-submitted automatically. Instead, the issuer +of the request must construct and submit a new request, if so desired. + +Note that this timeout, in combination with packet transmission and +acknowledgment timeouts, guarantees that the request layer will always make +progress, even if only through timing out packets, and never fully block. + +Concurrency and Locking +----------------------- + +Similar to the packet transport layer, there are two main locks in the +request transport layer: One guarding access to the request queue and one +guarding access to the pending set. These collections may only be accessed +and modified under the respective lock. + +Other parts of the request transport layer are guarded independently. State +flags are (again) managed by atomic bit operations and, if necessary, memory +barriers. Modifications to the timeout reaper work item and expiration date +are guarded by their own lock. + +Some request fields may be read outside of the respective locks guarding +them, specifically the state for tracing. In those cases, proper access is +ensured by employing ``WRITE_ONCE()`` and ``READ_ONCE()``. Such read-only +access is only allowed when stale values are not critical. + +With respect to the interface for higher layers, request submission +(|ssh_rtl_submit|), request cancellation (|ssh_rtl_cancel|), and layer +shutdown (|ssh_rtl_shutdown|) may always be executed concurrently with +respect to each other. Note that request submission may not run concurrently +with itself for the same request (and also may only be called once per +request). Equally, shutdown may also not run concurrently with itself. + + +Controller Layer +================ + +The controller layer extends on the request transport layer to provide an +easy-to-use interface for client drivers. It is represented by +|ssam_controller| and the SSH driver. While the lower level transport layers +take care of transmitting and handling packets and requests, the controller +layer takes on more of a management role. Specifically, it handles device +initialization, power management, and event handling, including event +delivery and registration via the (event) completion system (|ssam_cplt|). + +Event Registration +------------------ + +In general, an event (or rather a class of events) has to be explicitly +requested by the host before the EC will send it (HID input events seem to +be the exception). This is done via an event-enable request (similarly, +events should be disabled via an event-disable request once no longer +desired). + +The specific request used to enable (or disable) an event is given via an +event registry, i.e. the governing authority of this event (so to speak), +represented by |ssam_event_registry|. As parameters to this request, the +target category and, depending on the event registry, instance ID of the +event to be enabled must be provided. This (optional) instance ID must be +zero if the registry does not use it. Together, target category and instance +ID form the event ID, represented by |ssam_event_id|. In short, both, event +registry and event ID, are required to uniquely identify a respective class +of events. + +Note that a further *request ID* parameter must be provided for the +enable-event request. This parameter does not influence the class of events +being enabled, but instead is set as the request ID (RQID) on each event of +this class sent by the EC. It is used to identify events (as a limited +number of request IDs is reserved for use in events only, specifically one +to ``SSH_NUM_EVENTS`` inclusively) and also map events to their specific +class. Currently, the controller always sets this parameter to the target +category specified in |ssam_event_id|. + +As multiple client drivers may rely on the same (or overlapping) classes of +events and enable/disable calls are strictly binary (i.e. on/off), the +controller has to manage access to these events. It does so via reference +counting, storing the counter inside an RB-tree based mapping with event +registry and ID as key (there is no known list of valid event registry and +event ID combinations). See |ssam_nf|, |ssam_nf_refcount_inc|, and +|ssam_nf_refcount_dec| for details. + +This management is done together with notifier registration (described in +the next section) via the top-level |ssam_notifier_register| and +|ssam_notifier_unregister| functions. + +Event Delivery +-------------- + +To receive events, a client driver has to register an event notifier via +|ssam_notifier_register|. This increments the reference counter for that +specific class of events (as detailed in the previous section), enables the +class on the EC (if it has not been enabled already), and installs the +provided notifier callback. + +Notifier callbacks are stored in lists, with one (RCU) list per target +category (provided via the event ID; NB: there is a fixed known number of +target categories). There is no known association from the combination of +event registry and event ID to the command data (target ID, target category, +command ID, and instance ID) that can be provided by an event class, apart +from target category and instance ID given via the event ID. + +Note that due to the way notifiers are (or rather have to be) stored, client +drivers may receive events that they have not requested and need to account +for them. Specifically, they will, by default, receive all events from the +same target category. To simplify dealing with this, filtering of events by +target ID (provided via the event registry) and instance ID (provided via +the event ID) can be requested when registering a notifier. This filtering +is applied when iterating over the notifiers at the time they are executed. + +All notifier callbacks are executed on a dedicated workqueue, the so-called +completion workqueue. After an event has been received via the callback +installed in the request layer (running on the receiver thread of the packet +transport layer), it will be put on its respective event queue +(|ssam_event_queue|). From this event queue the completion work item of that +queue (running on the completion workqueue) will pick up the event and +execute the notifier callback. This is done to avoid blocking on the +receiver thread. + +There is one event queue per combination of target ID and target category. +This is done to ensure that notifier callbacks are executed in sequence for +events of the same target ID and target category. Callbacks can be executed +in parallel for events with a different combination of target ID and target +category. + +Concurrency and Locking +----------------------- + +Most of the concurrency related safety guarantees of the controller are +provided by the lower-level request transport layer. In addition to this, +event (un-)registration is guarded by its own lock. + +Access to the controller state is guarded by the state lock. This lock is a +read/write semaphore. The reader part can be used to ensure that the state +does not change while functions depending on the state to stay the same +(e.g. |ssam_notifier_register|, |ssam_notifier_unregister|, +|ssam_request_sync_submit|, and derivatives) are executed and this guarantee +is not already provided otherwise (e.g. through |ssam_client_bind| or +|ssam_client_link|). The writer part guards any transitions that will change +the state, i.e. initialization, destruction, suspension, and resumption. + +The controller state may be accessed (read-only) outside the state lock for +smoke-testing against invalid API usage (e.g. in |ssam_request_sync_submit|). +Note that such checks are not supposed to (and will not) protect against all +invalid usages, but rather aim to help catch them. In those cases, proper +variable access is ensured by employing ``WRITE_ONCE()`` and ``READ_ONCE()``. + +Assuming any preconditions on the state not changing have been satisfied, +all non-initialization and non-shutdown functions may run concurrently with +each other. This includes |ssam_notifier_register|, |ssam_notifier_unregister|, +|ssam_request_sync_submit|, as well as all functions building on top of those. diff --git a/Documentation/driver-api/surface_aggregator/overview.rst b/Documentation/driver-api/surface_aggregator/overview.rst new file mode 100644 index 0000000000000000000000000000000000000000..1e9d57e500637165f1a80ece4c2538e8ea2cb70c --- /dev/null +++ b/Documentation/driver-api/surface_aggregator/overview.rst @@ -0,0 +1,77 @@ +.. SPDX-License-Identifier: GPL-2.0+ + +======== +Overview +======== + +The Surface/System Aggregator Module (SAM, SSAM) is an (arguably *the*) +embedded controller (EC) on Microsoft Surface devices. It has been originally +introduced on 4th generation devices (Surface Pro 4, Surface Book 1), but +its responsibilities and feature-set have since been expanded significantly +with the following generations. + + +Features and Integration +======================== + +Not much is currently known about SAM on 4th generation devices (Surface Pro +4, Surface Book 1), due to the use of a different communication interface +between host and EC (as detailed below). On 5th (Surface Pro 2017, Surface +Book 2, Surface Laptop 1) and later generation devices, SAM is responsible +for providing battery information (both current status and static values, +such as maximum capacity etc.), as well as an assortment of temperature +sensors (e.g. skin temperature) and cooling/performance-mode setting to the +host. On the Surface Book 2, specifically, it additionally provides an +interface for properly handling clipboard detachment (i.e. separating the +display part from the keyboard part of the device), on the Surface Laptop 1 +and 2 it is required for keyboard HID input. This HID subsystem has been +restructured for 7th generation devices and on those, specifically Surface +Laptop 3 and Surface Book 3, is responsible for all major HID input (i.e. +keyboard and touchpad). + +While features have not changed much on a coarse level since the 5th +generation, internal interfaces have undergone some rather large changes. On +5th and 6th generation devices, both battery and temperature information is +exposed to ACPI via a shim driver (referred to as Surface ACPI Notify, or +SAN), translating ACPI generic serial bus write-/read-accesses to SAM +requests. On 7th generation devices, this additional layer is gone and these +devices require a driver hooking directly into the SAM interface. Equally, +on newer generations, less devices are declared in ACPI, making them a bit +harder to discover and requiring us to hard-code a sort of device registry. +Due to this, a SSAM bus and subsystem with client devices +(:c:type:`struct ssam_device `) has been implemented. + + +Communication +============= + +The type of communication interface between host and EC depends on the +generation of the Surface device. On 4th generation devices, host and EC +communicate via HID, specifically using a HID-over-I2C device, whereas on +5th and later generations, communication takes place via a USART serial +device. In accordance to the drivers found on other operating systems, we +refer to the serial device and its driver as Surface Serial Hub (SSH). When +needed, we differentiate between both types of SAM by referring to them as +SAM-over-SSH and SAM-over-HID. + +Currently, this subsystem only supports SAM-over-SSH. The SSH communication +interface is described in more detail below. The HID interface has not been +reverse engineered yet and it is, at the moment, unclear how many (and +which) concepts of the SSH interface detailed below can be transferred to +it. + +Surface Serial Hub +------------------ + +As already elaborated above, the Surface Serial Hub (SSH) is the +communication interface for SAM on 5th- and all later-generation Surface +devices. On the highest level, communication can be separated into two main +types: Requests, messages sent from host to EC that may trigger a direct +response from the EC (explicitly associated with the request), and events +(sometimes also referred to as notifications), sent from EC to host without +being a direct response to a previous request. We may also refer to requests +without response as commands. In general, events need to be enabled via one +of multiple dedicated requests before they are sent by the EC. + +See :doc:`ssh` for a more technical protocol documentation and +:doc:`internal` for an overview of the internal driver architecture. diff --git a/Documentation/driver-api/surface_aggregator/ssh.rst b/Documentation/driver-api/surface_aggregator/ssh.rst new file mode 100644 index 0000000000000000000000000000000000000000..bf007d6c987321edf9dee963dce8c56faef7810b --- /dev/null +++ b/Documentation/driver-api/surface_aggregator/ssh.rst @@ -0,0 +1,344 @@ +.. SPDX-License-Identifier: GPL-2.0+ + +.. |u8| replace:: :c:type:`u8 ` +.. |u16| replace:: :c:type:`u16 ` +.. |TYPE| replace:: ``TYPE`` +.. |LEN| replace:: ``LEN`` +.. |SEQ| replace:: ``SEQ`` +.. |SYN| replace:: ``SYN`` +.. |NAK| replace:: ``NAK`` +.. |ACK| replace:: ``ACK`` +.. |DATA| replace:: ``DATA`` +.. |DATA_SEQ| replace:: ``DATA_SEQ`` +.. |DATA_NSQ| replace:: ``DATA_NSQ`` +.. |TC| replace:: ``TC`` +.. |TID| replace:: ``TID`` +.. |IID| replace:: ``IID`` +.. |RQID| replace:: ``RQID`` +.. |CID| replace:: ``CID`` + +=========================== +Surface Serial Hub Protocol +=========================== + +The Surface Serial Hub (SSH) is the central communication interface for the +embedded Surface Aggregator Module controller (SAM or EC), found on newer +Surface generations. We will refer to this protocol and interface as +SAM-over-SSH, as opposed to SAM-over-HID for the older generations. + +On Surface devices with SAM-over-SSH, SAM is connected to the host via UART +and defined in ACPI as device with ID ``MSHW0084``. On these devices, +significant functionality is provided via SAM, including access to battery +and power information and events, thermal read-outs and events, and many +more. For Surface Laptops, keyboard input is handled via HID directed +through SAM, on the Surface Laptop 3 and Surface Book 3 this also includes +touchpad input. + +Note that the standard disclaimer for this subsystem also applies to this +document: All of this has been reverse-engineered and may thus be erroneous +and/or incomplete. + +All CRCs used in the following are two-byte ``crc_ccitt_false(0xffff, ...)``. +All multi-byte values are little-endian, there is no implicit padding between +values. + + +SSH Packet Protocol: Definitions +================================ + +The fundamental communication unit of the SSH protocol is a frame +(:c:type:`struct ssh_frame `). A frame consists of the following +fields, packed together and in order: + +.. flat-table:: SSH Frame + :widths: 1 1 4 + :header-rows: 1 + + * - Field + - Type + - Description + + * - |TYPE| + - |u8| + - Type identifier of the frame. + + * - |LEN| + - |u16| + - Length of the payload associated with the frame. + + * - |SEQ| + - |u8| + - Sequence ID (see explanation below). + +Each frame structure is followed by a CRC over this structure. The CRC over +the frame structure (|TYPE|, |LEN|, and |SEQ| fields) is placed directly +after the frame structure and before the payload. The payload is followed by +its own CRC (over all payload bytes). If the payload is not present (i.e. +the frame has ``LEN=0``), the CRC of the payload is still present and will +evaluate to ``0xffff``. The |LEN| field does not include any of the CRCs, it +equals the number of bytes inbetween the CRC of the frame and the CRC of the +payload. + +Additionally, the following fixed two-byte sequences are used: + +.. flat-table:: SSH Byte Sequences + :widths: 1 1 4 + :header-rows: 1 + + * - Name + - Value + - Description + + * - |SYN| + - ``[0xAA, 0x55]`` + - Synchronization bytes. + +A message consists of |SYN|, followed by the frame (|TYPE|, |LEN|, |SEQ| and +CRC) and, if specified in the frame (i.e. ``LEN > 0``), payload bytes, +followed finally, regardless if the payload is present, the payload CRC. The +messages corresponding to an exchange are, in part, identified by having the +same sequence ID (|SEQ|), stored inside the frame (more on this in the next +section). The sequence ID is a wrapping counter. + +A frame can have the following types +(:c:type:`enum ssh_frame_type `): + +.. flat-table:: SSH Frame Types + :widths: 1 1 4 + :header-rows: 1 + + * - Name + - Value + - Short Description + + * - |NAK| + - ``0x04`` + - Sent on error in previously received message. + + * - |ACK| + - ``0x40`` + - Sent to acknowledge receival of |DATA| frame. + + * - |DATA_SEQ| + - ``0x80`` + - Sent to transfer data. Sequenced. + + * - |DATA_NSQ| + - ``0x00`` + - Same as |DATA_SEQ|, but does not need to be ACKed. + +Both |NAK|- and |ACK|-type frames are used to control flow of messages and +thus do not carry a payload. |DATA_SEQ|- and |DATA_NSQ|-type frames on the +other hand must carry a payload. The flow sequence and interaction of +different frame types will be described in more depth in the next section. + + +SSH Packet Protocol: Flow Sequence +================================== + +Each exchange begins with |SYN|, followed by a |DATA_SEQ|- or +|DATA_NSQ|-type frame, followed by its CRC, payload, and payload CRC. In +case of a |DATA_NSQ|-type frame, the exchange is then finished. In case of a +|DATA_SEQ|-type frame, the receiving party has to acknowledge receival of +the frame by responding with a message containing an |ACK|-type frame with +the same sequence ID of the |DATA| frame. In other words, the sequence ID of +the |ACK| frame specifies the |DATA| frame to be acknowledged. In case of an +error, e.g. an invalid CRC, the receiving party responds with a message +containing an |NAK|-type frame. As the sequence ID of the previous data +frame, for which an error is indicated via the |NAK| frame, cannot be relied +upon, the sequence ID of the |NAK| frame should not be used and is set to +zero. After receival of an |NAK| frame, the sending party should re-send all +outstanding (non-ACKed) messages. + +Sequence IDs are not synchronized between the two parties, meaning that they +are managed independently for each party. Identifying the messages +corresponding to a single exchange thus relies on the sequence ID as well as +the type of the message, and the context. Specifically, the sequence ID is +used to associate an ``ACK`` with its ``DATA_SEQ``-type frame, but not +``DATA_SEQ``- or ``DATA_NSQ``-type frames with other ``DATA``- type frames. + +An example exchange might look like this: + +:: + + tx: -- SYN FRAME(D) CRC(F) PAYLOAD CRC(P) ----------------------------- + rx: ------------------------------------- SYN FRAME(A) CRC(F) CRC(P) -- + +where both frames have the same sequence ID (``SEQ``). Here, ``FRAME(D)`` +indicates a |DATA_SEQ|-type frame, ``FRAME(A)`` an ``ACK``-type frame, +``CRC(F)`` the CRC over the previous frame, ``CRC(P)`` the CRC over the +previous payload. In case of an error, the exchange would look like this: + +:: + + tx: -- SYN FRAME(D) CRC(F) PAYLOAD CRC(P) ----------------------------- + rx: ------------------------------------- SYN FRAME(N) CRC(F) CRC(P) -- + +upon which the sender should re-send the message. ``FRAME(N)`` indicates an +|NAK|-type frame. Note that the sequence ID of the |NAK|-type frame is fixed +to zero. For |DATA_NSQ|-type frames, both exchanges are the same: + +:: + + tx: -- SYN FRAME(DATA_NSQ) CRC(F) PAYLOAD CRC(P) ---------------------- + rx: ------------------------------------------------------------------- + +Here, an error can be detected, but not corrected or indicated to the +sending party. These exchanges are symmetric, i.e. switching ``rx`` and +``tx`` results again in a valid exchange. Currently, no longer exchanges are +known. + + +Commands: Requests, Responses, and Events +========================================= + +Commands are sent as payload inside a data frame. Currently, this is the +only known payload type of |DATA| frames, with a payload-type value of +``0x80`` (:c:type:`SSH_PLD_TYPE_CMD `). + +The command-type payload (:c:type:`struct ssh_command `) +consists of an eight-byte command structure, followed by optional and +variable length command data. The length of this optional data is derived +from the frame payload length given in the corresponding frame, i.e. it is +``frame.len - sizeof(struct ssh_command)``. The command struct contains the +following fields, packed together and in order: + +.. flat-table:: SSH Command + :widths: 1 1 4 + :header-rows: 1 + + * - Field + - Type + - Description + + * - |TYPE| + - |u8| + - Type of the payload. For commands always ``0x80``. + + * - |TC| + - |u8| + - Target category. + + * - |TID| (out) + - |u8| + - Target ID for outgoing (host to EC) commands. + + * - |TID| (in) + - |u8| + - Target ID for incoming (EC to host) commands. + + * - |IID| + - |u8| + - Instance ID. + + * - |RQID| + - |u16| + - Request ID. + + * - |CID| + - |u8| + - Command ID. + +The command struct and data, in general, does not contain any failure +detection mechanism (e.g. CRCs), this is solely done on the frame level. + +Command-type payloads are used by the host to send commands and requests to +the EC as well as by the EC to send responses and events back to the host. +We differentiate between requests (sent by the host), responses (sent by the +EC in response to a request), and events (sent by the EC without a preceding +request). + +Commands and events are uniquely identified by their target category +(``TC``) and command ID (``CID``). The target category specifies a general +category for the command (e.g. system in general, vs. battery and AC, vs. +temperature, and so on), while the command ID specifies the command inside +that category. Only the combination of |TC| + |CID| is unique. Additionally, +commands have an instance ID (``IID``), which is used to differentiate +between different sub-devices. For example ``TC=3`` ``CID=1`` is a +request to get the temperature on a thermal sensor, where |IID| specifies +the respective sensor. If the instance ID is not used, it should be set to +zero. If instance IDs are used, they, in general, start with a value of one, +whereas zero may be used for instance independent queries, if applicable. A +response to a request should have the same target category, command ID, and +instance ID as the corresponding request. + +Responses are matched to their corresponding request via the request ID +(``RQID``) field. This is a 16 bit wrapping counter similar to the sequence +ID on the frames. Note that the sequence ID of the frames for a +request-response pair does not match. Only the request ID has to match. +Frame-protocol wise these are two separate exchanges, and may even be +separated, e.g. by an event being sent after the request but before the +response. Not all commands produce a response, and this is not detectable by +|TC| + |CID|. It is the responsibility of the issuing party to wait for a +response (or signal this to the communication framework, as is done in +SAN/ACPI via the ``SNC`` flag). + +Events are identified by unique and reserved request IDs. These IDs should +not be used by the host when sending a new request. They are used on the +host to, first, detect events and, second, match them with a registered +event handler. Request IDs for events are chosen by the host and directed to +the EC when setting up and enabling an event source (via the +enable-event-source request). The EC then uses the specified request ID for +events sent from the respective source. Note that an event should still be +identified by its target category, command ID, and, if applicable, instance +ID, as a single event source can send multiple different event types. In +general, however, a single target category should map to a single reserved +event request ID. + +Furthermore, requests, responses, and events have an associated target ID +(``TID``). This target ID is split into output (host to EC) and input (EC to +host) fields, with the respecting other field (e.g. output field on incoming +messages) set to zero. Two ``TID`` values are known: Primary (``0x01``) and +secondary (``0x02``). In general, the response to a request should have the +same ``TID`` value, however, the field (output vs. input) should be used in +accordance to the direction in which the response is sent (i.e. on the input +field, as responses are generally sent from the EC to the host). + +Note that, even though requests and events should be uniquely identifiable +by target category and command ID alone, the EC may require specific +target ID and instance ID values to accept a command. A command that is +accepted for ``TID=1``, for example, may not be accepted for ``TID=2`` +and vice versa. + + +Limitations and Observations +============================ + +The protocol can, in theory, handle up to ``U8_MAX`` frames in parallel, +with up to ``U16_MAX`` pending requests (neglecting request IDs reserved for +events). In practice, however, this is more limited. From our testing +(although via a python and thus a user-space program), it seems that the EC +can handle up to four requests (mostly) reliably in parallel at a certain +time. With five or more requests in parallel, consistent discarding of +commands (ACKed frame but no command response) has been observed. For five +simultaneous commands, this reproducibly resulted in one command being +dropped and four commands being handled. + +However, it has also been noted that, even with three requests in parallel, +occasional frame drops happen. Apart from this, with a limit of three +pending requests, no dropped commands (i.e. command being dropped but frame +carrying command being ACKed) have been observed. In any case, frames (and +possibly also commands) should be re-sent by the host if a certain timeout +is exceeded. This is done by the EC for frames with a timeout of one second, +up to two re-tries (i.e. three transmissions in total). The limit of +re-tries also applies to received NAKs, and, in a worst case scenario, can +lead to entire messages being dropped. + +While this also seems to work fine for pending data frames as long as no +transmission failures occur, implementation and handling of these seems to +depend on the assumption that there is only one non-acknowledged data frame. +In particular, the detection of repeated frames relies on the last sequence +number. This means that, if a frame that has been successfully received by +the EC is sent again, e.g. due to the host not receiving an |ACK|, the EC +will only detect this if it has the sequence ID of the last frame received +by the EC. As an example: Sending two frames with ``SEQ=0`` and ``SEQ=1`` +followed by a repetition of ``SEQ=0`` will not detect the second ``SEQ=0`` +frame as such, and thus execute the command in this frame each time it has +been received, i.e. twice in this example. Sending ``SEQ=0``, ``SEQ=1`` and +then repeating ``SEQ=1`` will detect the second ``SEQ=1`` as repetition of +the first one and ignore it, thus executing the contained command only once. + +In conclusion, this suggests a limit of at most one pending un-ACKed frame +(per party, effectively leading to synchronous communication regarding +frames) and at most three pending commands. The limit to synchronous frame +transfers seems to be consistent with behavior observed on Windows. diff --git a/Documentation/driver-api/thermal/sysfs-api.rst b/Documentation/driver-api/thermal/sysfs-api.rst index e7520cb439acdfcd7c48d1ccc4ae0d225e20a164..29fdd817ddb004587394cea730b636e139971703 100644 --- a/Documentation/driver-api/thermal/sysfs-api.rst +++ b/Documentation/driver-api/thermal/sysfs-api.rst @@ -54,7 +54,7 @@ temperature) and throttle appropriate devices. trips: the total number of trip points this thermal zone supports. mask: - Bit string: If 'n'th bit is set, then trip point 'n' is writeable. + Bit string: If 'n'th bit is set, then trip point 'n' is writable. devdata: device private data ops: @@ -406,7 +406,7 @@ Thermal cooling device sys I/F, created once it's registered:: |---stats/reset: Writing any value resets the statistics |---stats/time_in_state_ms: Time (msec) spent in various cooling states |---stats/total_trans: Total number of times cooling state is changed - |---stats/trans_table: Cooing state transition table + |---stats/trans_table: Cooling state transition table Then next two dynamic attributes are created/removed in pairs. They represent @@ -520,19 +520,6 @@ available_policies RW, Optional -passive - Attribute is only present for zones in which the passive cooling - policy is not supported by native thermal driver. Default is zero - and can be set to a temperature (in millidegrees) to enable a - passive trip point for the zone. Activation is done by polling with - an interval of 1 second. - - Unit: millidegrees Celsius - - Valid values: 0 (disabled) or greater than 1000 - - RW, Optional - emul_temp Interface to set the emulated temperature method in thermal zone (sensor). After setting this temperature, the thermal zone may pass @@ -779,5 +766,5 @@ emergency poweroff kicks in after the delay has elapsed and shuts down the system. If set to 0 emergency poweroff will not be supported. So a carefully -profiled non-zero positive value is a must for emergerncy poweroff to be +profiled non-zero positive value is a must for emergency poweroff to be triggered. diff --git a/Documentation/features/core/cBPF-JIT/arch-support.txt b/Documentation/features/core/cBPF-JIT/arch-support.txt index 3999356168133920b6adb1598d52494b1cdb4046..e59b5215402d0f97936aaec159e3e8564e092531 100644 --- a/Documentation/features/core/cBPF-JIT/arch-support.txt +++ b/Documentation/features/core/cBPF-JIT/arch-support.txt @@ -10,7 +10,6 @@ | arc: | TODO | | arm: | TODO | | arm64: | TODO | - | c6x: | TODO | | csky: | TODO | | h8300: | TODO | | hexagon: | TODO | diff --git a/Documentation/features/core/eBPF-JIT/arch-support.txt b/Documentation/features/core/eBPF-JIT/arch-support.txt index 79409bfe0263a754c7af86a685e135445a8392ff..dcbd8679f514a329d4658a3b9892f9977ea621b9 100644 --- a/Documentation/features/core/eBPF-JIT/arch-support.txt +++ b/Documentation/features/core/eBPF-JIT/arch-support.txt @@ -10,7 +10,6 @@ | arc: | TODO | | arm: | ok | | arm64: | ok | - | c6x: | TODO | | csky: | TODO | | h8300: | TODO | | hexagon: | TODO | diff --git a/Documentation/features/core/generic-idle-thread/arch-support.txt b/Documentation/features/core/generic-idle-thread/arch-support.txt index 9ea60e416efdaabbb3867999ef7ca3aaff087db6..4efcba7b523901b72dccaa2db0402bd883e3d39c 100644 --- a/Documentation/features/core/generic-idle-thread/arch-support.txt +++ b/Documentation/features/core/generic-idle-thread/arch-support.txt @@ -10,7 +10,6 @@ | arc: | ok | | arm: | ok | | arm64: | ok | - | c6x: | TODO | | csky: | ok | | h8300: | TODO | | hexagon: | ok | diff --git a/Documentation/features/core/jump-labels/arch-support.txt b/Documentation/features/core/jump-labels/arch-support.txt index 894d9693b3809ec9b7627230045228a4c048abb9..0c801d1bd2daaabe2c51dd68d719f4ef302eb2f2 100644 --- a/Documentation/features/core/jump-labels/arch-support.txt +++ b/Documentation/features/core/jump-labels/arch-support.txt @@ -10,7 +10,6 @@ | arc: | ok | | arm: | ok | | arm64: | ok | - | c6x: | TODO | | csky: | TODO | | h8300: | TODO | | hexagon: | TODO | diff --git a/Documentation/features/core/tracehook/arch-support.txt b/Documentation/features/core/tracehook/arch-support.txt index cd3510e2eedba3128916714fcb18cdac656537ff..af34308fce7fdba48f21d47df0e6f50c5d981507 100644 --- a/Documentation/features/core/tracehook/arch-support.txt +++ b/Documentation/features/core/tracehook/arch-support.txt @@ -10,7 +10,6 @@ | arc: | ok | | arm: | ok | | arm64: | ok | - | c6x: | ok | | csky: | ok | | h8300: | TODO | | hexagon: | ok | diff --git a/Documentation/features/debug/KASAN/arch-support.txt b/Documentation/features/debug/KASAN/arch-support.txt index b2288dc14b72674822dc793f70cb1069bb99994b..c244ac7eee2687e6bb02c55c7d8082b2907c6190 100644 --- a/Documentation/features/debug/KASAN/arch-support.txt +++ b/Documentation/features/debug/KASAN/arch-support.txt @@ -10,7 +10,6 @@ | arc: | TODO | | arm: | ok | | arm64: | ok | - | c6x: | TODO | | csky: | TODO | | h8300: | TODO | | hexagon: | TODO | diff --git a/Documentation/features/debug/debug-vm-pgtable/arch-support.txt b/Documentation/features/debug/debug-vm-pgtable/arch-support.txt index 1c49723e7534577b770a0a16507a3a71a6b2524b..7aff505af706d7b844307d418b5082bd23544542 100644 --- a/Documentation/features/debug/debug-vm-pgtable/arch-support.txt +++ b/Documentation/features/debug/debug-vm-pgtable/arch-support.txt @@ -10,7 +10,6 @@ | arc: | ok | | arm: | TODO | | arm64: | ok | - | c6x: | TODO | | csky: | TODO | | h8300: | TODO | | hexagon: | TODO | diff --git a/Documentation/features/debug/gcov-profile-all/arch-support.txt b/Documentation/features/debug/gcov-profile-all/arch-support.txt index 7563a494ddb85d67c9bf23e25615795a3783cb20..b39c1a5de3f36be0c6394f627fed5bc5ead2665a 100644 --- a/Documentation/features/debug/gcov-profile-all/arch-support.txt +++ b/Documentation/features/debug/gcov-profile-all/arch-support.txt @@ -10,14 +10,13 @@ | arc: | TODO | | arm: | ok | | arm64: | ok | - | c6x: | TODO | | csky: | ok | | h8300: | TODO | | hexagon: | TODO | | ia64: | TODO | | m68k: | TODO | | microblaze: | ok | - | mips: | TODO | + | mips: | ok | | nds32: | TODO | | nios2: | TODO | | openrisc: | TODO | diff --git a/Documentation/features/debug/kcov/arch-support.txt b/Documentation/features/debug/kcov/arch-support.txt index ab0ee1c933c238a9a27d8a53d0259f62469b8549..7e44013cc32090ae6871c55f159376ca7f112743 100644 --- a/Documentation/features/debug/kcov/arch-support.txt +++ b/Documentation/features/debug/kcov/arch-support.txt @@ -10,7 +10,6 @@ | arc: | TODO | | arm: | ok | | arm64: | ok | - | c6x: | TODO | | csky: | TODO | | h8300: | TODO | | hexagon: | TODO | diff --git a/Documentation/features/debug/kgdb/arch-support.txt b/Documentation/features/debug/kgdb/arch-support.txt index bc45bac2044264bd6e2b5b74120b31a115f0b888..2cb0576f9180c4bb37181395d4b662cb04afd337 100644 --- a/Documentation/features/debug/kgdb/arch-support.txt +++ b/Documentation/features/debug/kgdb/arch-support.txt @@ -10,7 +10,6 @@ | arc: | ok | | arm: | ok | | arm64: | ok | - | c6x: | TODO | | csky: | TODO | | h8300: | ok | | hexagon: | ok | diff --git a/Documentation/features/debug/kmemleak/arch-support.txt b/Documentation/features/debug/kmemleak/arch-support.txt index 2db76807ec6f6fb8b813f1e57397a62f7ea3414a..e9ac415f8aecfe4c6f4a291946e841be4ce8a053 100644 --- a/Documentation/features/debug/kmemleak/arch-support.txt +++ b/Documentation/features/debug/kmemleak/arch-support.txt @@ -10,8 +10,7 @@ | arc: | ok | | arm: | ok | | arm64: | ok | - | c6x: | TODO | - | csky: | TODO | + | csky: | ok | | h8300: | TODO | | hexagon: | TODO | | ia64: | TODO | diff --git a/Documentation/features/debug/kprobes-on-ftrace/arch-support.txt b/Documentation/features/debug/kprobes-on-ftrace/arch-support.txt index 6225cfe0c5bf8aab1e41bdc1ab589ffae463b450..96156e8802a77d6834eb51d2478d362106454ec9 100644 --- a/Documentation/features/debug/kprobes-on-ftrace/arch-support.txt +++ b/Documentation/features/debug/kprobes-on-ftrace/arch-support.txt @@ -10,7 +10,6 @@ | arc: | TODO | | arm: | TODO | | arm64: | TODO | - | c6x: | TODO | | csky: | ok | | h8300: | TODO | | hexagon: | TODO | @@ -23,7 +22,7 @@ | openrisc: | TODO | | parisc: | ok | | powerpc: | ok | - | riscv: | TODO | + | riscv: | ok | | s390: | ok | | sh: | TODO | | sparc: | TODO | diff --git a/Documentation/features/debug/kprobes/arch-support.txt b/Documentation/features/debug/kprobes/arch-support.txt index 371f0ac488f54bb0cab313d63116f855fa8d7322..ee95ed61909afae96f173ac7c7650cdb5e7c9bc4 100644 --- a/Documentation/features/debug/kprobes/arch-support.txt +++ b/Documentation/features/debug/kprobes/arch-support.txt @@ -10,7 +10,6 @@ | arc: | ok | | arm: | ok | | arm64: | ok | - | c6x: | TODO | | csky: | ok | | h8300: | TODO | | hexagon: | TODO | @@ -23,7 +22,7 @@ | openrisc: | TODO | | parisc: | ok | | powerpc: | ok | - | riscv: | TODO | + | riscv: | ok | | s390: | ok | | sh: | ok | | sparc: | ok | diff --git a/Documentation/features/debug/kretprobes/arch-support.txt b/Documentation/features/debug/kretprobes/arch-support.txt index 38e95251deedf09c7bfbc481432431f9bc362bfd..612cb97d47b84d6aa35990731e219fc8261a7980 100644 --- a/Documentation/features/debug/kretprobes/arch-support.txt +++ b/Documentation/features/debug/kretprobes/arch-support.txt @@ -10,7 +10,6 @@ | arc: | ok | | arm: | ok | | arm64: | ok | - | c6x: | TODO | | csky: | ok | | h8300: | TODO | | hexagon: | TODO | @@ -23,7 +22,7 @@ | openrisc: | TODO | | parisc: | ok | | powerpc: | ok | - | riscv: | TODO | + | riscv: | ok | | s390: | ok | | sh: | ok | | sparc: | ok | diff --git a/Documentation/features/debug/optprobes/arch-support.txt b/Documentation/features/debug/optprobes/arch-support.txt index 7f4a20e6a12b3585239e328c3d8500cf006a54fb..d6ff141a6122e8fecb7b02abd6ba850dd5c28f11 100644 --- a/Documentation/features/debug/optprobes/arch-support.txt +++ b/Documentation/features/debug/optprobes/arch-support.txt @@ -10,7 +10,6 @@ | arc: | TODO | | arm: | ok | | arm64: | TODO | - | c6x: | TODO | | csky: | TODO | | h8300: | TODO | | hexagon: | TODO | diff --git a/Documentation/features/debug/stackprotector/arch-support.txt b/Documentation/features/debug/stackprotector/arch-support.txt index 3329559c8207dd85e31b03cddad85f5867f636fd..ad4de22a71ab9b614d71fc25efd49eba2f5252f2 100644 --- a/Documentation/features/debug/stackprotector/arch-support.txt +++ b/Documentation/features/debug/stackprotector/arch-support.txt @@ -10,7 +10,6 @@ | arc: | TODO | | arm: | ok | | arm64: | ok | - | c6x: | TODO | | csky: | ok | | h8300: | TODO | | hexagon: | TODO | diff --git a/Documentation/features/debug/uprobes/arch-support.txt b/Documentation/features/debug/uprobes/arch-support.txt index 43cac6ee0c6842879a439c04543d08971c25f77a..8bd5548a448575ad193b1d5e64ff78853a7eeefd 100644 --- a/Documentation/features/debug/uprobes/arch-support.txt +++ b/Documentation/features/debug/uprobes/arch-support.txt @@ -10,7 +10,6 @@ | arc: | TODO | | arm: | ok | | arm64: | ok | - | c6x: | TODO | | csky: | ok | | h8300: | TODO | | hexagon: | TODO | @@ -23,7 +22,7 @@ | openrisc: | TODO | | parisc: | TODO | | powerpc: | ok | - | riscv: | TODO | + | riscv: | ok | | s390: | ok | | sh: | TODO | | sparc: | ok | diff --git a/Documentation/features/debug/user-ret-profiler/arch-support.txt b/Documentation/features/debug/user-ret-profiler/arch-support.txt index d636ed0e679f1d26542b0c82860d7d0e7c2dc6de..2a3fe812a5fa0806b13d312fe8fe6e1572918102 100644 --- a/Documentation/features/debug/user-ret-profiler/arch-support.txt +++ b/Documentation/features/debug/user-ret-profiler/arch-support.txt @@ -10,7 +10,6 @@ | arc: | TODO | | arm: | TODO | | arm64: | TODO | - | c6x: | TODO | | csky: | TODO | | h8300: | TODO | | hexagon: | TODO | diff --git a/Documentation/features/io/dma-contiguous/arch-support.txt b/Documentation/features/io/dma-contiguous/arch-support.txt index dfc93d074e3d1a06ca2cbba7c0fc493629ae7f38..bece89586efacd027957730ecb48d3bf72a93bd2 100644 --- a/Documentation/features/io/dma-contiguous/arch-support.txt +++ b/Documentation/features/io/dma-contiguous/arch-support.txt @@ -10,7 +10,6 @@ | arc: | TODO | | arm: | ok | | arm64: | ok | - | c6x: | TODO | | csky: | ok | | h8300: | TODO | | hexagon: | TODO | diff --git a/Documentation/features/locking/cmpxchg-local/arch-support.txt b/Documentation/features/locking/cmpxchg-local/arch-support.txt index 1815c7fed06d18438fdf908289b1fc85f3934c0f..52bdda004f5c1d23972b5292bcbe628c0c9fa303 100644 --- a/Documentation/features/locking/cmpxchg-local/arch-support.txt +++ b/Documentation/features/locking/cmpxchg-local/arch-support.txt @@ -10,7 +10,6 @@ | arc: | TODO | | arm: | TODO | | arm64: | ok | - | c6x: | TODO | | csky: | TODO | | h8300: | TODO | | hexagon: | TODO | diff --git a/Documentation/features/locking/lockdep/arch-support.txt b/Documentation/features/locking/lockdep/arch-support.txt index 940b0bd02957542c89856e7d73fd254f395e089a..a8cd163c8b7ee25f61c2cbbd009712d0f3a79e2a 100644 --- a/Documentation/features/locking/lockdep/arch-support.txt +++ b/Documentation/features/locking/lockdep/arch-support.txt @@ -10,7 +10,6 @@ | arc: | ok | | arm: | ok | | arm64: | ok | - | c6x: | TODO | | csky: | ok | | h8300: | TODO | | hexagon: | ok | diff --git a/Documentation/features/locking/queued-rwlocks/arch-support.txt b/Documentation/features/locking/queued-rwlocks/arch-support.txt index 4dd5e554873fe93ea1ac7a039ab1f9865d8e278b..8c85949752b3b1013d396bcf528198021b3fd3ce 100644 --- a/Documentation/features/locking/queued-rwlocks/arch-support.txt +++ b/Documentation/features/locking/queued-rwlocks/arch-support.txt @@ -10,7 +10,6 @@ | arc: | TODO | | arm: | TODO | | arm64: | ok | - | c6x: | TODO | | csky: | ok | | h8300: | TODO | | hexagon: | TODO | diff --git a/Documentation/features/locking/queued-spinlocks/arch-support.txt b/Documentation/features/locking/queued-spinlocks/arch-support.txt index b16d4f71e5cecf77cb5e9a6df7204e1fd7b60e79..5f4e1b3841afab68c658f02b38efedc250648591 100644 --- a/Documentation/features/locking/queued-spinlocks/arch-support.txt +++ b/Documentation/features/locking/queued-spinlocks/arch-support.txt @@ -10,7 +10,6 @@ | arc: | TODO | | arm: | TODO | | arm64: | ok | - | c6x: | TODO | | csky: | TODO | | h8300: | TODO | | hexagon: | TODO | diff --git a/Documentation/features/perf/kprobes-event/arch-support.txt b/Documentation/features/perf/kprobes-event/arch-support.txt index 04c17c2106a48f90d7d1c1d526fc6214c981aee3..78f3fe080f0e9a0817d0294c3b59b4b0463868c4 100644 --- a/Documentation/features/perf/kprobes-event/arch-support.txt +++ b/Documentation/features/perf/kprobes-event/arch-support.txt @@ -10,7 +10,6 @@ | arc: | TODO | | arm: | ok | | arm64: | ok | - | c6x: | TODO | | csky: | ok | | h8300: | TODO | | hexagon: | ok | @@ -23,7 +22,7 @@ | openrisc: | TODO | | parisc: | ok | | powerpc: | ok | - | riscv: | TODO | + | riscv: | ok | | s390: | ok | | sh: | ok | | sparc: | ok | diff --git a/Documentation/features/perf/perf-regs/arch-support.txt b/Documentation/features/perf/perf-regs/arch-support.txt index e7450fbb825376400e0c682d0aeac0f097e541d7..5bf3b1854a1f5fac2b1a31d59d7a4a371d119a08 100644 --- a/Documentation/features/perf/perf-regs/arch-support.txt +++ b/Documentation/features/perf/perf-regs/arch-support.txt @@ -10,14 +10,13 @@ | arc: | TODO | | arm: | ok | | arm64: | ok | - | c6x: | TODO | | csky: | ok | | h8300: | TODO | | hexagon: | TODO | | ia64: | TODO | | m68k: | TODO | | microblaze: | TODO | - | mips: | TODO | + | mips: | ok | | nds32: | TODO | | nios2: | TODO | | openrisc: | TODO | diff --git a/Documentation/features/perf/perf-stackdump/arch-support.txt b/Documentation/features/perf/perf-stackdump/arch-support.txt index 98e79d128d9b469b00f2400ba358c9a7532f0f29..d88659bb4fc1c3479f5e688fce267da63e8704e1 100644 --- a/Documentation/features/perf/perf-stackdump/arch-support.txt +++ b/Documentation/features/perf/perf-stackdump/arch-support.txt @@ -10,14 +10,13 @@ | arc: | TODO | | arm: | ok | | arm64: | ok | - | c6x: | TODO | | csky: | ok | | h8300: | TODO | | hexagon: | TODO | | ia64: | TODO | | m68k: | TODO | | microblaze: | TODO | - | mips: | TODO | + | mips: | ok | | nds32: | TODO | | nios2: | TODO | | openrisc: | TODO | diff --git a/Documentation/features/sched/membarrier-sync-core/arch-support.txt b/Documentation/features/sched/membarrier-sync-core/arch-support.txt index 47e6903f47a5b722200e326254720ff1439cd2f6..883d33b265d6952e2bea8a014d108946dac45bdf 100644 --- a/Documentation/features/sched/membarrier-sync-core/arch-support.txt +++ b/Documentation/features/sched/membarrier-sync-core/arch-support.txt @@ -33,7 +33,6 @@ | arc: | TODO | | arm: | ok | | arm64: | ok | - | c6x: | TODO | | csky: | TODO | | h8300: | TODO | | hexagon: | TODO | diff --git a/Documentation/features/sched/numa-balancing/arch-support.txt b/Documentation/features/sched/numa-balancing/arch-support.txt index 964457ad26c1b6cb106591f302fee39391ca84e0..9affb7c2c500fdad120a143278dc57b39e07d2f6 100644 --- a/Documentation/features/sched/numa-balancing/arch-support.txt +++ b/Documentation/features/sched/numa-balancing/arch-support.txt @@ -10,7 +10,6 @@ | arc: | .. | | arm: | .. | | arm64: | ok | - | c6x: | .. | | csky: | .. | | h8300: | .. | | hexagon: | .. | @@ -23,7 +22,7 @@ | openrisc: | .. | | parisc: | .. | | powerpc: | ok | - | riscv: | TODO | + | riscv: | ok | | s390: | ok | | sh: | .. | | sparc: | TODO | diff --git a/Documentation/features/seccomp/seccomp-filter/arch-support.txt b/Documentation/features/seccomp/seccomp-filter/arch-support.txt index eb3d74092c61ea8538a58410cdd7289fc771ecc3..26eec58ab8197375f09e1c73ac0498d64992cec8 100644 --- a/Documentation/features/seccomp/seccomp-filter/arch-support.txt +++ b/Documentation/features/seccomp/seccomp-filter/arch-support.txt @@ -10,7 +10,6 @@ | arc: | TODO | | arm: | ok | | arm64: | ok | - | c6x: | TODO | | csky: | ok | | h8300: | TODO | | hexagon: | TODO | diff --git a/Documentation/features/time/arch-tick-broadcast/arch-support.txt b/Documentation/features/time/arch-tick-broadcast/arch-support.txt index 4d11cbb3c09b2497a1fa04944a029d0c6a327c6b..8639fe8315f50ed81ee96b23219c781f6b02459e 100644 --- a/Documentation/features/time/arch-tick-broadcast/arch-support.txt +++ b/Documentation/features/time/arch-tick-broadcast/arch-support.txt @@ -10,7 +10,6 @@ | arc: | TODO | | arm: | ok | | arm64: | ok | - | c6x: | TODO | | csky: | TODO | | h8300: | TODO | | hexagon: | TODO | diff --git a/Documentation/features/time/clockevents/arch-support.txt b/Documentation/features/time/clockevents/arch-support.txt index 6863a3fbddad64d2cfb69ce511c632871b708c18..9a81cb03b1fd50a0367cabc752ab6c76b52a7d05 100644 --- a/Documentation/features/time/clockevents/arch-support.txt +++ b/Documentation/features/time/clockevents/arch-support.txt @@ -10,7 +10,6 @@ | arc: | ok | | arm: | TODO | | arm64: | ok | - | c6x: | ok | | csky: | ok | | h8300: | ok | | hexagon: | ok | diff --git a/Documentation/features/time/context-tracking/arch-support.txt b/Documentation/features/time/context-tracking/arch-support.txt index 52aea275aab783512c75efb896177d21b4ffb3c5..4ed116c2ec391ee8963ba1e7a53767208e1b2a4d 100644 --- a/Documentation/features/time/context-tracking/arch-support.txt +++ b/Documentation/features/time/context-tracking/arch-support.txt @@ -10,7 +10,6 @@ | arc: | TODO | | arm: | ok | | arm64: | ok | - | c6x: | TODO | | csky: | ok | | h8300: | TODO | | hexagon: | TODO | diff --git a/Documentation/features/time/irq-time-acct/arch-support.txt b/Documentation/features/time/irq-time-acct/arch-support.txt index 6fc03deb1c3883e6471487bbac286131d4a03715..bc30c15557c7d703417e669698e720b35a74c45c 100644 --- a/Documentation/features/time/irq-time-acct/arch-support.txt +++ b/Documentation/features/time/irq-time-acct/arch-support.txt @@ -10,7 +10,6 @@ | arc: | TODO | | arm: | ok | | arm64: | ok | - | c6x: | TODO | | csky: | TODO | | h8300: | TODO | | hexagon: | TODO | diff --git a/Documentation/features/time/virt-cpuacct/arch-support.txt b/Documentation/features/time/virt-cpuacct/arch-support.txt index e51f3af38e3129dfda94fcf739522a111cd99561..050de43bbbb938fb4d2ddc9439ec17836212c601 100644 --- a/Documentation/features/time/virt-cpuacct/arch-support.txt +++ b/Documentation/features/time/virt-cpuacct/arch-support.txt @@ -10,7 +10,6 @@ | arc: | TODO | | arm: | ok | | arm64: | ok | - | c6x: | TODO | | csky: | ok | | h8300: | TODO | | hexagon: | TODO | diff --git a/Documentation/features/vm/ELF-ASLR/arch-support.txt b/Documentation/features/vm/ELF-ASLR/arch-support.txt index eccda07324742a72f850cbad88e2fd148654560e..99cb6d7f50051cab495d937747f733eaece9730e 100644 --- a/Documentation/features/vm/ELF-ASLR/arch-support.txt +++ b/Documentation/features/vm/ELF-ASLR/arch-support.txt @@ -10,7 +10,6 @@ | arc: | TODO | | arm: | ok | | arm64: | ok | - | c6x: | TODO | | csky: | TODO | | h8300: | TODO | | hexagon: | TODO | diff --git a/Documentation/features/vm/PG_uncached/arch-support.txt b/Documentation/features/vm/PG_uncached/arch-support.txt index c74e3f8040e1b7aed021fddf77d753490bb4e53d..6cde384585965a374f70bd56264776ba91b6a930 100644 --- a/Documentation/features/vm/PG_uncached/arch-support.txt +++ b/Documentation/features/vm/PG_uncached/arch-support.txt @@ -10,7 +10,6 @@ | arc: | TODO | | arm: | TODO | | arm64: | TODO | - | c6x: | TODO | | csky: | TODO | | h8300: | TODO | | hexagon: | TODO | diff --git a/Documentation/features/vm/THP/arch-support.txt b/Documentation/features/vm/THP/arch-support.txt index 1c0b95f2b40d917319655fc3ead67080617d5bb2..e8238cb2a4dabb7eb7a8d54a05ce96cbd3294400 100644 --- a/Documentation/features/vm/THP/arch-support.txt +++ b/Documentation/features/vm/THP/arch-support.txt @@ -10,7 +10,6 @@ | arc: | ok | | arm: | ok | | arm64: | ok | - | c6x: | .. | | csky: | .. | | h8300: | .. | | hexagon: | .. | diff --git a/Documentation/features/vm/TLB/arch-support.txt b/Documentation/features/vm/TLB/arch-support.txt index 30f75a79ce014a12b0fd53a18979cfff557f7f2f..48a5ca54839935e4f7721824de4358491cf738e2 100644 --- a/Documentation/features/vm/TLB/arch-support.txt +++ b/Documentation/features/vm/TLB/arch-support.txt @@ -10,7 +10,6 @@ | arc: | TODO | | arm: | TODO | | arm64: | TODO | - | c6x: | .. | | csky: | TODO | | h8300: | .. | | hexagon: | TODO | diff --git a/Documentation/features/vm/huge-vmap/arch-support.txt b/Documentation/features/vm/huge-vmap/arch-support.txt index c5ff3a427722bb2b530251905e10dd81ab046c5c..439fd9069b8b1d0b42d29c95eb8e77236d73d289 100644 --- a/Documentation/features/vm/huge-vmap/arch-support.txt +++ b/Documentation/features/vm/huge-vmap/arch-support.txt @@ -10,7 +10,6 @@ | arc: | TODO | | arm: | TODO | | arm64: | ok | - | c6x: | TODO | | csky: | TODO | | h8300: | TODO | | hexagon: | TODO | diff --git a/Documentation/features/vm/ioremap_prot/arch-support.txt b/Documentation/features/vm/ioremap_prot/arch-support.txt index b5fb37c28cc67b6a7b6e82f46233bbe879ce8e18..9a0c8783b84d423c2a9b3b2b3292fd9b0599ca12 100644 --- a/Documentation/features/vm/ioremap_prot/arch-support.txt +++ b/Documentation/features/vm/ioremap_prot/arch-support.txt @@ -10,7 +10,6 @@ | arc: | ok | | arm: | TODO | | arm64: | TODO | - | c6x: | TODO | | csky: | TODO | | h8300: | TODO | | hexagon: | TODO | diff --git a/Documentation/features/vm/pte_special/arch-support.txt b/Documentation/features/vm/pte_special/arch-support.txt index 13d0e1e17001c25e7b9cdfc722599361fe0b7f45..40b969f3a6bb3c87f92e43d5369da478af8678a0 100644 --- a/Documentation/features/vm/pte_special/arch-support.txt +++ b/Documentation/features/vm/pte_special/arch-support.txt @@ -10,7 +10,6 @@ | arc: | ok | | arm: | ok | | arm64: | ok | - | c6x: | TODO | | csky: | TODO | | h8300: | TODO | | hexagon: | TODO | diff --git a/Documentation/filesystems/afs.rst b/Documentation/filesystems/afs.rst index 0abb155ac666bb3e74dbec855cf7022fa3db1e3a..ca062a7f8ee2d0e6a61c9d7a5bde1e6c25d32ab8 100644 --- a/Documentation/filesystems/afs.rst +++ b/Documentation/filesystems/afs.rst @@ -109,7 +109,7 @@ Mountpoints AFS has a concept of mountpoints. In AFS terms, these are specially formatted symbolic links (of the same form as the "device name" passed to mount). kAFS presents these to the user as directories that have a follow-link capability -(ie: symbolic link semantics). If anyone attempts to access them, they will +(i.e.: symbolic link semantics). If anyone attempts to access them, they will automatically cause the target volume to be mounted (if possible) on that site. Automatically mounted filesystems will be automatically unmounted approximately @@ -144,7 +144,7 @@ looks up a cell of the same name, for example:: Proc Filesystem =============== -The AFS modules creates a "/proc/fs/afs/" directory and populates it: +The AFS module creates a "/proc/fs/afs/" directory and populates it: (*) A "cells" file that lists cells currently known to the afs module and their usage counts:: @@ -201,7 +201,7 @@ And then run as:: ./klog Assuming it's successful, this adds a key of type RxRPC, named for the service -and cell, eg: "afs@". This can be viewed with the keyctl program or +and cell, e.g.: "afs@". This can be viewed with the keyctl program or by cat'ing /proc/keys:: [root@andromeda ~]# keyctl show @@ -211,7 +211,7 @@ by cat'ing /proc/keys:: 111416553 --als--v 0 0 \_ rxrpc: afs@CAMBRIDGE.REDHAT.COM Currently the username, realm, password and proposed ticket lifetime are -compiled in to the program. +compiled into the program. It is not required to acquire a key before using AFS facilities, but if one is not acquired then all operations will be governed by the anonymous user parts diff --git a/Documentation/filesystems/dax.txt b/Documentation/filesystems/dax.txt index 8fdb78f3c6c99979180ac845bf492a2c81d5aec6..e03c20564f3a73211f246ac44c85b196ddf07345 100644 --- a/Documentation/filesystems/dax.txt +++ b/Documentation/filesystems/dax.txt @@ -83,20 +83,9 @@ Summary directories. This has runtime constraints and limitations that are described in 6) below. - 6. When changing the S_DAX policy via toggling the persistent FS_XFLAG_DAX flag, - the change in behaviour for existing regular files may not occur - immediately. If the change must take effect immediately, the administrator - needs to: - - a) stop the application so there are no active references to the data set - the policy change will affect - - b) evict the data set from kernel caches so it will be re-instantiated when - the application is restarted. This can be achieved by: - - i. drop-caches - ii. a filesystem unmount and mount cycle - iii. a system reboot + 6. When changing the S_DAX policy via toggling the persistent FS_XFLAG_DAX + flag, the change to existing regular files won't take effect until the + files are closed by all processes. Details diff --git a/Documentation/filesystems/index.rst b/Documentation/filesystems/index.rst index 7be9b46d85d9bfdc55c61dd11fea3a31e8175094..1f76b1cb33484361625c4460b6356906a88bc73e 100644 --- a/Documentation/filesystems/index.rst +++ b/Documentation/filesystems/index.rst @@ -83,6 +83,7 @@ Documentation for filesystem implementations. erofs ext2 ext3 + ext4/index f2fs gfs2 gfs2-uevents diff --git a/Documentation/filesystems/locking.rst b/Documentation/filesystems/locking.rst index c0f2c7586531b0a562d7d5e62f44a456989a57c2..b7dcc86c92a45f5ed47cd17cbc5a8b82855927c2 100644 --- a/Documentation/filesystems/locking.rst +++ b/Documentation/filesystems/locking.rst @@ -126,9 +126,10 @@ prototypes:: int (*get)(const struct xattr_handler *handler, struct dentry *dentry, struct inode *inode, const char *name, void *buffer, size_t size); - int (*set)(const struct xattr_handler *handler, struct dentry *dentry, - struct inode *inode, const char *name, const void *buffer, - size_t size, int flags); + int (*set)(const struct xattr_handler *handler, + struct user_namespace *mnt_userns, + struct dentry *dentry, struct inode *inode, const char *name, + const void *buffer, size_t size, int flags); locking rules: all may block diff --git a/Documentation/filesystems/porting.rst b/Documentation/filesystems/porting.rst index 1f8cf8e10b340acc7e315c3c9e166896567f5d2c..0302035781beb0bc1af82ad51bcda83c57428d70 100644 --- a/Documentation/filesystems/porting.rst +++ b/Documentation/filesystems/porting.rst @@ -717,6 +717,8 @@ be removed. Switch while you still can; the old one won't stay. **mandatory** ->setxattr() and xattr_handler.set() get dentry and inode passed separately. +The xattr_handler.set() gets passed the user namespace of the mount the inode +is seen from so filesystems can idmap the i_uid and i_gid accordingly. dentry might be yet to be attached to inode, so do _not_ use its ->d_inode in the instances. Rationale: !@#!@# security_d_instantiate() needs to be called before we attach dentry to inode and !@#!@##!@$!$#!@#$!@$!@$ smack @@ -881,3 +883,10 @@ For bvec based itererators bio_iov_iter_get_pages() now doesn't copy bvecs but uses the one provided. Anyone issuing kiocb-I/O should ensure that the bvec and page references stay until I/O has completed, i.e. until ->ki_complete() has been called or returned with non -EIOCBQUEUED code. + +--- + +**mandatory** + +mnt_want_write_file() can now only be paired with mnt_drop_write_file(), +whereas previously it could be paired with mnt_drop_write() as well. diff --git a/Documentation/filesystems/proc.rst b/Documentation/filesystems/proc.rst index 2fa69f710e2a2dd93e3241258ca8dd2fc8afd45b..48fbfc336ebfd420ac37d27064e2eaa787b48371 100644 --- a/Documentation/filesystems/proc.rst +++ b/Documentation/filesystems/proc.rst @@ -687,7 +687,14 @@ files are there, and which are missing. kcore Kernel core image (can be ELF or A.OUT(deprecated in 2.4)) kmsg Kernel messages ksyms Kernel symbol table - loadavg Load average of last 1, 5 & 15 minutes + loadavg Load average of last 1, 5 & 15 minutes; + number of processes currently runnable (running or on ready queue); + total number of processes in system; + last pid created. + All fields are separated by one space except "number of + processes currently runnable" and "total number of processes + in system", which are separated by a slash ('/'). Example: + 0.61 0.61 0.55 3/828 22084 locks Kernel locks meminfo Memory info misc Miscellaneous diff --git a/Documentation/filesystems/seq_file.rst b/Documentation/filesystems/seq_file.rst index 56856481dc8d8e2ee3abf2652bc6485c69ef8263..a6726082a7c2571061a55b4577b320514e59c12d 100644 --- a/Documentation/filesystems/seq_file.rst +++ b/Documentation/filesystems/seq_file.rst @@ -217,6 +217,12 @@ between the calls to start() and stop(), so holding a lock during that time is a reasonable thing to do. The seq_file code will also avoid taking any other locks while the iterator is active. +The iterater value returned by start() or next() is guaranteed to be +passed to a subsequent next() or stop() call. This allows resources +such as locks that were taken to be reliably released. There is *no* +guarantee that the iterator will be passed to show(), though in practice +it often will be. + Formatted output ================ diff --git a/Documentation/filesystems/vfs.rst b/Documentation/filesystems/vfs.rst index ca52c82e5bb5494d0ac9721995226d63fc5cd012..2049bbf5e3885a32b533e8e70c14103f2b09f2e4 100644 --- a/Documentation/filesystems/vfs.rst +++ b/Documentation/filesystems/vfs.rst @@ -112,7 +112,7 @@ members are defined: .. code-block:: c - struct file_system_operations { + struct file_system_type { const char *name; int fs_flags; struct dentry *(*mount) (struct file_system_type *, int, @@ -270,7 +270,10 @@ or bottom half). ->alloc_inode. ``dirty_inode`` - this method is called by the VFS to mark an inode dirty. + this method is called by the VFS when an inode is marked dirty. + This is specifically for the inode itself being marked dirty, + not its data. If the update needs to be persisted by fdatasync(), + then I_DIRTY_DATASYNC will be set in the flags argument. ``write_inode`` this method is called when the VFS needs to write an inode to @@ -415,28 +418,29 @@ As of kernel 2.6.22, the following members are defined: .. code-block:: c struct inode_operations { - int (*create) (struct inode *,struct dentry *, umode_t, bool); + int (*create) (struct user_namespace *, struct inode *,struct dentry *, umode_t, bool); struct dentry * (*lookup) (struct inode *,struct dentry *, unsigned int); int (*link) (struct dentry *,struct inode *,struct dentry *); int (*unlink) (struct inode *,struct dentry *); - int (*symlink) (struct inode *,struct dentry *,const char *); - int (*mkdir) (struct inode *,struct dentry *,umode_t); + int (*symlink) (struct user_namespace *, struct inode *,struct dentry *,const char *); + int (*mkdir) (struct user_namespace *, struct inode *,struct dentry *,umode_t); int (*rmdir) (struct inode *,struct dentry *); - int (*mknod) (struct inode *,struct dentry *,umode_t,dev_t); - int (*rename) (struct inode *, struct dentry *, + int (*mknod) (struct user_namespace *, struct inode *,struct dentry *,umode_t,dev_t); + int (*rename) (struct user_namespace *, struct inode *, struct dentry *, struct inode *, struct dentry *, unsigned int); int (*readlink) (struct dentry *, char __user *,int); const char *(*get_link) (struct dentry *, struct inode *, struct delayed_call *); - int (*permission) (struct inode *, int); + int (*permission) (struct user_namespace *, struct inode *, int); int (*get_acl)(struct inode *, int); - int (*setattr) (struct dentry *, struct iattr *); - int (*getattr) (const struct path *, struct kstat *, u32, unsigned int); + int (*setattr) (struct user_namespace *, struct dentry *, struct iattr *); + int (*getattr) (struct user_namespace *, const struct path *, struct kstat *, u32, unsigned int); ssize_t (*listxattr) (struct dentry *, char *, size_t); void (*update_time)(struct inode *, struct timespec *, int); int (*atomic_open)(struct inode *, struct dentry *, struct file *, unsigned open_flag, umode_t create_mode); - int (*tmpfile) (struct inode *, struct dentry *, umode_t); + int (*tmpfile) (struct user_namespace *, struct inode *, struct dentry *, umode_t); + int (*set_acl)(struct user_namespace *, struct inode *, struct posix_acl *, int); }; Again, all methods are called without any locks being held, unless diff --git a/Documentation/fpga/dfl.rst b/Documentation/fpga/dfl.rst index 0404fe6ffc7430bcd8e11aaaddaf7351abcc6b58..c41ac76ffaae110368ab7126d7fe9d01320eaf55 100644 --- a/Documentation/fpga/dfl.rst +++ b/Documentation/fpga/dfl.rst @@ -501,6 +501,34 @@ Developer only needs to provide a sub feature driver with matched feature id. FME Partial Reconfiguration Sub Feature driver (see drivers/fpga/dfl-fme-pr.c) could be a reference. +Location of DFLs on a PCI Device +================================ +The original method for finding a DFL on a PCI device assumed the start of the +first DFL to offset 0 of bar 0. If the first node of the DFL is an FME, +then further DFLs in the port(s) are specified in FME header registers. +Alternatively, a PCIe vendor specific capability structure can be used to +specify the location of all the DFLs on the device, providing flexibility +for the type of starting node in the DFL. Intel has reserved the +VSEC ID of 0x43 for this purpose. The vendor specific +data begins with a 4 byte vendor specific register for the number of DFLs followed 4 byte +Offset/BIR vendor specific registers for each DFL. Bits 2:0 of Offset/BIR register +indicates the BAR, and bits 31:3 form the 8 byte aligned offset where bits 2:0 are +zero. +:: + + +----------------------------+ + |31 Number of DFLS 0| + +----------------------------+ + |31 Offset 3|2 BIR 0| + +----------------------------+ + . . . + +----------------------------+ + |31 Offset 3|2 BIR 0| + +----------------------------+ + +Being able to specify more than one DFL per BAR has been considered, but it +was determined the use case did not provide value. Specifying a single DFL +per BAR simplifies the implementation and allows for extra error checking. Open discussion =============== diff --git a/Documentation/gpu/drm-kms.rst b/Documentation/gpu/drm-kms.rst index 3c5ae4f6dfd2340f1de3b6fc68be2aac0b39f284..87e5023e3f5508e047cc1025df05009acdf2b5b7 100644 --- a/Documentation/gpu/drm-kms.rst +++ b/Documentation/gpu/drm-kms.rst @@ -319,6 +319,15 @@ CRTC Functions Reference .. kernel-doc:: drivers/gpu/drm/drm_crtc.c :export: +Color Management Functions Reference +------------------------------------ + +.. kernel-doc:: drivers/gpu/drm/drm_color_mgmt.c + :export: + +.. kernel-doc:: include/drm/drm_color_mgmt.h + :internal: + Frame Buffer Abstraction ======================== @@ -370,6 +379,21 @@ Plane Functions Reference .. kernel-doc:: drivers/gpu/drm/drm_plane.c :export: +Plane Composition Functions Reference +------------------------------------- + +.. kernel-doc:: drivers/gpu/drm/drm_blend.c + :export: + +Plane Damage Tracking Functions Reference +----------------------------------------- + +.. kernel-doc:: drivers/gpu/drm/drm_damage_helper.c + :export: + +.. kernel-doc:: include/drm/drm_damage_helper.h + :internal: + Display Modes Function Reference ================================ @@ -436,6 +460,9 @@ KMS Locking KMS Properties ============== +This section of the documentation is primarily aimed at user-space developers. +For the driver APIs, see the other sections. + Property Types and Blob Property Support ---------------------------------------- @@ -466,39 +493,30 @@ Standard CRTC Properties .. kernel-doc:: drivers/gpu/drm/drm_crtc.c :doc: standard CRTC properties +Standard Plane Properties +------------------------- + +.. kernel-doc:: drivers/gpu/drm/drm_plane.c + :doc: standard plane properties + Plane Composition Properties ---------------------------- .. kernel-doc:: drivers/gpu/drm/drm_blend.c :doc: overview -.. kernel-doc:: drivers/gpu/drm/drm_blend.c - :export: - -FB_DAMAGE_CLIPS -~~~~~~~~~~~~~~~ +Damage Tracking Properties +-------------------------- .. kernel-doc:: drivers/gpu/drm/drm_damage_helper.c :doc: overview -.. kernel-doc:: drivers/gpu/drm/drm_damage_helper.c - :export: - -.. kernel-doc:: include/drm/drm_damage_helper.h - :internal: - Color Management Properties --------------------------- .. kernel-doc:: drivers/gpu/drm/drm_color_mgmt.c :doc: overview -.. kernel-doc:: drivers/gpu/drm/drm_color_mgmt.c - :export: - -.. kernel-doc:: include/drm/drm_color_mgmt.h - :internal: - Tile Group Property ------------------- diff --git a/Documentation/gpu/drm-uapi.rst b/Documentation/gpu/drm-uapi.rst index 7dce175f6d75bc13745d52f3ebea992d8c212698..04bdc7a91d5371fe7df901361d454d9be48c4777 100644 --- a/Documentation/gpu/drm-uapi.rst +++ b/Documentation/gpu/drm-uapi.rst @@ -457,5 +457,8 @@ Userspace API Structures .. kernel-doc:: include/uapi/drm/drm_mode.h :doc: overview +.. kernel-doc:: include/uapi/drm/drm.h + :internal: + .. kernel-doc:: include/uapi/drm/drm_mode.h :internal: diff --git a/Documentation/gpu/i915.rst b/Documentation/gpu/i915.rst index 20868f3d0123606291b3fc01cb593e9acf6c6ad1..486c720f38907f8629a3abd53325ff3042a1c9c1 100644 --- a/Documentation/gpu/i915.rst +++ b/Documentation/gpu/i915.rst @@ -428,7 +428,7 @@ User Batchbuffer Execution Logical Rings, Logical Ring Contexts and Execlists -------------------------------------------------- -.. kernel-doc:: drivers/gpu/drm/i915/gt/intel_lrc.c +.. kernel-doc:: drivers/gpu/drm/i915/gt/intel_execlists_submission.c :doc: Logical Rings, Logical Ring Contexts and Execlists Global GTT views diff --git a/Documentation/gpu/todo.rst b/Documentation/gpu/todo.rst index 009d8e6c7e3c3746b61985333c2cf50f5575be04..40ccac61137e27f936128d037e1fe4c52dc721fd 100644 --- a/Documentation/gpu/todo.rst +++ b/Documentation/gpu/todo.rst @@ -23,6 +23,9 @@ Advanced: Tricky tasks that need fairly good understanding of the DRM subsystem and graphics topics. Generally need the relevant hardware for development and testing. +Expert: Only attempt these if you've successfully completed some tricky +refactorings already and are an expert in the specific area + Subsystem-wide refactorings =========================== @@ -168,6 +171,22 @@ Contact: Daniel Vetter, respective driver maintainers Level: Advanced +Move Buffer Object Locking to dma_resv_lock() +--------------------------------------------- + +Many drivers have their own per-object locking scheme, usually using +mutex_lock(). This causes all kinds of trouble for buffer sharing, since +depending which driver is the exporter and importer, the locking hierarchy is +reversed. + +To solve this we need one standard per-object locking mechanism, which is +dma_resv_lock(). This lock needs to be called as the outermost lock, with all +other driver specific per-object locks removed. The problem is tha rolling out +the actual change to the locking contract is a flag day, due to struct dma_buf +buffer sharing. + +Level: Expert + Convert logging to drm_* functions with drm_device paramater ------------------------------------------------------------ @@ -669,7 +688,7 @@ for fbdev. https://patchwork.freedesktop.org/patch/306579/ - [RFC PATCH v2 00/13] Kernel based bootsplash - https://lkml.org/lkml/2017/12/13/764 + https://lore.kernel.org/r/20171213194755.3409-1-mstaudt@suse.de Contact: Sam Ravnborg diff --git a/Documentation/gpu/vkms.rst b/Documentation/gpu/vkms.rst index 13bab1d93bb3bb20a9ccb9ae7535071ebd82dc0e..2c9b376da5ca877d27a2672dd9e29d4f55d564dd 100644 --- a/Documentation/gpu/vkms.rst +++ b/Documentation/gpu/vkms.rst @@ -7,6 +7,88 @@ .. kernel-doc:: drivers/gpu/drm/vkms/vkms_drv.c :doc: vkms (Virtual Kernel Modesetting) +Setup +===== + +The VKMS driver can be setup with the following steps: + +To check if VKMS is loaded, run:: + + lsmod | grep vkms + +This should list the VKMS driver. If no output is obtained, then +you need to enable and/or load the VKMS driver. +Ensure that the VKMS driver has been set as a loadable module in your +kernel config file. Do:: + + make nconfig + + Go to `Device Drivers> Graphics support` + + Enable `Virtual KMS (EXPERIMENTAL)` + +Compile and build the kernel for the changes to get reflected. +Now, to load the driver, use:: + + sudo modprobe vkms + +On running the lsmod command now, the VKMS driver will appear listed. +You can also observe the driver being loaded in the dmesg logs. + +The VKMS driver has optional features to simulate different kinds of hardware, +which are exposed as module options. You can use the `modinfo` command +to see the module options for vkms:: + + modinfo vkms + +Module options are helpful when testing, and enabling modules +can be done while loading vkms. For example, to load vkms with cursor enabled, +use:: + + sudo modprobe vkms enable_cursor=1 + +To disable the driver, use :: + + sudo modprobe -r vkms + +Testing With IGT +================ + +The IGT GPU Tools is a test suite used specifically for debugging and +development of the DRM drivers. +The IGT Tools can be installed from +`here `_ . + +The tests need to be run without a compositor, so you need to switch to text +only mode. You can do this by:: + + sudo systemctl isolate multi-user.target + +To return to graphical mode, do:: + + sudo systemctl isolate graphical.target + +Once you are in text only mode, you can run tests using the --device switch +or IGT_DEVICE variable to specify the device filter for the driver we want +to test. IGT_DEVICE can also be used with the run-test.sh script to run the +tests for a specific driver:: + + sudo ./build/tests/ --device "sys:/sys/devices/platform/vkms" + sudo IGT_DEVICE="sys:/sys/devices/platform/vkms" ./build/tests/ + sudo IGT_DEVICE="sys:/sys/devices/platform/vkms" ./scripts/run-tests.sh -t + +For example, to test the functionality of the writeback library, +we can run the kms_writeback test:: + + sudo ./build/tests/kms_writeback --device "sys:/sys/devices/platform/vkms" + sudo IGT_DEVICE="sys:/sys/devices/platform/vkms" ./build/tests/kms_writeback + sudo IGT_DEVICE="sys:/sys/devices/platform/vkms" ./scripts/run-tests.sh -t kms_writeback + +You can also run subtests if you do not want to run the entire test:: + + sudo ./build/tests/kms_flip --run-subtest basic-plain-flip --device "sys:/sys/devices/platform/vkms" + sudo IGT_DEVICE="sys:/sys/devices/platform/vkms" ./build/tests/kms_flip --run-subtest basic-plain-flip + TODO ==== diff --git a/Documentation/hid/amd-sfh-hid.rst b/Documentation/hid/amd-sfh-hid.rst index 1f2fe29ccd4fb9f48d563dcb41a79081d404cf55..19ae94cde3b4e96344514b65866b3eaa19c0f00d 100644 --- a/Documentation/hid/amd-sfh-hid.rst +++ b/Documentation/hid/amd-sfh-hid.rst @@ -3,13 +3,13 @@ AMD Sensor Fusion Hub ===================== -AMD Sensor Fusion Hub (SFH) is part of an SOC starting from Ryzen based platforms. +AMD Sensor Fusion Hub (SFH) is part of an SOC starting from Ryzen-based platforms. The solution is working well on several OEM products. AMD SFH uses HID over PCIe bus. In terms of architecture it resembles ISH, however the major difference is all the HID reports are generated as part of the kernel driver. -1. Block Diagram -================ +Block Diagram +------------- :: @@ -45,20 +45,20 @@ the HID reports are generated as part of the kernel driver. AMD HID Transport Layer ----------------------- AMD SFH transport is also implemented as a bus. Each client application executing in the AMD MP2 is -registered as a device on this bus. Here: MP2 which is an ARM core connected to x86 for processing +registered as a device on this bus. Here, MP2 is an ARM core connected to x86 for processing sensor data. The layer, which binds each device (AMD SFH HID driver) identifies the device type and -registers with the hid core. Transport layer attach a constant "struct hid_ll_driver" object with +registers with the HID core. Transport layer attaches a constant "struct hid_ll_driver" object with each device. Once a device is registered with HID core, the callbacks provided via this struct are used by HID core to communicate with the device. AMD HID Transport layer implements the synchronous calls. AMD HID Client Layer -------------------- -This layer is responsible to implement HID request and descriptors. As firmware is OS agnostic, HID +This layer is responsible to implement HID requests and descriptors. As firmware is OS agnostic, HID client layer fills the HID request structure and descriptors. HID client layer is complex as it is -interface between MP2 PCIe layer and HID. HID client layer initialized the MP2 PCIe layer and holds -the instance of MP2 layer. It identifies the number of sensors connected using MP2-PCIe layer. Base -on that allocates the DRAM address for each and every sensor and pass it to MP2-PCIe driver.On -enumeration of each the sensor, client layer fills the HID Descriptor structure and HID input repor +interface between MP2 PCIe layer and HID. HID client layer initializes the MP2 PCIe layer and holds +the instance of MP2 layer. It identifies the number of sensors connected using MP2-PCIe layer. Based +on that allocates the DRAM address for each and every sensor and passes it to MP2-PCIe driver. On +enumeration of each sensor, client layer fills the HID Descriptor structure and HID input report structure. HID Feature report structure is optional. The report descriptor structure varies from sensor to sensor. @@ -72,7 +72,7 @@ The communication between X86 and MP2 is split into three parts. 2. Data transfer via DRAM. 3. Supported sensor info via P2C registers. -Commands are sent to MP2 using C2P Mailbox registers. Writing into C2P Message registers generate +Commands are sent to MP2 using C2P Mailbox registers. Writing into C2P Message registers generates interrupt to MP2. The client layer allocates the physical memory and the same is sent to MP2 via the PCI layer. MP2 firmware writes the command output to the access DRAM memory which the client layer has allocated. Firmware always writes minimum of 32 bytes into DRAM. So as a protocol driver diff --git a/Documentation/hid/hid-alps.rst b/Documentation/hid/hid-alps.rst index e2f4c4c11e3f9bff70a5b84ed34bab9ca25e43ba..767c96bcbb7cdfdbcbea2bd795b9943d3f7c5b5c 100644 --- a/Documentation/hid/hid-alps.rst +++ b/Documentation/hid/hid-alps.rst @@ -64,7 +64,7 @@ Case2 ReportID_3 TP Absolute Command Read/Write ------------------ -To read/write to RAM, need to send a commands to the device. +To read/write to RAM, need to send a command to the device. The command format is as below. @@ -80,7 +80,7 @@ Byte6 Value Byte Byte7 Checksum ===== ====================== -Command Byte is read=0xD1/write=0xD2 . +Command Byte is read=0xD1/write=0xD2. Address is read/write RAM address. diff --git a/Documentation/hid/hid-sensor.rst b/Documentation/hid/hid-sensor.rst index 758972e349711078bcea435e8a0e3d6a980bfc1e..c1c9b8d8dca61563969b90a85fcbca1dd72238f8 100644 --- a/Documentation/hid/hid-sensor.rst +++ b/Documentation/hid/hid-sensor.rst @@ -48,12 +48,12 @@ for different sensors. For example an accelerometer can send X,Y and Z data, whe an ambient light sensor can send illumination data. So the implementation has two parts: -- Core hid driver +- Core HID driver - Individual sensor processing part (sensor drivers) Core driver ----------- -The core driver registers (hid-sensor-hub) registers as a HID driver. It parses +The core driver (hid-sensor-hub) registers as a HID driver. It parses report descriptors and identifies all the sensors present. It adds an MFD device with name HID-SENSOR-xxxx (where xxxx is usage id from the specification). @@ -95,14 +95,14 @@ Registration functions:: u32 usage_id, struct hid_sensor_hub_callbacks *usage_callback): -Registers callbacks for an usage id. The callback functions are not allowed +Registers callbacks for a usage id. The callback functions are not allowed to sleep:: int sensor_hub_remove_callback(struct hid_sensor_hub_device *hsdev, u32 usage_id): -Removes callbacks for an usage id. +Removes callbacks for a usage id. Parsing function:: @@ -166,7 +166,7 @@ This allows some differentiating use cases, where vendor can provide application Some common use cases are debug other sensors or to provide some events like keyboard attached/detached or lid open/close. -To allow application to utilize these sensors, here they are exported uses sysfs +To allow application to utilize these sensors, here they are exported using sysfs attribute groups, attributes and misc device interface. An example of this representation on sysfs:: @@ -207,9 +207,9 @@ An example of this representation on sysfs:: │   │   │   ├── input-1-200202-units │   │   │   ├── input-1-200202-value -Here there is a custom sensors with four fields, two feature and two inputs. +Here there is a custom sensor with four fields: two feature and two inputs. Each field is represented by a set of attributes. All fields except the "value" -are read only. The value field is a RW field. +are read only. The value field is a read-write field. Example:: @@ -237,6 +237,6 @@ These reports are pushed using misc device interface in a FIFO order:: │   │   │   ├── 10:53 -> ../HID-SENSOR-2000e1.6.auto │   ├── HID-SENSOR-2000e1.6.auto -Each reports can be of variable length preceded by a header. This header -consist of a 32 bit usage id, 64 bit time stamp and 32 bit length field of raw +Each report can be of variable length preceded by a header. This header +consists of a 32-bit usage id, 64-bit time stamp and 32-bit length field of raw data. diff --git a/Documentation/hid/hid-transport.rst b/Documentation/hid/hid-transport.rst index 0fe526f36db64313aa88754dde5e8202f8f3035c..6f1692da296cf2f6b49cbf96c222e022ffc27162 100644 --- a/Documentation/hid/hid-transport.rst +++ b/Documentation/hid/hid-transport.rst @@ -12,8 +12,8 @@ Bluetooth, I2C and user-space I/O drivers. The HID subsystem is designed as a bus. Any I/O subsystem may provide HID devices and register them with the HID bus. HID core then loads generic device -drivers on top of it. The transport drivers are responsible of raw data -transport and device setup/management. HID core is responsible of +drivers on top of it. The transport drivers are responsible for raw data +transport and device setup/management. HID core is responsible for report-parsing, report interpretation and the user-space API. Device specifics and quirks are handled by all layers depending on the quirk. @@ -67,7 +67,7 @@ Transport drivers attach a constant "struct hid_ll_driver" object with each device. Once a device is registered with HID core, the callbacks provided via this struct are used by HID core to communicate with the device. -Transport drivers are responsible of detecting device failures and unplugging. +Transport drivers are responsible for detecting device failures and unplugging. HID core will operate a device as long as it is registered regardless of any device failures. Once transport drivers detect unplug or failure events, they must unregister the device from HID core and HID core will stop using the @@ -101,7 +101,7 @@ properties in common. channel. Any unrequested incoming or outgoing data report must be sent on this channel and is never acknowledged by the remote side. Devices usually send their input events on this channel. Outgoing events are normally - not send via intr, except if high throughput is required. + not sent via intr, except if high throughput is required. - Control Channel (ctrl): The ctrl channel is used for synchronous requests and device management. Unrequested data input events must not be sent on this channel and are normally ignored. Instead, devices only send management @@ -161,7 +161,7 @@ allowed on the intr channel and are the only means of data there. payload may be blocked by the underlying transport driver if the specification does not allow them. - SET_REPORT: A SET_REPORT request has a report ID plus data as payload. It is - sent from host to device and a device must update it's current report state + sent from host to device and a device must update its current report state according to the given data. Any of the 3 report types can be used. However, INPUT reports as payload might be blocked by the underlying transport driver if the specification does not allow them. @@ -294,7 +294,7 @@ The available HID callbacks are: void (*request) (struct hid_device *hdev, struct hid_report *report, int reqtype) - Send an HID request on the ctrl channel. "report" contains the report that + Send a HID request on the ctrl channel. "report" contains the report that should be sent and "reqtype" the request type. Request-type can be HID_REQ_SET_REPORT or HID_REQ_GET_REPORT. diff --git a/Documentation/hid/hiddev.rst b/Documentation/hid/hiddev.rst index 9b28a97c03e6d7517cc55f51437f8b37b0396d1d..caebc6266603c1a27b217471e07fcdf8ef1c0b8f 100644 --- a/Documentation/hid/hiddev.rst +++ b/Documentation/hid/hiddev.rst @@ -27,7 +27,7 @@ the following:: --> hiddev.c ----> POWER / MONITOR CONTROL In addition, other subsystems (apart from USB) can potentially feed -events into the input subsystem, but these have no effect on the hid +events into the input subsystem, but these have no effect on the HID device interface. Using the HID Device Interface @@ -73,7 +73,7 @@ The hiddev API uses a read() interface, and a set of ioctl() calls. HID devices exchange data with the host computer using data bundles called "reports". Each report is divided into "fields", each of which can have one or more "usages". In the hid-core, -each one of these usages has a single signed 32 bit value. +each one of these usages has a single signed 32-bit value. read(): ------- @@ -113,7 +113,7 @@ HIDIOCAPPLICATION - (none) This ioctl call returns the HID application usage associated with the -hid device. The third argument to ioctl() specifies which application +HID device. The third argument to ioctl() specifies which application index to get. This is useful when the device has more than one application collection. If the index is invalid (greater or equal to the number of application collections this device has) the ioctl @@ -181,7 +181,7 @@ looked up by type (input, output or feature) and id, so these fields must be filled in by the user. The ID can be absolute -- the actual report id as reported by the device -- or relative -- HID_REPORT_ID_FIRST for the first report, and (HID_REPORT_ID_NEXT | -report_id) for the next report after report_id. Without a-priori +report_id) for the next report after report_id. Without a priori information about report ids, the right way to use this ioctl is to use the relative IDs above to enumerate the valid IDs. The ioctl returns non-zero when there is no more next ID. The real report ID is @@ -200,7 +200,7 @@ HIDIOCGUCODE - struct hiddev_usage_ref (read/write) Returns the usage_code in a hiddev_usage_ref structure, given that -given its report type, report id, field index, and index within the +its report type, report id, field index, and index within the field have already been filled into the structure. HIDIOCGUSAGE diff --git a/Documentation/hid/hidraw.rst b/Documentation/hid/hidraw.rst index f41c1f0f6252c0b6323d64c89c217745c0487277..b717ee5cdaef7cbba515d604e4f34eb532b8736a 100644 --- a/Documentation/hid/hidraw.rst +++ b/Documentation/hid/hidraw.rst @@ -21,7 +21,7 @@ Hidraw is the only alternative, short of writing a custom kernel driver, for these non-conformant devices. A benefit of hidraw is that its use by userspace applications is independent -of the underlying hardware type. Currently, Hidraw is implemented for USB +of the underlying hardware type. Currently, hidraw is implemented for USB and Bluetooth. In the future, as new hardware bus types are developed which use the HID specification, hidraw will be expanded to add support for these new bus types. @@ -31,9 +31,10 @@ create hidraw device nodes. Udev will typically create the device nodes directly under /dev (eg: /dev/hidraw0). As this location is distribution- and udev rule-dependent, applications should use libudev to locate hidraw devices attached to the system. There is a tutorial on libudev with a -working example at: +working example at:: http://www.signal11.us/oss/udev/ + https://web.archive.org/web/2019*/www.signal11.us The HIDRAW API --------------- diff --git a/Documentation/hid/intel-ish-hid.rst b/Documentation/hid/intel-ish-hid.rst index d4785cf6eefd08e0b1192e77b77af88ad2f7adea..f6ce44ff611df775f0ae0ac33f78a7b0159201b7 100644 --- a/Documentation/hid/intel-ish-hid.rst +++ b/Documentation/hid/intel-ish-hid.rst @@ -4,19 +4,19 @@ Intel Integrated Sensor Hub (ISH) A sensor hub enables the ability to offload sensor polling and algorithm processing to a dedicated low power co-processor. This allows the core -processor to go into low power modes more often, resulting in the increased +processor to go into low power modes more often, resulting in increased battery life. -There are many vendors providing external sensor hubs confirming to HID -Sensor usage tables, and used in several tablets, 2 in 1 convertible laptops -and embedded products. Linux had this support since Linux 3.9. +There are many vendors providing external sensor hubs conforming to HID +Sensor usage tables. These may be found in tablets, 2-in-1 convertible laptops +and embedded products. Linux has had this support since Linux 3.9. Intel® introduced integrated sensor hubs as a part of the SoC starting from Cherry Trail and now supported on multiple generations of CPU packages. There are many commercial devices already shipped with Integrated Sensor Hubs (ISH). -These ISH also comply to HID sensor specification, but the difference is the +These ISH also comply to HID sensor specification, but the difference is the transport protocol used for communication. The current external sensor hubs -mainly use HID over i2C or USB. But ISH doesn't use either i2c or USB. +mainly use HID over I2C or USB. But ISH doesn't use either I2C or USB. 1. Overview =========== @@ -35,7 +35,7 @@ for a very high speed communication:: ----------------- ---------------------- PCI PCI ----------------- ---------------------- - |Host controller| --> | ISH processor | + |Host controller| --> | ISH processor | ----------------- ---------------------- USB Link ----------------- ---------------------- @@ -50,13 +50,13 @@ applications implemented in the firmware. The ISH allows multiple sensor management applications executing in the firmware. Like USB endpoints the messaging can be to/from a client. As part of enumeration process, these clients are identified. These clients can be simple -HID sensor applications, sensor calibration application or senor firmware -update application. +HID sensor applications, sensor calibration applications or sensor firmware +update applications. The implementation model is similar, like USB bus, ISH transport is also implemented as a bus. Each client application executing in the ISH processor is registered as a device on this bus. The driver, which binds each device -(ISH HID driver) identifies the device type and registers with the hid core. +(ISH HID driver) identifies the device type and registers with the HID core. 2. ISH Implementation: Block Diagram ==================================== @@ -104,7 +104,7 @@ is registered as a device on this bus. The driver, which binds each device The ISH is exposed as "Non-VGA unclassified PCI device" to the host. The PCI product and vendor IDs are changed from different generations of processors. So -the source code which enumerate drivers needs to update from generation to +the source code which enumerates drivers needs to update from generation to generation. 3.2 Inter Processor Communication (IPC) driver @@ -112,41 +112,42 @@ generation. Location: drivers/hid/intel-ish-hid/ipc -The IPC message used memory mapped I/O. The registers are defined in +The IPC message uses memory mapped I/O. The registers are defined in hw-ish-regs.h. 3.2.1 IPC/FW message types ^^^^^^^^^^^^^^^^^^^^^^^^^^ -There are two types of messages, one for management of link and other messages -are to and from transport layers. +There are two types of messages, one for management of link and another for +messages to and from transport layers. TX and RX of Transport messages ............................... -A set of memory mapped register offers support of multi byte messages TX and -RX (E.g.IPC_REG_ISH2HOST_MSG, IPC_REG_HOST2ISH_MSG). The IPC layer maintains -internal queues to sequence messages and send them in order to the FW. +A set of memory mapped register offers support of multi-byte messages TX and +RX (e.g. IPC_REG_ISH2HOST_MSG, IPC_REG_HOST2ISH_MSG). The IPC layer maintains +internal queues to sequence messages and send them in order to the firmware. Optionally the caller can register handler to get notification of completion. -A door bell mechanism is used in messaging to trigger processing in host and +A doorbell mechanism is used in messaging to trigger processing in host and client firmware side. When ISH interrupt handler is called, the ISH2HOST doorbell register is used by host drivers to determine that the interrupt is for ISH. Each side has 32 32-bit message registers and a 32-bit doorbell. Doorbell -register has the following format: -Bits 0..6: fragment length (7 bits are used) -Bits 10..13: encapsulated protocol -Bits 16..19: management command (for IPC management protocol) -Bit 31: doorbell trigger (signal H/W interrupt to the other side) -Other bits are reserved, should be 0. +register has the following format:: + + Bits 0..6: fragment length (7 bits are used) + Bits 10..13: encapsulated protocol + Bits 16..19: management command (for IPC management protocol) + Bit 31: doorbell trigger (signal H/W interrupt to the other side) + Other bits are reserved, should be 0. 3.2.2 Transport layer interface ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -To abstract HW level IPC communication, a set of callbacks are registered. +To abstract HW level IPC communication, a set of callbacks is registered. The transport layer uses them to send and receive messages. -Refer to struct ishtp_hw_ops for callbacks. +Refer to struct ishtp_hw_ops for callbacks. 3.3 ISH Transport layer ----------------------- @@ -158,7 +159,7 @@ Location: drivers/hid/intel-ish-hid/ishtp/ The transport layer is a bi-directional protocol, which defines: - Set of commands to start, stop, connect, disconnect and flow control -(ishtp/hbm.h) for details +(see ishtp/hbm.h for details) - A flow control mechanism to avoid buffer overflows This protocol resembles bus messages described in the following document: @@ -168,14 +169,14 @@ specifications/dcmi-hi-1-0-spec.pdf "Chapter 7: Bus Message Layer" 3.3.2 Connection and Flow Control Mechanism ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Each FW client and a protocol is identified by an UUID. In order to communicate +Each FW client and a protocol is identified by a UUID. In order to communicate to a FW client, a connection must be established using connect request and response bus messages. If successful, a pair (host_client_id and fw_client_id) will identify the connection. Once connection is established, peers send each other flow control bus messages independently. Every peer may send a message only if it has received a -flow-control credit before. Once it sent a message, it may not send another one +flow-control credit before. Once it has sent a message, it may not send another one before receiving the next flow control credit. Either side can send disconnect request bus message to end communication. Also the link will be dropped if major FW reset occurs. @@ -209,7 +210,7 @@ and DMA_XFER_ACK act as ownership indicators. At initial state all outgoing memory belongs to the sender (TX to host, RX to FW), DMA_XFER transfers ownership on the region that contains ISHTP message to the receiving side, DMA_XFER_ACK returns ownership to the sender. A sender -needs not wait for previous DMA_XFER to be ack'ed, and may send another message +need not wait for previous DMA_XFER to be ack'ed, and may send another message as long as remaining continuous memory in its ownership is enough. In principle, multiple DMA_XFER and DMA_XFER_ACK messages may be sent at once (up to IPC MTU), thus allowing for interrupt throttling. @@ -219,8 +220,8 @@ fragments and via IPC otherwise. 3.3.4 Ring Buffers ^^^^^^^^^^^^^^^^^^ -When a client initiate a connection, a ring or RX and TX buffers are allocated. -The size of ring can be specified by the client. HID client set 16 and 32 for +When a client initiates a connection, a ring of RX and TX buffers is allocated. +The size of ring can be specified by the client. HID client sets 16 and 32 for TX and RX buffers respectively. On send request from client, the data to be sent is copied to one of the send ring buffer and scheduled to be sent using bus message protocol. These buffers are required because the FW may have not @@ -230,10 +231,10 @@ to send. Same thing holds true on receive side and flow control is required. 3.3.5 Host Enumeration ^^^^^^^^^^^^^^^^^^^^^^ -The host enumeration bus command allow discovery of clients present in the FW. +The host enumeration bus command allows discovery of clients present in the FW. There can be multiple sensor clients and clients for calibration function. -To ease in implantation and allow independent driver handle each client +To ease implementation and allow independent drivers to handle each client, this transport layer takes advantage of Linux Bus driver model. Each client is registered as device on the transport bus (ishtp bus). @@ -270,7 +271,7 @@ The ISHTP client driver is responsible for: The functionality in these drivers is the same as an external sensor hub. Refer to Documentation/hid/hid-sensor.rst for HID sensor -Documentation/ABI/testing/sysfs-bus-iio for IIO ABIs to user space +Documentation/ABI/testing/sysfs-bus-iio for IIO ABIs to user space. 3.6 End to End HID transport Sequence Diagram --------------------------------------------- @@ -341,9 +342,10 @@ Documentation/ABI/testing/sysfs-bus-iio for IIO ABIs to user space 3.7 ISH Debugging ----------------- -To debug ISH, event tracing mechanism is used. To enable debug logs -echo 1 > /sys/kernel/debug/tracing/events/intel_ish/enable -cat sys/kernel/debug/tracing/trace +To debug ISH, event tracing mechanism is used. To enable debug logs:: + + echo 1 > /sys/kernel/debug/tracing/events/intel_ish/enable + cat sys/kernel/debug/tracing/trace 3.8 ISH IIO sysfs Example on Lenovo thinkpad Yoga 260 ----------------------------------------------------- diff --git a/Documentation/hid/uhid.rst b/Documentation/hid/uhid.rst index b18cb96c885fee559d287101e64c04192f247c51..2243a6b75914a5cb63808b3e660f1e9eceb89f5a 100644 --- a/Documentation/hid/uhid.rst +++ b/Documentation/hid/uhid.rst @@ -3,7 +3,7 @@ UHID - User-space I/O driver support for HID subsystem ====================================================== UHID allows user-space to implement HID transport drivers. Please see -hid-transport.txt for an introduction into HID transport drivers. This document +hid-transport.rst for an introduction into HID transport drivers. This document relies heavily on the definitions declared there. With UHID, a user-space transport driver can create kernel hid-devices for each @@ -15,7 +15,7 @@ There is an example user-space application in ./samples/uhid/uhid-example.c The UHID API ------------ -UHID is accessed through a character misc-device. The minor-number is allocated +UHID is accessed through a character misc-device. The minor number is allocated dynamically so you need to rely on udev (or similar) to create the device node. This is /dev/uhid by default. @@ -45,23 +45,23 @@ The "type" field defines the payload. For each type, there is a payload-structure available in the union "u" (except for empty payloads). This payload contains management and/or device data. -The first thing you should do is sending an UHID_CREATE2 event. This will -register the device. UHID will respond with an UHID_START event. You can now +The first thing you should do is send a UHID_CREATE2 event. This will +register the device. UHID will respond with a UHID_START event. You can now start sending data to and reading data from UHID. However, unless UHID sends the UHID_OPEN event, the internally attached HID Device Driver has no user attached. That is, you might put your device asleep unless you receive the UHID_OPEN event. If you receive the UHID_OPEN event, you should start I/O. If the last -user closes the HID device, you will receive an UHID_CLOSE event. This may be -followed by an UHID_OPEN event again and so on. There is no need to perform +user closes the HID device, you will receive a UHID_CLOSE event. This may be +followed by a UHID_OPEN event again and so on. There is no need to perform reference-counting in user-space. That is, you will never receive multiple -UHID_OPEN events without an UHID_CLOSE event. The HID subsystem performs +UHID_OPEN events without a UHID_CLOSE event. The HID subsystem performs ref-counting for you. You may decide to ignore UHID_OPEN/UHID_CLOSE, though. I/O is allowed even though the device may have no users. If you want to send data on the interrupt channel to the HID subsystem, you send -an HID_INPUT2 event with your raw data payload. If the kernel wants to send data -on the interrupt channel to the device, you will read an UHID_OUTPUT event. +a HID_INPUT2 event with your raw data payload. If the kernel wants to send data +on the interrupt channel to the device, you will read a UHID_OUTPUT event. Data requests on the control channel are currently limited to GET_REPORT and SET_REPORT (no other data reports on the control channel are defined so far). Those requests are always synchronous. That means, the kernel sends @@ -71,7 +71,7 @@ the response via UHID_GET_REPORT_REPLY and UHID_SET_REPORT_REPLY to the kernel. The kernel blocks internal driver-execution during such round-trips (times out after a hard-coded period). -If your device disconnects, you should send an UHID_DESTROY event. This will +If your device disconnects, you should send a UHID_DESTROY event. This will unregister the device. You can now send UHID_CREATE2 again to register a new device. If you close() the fd, the device is automatically unregistered and destroyed @@ -125,7 +125,7 @@ UHID_START: This is sent when the HID device is started. Consider this as an answer to UHID_CREATE2. This is always the first event that is sent. Note that this event might not be available immediately after write(UHID_CREATE2) returns. - Device drivers might required delayed setups. + Device drivers might require delayed setups. This event contains a payload of type uhid_start_req. The "dev_flags" field describes special behaviors of a device. The following flags are defined: @@ -149,7 +149,7 @@ UHID_STOP: reloaded/changed the device driver loaded on your HID device (or some other maintenance actions happened). - You can usually ignored any UHID_STOP events safely. + You can usually ignore any UHID_STOP events safely. UHID_OPEN: This is sent when the HID device is opened. That is, the data that the HID @@ -166,17 +166,17 @@ UHID_OUTPUT: This is sent if the HID device driver wants to send raw data to the I/O device on the interrupt channel. You should read the payload and forward it to the device. The payload is of type "struct uhid_output_req". - This may be received even though you haven't received UHID_OPEN, yet. + This may be received even though you haven't received UHID_OPEN yet. UHID_GET_REPORT: This event is sent if the kernel driver wants to perform a GET_REPORT request - on the control channeld as described in the HID specs. The report-type and + on the control channel as described in the HID specs. The report-type and report-number are available in the payload. The kernel serializes GET_REPORT requests so there will never be two in parallel. However, if you fail to respond with a UHID_GET_REPORT_REPLY, the request might silently time out. - Once you read a GET_REPORT request, you shall forward it to the hid device and - remember the "id" field in the payload. Once your hid device responds to the + Once you read a GET_REPORT request, you shall forward it to the HID device and + remember the "id" field in the payload. Once your HID device responds to the GET_REPORT (or if it fails), you must send a UHID_GET_REPORT_REPLY to the kernel with the exact same "id" as in the request. If the request already timed out, the kernel will ignore the response silently. The "id" field is @@ -184,7 +184,7 @@ UHID_GET_REPORT: UHID_SET_REPORT: This is the SET_REPORT equivalent of UHID_GET_REPORT. On receipt, you shall - send a SET_REPORT request to your hid device. Once it replies, you must tell + send a SET_REPORT request to your HID device. Once it replies, you must tell the kernel about it via UHID_SET_REPORT_REPLY. The same restrictions as for UHID_GET_REPORT apply. diff --git a/Documentation/hwmon/ab8500.rst b/Documentation/hwmon/ab8500.rst deleted file mode 100644 index 33f93a9cec04e7f40deadf70e1bf730308a24d3b..0000000000000000000000000000000000000000 --- a/Documentation/hwmon/ab8500.rst +++ /dev/null @@ -1,26 +0,0 @@ -Kernel driver ab8500 -==================== - -Supported chips: - - * ST-Ericsson AB8500 - - Prefix: 'ab8500' - - Addresses scanned: - - - Datasheet: http://www.stericsson.com/developers/documentation.jsp - -Authors: - - Martin Persson - - Hongbo Zhang - -Description ------------ - -See also Documentation/hwmon/abx500.rst. This is the ST-Ericsson AB8500 specific -driver. - -Currently only the AB8500 internal sensor and one external sensor for battery -temperature are monitored. Other GPADC channels can also be monitored if needed -in future. diff --git a/Documentation/hwmon/abx500.rst b/Documentation/hwmon/abx500.rst deleted file mode 100644 index 3d88b2ce0f00b464182401de40cd0c4ace74515d..0000000000000000000000000000000000000000 --- a/Documentation/hwmon/abx500.rst +++ /dev/null @@ -1,32 +0,0 @@ -Kernel driver abx500 -==================== - -Supported chips: - - * ST-Ericsson ABx500 series - - Prefix: 'abx500' - - Addresses scanned: - - - Datasheet: http://www.stericsson.com/developers/documentation.jsp - -Authors: - Martin Persson - Hongbo Zhang - -Description ------------ - -Every ST-Ericsson Ux500 SOC consists of both ABx500 and DBx500 physically, -this is kernel hwmon driver for ABx500. - -There are some GPADCs inside ABx500 which are designed for connecting to -thermal sensors, and there is also a thermal sensor inside ABx500 too, which -raises interrupt when critical temperature reached. - -This abx500 is a common layer which can monitor all of the sensors, every -specific abx500 chip has its special configurations in its own file, e.g. some -sensors can be configured invisible if they are not available on that chip, and -the corresponding gpadc_addr should be set to 0, thus this sensor won't be -polled. diff --git a/Documentation/hwmon/aht10.rst b/Documentation/hwmon/aht10.rst new file mode 100644 index 0000000000000000000000000000000000000000..482262ca117c5a44cca08740c067a46a1e9093f8 --- /dev/null +++ b/Documentation/hwmon/aht10.rst @@ -0,0 +1,46 @@ +.. SPDX-License-Identifier: GPL-2.0 + +Kernel driver aht10 +===================== + +Supported chips: + + * Aosong AHT10 + + Prefix: 'aht10' + + Addresses scanned: None + + Datasheet: + + Chinese: http://www.aosong.com/userfiles/files/media/AHT10%E4%BA%A7%E5%93%81%E6%89%8B%E5%86%8C%20A3%2020201210.pdf + English: https://server4.eca.ir/eshop/AHT10/Aosong_AHT10_en_draft_0c.pdf + +Author: Johannes Cornelis Draaijer + + +Description +----------- + +The AHT10 is a Temperature and Humidity sensor + +The address of this i2c device may only be 0x38 + +Usage Notes +----------- + +This driver does not probe for AHT10 devices, as there is no reliable +way to determine if an i2c chip is or isn't an AHT10. The device has +to be instantiated explicitly with the address 0x38. See +Documentation/i2c/instantiating-devices.rst for details. + +Sysfs entries +------------- + +=============== ============================================ +temp1_input Measured temperature in millidegrees Celcius +humidity1_input Measured humidity in %H +update_interval The minimum interval for polling the sensor, + in milliseconds. Writable. Must be at + least 2000. +=============== ============================================ diff --git a/Documentation/hwmon/ina2xx.rst b/Documentation/hwmon/ina2xx.rst index f78a5cd44c4ca9a5a5a74b903acf301d424a973f..27d2e39bc8ac71d585cd0c73898a1b61f34cb3ba 100644 --- a/Documentation/hwmon/ina2xx.rst +++ b/Documentation/hwmon/ina2xx.rst @@ -74,7 +74,7 @@ bus supply voltage. The shunt value in micro-ohms can be set via platform data or device tree at compile-time or via the shunt_resistor attribute in sysfs at run-time. Please -refer to the Documentation/devicetree/bindings/hwmon/ina2xx.txt for bindings +refer to the Documentation/devicetree/bindings/hwmon/ti,ina2xx.yaml for bindings if the device tree is used. Additionally ina226 supports update_interval attribute as described in diff --git a/Documentation/hwmon/index.rst b/Documentation/hwmon/index.rst index fcb870ce628601ef2cdd13c95f7953fe43a75a74..8d5a2df1ecb642b31345b07e3caee8f13db6efcf 100644 --- a/Documentation/hwmon/index.rst +++ b/Documentation/hwmon/index.rst @@ -18,10 +18,8 @@ Hardware Monitoring Kernel Drivers .. toctree:: :maxdepth: 1 - ab8500 abituguru abituguru3 - abx500 acpi_power_meter ad7314 adc128d818 @@ -39,6 +37,7 @@ Hardware Monitoring Kernel Drivers adt7462 adt7470 adt7475 + aht10 amc6821 amd_energy asb100 @@ -178,6 +177,7 @@ Hardware Monitoring Kernel Drivers tmp401 tmp421 tmp513 + tps23861 tps40422 tps53679 twl4030-madc-hwmon diff --git a/Documentation/hwmon/max16601.rst b/Documentation/hwmon/max16601.rst index 346e74674c51743abf91a4d8312aa8e18f1c07d7..92c0a7d7808c1176c7e84c29ca060cf5ab59a59c 100644 --- a/Documentation/hwmon/max16601.rst +++ b/Documentation/hwmon/max16601.rst @@ -5,6 +5,14 @@ Kernel driver max16601 Supported chips: + * Maxim MAX16508 + + Prefix: 'max16508' + + Addresses scanned: - + + Datasheet: Not published + * Maxim MAX16601 Prefix: 'max16601' @@ -19,8 +27,8 @@ Author: Guenter Roeck Description ----------- -This driver supports the MAX16601 VR13.HC Dual-Output Voltage Regulator -Chipset. +This driver supports the MAX16508 VR13 Dual-Output Voltage Regulator +as well as the MAX16601 VR13.HC Dual-Output Voltage Regulator chipsets. The driver is a client driver to the core PMBus driver. Please see Documentation/hwmon/pmbus.rst for details on PMBus client drivers. @@ -45,115 +53,76 @@ Sysfs entries The following attributes are supported. -======================= ======================================================= -in1_label "vin1" -in1_input VCORE input voltage. -in1_alarm Input voltage alarm. - -in2_label "vout1" -in2_input VCORE output voltage. -in2_alarm Output voltage alarm. - -curr1_label "iin1" -curr1_input VCORE input current, derived from duty cycle and output - current. -curr1_max Maximum input current. -curr1_max_alarm Current high alarm. - -curr2_label "iin1.0" -curr2_input VCORE phase 0 input current. - -curr3_label "iin1.1" -curr3_input VCORE phase 1 input current. - -curr4_label "iin1.2" -curr4_input VCORE phase 2 input current. - -curr5_label "iin1.3" -curr5_input VCORE phase 3 input current. - -curr6_label "iin1.4" -curr6_input VCORE phase 4 input current. - -curr7_label "iin1.5" -curr7_input VCORE phase 5 input current. - -curr8_label "iin1.6" -curr8_input VCORE phase 6 input current. - -curr9_label "iin1.7" -curr9_input VCORE phase 7 input current. - -curr10_label "iin2" -curr10_input VCORE input current, derived from sensor element. - -curr11_label "iin3" -curr11_input VSA input current. - -curr12_label "iout1" -curr12_input VCORE output current. -curr12_crit Critical output current. -curr12_crit_alarm Output current critical alarm. -curr12_max Maximum output current. -curr12_max_alarm Output current high alarm. - -curr13_label "iout1.0" -curr13_input VCORE phase 0 output current. - -curr14_label "iout1.1" -curr14_input VCORE phase 1 output current. - -curr15_label "iout1.2" -curr15_input VCORE phase 2 output current. - -curr16_label "iout1.3" -curr16_input VCORE phase 3 output current. - -curr17_label "iout1.4" -curr17_input VCORE phase 4 output current. - -curr18_label "iout1.5" -curr18_input VCORE phase 5 output current. - -curr19_label "iout1.6" -curr19_input VCORE phase 6 output current. - -curr20_label "iout1.7" -curr20_input VCORE phase 7 output current. - -curr21_label "iout3" -curr21_input VSA output current. -curr21_highest Historical maximum VSA output current. -curr21_reset_history Write any value to reset curr21_highest. -curr21_crit Critical output current. -curr21_crit_alarm Output current critical alarm. -curr21_max Maximum output current. -curr21_max_alarm Output current high alarm. - -power1_label "pin1" -power1_input Input power, derived from duty cycle and output current. -power1_alarm Input power alarm. - -power2_label "pin2" -power2_input Input power, derived from input current sensor. - -power3_label "pout" -power3_input Output power. - -temp1_input VCORE temperature. -temp1_crit Critical high temperature. -temp1_crit_alarm Chip temperature critical high alarm. -temp1_max Maximum temperature. -temp1_max_alarm Chip temperature high alarm. - -temp2_input TSENSE_0 temperature -temp3_input TSENSE_1 temperature -temp4_input TSENSE_2 temperature -temp5_input TSENSE_3 temperature - -temp6_input VSA temperature. -temp6_crit Critical high temperature. -temp6_crit_alarm Chip temperature critical high alarm. -temp6_max Maximum temperature. -temp6_max_alarm Chip temperature high alarm. -======================= ======================================================= +=============================== =============================================== +in1_label "vin1" +in1_input VCORE input voltage. +in1_alarm Input voltage alarm. + +in2_label "vout1" +in2_input VCORE output voltage. +in2_alarm Output voltage alarm. + +curr1_label "iin1" +curr1_input VCORE input current, derived from duty cycle + and output current. +curr1_max Maximum input current. +curr1_max_alarm Current high alarm. + +curr[P+2]_label "iin1.P" +curr[P+2]_input VCORE phase P input current. + +curr[N+2]_label "iin2" +curr[N+2]_input VCORE input current, derived from sensor + element. + 'N' is the number of enabled/populated phases. + +curr[N+3]_label "iin3" +curr[N+3]_input VSA input current. + +curr[N+4]_label "iout1" +curr[N+4]_input VCORE output current. +curr[N+4]_crit Critical output current. +curr[N+4]_crit_alarm Output current critical alarm. +curr[N+4]_max Maximum output current. +curr[N+4]_max_alarm Output current high alarm. + +curr[N+P+5]_label "iout1.P" +curr[N+P+5]_input VCORE phase P output current. + +curr[2*N+5]_label "iout3" +curr[2*N+5]_input VSA output current. +curr[2*N+5]_highest Historical maximum VSA output current. +curr[2*N+5]_reset_history Write any value to reset curr21_highest. +curr[2*N+5]_crit Critical output current. +curr[2*N+5]_crit_alarm Output current critical alarm. +curr[2*N+5]_max Maximum output current. +curr[2*N+5]_max_alarm Output current high alarm. + +power1_label "pin1" +power1_input Input power, derived from duty cycle and output + current. +power1_alarm Input power alarm. + +power2_label "pin2" +power2_input Input power, derived from input current sensor. + +power3_label "pout" +power3_input Output power. + +temp1_input VCORE temperature. +temp1_crit Critical high temperature. +temp1_crit_alarm Chip temperature critical high alarm. +temp1_max Maximum temperature. +temp1_max_alarm Chip temperature high alarm. + +temp2_input TSENSE_0 temperature +temp3_input TSENSE_1 temperature +temp4_input TSENSE_2 temperature +temp5_input TSENSE_3 temperature + +temp6_input VSA temperature. +temp6_crit Critical high temperature. +temp6_crit_alarm Chip temperature critical high alarm. +temp6_max Maximum temperature. +temp6_max_alarm Chip temperature high alarm. +=============================== =============================================== diff --git a/Documentation/hwmon/nct6683.rst b/Documentation/hwmon/nct6683.rst index 8646ad519fcd09f01f1b7293de1aebf32edcc105..2e1408d174bd3acc6d3a9120393a2b981efd197e 100644 --- a/Documentation/hwmon/nct6683.rst +++ b/Documentation/hwmon/nct6683.rst @@ -61,5 +61,6 @@ Board Firmware version Intel DH87RL NCT6683D EC firmware version 1.0 build 04/03/13 Intel DH87MC NCT6683D EC firmware version 1.0 build 04/03/13 Intel DB85FL NCT6683D EC firmware version 1.0 build 04/03/13 +ASRock X570 NCT6683D EC firmware version 1.0 build 06/28/19 MSI B550 NCT6687D EC firmware version 1.0 build 05/07/20 =============== =============================================== diff --git a/Documentation/hwmon/tps23861.rst b/Documentation/hwmon/tps23861.rst new file mode 100644 index 0000000000000000000000000000000000000000..46d121ff3f316d977ab3d1ab4e0d7b7736a57c60 --- /dev/null +++ b/Documentation/hwmon/tps23861.rst @@ -0,0 +1,41 @@ +.. SPDX-License-Identifier: GPL-2.0-only + +Kernel driver tps23861 +====================== + +Supported chips: + * Texas Instruments TPS23861 + + Prefix: 'tps23861' + + Datasheet: https://www.ti.com/lit/gpn/tps23861 + +Author: Robert Marko + +Description +----------- + +This driver supports hardware monitoring for Texas Instruments TPS23861 PoE PSE. + +TPS23861 is a quad port IEEE802.3at PSE controller with optional I2C control +and monitoring capabilities. + +TPS23861 offers three modes of operation: Auto, Semi-Auto and Manual. + +This driver only supports the Auto mode of operation providing monitoring +as well as enabling/disabling the four ports. + +Sysfs entries +------------- + +======================= ===================================================================== +in[0-3]_input Voltage on ports [1-4] +in[0-3]_label "Port[1-4]" +in4_input IC input voltage +in4_label "Input" +temp1_input IC die temperature +temp1_label "Die" +curr[1-4]_input Current on ports [1-4] +in[1-4]_label "Port[1-4]" +in[0-3]_enable Enable/disable ports [1-4] +======================= ===================================================================== diff --git a/Documentation/i2c/slave-testunit-backend.rst b/Documentation/i2c/slave-testunit-backend.rst index 2c38e64f0bac4e12734570ad2708e6110112c3e1..ecfc2abec32d7299ee5f612e17125df934ab72fc 100644 --- a/Documentation/i2c/slave-testunit-backend.rst +++ b/Documentation/i2c/slave-testunit-backend.rst @@ -22,8 +22,9 @@ Instantiating the device is regular. Example for bus 0, address 0x30: After that, you will have a write-only device listening. Reads will just return an 8-bit version number of the testunit. When writing, the device consists of 4 -8-bit registers and all must be written to start a testcase, i.e. you must -always write 4 bytes to the device. The registers are: +8-bit registers and, except for some "partial" commands, all registers must be +written to start a testcase, i.e. you usually write 4 bytes to the device. The +registers are: 0x00 CMD - which test to trigger 0x01 DATAL - configuration byte 1 for the test @@ -67,3 +68,21 @@ status word is currently ignored in the Linux Kernel. Example to send a notification after 10ms: # i2cset -y 0 0x30 0x02 0x42 0x64 0x01 i + +0x03 SMBUS_BLOCK_PROC_CALL (partial command) + DATAL - must be '1', i.e. one further byte will be written + DATAH - number of bytes to be sent back + DELAY - not applicable, partial command! + +This test will respond to a block process call as defined by the SMBus +specification. The one data byte written specifies how many bytes will be sent +back in the following read transfer. Note that in this read transfer, the +testunit will prefix the length of the bytes to follow. So, if your host bus +driver emulates SMBus calls like the majority does, it needs to support the +I2C_M_RECV_LEN flag of an i2c_msg. This is a good testcase for it. The returned +data consists of the length first, and then of an array of bytes from length-1 +to 0. Here is an example which emulates i2c_smbus_block_process_call() using +i2ctransfer (you need i2c-tools v4.2 or later): + +# i2ctransfer -y 0 w3@0x30 0x03 0x01 0x10 r? +0x10 0x0f 0x0e 0x0d 0x0c 0x0b 0x0a 0x09 0x08 0x07 0x06 0x05 0x04 0x03 0x02 0x01 0x00 diff --git a/Documentation/iio/ep93xx_adc.rst b/Documentation/iio/ep93xx_adc.rst index 4fd8dea3f6b8c7f26c8aac8ae5b1fcc335d49719..0af0e90404578fb1c29c548f51fe9d201f2b107c 100644 --- a/Documentation/iio/ep93xx_adc.rst +++ b/Documentation/iio/ep93xx_adc.rst @@ -13,7 +13,7 @@ touchscreen/ADC module. ==================== Numbering scheme for channels 0..4 is defined in EP9301 and EP9302 datasheets. -EP9307, EP9312 and EP9312 have 3 channels more (total 8), but the numbering is +EP9307, EP9312 and EP9315 have 3 channels more (total 8), but the numbering is not defined. So the last three are numbered randomly, let's say. Assuming ep93xx_adc is IIO device0, you'd find the following entries under diff --git a/Documentation/index.rst b/Documentation/index.rst index 5888e8a7272fdfd9dab329641a8acf2d8f9cc7e0..31f2adc8542dcef0a76995a3fad2ddf473919b71 100644 --- a/Documentation/index.rst +++ b/Documentation/index.rst @@ -171,17 +171,6 @@ implementation. x86/index xtensa/index -Filesystem Documentation ------------------------- - -The documentation in this section are provided by specific filesystem -subprojects. - -.. toctree:: - :maxdepth: 2 - - filesystems/ext4/index - Other documentation ------------------- diff --git a/Documentation/input/event-codes.rst b/Documentation/input/event-codes.rst index b24b5343f5eb3a5e527f55b0a4dd0ff0095606cd..3118fc1c1e26bc50640ec478d65f3ca1c9e39868 100644 --- a/Documentation/input/event-codes.rst +++ b/Documentation/input/event-codes.rst @@ -236,6 +236,21 @@ A few EV_ABS codes have special meanings: - Used to describe multitouch input events. Please see multi-touch-protocol.txt for details. +* ABS_PRESSURE/ABS_MT_PRESSURE: + + - For touch devices, many devices converted contact size into pressure. + A finger flattens with pressure, causing a larger contact area and thus + pressure and contact size are directly related. This is not the case + for other devices, for example digitizers and touchpads with a true + pressure sensor ("pressure pads"). + + A device should set the resolution of the axis to indicate whether the + pressure is in measurable units. If the resolution is zero, the + pressure data is in arbitrary units. If the resolution is nonzero, the + pressure data is in units/gram. For example, a value of 10 with a + resolution of 1 represents 10 gram, a value of 10 with a resolution on + 1000 represents 10 microgram. + EV_SW ----- diff --git a/Documentation/input/multi-touch-protocol.rst b/Documentation/input/multi-touch-protocol.rst index 307fe22d96686464be5705952eec370444309f34..21c1e6a228886136e4f126e88dedc844420c3a90 100644 --- a/Documentation/input/multi-touch-protocol.rst +++ b/Documentation/input/multi-touch-protocol.rst @@ -260,6 +260,10 @@ ABS_MT_PRESSURE of TOUCH and WIDTH for pressure-based devices or any device with a spatial signal intensity distribution. + If the resolution is zero, the pressure data is in arbitrary units. + If the resolution is nonzero, the pressure data is in units/gram. See + :ref:`input-event-codes` for details. + ABS_MT_DISTANCE The distance, in surface units, between the contact and the surface. Zero distance means the contact is touching the surface. A positive number means diff --git a/Documentation/kbuild/makefiles.rst b/Documentation/kbuild/makefiles.rst index 910c6303c7eaf2711c62b8f71d63acd4fc5fb18e..db3af0b45bafa7872f54a3b5fd91753396ea89a6 100644 --- a/Documentation/kbuild/makefiles.rst +++ b/Documentation/kbuild/makefiles.rst @@ -12,7 +12,7 @@ This document describes the Linux kernel Makefiles. --- 3.1 Goal definitions --- 3.2 Built-in object goals - obj-y --- 3.3 Loadable module goals - obj-m - --- 3.4 Objects which export symbols + --- 3.4 --- 3.5 Library file goals - lib-y --- 3.6 Descending down in directories --- 3.7 Non-builtin vmlinux targets - extra-y @@ -247,12 +247,6 @@ more details, with real examples. kbuild will build an ext2.o file for you out of the individual parts and then link this into built-in.a, as you would expect. -3.4 Objects which export symbols --------------------------------- - - No special notation is required in the makefiles for - modules exporting symbols. - 3.5 Library file goals - lib-y ------------------------------ @@ -461,10 +455,8 @@ more details, with real examples. # drivers/scsi/Makefile CFLAGS_aha152x.o = -DAHA152X_STAT -DAUTOCONF - CFLAGS_gdth.o = # -DDEBUG_GDTH=2 -D__SERIAL__ -D__COM2__ \ - -DGDTH_STATISTICS - These two lines specify compilation flags for aha152x.o and gdth.o. + This line specify compilation flags for aha152x.o. $(AFLAGS_$@) is a similar feature for source files in assembly languages. diff --git a/Documentation/kernel-hacking/hacking.rst b/Documentation/kernel-hacking/hacking.rst index eed2136d847fe0315b91db9d25f6598d3b16372a..451523424942bdf9ce6f4fa7f427ea098c6d4617 100644 --- a/Documentation/kernel-hacking/hacking.rst +++ b/Documentation/kernel-hacking/hacking.rst @@ -346,8 +346,8 @@ routine. Before inventing your own cache of often-used objects consider using a slab cache in ``include/linux/slab.h`` -:c:func:`current()` -------------------- +:c:macro:`current` +------------------ Defined in ``include/asm/current.h`` diff --git a/Documentation/kernel-hacking/locking.rst b/Documentation/kernel-hacking/locking.rst index c3448929a824ab2373cbc64cb894ee70177f3ac8..ed1284c6f078a232dd90b92f4a94672dbc1d6e2b 100644 --- a/Documentation/kernel-hacking/locking.rst +++ b/Documentation/kernel-hacking/locking.rst @@ -958,7 +958,7 @@ grabs a read lock, searches a list, fails to find what it wants, drops the read lock, grabs a write lock and inserts the object has a race condition. -If you don't see why, please stay the fuck away from my code. +If you don't see why, please stay away from my code. Racing Timers: A Kernel Pastime ------------------------------- diff --git a/Documentation/livepatch/index.rst b/Documentation/livepatch/index.rst index 525944063be7aac9f4b20766830fefe6ff9d5796..43cce5fad705f58b956b0e39145b64f6f3901581 100644 --- a/Documentation/livepatch/index.rst +++ b/Documentation/livepatch/index.rst @@ -13,6 +13,7 @@ Kernel Livepatching module-elf-format shadow-vars system-state + reliable-stacktrace .. only:: subproject and html diff --git a/Documentation/livepatch/livepatch.rst b/Documentation/livepatch/livepatch.rst index c2c598c4ead89779709f760de2afc4d5902169b2..68e3651e8af92521dbea3fd62effe2c87907541b 100644 --- a/Documentation/livepatch/livepatch.rst +++ b/Documentation/livepatch/livepatch.rst @@ -6,20 +6,7 @@ This document outlines basic information about kernel livepatching. .. Table of Contents: - 1. Motivation - 2. Kprobes, Ftrace, Livepatching - 3. Consistency model - 4. Livepatch module - 4.1. New functions - 4.2. Metadata - 5. Livepatch life-cycle - 5.1. Loading - 5.2. Enabling - 5.3. Replacing - 5.4. Disabling - 5.5. Removing - 6. Sysfs - 7. Limitations +.. contents:: :local: 1. Motivation diff --git a/Documentation/livepatch/module-elf-format.rst b/Documentation/livepatch/module-elf-format.rst index 8c6b894c4661069a8464fe58dd28cf559dd5992e..dbe9b400e39fe041bd36ce038a8ed7fe58e43650 100644 --- a/Documentation/livepatch/module-elf-format.rst +++ b/Documentation/livepatch/module-elf-format.rst @@ -7,14 +7,8 @@ This document outlines the Elf format requirements that livepatch modules must f .. Table of Contents - 1. Background and motivation - 2. Livepatch modinfo field - 3. Livepatch relocation sections - 3.1 Livepatch relocation section format - 4. Livepatch symbols - 4.1 A livepatch module's symbol table - 4.2 Livepatch symbol format - 5. Symbol table and Elf section access +.. contents:: :local: + 1. Background and motivation ============================ diff --git a/Documentation/livepatch/reliable-stacktrace.rst b/Documentation/livepatch/reliable-stacktrace.rst new file mode 100644 index 0000000000000000000000000000000000000000..67459d2ca2af9e1659ec534e3a3a1c1832775e59 --- /dev/null +++ b/Documentation/livepatch/reliable-stacktrace.rst @@ -0,0 +1,309 @@ +=================== +Reliable Stacktrace +=================== + +This document outlines basic information about reliable stacktracing. + +.. Table of Contents: + +.. contents:: :local: + +1. Introduction +=============== + +The kernel livepatch consistency model relies on accurately identifying which +functions may have live state and therefore may not be safe to patch. One way +to identify which functions are live is to use a stacktrace. + +Existing stacktrace code may not always give an accurate picture of all +functions with live state, and best-effort approaches which can be helpful for +debugging are unsound for livepatching. Livepatching depends on architectures +to provide a *reliable* stacktrace which ensures it never omits any live +functions from a trace. + + +2. Requirements +=============== + +Architectures must implement one of the reliable stacktrace functions. +Architectures using CONFIG_ARCH_STACKWALK must implement +'arch_stack_walk_reliable', and other architectures must implement +'save_stack_trace_tsk_reliable'. + +Principally, the reliable stacktrace function must ensure that either: + +* The trace includes all functions that the task may be returned to, and the + return code is zero to indicate that the trace is reliable. + +* The return code is non-zero to indicate that the trace is not reliable. + +.. note:: + In some cases it is legitimate to omit specific functions from the trace, + but all other functions must be reported. These cases are described in + futher detail below. + +Secondly, the reliable stacktrace function must be robust to cases where +the stack or other unwind state is corrupt or otherwise unreliable. The +function should attempt to detect such cases and return a non-zero error +code, and should not get stuck in an infinite loop or access memory in +an unsafe way. Specific cases are described in further detail below. + + +3. Compile-time analysis +======================== + +To ensure that kernel code can be correctly unwound in all cases, +architectures may need to verify that code has been compiled in a manner +expected by the unwinder. For example, an unwinder may expect that +functions manipulate the stack pointer in a limited way, or that all +functions use specific prologue and epilogue sequences. Architectures +with such requirements should verify the kernel compilation using +objtool. + +In some cases, an unwinder may require metadata to correctly unwind. +Where necessary, this metadata should be generated at build time using +objtool. + + +4. Considerations +================= + +The unwinding process varies across architectures, their respective procedure +call standards, and kernel configurations. This section describes common +details that architectures should consider. + +4.1 Identifying successful termination +-------------------------------------- + +Unwinding may terminate early for a number of reasons, including: + +* Stack or frame pointer corruption. + +* Missing unwind support for an uncommon scenario, or a bug in the unwinder. + +* Dynamically generated code (e.g. eBPF) or foreign code (e.g. EFI runtime + services) not following the conventions expected by the unwinder. + +To ensure that this does not result in functions being omitted from the trace, +even if not caught by other checks, it is strongly recommended that +architectures verify that a stacktrace ends at an expected location, e.g. + +* Within a specific function that is an entry point to the kernel. + +* At a specific location on a stack expected for a kernel entry point. + +* On a specific stack expected for a kernel entry point (e.g. if the + architecture has separate task and IRQ stacks). + +4.2 Identifying unwindable code +------------------------------- + +Unwinding typically relies on code following specific conventions (e.g. +manipulating a frame pointer), but there can be code which may not follow these +conventions and may require special handling in the unwinder, e.g. + +* Exception vectors and entry assembly. + +* Procedure Linkage Table (PLT) entries and veneer functions. + +* Trampoline assembly (e.g. ftrace, kprobes). + +* Dynamically generated code (e.g. eBPF, optprobe trampolines). + +* Foreign code (e.g. EFI runtime services). + +To ensure that such cases do not result in functions being omitted from a +trace, it is strongly recommended that architectures positively identify code +which is known to be reliable to unwind from, and reject unwinding from all +other code. + +Kernel code including modules and eBPF can be distinguished from foreign code +using '__kernel_text_address()'. Checking for this also helps to detect stack +corruption. + +There are several ways an architecture may identify kernel code which is deemed +unreliable to unwind from, e.g. + +* Placing such code into special linker sections, and rejecting unwinding from + any code in these sections. + +* Identifying specific portions of code using bounds information. + +4.3 Unwinding across interrupts and exceptions +---------------------------------------------- + +At function call boundaries the stack and other unwind state is expected to be +in a consistent state suitable for reliable unwinding, but this may not be the +case part-way through a function. For example, during a function prologue or +epilogue a frame pointer may be transiently invalid, or during the function +body the return address may be held in an arbitrary general purpose register. +For some architectures this may change at runtime as a result of dynamic +instrumentation. + +If an interrupt or other exception is taken while the stack or other unwind +state is in an inconsistent state, it may not be possible to reliably unwind, +and it may not be possible to identify whether such unwinding will be reliable. +See below for examples. + +Architectures which cannot identify when it is reliable to unwind such cases +(or where it is never reliable) must reject unwinding across exception +boundaries. Note that it may be reliable to unwind across certain +exceptions (e.g. IRQ) but unreliable to unwind across other exceptions +(e.g. NMI). + +Architectures which can identify when it is reliable to unwind such cases (or +have no such cases) should attempt to unwind across exception boundaries, as +doing so can prevent unnecessarily stalling livepatch consistency checks and +permits livepatch transitions to complete more quickly. + +4.4 Rewriting of return addresses +--------------------------------- + +Some trampolines temporarily modify the return address of a function in order +to intercept when that function returns with a return trampoline, e.g. + +* An ftrace trampoline may modify the return address so that function graph + tracing can intercept returns. + +* A kprobes (or optprobes) trampoline may modify the return address so that + kretprobes can intercept returns. + +When this happens, the original return address will not be in its usual +location. For trampolines which are not subject to live patching, where an +unwinder can reliably determine the original return address and no unwind state +is altered by the trampoline, the unwinder may report the original return +address in place of the trampoline and report this as reliable. Otherwise, an +unwinder must report these cases as unreliable. + +Special care is required when identifying the original return address, as this +information is not in a consistent location for the duration of the entry +trampoline or return trampoline. For example, considering the x86_64 +'return_to_handler' return trampoline: + +.. code-block:: none + + SYM_CODE_START(return_to_handler) + UNWIND_HINT_EMPTY + subq $24, %rsp + + /* Save the return values */ + movq %rax, (%rsp) + movq %rdx, 8(%rsp) + movq %rbp, %rdi + + call ftrace_return_to_handler + + movq %rax, %rdi + movq 8(%rsp), %rdx + movq (%rsp), %rax + addq $24, %rsp + JMP_NOSPEC rdi + SYM_CODE_END(return_to_handler) + +While the traced function runs its return address on the stack points to +the start of return_to_handler, and the original return address is stored in +the task's cur_ret_stack. During this time the unwinder can find the return +address using ftrace_graph_ret_addr(). + +When the traced function returns to return_to_handler, there is no longer a +return address on the stack, though the original return address is still stored +in the task's cur_ret_stack. Within ftrace_return_to_handler(), the original +return address is removed from cur_ret_stack and is transiently moved +arbitrarily by the compiler before being returned in rax. The return_to_handler +trampoline moves this into rdi before jumping to it. + +Architectures might not always be able to unwind such sequences, such as when +ftrace_return_to_handler() has removed the address from cur_ret_stack, and the +location of the return address cannot be reliably determined. + +It is recommended that architectures unwind cases where return_to_handler has +not yet been returned to, but architectures are not required to unwind from the +middle of return_to_handler and can report this as unreliable. Architectures +are not required to unwind from other trampolines which modify the return +address. + +4.5 Obscuring of return addresses +--------------------------------- + +Some trampolines do not rewrite the return address in order to intercept +returns, but do transiently clobber the return address or other unwind state. + +For example, the x86_64 implementation of optprobes patches the probed function +with a JMP instruction which targets the associated optprobe trampoline. When +the probe is hit, the CPU will branch to the optprobe trampoline, and the +address of the probed function is not held in any register or on the stack. + +Similarly, the arm64 implementation of DYNAMIC_FTRACE_WITH_REGS patches traced +functions with the following: + +.. code-block:: none + + MOV X9, X30 + BL + +The MOV saves the link register (X30) into X9 to preserve the return address +before the BL clobbers the link register and branches to the trampoline. At the +start of the trampoline, the address of the traced function is in X9 rather +than the link register as would usually be the case. + +Architectures must either ensure that unwinders either reliably unwind +such cases, or report the unwinding as unreliable. + +4.6 Link register unreliability +------------------------------- + +On some other architectures, 'call' instructions place the return address into a +link register, and 'return' instructions consume the return address from the +link register without modifying the register. On these architectures software +must save the return address to the stack prior to making a function call. Over +the duration of a function call, the return address may be held in the link +register alone, on the stack alone, or in both locations. + +Unwinders typically assume the link register is always live, but this +assumption can lead to unreliable stack traces. For example, consider the +following arm64 assembly for a simple function: + +.. code-block:: none + + function: + STP X29, X30, [SP, -16]! + MOV X29, SP + BL + LDP X29, X30, [SP], #16 + RET + +At entry to the function, the link register (x30) points to the caller, and the +frame pointer (X29) points to the caller's frame including the caller's return +address. The first two instructions create a new stackframe and update the +frame pointer, and at this point the link register and the frame pointer both +describe this function's return address. A trace at this point may describe +this function twice, and if the function return is being traced, the unwinder +may consume two entries from the fgraph return stack rather than one entry. + +The BL invokes 'other_function' with the link register pointing to this +function's LDR and the frame pointer pointing to this function's stackframe. +When 'other_function' returns, the link register is left pointing at the BL, +and so a trace at this point could result in 'function' appearing twice in the +backtrace. + +Similarly, a function may deliberately clobber the LR, e.g. + +.. code-block:: none + + caller: + STP X29, X30, [SP, -16]! + MOV X29, SP + ADR LR, + BLR LR + LDP X29, X30, [SP], #16 + RET + +The ADR places the address of 'callee' into the LR, before the BLR branches to +this address. If a trace is made immediately after the ADR, 'callee' will +appear to be the parent of 'caller', rather than the child. + +Due to cases such as the above, it may only be possible to reliably consume a +link register value at a function call boundary. Architectures where this is +the case must reject unwinding across exception boundaries unless they can +reliably identify when the LR or stack value should be used (e.g. using +metadata generated by objtool). diff --git a/Documentation/power/freezing-of-tasks.rst b/Documentation/power/freezing-of-tasks.rst index 8bd693399834784be059839683c12d89b0c9ea58..53b6a56c46354f8f84991e640b3a4315b42f9a57 100644 --- a/Documentation/power/freezing-of-tasks.rst +++ b/Documentation/power/freezing-of-tasks.rst @@ -134,7 +134,7 @@ Generally speaking, there is a couple of reasons to use the freezing of tasks: safeguards against race conditions that might occur in such a case. Although Linus Torvalds doesn't like the freezing of tasks, he said this in one -of the discussions on LKML (http://lkml.org/lkml/2007/4/27/608): +of the discussions on LKML (https://lore.kernel.org/r/alpine.LFD.0.98.0704271801020.9964@woody.linux-foundation.org): "RJW:> Why we freeze tasks at all or why we freeze kernel threads? diff --git a/Documentation/powerpc/syscall64-abi.rst b/Documentation/powerpc/syscall64-abi.rst index cf9b2857c72aa3ffbcda31b726d177b6c11c2730..dabee3729e5a5b1a12ed4c4274c2616e48660602 100644 --- a/Documentation/powerpc/syscall64-abi.rst +++ b/Documentation/powerpc/syscall64-abi.rst @@ -46,25 +46,38 @@ stack frame LR and CR save fields are not used. Register preservation rules --------------------------- -Register preservation rules match the ELF ABI calling sequence with the -following differences: - -+------------------------------------------------------------------------+ -| For the sc instruction, differences with the ELF ABI | -+--------------+--------------+------------------------------------------+ -| r0 | Volatile | (System call number.) | -| rr3 | Volatile | (Parameter 1, and return value.) | -| rr4-r8 | Volatile | (Parameters 2-6.) | -| rcr0 | Volatile | (cr0.SO is the return error condition.) | -| rcr1, cr5-7 | Nonvolatile | | -| rlr | Nonvolatile | | -+--------------+--------------+------------------------------------------+ -| For the scv 0 instruction, differences with the ELF ABI | -+--------------+--------------+------------------------------------------+ -| r0 | Volatile | (System call number.) | -| r3 | Volatile | (Parameter 1, and return value.) | -| r4-r8 | Volatile | (Parameters 2-6.) | -+--------------+--------------+------------------------------------------+ +Register preservation rules match the ELF ABI calling sequence with some +differences. + +For the sc instruction, the differences from the ELF ABI are as follows: + ++--------------+--------------------+-----------------------------------------+ +| Register | Preservation Rules | Purpose | ++==============+====================+=========================================+ +| r0 | Volatile | (System call number.) | ++--------------+--------------------+-----------------------------------------+ +| r3 | Volatile | (Parameter 1, and return value.) | ++--------------+--------------------+-----------------------------------------+ +| r4-r8 | Volatile | (Parameters 2-6.) | ++--------------+--------------------+-----------------------------------------+ +| cr0 | Volatile | (cr0.SO is the return error condition.) | ++--------------+--------------------+-----------------------------------------+ +| cr1, cr5-7 | Nonvolatile | | ++--------------+--------------------+-----------------------------------------+ +| lr | Nonvolatile | | ++--------------+--------------------+-----------------------------------------+ + +For the scv 0 instruction, the differences from the ELF ABI are as follows: + ++--------------+--------------------+-----------------------------------------+ +| Register | Preservation Rules | Purpose | ++==============+====================+=========================================+ +| r0 | Volatile | (System call number.) | ++--------------+--------------------+-----------------------------------------+ +| r3 | Volatile | (Parameter 1, and return value.) | ++--------------+--------------------+-----------------------------------------+ +| r4-r8 | Volatile | (Parameters 2-6.) | ++--------------+--------------------+-----------------------------------------+ All floating point and vector data registers as well as control and status registers are nonvolatile. diff --git a/Documentation/process/4.Coding.rst b/Documentation/process/4.Coding.rst index 0825dc496f22d7e75de88ec700704fa81f302972..1f0d81f44e14b25981dbb8c65c972fa9f20b55ce 100644 --- a/Documentation/process/4.Coding.rst +++ b/Documentation/process/4.Coding.rst @@ -242,7 +242,7 @@ and try to avoid "fixes" which make the warning go away without addressing its cause. Note that not all compiler warnings are enabled by default. Build the -kernel with "make EXTRA_CFLAGS=-W" to get the full set. +kernel with "make KCFLAGS=-W" to get the full set. The kernel provides several configuration options which turn on debugging features; most of these are found in the "kernel hacking" submenu. Several diff --git a/Documentation/process/adding-syscalls.rst b/Documentation/process/adding-syscalls.rst index a3ecb236576cabd8c0cb4f7a8288b97861b0bd36..906c47f1a9e5086a1c5f451b1d8bebe19b0c6ff0 100644 --- a/Documentation/process/adding-syscalls.rst +++ b/Documentation/process/adding-syscalls.rst @@ -501,7 +501,7 @@ table, but not from elsewhere in the kernel. If the syscall functionality is useful to be used within the kernel, needs to be shared between an old and a new syscall, or needs to be shared between a syscall and its compatibility variant, it should be implemented by means of a "helper" function (such as -``kern_xyzzy()``). This kernel function may then be called within the +``ksys_xyzzy()``). This kernel function may then be called within the syscall stub (``sys_xyzzy()``), the compatibility syscall stub (``compat_sys_xyzzy()``), and/or other kernel code. @@ -548,18 +548,18 @@ References and Sources https://lwn.net/Articles/486306/ - Recommendation from Andrew Morton that all related information for a new system call should come in the same email thread: - https://lkml.org/lkml/2014/7/24/641 + https://lore.kernel.org/r/20140724144747.3041b208832bbdf9fbce5d96@linux-foundation.org - Recommendation from Michael Kerrisk that a new system call should come with - a man page: https://lkml.org/lkml/2014/6/13/309 + a man page: https://lore.kernel.org/r/CAKgNAkgMA39AfoSoA5Pe1r9N+ZzfYQNvNPvcRN7tOvRb8+v06Q@mail.gmail.com - Suggestion from Thomas Gleixner that x86 wire-up should be in a separate - commit: https://lkml.org/lkml/2014/11/19/254 + commit: https://lore.kernel.org/r/alpine.DEB.2.11.1411191249560.3909@nanos - Suggestion from Greg Kroah-Hartman that it's good for new system calls to - come with a man-page & selftest: https://lkml.org/lkml/2014/3/19/710 + come with a man-page & selftest: https://lore.kernel.org/r/20140320025530.GA25469@kroah.com - Discussion from Michael Kerrisk of new system call vs. :manpage:`prctl(2)` extension: - https://lkml.org/lkml/2014/6/3/411 + https://lore.kernel.org/r/CAHO5Pa3F2MjfTtfNxa8LbnkeeU8=YJ+9tDqxZpw7Gz59E-4AUg@mail.gmail.com - Suggestion from Ingo Molnar that system calls that involve multiple arguments should encapsulate those arguments in a struct, which includes a - size field for future extensibility: https://lkml.org/lkml/2015/7/30/117 + size field for future extensibility: https://lore.kernel.org/r/20150730083831.GA22182@gmail.com - Numbering oddities arising from (re-)use of O_* numbering space flags: - commit 75069f2b5bfb ("vfs: renumber FMODE_NONOTIFY and add to uniqueness @@ -569,9 +569,9 @@ References and Sources - commit bb458c644a59 ("Safer ABI for O_TMPFILE") - Discussion from Matthew Wilcox about restrictions on 64-bit arguments: - https://lkml.org/lkml/2008/12/12/187 + https://lore.kernel.org/r/20081212152929.GM26095@parisc-linux.org - Recommendation from Greg Kroah-Hartman that unknown flags should be - policed: https://lkml.org/lkml/2014/7/17/577 + policed: https://lore.kernel.org/r/20140717193330.GB4703@kroah.com - Recommendation from Linus Torvalds that x32 system calls should prefer compatibility with 64-bit versions rather than 32-bit versions: - https://lkml.org/lkml/2011/8/31/244 + https://lore.kernel.org/r/CA+55aFxfmwfB7jbbrXxa=K7VBYPfAvmu3XOkGrLbB1UFjX1+Ew@mail.gmail.com diff --git a/Documentation/process/coding-style.rst b/Documentation/process/coding-style.rst index 98227226c4e5f24dfb45550d3ccb718fa7e2208e..42969ab37b341ca45a825e97581edc1e5a1a678e 100644 --- a/Documentation/process/coding-style.rst +++ b/Documentation/process/coding-style.rst @@ -69,9 +69,26 @@ something to hide: if (condition) do_this; do_something_everytime; +Don't use commas to avoid using braces: + +.. code-block:: c + + if (condition) + do_this(), do_that(); + +Always uses braces for multiple statements: + +.. code-block:: c + + if (condition) { + do_this(); + do_that(); + } + Don't put multiple assignments on a single line either. Kernel coding style is super simple. Avoid tricky expressions. + Outside of comments, documentation and except in Kconfig, spaces are never used for indentation, and the above example is deliberately broken. @@ -306,8 +323,7 @@ that counts the number of active users, you should call that Encoding the type of a function into the name (so-called Hungarian notation) is asinine - the compiler knows the types anyway and can check -those, and it only confuses the programmer. No wonder Microsoft makes buggy -programs. +those, and it only confuses the programmer. LOCAL variable names should be short, and to the point. If you have some random integer loop counter, it should probably be called ``i``. diff --git a/Documentation/process/howto.rst b/Documentation/process/howto.rst index 7a5c105e34d4cea860486578e74d39d98121f3a4..e4beeca57e5f281e64c41bf004a6d8ad632ccad8 100644 --- a/Documentation/process/howto.rst +++ b/Documentation/process/howto.rst @@ -342,16 +342,10 @@ Adventurous testers are very welcome to runtime-test the linux-next. Bug Reporting ------------- -https://bugzilla.kernel.org is where the Linux kernel developers track kernel -bugs. Users are encouraged to report all bugs that they find in this -tool. For details on how to use the kernel bugzilla, please see: - - https://bugzilla.kernel.org/page.cgi?id=faq.html - The file 'Documentation/admin-guide/reporting-issues.rst' in the main kernel -source directory has a good template for how to report a possible kernel bug, -and details what kind of information is needed by the kernel developers to help -track down the problem. +source directory describes how to report a possible kernel bug, and details +what kind of information is needed by the kernel developers to help track +down the problem. Managing bug reports @@ -364,7 +358,13 @@ improve your skills, and other developers will be aware of your presence. Fixing bugs is one of the best ways to get merits among other developers, because not many people like wasting time fixing other people's bugs. -To work in the already reported bug reports, go to https://bugzilla.kernel.org. +To work on already reported bug reports, find a subsystem you are interested in. +Check the MAINTAINERS file where bugs for that subsystem get reported to; often +it will be a mailing list, rarely a bugtracker. Search the archives of said +place for recent reports and help where you see fit. You may also want to check +https://bugzilla.kernel.org for bug reports; only a handful of kernel subsystems +use it actively for reporting or tracking, nevertheless bugs for the whole +kernel get filed there. Mailing lists diff --git a/Documentation/process/magic-number.rst b/Documentation/process/magic-number.rst index c6dfe060ec2f90d229f9129130753536f4502a8c..fa5a62f4150cc8e9cfd90fb0b9aef059d9a66458 100644 --- a/Documentation/process/magic-number.rst +++ b/Documentation/process/magic-number.rst @@ -99,7 +99,6 @@ USB_SERIAL_PORT_MAGIC 0x7301 usb_serial_port ``drivers/usb/se CG_MAGIC 0x00090255 ufs_cylinder_group ``include/linux/ufs_fs.h`` RPORT_MAGIC 0x00525001 r_port ``drivers/char/rocket_int.h`` LSEMAGIC 0x05091998 lse ``drivers/fc4/fc.c`` -GDTIOCTL_MAGIC 0x06030f07 gdth_iowr_str ``drivers/scsi/gdth_ioctl.h`` RIEBL_MAGIC 0x09051990 ``drivers/net/atarilance.c`` NBD_REQUEST_MAGIC 0x12560953 nbd_request ``include/linux/nbd.h`` RED_MAGIC2 0x170fc2a5 (any) ``mm/slab.c`` @@ -142,7 +141,6 @@ PWC_MAGIC 0x89DC10AB pwc_device ``drivers/usb/me NBD_REPLY_MAGIC 0x96744668 nbd_reply ``include/linux/nbd.h`` ENI155_MAGIC 0xa54b872d midway_eprom ``drivers/atm/eni.h`` CODA_MAGIC 0xC0DAC0DA coda_file_info ``fs/coda/coda_fs_i.h`` -DPMEM_MAGIC 0xc0ffee11 gdt_pci_sram ``drivers/scsi/gdth.h`` YAM_MAGIC 0xF10A7654 yam_port ``drivers/net/hamradio/yam.c`` CCB_MAGIC 0xf2691ad2 ccb ``drivers/scsi/ncr53c8xx.c`` QUEUE_MAGIC_FREE 0xf7e1c9a3 queue_entry ``drivers/scsi/arm/queue.c`` diff --git a/Documentation/process/submit-checklist.rst b/Documentation/process/submit-checklist.rst index 230ee42f872f13f93d4470ba8eef11fff2d68080..b1bc2d37bd0a1d46df5e3f2c6be973ec3daa5469 100644 --- a/Documentation/process/submit-checklist.rst +++ b/Documentation/process/submit-checklist.rst @@ -89,30 +89,28 @@ and elsewhere regarding submitting Linux kernel patches. Patches that change userspace interfaces should be CCed to linux-api@vger.kernel.org. -19) Check that it all passes ``make headers_check``. - -20) Has been checked with injection of at least slab and page-allocation +19) Has been checked with injection of at least slab and page-allocation failures. See ``Documentation/fault-injection/``. If the new code is substantial, addition of subsystem-specific fault injection might be appropriate. -21) Newly-added code has been compiled with ``gcc -W`` (use - ``make EXTRA_CFLAGS=-W``). This will generate lots of noise, but is good +20) Newly-added code has been compiled with ``gcc -W`` (use + ``make KCFLAGS=-W``). This will generate lots of noise, but is good for finding bugs like "warning: comparison between signed and unsigned". -22) Tested after it has been merged into the -mm patchset to make sure +21) Tested after it has been merged into the -mm patchset to make sure that it still works with all of the other queued patches and various changes in the VM, VFS, and other subsystems. -23) All memory barriers {e.g., ``barrier()``, ``rmb()``, ``wmb()``} need a +22) All memory barriers {e.g., ``barrier()``, ``rmb()``, ``wmb()``} need a comment in the source code that explains the logic of what they are doing and why. -24) If any ioctl's are added by the patch, then also update +23) If any ioctl's are added by the patch, then also update ``Documentation/userspace-api/ioctl/ioctl-number.rst``. -25) If your modified source code depends on or uses any of the kernel +24) If your modified source code depends on or uses any of the kernel APIs or features that are related to the following ``Kconfig`` symbols, then test multiple builds with the related ``Kconfig`` symbols disabled and/or ``=m`` (if that option is available) [not all of these at the diff --git a/Documentation/process/submitting-patches.rst b/Documentation/process/submitting-patches.rst index 5ba54120bef7ecfc11a17803edc33df5d43671a1..8c991c8636280e47fc80be27f08479b3e13f5b56 100644 --- a/Documentation/process/submitting-patches.rst +++ b/Documentation/process/submitting-patches.rst @@ -556,6 +556,11 @@ which stable kernel versions should receive your fix. This is the preferred method for indicating a bug fixed by the patch. See :ref:`describe_changes` for more details. +Note: Attaching a Fixes: tag does not subvert the stable kernel rules +process nor the requirement to Cc: stable@vger.kernel.org on all stable +patch candidates. For more information, please read +:ref:`Documentation/process/stable-kernel-rules.rst ` + .. _the_canonical_patch_format: The canonical patch format @@ -679,6 +684,26 @@ generates appropriate diffstats by default.) See more details on the proper patch format in the following references. +Backtraces in commit mesages +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Backtraces help document the call chain leading to a problem. However, +not all backtraces are helpful. For example, early boot call chains are +unique and obvious. Copying the full dmesg output verbatim, however, +adds distracting information like timestamps, module lists, register and +stack dumps. + +Therefore, the most useful backtraces should distill the relevant +information from the dump, which makes it easier to focus on the real +issue. Here is an example of a well-trimmed backtrace:: + + unchecked MSR access error: WRMSR to 0xd51 (tried to write 0x0000000000000064) + at rIP: 0xffffffffae059994 (native_write_msr+0x4/0x20) + Call Trace: + mba_wrmsr + update_domains + rdtgroup_mkdir + .. _explicit_in_reply_to: Explicit In-Reply-To headers @@ -769,13 +794,13 @@ Greg Kroah-Hartman, "How to piss off a kernel subsystem maintainer". NO!!!! No more huge patch bombs to linux-kernel@vger.kernel.org people! - + Kernel Documentation/process/coding-style.rst: :ref:`Documentation/process/coding-style.rst ` Linus Torvalds's mail on the canonical patch format: - + Andi Kleen, "On submitting kernel patches" Some strategies to get difficult or controversial changes in. diff --git a/Documentation/scheduler/sched-bwc.rst b/Documentation/scheduler/sched-bwc.rst index 9801d6b284b1ecfdd51b7e1ab5ab5f9bc6da3b40..845eee659199c23ac9a5e0f6dc9b9e757718af69 100644 --- a/Documentation/scheduler/sched-bwc.rst +++ b/Documentation/scheduler/sched-bwc.rst @@ -2,8 +2,9 @@ CFS Bandwidth Control ===================== -[ This document only discusses CPU bandwidth control for SCHED_NORMAL. - The SCHED_RT case is covered in Documentation/scheduler/sched-rt-group.rst ] +.. note:: + This document only discusses CPU bandwidth control for SCHED_NORMAL. + The SCHED_RT case is covered in Documentation/scheduler/sched-rt-group.rst CFS bandwidth control is a CONFIG_FAIR_GROUP_SCHED extension which allows the specification of the maximum CPU bandwidth available to a group or hierarchy. @@ -25,9 +26,15 @@ Management ---------- Quota and period are managed within the cpu subsystem via cgroupfs. -cpu.cfs_quota_us: the total available run-time within a period (in microseconds) -cpu.cfs_period_us: the length of a period (in microseconds) -cpu.stat: exports throttling statistics [explained further below] +.. note:: + The cgroupfs files described in this section are only applicable + to cgroup v1. For cgroup v2, see + :ref:`Documentation/admin-guide/cgroupv2.rst `. + +- cpu.cfs_quota_us: the total available run-time within a period (in + microseconds) +- cpu.cfs_period_us: the length of a period (in microseconds) +- cpu.stat: exports throttling statistics [explained further below] The default values are:: diff --git a/Documentation/scheduler/sched-deadline.rst b/Documentation/scheduler/sched-deadline.rst index 14a2f7bf63fe1f5ac57371d5799e649d85b9dbf9..9d9be52f221a5a165bb206e6c3e6456c6b6916fd 100644 --- a/Documentation/scheduler/sched-deadline.rst +++ b/Documentation/scheduler/sched-deadline.rst @@ -707,7 +707,7 @@ Deadline Task Scheduling and how to prevent non-root users "cheat" the system? As already discussed, we are planning also to merge this work with the EDF - throttling patches [https://lkml.org/lkml/2010/2/23/239] but we still are in + throttling patches [https://lore.kernel.org/r/cover.1266931410.git.fabio@helm.retis] but we still are in the preliminary phases of the merge and we really seek feedback that would help us decide on the direction it should take. diff --git a/Documentation/scheduler/sched-design-CFS.rst b/Documentation/scheduler/sched-design-CFS.rst index a96c726518771000c7805f2a9c18edbc7f0e0b14..59b2d1fb4dc4779382a7487bad7a829f9a6c6bef 100644 --- a/Documentation/scheduler/sched-design-CFS.rst +++ b/Documentation/scheduler/sched-design-CFS.rst @@ -34,9 +34,9 @@ In CFS the virtual runtime is expressed and tracked via the per-task p->se.vruntime (nanosec-unit) value. This way, it's possible to accurately timestamp and measure the "expected CPU time" a task should have gotten. -[ small detail: on "ideal" hardware, at any time all tasks would have the same - p->se.vruntime value --- i.e., tasks would execute simultaneously and no task - would ever get "out of balance" from the "ideal" share of CPU time. ] + Small detail: on "ideal" hardware, at any time all tasks would have the same + p->se.vruntime value --- i.e., tasks would execute simultaneously and no task + would ever get "out of balance" from the "ideal" share of CPU time. CFS's task picking logic is based on this p->se.vruntime value and it is thus very simple: it always tries to run the task with the smallest p->se.vruntime diff --git a/Documentation/scsi/libsas.rst b/Documentation/scsi/libsas.rst index 7216b5d25800112efcd81a2985c49e0cdf77b6ba..6589dfefbc02ad24d1f87e7f9906e427e29faef0 100644 --- a/Documentation/scsi/libsas.rst +++ b/Documentation/scsi/libsas.rst @@ -189,13 +189,8 @@ num_phys The event interface:: /* LLDD calls these to notify the class of an event. */ - void (*notify_ha_event)(struct sas_ha_struct *, enum ha_event); - void (*notify_port_event)(struct sas_phy *, enum port_event); - void (*notify_phy_event)(struct sas_phy *, enum phy_event); - -When sas_register_ha() returns, those are set and can be -called by the LLDD to notify the SAS layer of such events -the SAS layer. + void sas_notify_port_event(struct sas_phy *, enum port_event, gfp_t); + void sas_notify_phy_event(struct sas_phy *, enum phy_event, gfp_t); The port notification:: diff --git a/Documentation/scsi/scsi-parameters.rst b/Documentation/scsi/scsi-parameters.rst index dea5803f5c623e5f032d9880e68b2334e6749fb0..c42c55e1e25e7292420d0a13d2da2de977528392 100644 --- a/Documentation/scsi/scsi-parameters.rst +++ b/Documentation/scsi/scsi-parameters.rst @@ -38,9 +38,6 @@ parameters may be changed at runtime by the command See drivers/scsi/BusLogic.c, comment before function BusLogic_ParseDriverOptions(). - gdth= [HW,SCSI] - See header of drivers/scsi/gdth.c. - gvp11= [HW,SCSI] ips= [HW,SCSI] Adaptec / IBM ServeRAID controller diff --git a/Documentation/security/keys/core.rst b/Documentation/security/keys/core.rst index aa0081685ee11fd7c5072e6fbd1a15d34a4e42a3..b3ed5c581034c74104ab82a32bb8a70c5c275658 100644 --- a/Documentation/security/keys/core.rst +++ b/Documentation/security/keys/core.rst @@ -1040,8 +1040,8 @@ The keyctl syscall functions are: "key" is the ID of the key to be watched. - "queue_fd" is a file descriptor referring to an open "/dev/watch_queue" - which manages the buffer into which notifications will be delivered. + "queue_fd" is a file descriptor referring to an open pipe which + manages the buffer into which notifications will be delivered. "filter" is either NULL to remove a watch or a filter specification to indicate what events are required from the key. diff --git a/Documentation/security/lsm-development.rst b/Documentation/security/lsm-development.rst index 31d92bc5fdd2ef893c701d217c97faf61806b390..ac53e5065f79ed209d92da38382cf0eea281cb31 100644 --- a/Documentation/security/lsm-development.rst +++ b/Documentation/security/lsm-development.rst @@ -2,7 +2,7 @@ Linux Security Module Development ================================= -Based on https://lkml.org/lkml/2007/10/26/215, +Based on https://lore.kernel.org/r/20071026073721.618b4778@laptopd505.fenrus.org, a new LSM is accepted into the kernel when its intent (a description of what it tries to protect against and in what cases one would expect to use it) has been appropriately documented in ``Documentation/admin-guide/LSM/``. diff --git a/Documentation/sphinx/automarkup.py b/Documentation/sphinx/automarkup.py index 953b24b6e2b45ce3f4cb88308b3b60c9bbc840b3..acf5473002f3770d623ca0a54e67acd433247a1f 100644 --- a/Documentation/sphinx/automarkup.py +++ b/Documentation/sphinx/automarkup.py @@ -51,7 +51,7 @@ RE_typedef = re.compile(r'\b(typedef)\s+([a-zA-Z_]\w+)', flags=ascii_p3) # Detects a reference to a documentation page of the form Documentation/... with # an optional extension # -RE_doc = re.compile(r'\bDocumentation(/[\w\-_/]+)(\.\w+)*') +RE_doc = re.compile(r'(\bDocumentation/)?((\.\./)*[\w\-/]+)\.(rst|txt)') RE_namespace = re.compile(r'^\s*..\s*c:namespace::\s*(\S+)\s*$') @@ -234,7 +234,10 @@ def markup_doc_ref(docname, app, match): # # Go through the dance of getting an xref out of the std domain # - target = match.group(1) + absolute = match.group(1) + target = match.group(2) + if absolute: + target = "/" + target xref = None pxref = addnodes.pending_xref('', refdomain = 'std', reftype = 'doc', reftarget = target, modname = None, diff --git a/Documentation/sphinx/cdomain.py b/Documentation/sphinx/cdomain.py index 014a5229e57af679942e9ced9b0f3e9a2a0af1a4..ca8ac9e59ded4bd4bbaa7534874ceb7b08228236 100644 --- a/Documentation/sphinx/cdomain.py +++ b/Documentation/sphinx/cdomain.py @@ -236,13 +236,7 @@ class CObject(Base_CObject): indextext = self.get_index_text(name) if indextext: - if major == 1 and minor < 4: - # indexnode's tuple changed in 1.4 - # https://github.com/sphinx-doc/sphinx/commit/e6a5a3a92e938fcd75866b4227db9e0524d58f7c - self.indexnode['entries'].append( - ('single', indextext, targetname, '')) - else: - self.indexnode['entries'].append( + self.indexnode['entries'].append( ('single', indextext, targetname, '', None)) class CDomain(Base_CDomain): diff --git a/Documentation/sphinx/kernel_abi.py b/Documentation/sphinx/kernel_abi.py index f3da859c9878e09ec4da011a852c8644752bfc91..efe760e410c457dac2e99f5bff73cfbc4f2a4a78 100644 --- a/Documentation/sphinx/kernel_abi.py +++ b/Documentation/sphinx/kernel_abi.py @@ -45,17 +45,7 @@ from docutils import nodes, statemachine from docutils.statemachine import ViewList from docutils.parsers.rst import directives, Directive from docutils.utils.error_reporting import ErrorString - -# -# AutodocReporter is only good up to Sphinx 1.7 -# -import sphinx - -Use_SSI = sphinx.__version__[:3] >= '1.7' -if Use_SSI: - from sphinx.util.docutils import switch_source_input -else: - from sphinx.ext.autodoc import AutodocReporter +from sphinx.util.docutils import switch_source_input __version__ = '1.0' @@ -179,16 +169,5 @@ class KernelCmd(Directive): return node.children def do_parse(self, content, node): - if Use_SSI: - with switch_source_input(self.state, content): - self.state.nested_parse(content, 0, node, match_titles=1) - else: - buf = self.state.memo.title_styles, self.state.memo.section_level, self.state.memo.reporter - - self.state.memo.title_styles = [] - self.state.memo.section_level = 0 - self.state.memo.reporter = AutodocReporter(content, self.state.memo.reporter) - try: - self.state.nested_parse(content, 0, node, match_titles=1) - finally: - self.state.memo.title_styles, self.state.memo.section_level, self.state.memo.reporter = buf + with switch_source_input(self.state, content): + self.state.nested_parse(content, 0, node, match_titles=1) diff --git a/Documentation/sphinx/kernel_feat.py b/Documentation/sphinx/kernel_feat.py index 2fee04f1dedd885aad9876cb47f50e801acef6c5..c91ea2b27697a2af430b9a6bb00035955e9669b8 100644 --- a/Documentation/sphinx/kernel_feat.py +++ b/Documentation/sphinx/kernel_feat.py @@ -42,17 +42,7 @@ from docutils import nodes, statemachine from docutils.statemachine import ViewList from docutils.parsers.rst import directives, Directive from docutils.utils.error_reporting import ErrorString - -# -# AutodocReporter is only good up to Sphinx 1.7 -# -import sphinx - -Use_SSI = sphinx.__version__[:3] >= '1.7' -if Use_SSI: - from sphinx.util.docutils import switch_source_input -else: - from sphinx.ext.autodoc import AutodocReporter +from sphinx.util.docutils import switch_source_input __version__ = '1.0' @@ -154,16 +144,7 @@ class KernelFeat(Directive): buf = self.state.memo.title_styles, self.state.memo.section_level, self.state.memo.reporter - if Use_SSI: - with switch_source_input(self.state, content): - self.state.nested_parse(content, 0, node, match_titles=1) - else: - self.state.memo.title_styles = [] - self.state.memo.section_level = 0 - self.state.memo.reporter = AutodocReporter(content, self.state.memo.reporter) - try: - self.state.nested_parse(content, 0, node, match_titles=1) - finally: - self.state.memo.title_styles, self.state.memo.section_level, self.state.memo.reporter = buf + with switch_source_input(self.state, content): + self.state.nested_parse(content, 0, node, match_titles=1) return node.children diff --git a/Documentation/sphinx/kerneldoc.py b/Documentation/sphinx/kerneldoc.py index e9857ab904f1858566f4d0f10e1eb6c73794dad3..8189c33b9ddafd540d38c58290d07ce9248a1b2a 100644 --- a/Documentation/sphinx/kerneldoc.py +++ b/Documentation/sphinx/kerneldoc.py @@ -37,18 +37,8 @@ import glob from docutils import nodes, statemachine from docutils.statemachine import ViewList from docutils.parsers.rst import directives, Directive - -# -# AutodocReporter is only good up to Sphinx 1.7 -# import sphinx - -Use_SSI = sphinx.__version__[:3] >= '1.7' -if Use_SSI: - from sphinx.util.docutils import switch_source_input -else: - from sphinx.ext.autodoc import AutodocReporter - +from sphinx.util.docutils import switch_source_input import kernellog __version__ = '1.0' @@ -163,18 +153,8 @@ class KernelDocDirective(Directive): return [nodes.error(None, nodes.paragraph(text = "kernel-doc missing"))] def do_parse(self, result, node): - if Use_SSI: - with switch_source_input(self.state, result): - self.state.nested_parse(result, 0, node, match_titles=1) - else: - save = self.state.memo.title_styles, self.state.memo.section_level, self.state.memo.reporter - self.state.memo.reporter = AutodocReporter(result, self.state.memo.reporter) - self.state.memo.title_styles, self.state.memo.section_level = [], 0 - try: - self.state.nested_parse(result, 0, node, match_titles=1) - finally: - self.state.memo.title_styles, self.state.memo.section_level, self.state.memo.reporter = save - + with switch_source_input(self.state, result): + self.state.nested_parse(result, 0, node, match_titles=1) def setup(app): app.add_config_value('kerneldoc_bin', None, 'env') diff --git a/Documentation/sphinx/kernellog.py b/Documentation/sphinx/kernellog.py index 8ac7d274f542b78d9e6061b4f6900e309d817613..0bc00c138cadeec6509c230ff2181338c2472ce6 100644 --- a/Documentation/sphinx/kernellog.py +++ b/Documentation/sphinx/kernellog.py @@ -4,29 +4,19 @@ # only goes back to 1.6. So here's a wrapper layer to keep around for # as long as we support 1.4. # +# We don't support 1.4 anymore, but we'll keep the wrappers around until +# we change all the code to not use them anymore :) +# import sphinx +from sphinx.util import logging -if sphinx.__version__[:3] >= '1.6': - UseLogging = True - from sphinx.util import logging - logger = logging.getLogger('kerneldoc') -else: - UseLogging = False +logger = logging.getLogger('kerneldoc') def warn(app, message): - if UseLogging: - logger.warning(message) - else: - app.warn(message) + logger.warning(message) def verbose(app, message): - if UseLogging: - logger.verbose(message) - else: - app.verbose(message) + logger.verbose(message) def info(app, message): - if UseLogging: - logger.info(message) - else: - app.info(message) + logger.info(message) diff --git a/Documentation/sphinx/kfigure.py b/Documentation/sphinx/kfigure.py index 788704886eec96abfc1affd6219eb5a3d5fa9bbc..3c78828330be0ab61c7aeb304df4e75fc0dc2cfd 100644 --- a/Documentation/sphinx/kfigure.py +++ b/Documentation/sphinx/kfigure.py @@ -49,26 +49,14 @@ import os from os import path import subprocess from hashlib import sha1 -import sys - from docutils import nodes from docutils.statemachine import ViewList from docutils.parsers.rst import directives from docutils.parsers.rst.directives import images import sphinx - from sphinx.util.nodes import clean_astext -from six import iteritems - import kernellog -PY3 = sys.version_info[0] == 3 - -if PY3: - _unicode = str -else: - _unicode = unicode - # Get Sphinx version major, minor, patch = sphinx.version_info[:3] if major == 1 and minor > 3: @@ -540,7 +528,7 @@ def add_kernel_figure_to_std_domain(app, doctree): docname = app.env.docname labels = std.data["labels"] - for name, explicit in iteritems(doctree.nametypes): + for name, explicit in doctree.nametypes.items(): if not explicit: continue labelid = doctree.nameids[name] diff --git a/Documentation/sphinx/maintainers_include.py b/Documentation/sphinx/maintainers_include.py index dc8fed48d3c275189d0bdb2bb529d4bffecb3846..328b3631a585cd6d0b12b4376314892a083efc15 100755 --- a/Documentation/sphinx/maintainers_include.py +++ b/Documentation/sphinx/maintainers_include.py @@ -61,8 +61,6 @@ class MaintainersInclude(Include): field_content = "" for line in open(path): - if sys.version_info.major == 2: - line = unicode(line, 'utf-8') # Have we reached the end of the preformatted Descriptions text? if descriptions and line.startswith('Maintainers'): descriptions = False diff --git a/Documentation/sphinx/requirements.txt b/Documentation/sphinx/requirements.txt index 5030d346d23bc8f178b5b2c11458c52543a855ed..489f6626de6740374b0fdbb9f14d24243fecfdf2 100644 --- a/Documentation/sphinx/requirements.txt +++ b/Documentation/sphinx/requirements.txt @@ -1,4 +1,3 @@ docutils Sphinx==2.4.4 sphinx_rtd_theme -six diff --git a/Documentation/sphinx/rstFlatTable.py b/Documentation/sphinx/rstFlatTable.py index 2019a55f6b182a4a5306b204b096739bfae143bb..a3eea0bbe6ba6dc8ead548a6ad20c4a14f25cc43 100755 --- a/Documentation/sphinx/rstFlatTable.py +++ b/Documentation/sphinx/rstFlatTable.py @@ -42,8 +42,6 @@ u""" # imports # ============================================================================== -import sys - from docutils import nodes from docutils.parsers.rst import directives, roles from docutils.parsers.rst.directives.tables import Table @@ -55,14 +53,6 @@ from docutils.utils import SystemMessagePropagation __version__ = '1.0' -PY3 = sys.version_info[0] == 3 -PY2 = sys.version_info[0] == 2 - -if PY3: - # pylint: disable=C0103, W0622 - unicode = str - basestring = str - # ============================================================================== def setup(app): # ============================================================================== diff --git a/Documentation/timers/timers-howto.rst b/Documentation/timers/timers-howto.rst index afb0a43b8cdfb69853664bd5544860265ddfeb0c..5c169e3d29a8e586c3933b3a1bec1dbac686fda9 100644 --- a/Documentation/timers/timers-howto.rst +++ b/Documentation/timers/timers-howto.rst @@ -75,7 +75,7 @@ NON-ATOMIC CONTEXT: - Why not msleep for (1ms - 20ms)? Explained originally here: - http://lkml.org/lkml/2007/8/3/250 + https://lore.kernel.org/r/15327.1186166232@lwn.net msleep(1~20) may not do what the caller intends, and will often sleep longer (~20 ms actual sleep for any diff --git a/Documentation/trace/coresight/coresight.rst b/Documentation/trace/coresight/coresight.rst index 0b73acb44efa8906a18701c8733a5e86335691a1..169749efd8d1a438a79b3b0b5898206880db87f2 100644 --- a/Documentation/trace/coresight/coresight.rst +++ b/Documentation/trace/coresight/coresight.rst @@ -512,6 +512,38 @@ The --itrace option controls the type and frequency of synthesized events Note that only 64-bit programs are currently supported - further work is required to support instruction decode of 32-bit Arm programs. +2.2) Tracing PID + +The kernel can be built to write the PID value into the PE ContextID registers. +For a kernel running at EL1, the PID is stored in CONTEXTIDR_EL1. A PE may +implement Arm Virtualization Host Extensions (VHE), which the kernel can +run at EL2 as a virtualisation host; in this case, the PID value is stored in +CONTEXTIDR_EL2. + +perf provides PMU formats that program the ETM to insert these values into the +trace data; the PMU formats are defined as below: + + "contextid1": Available on both EL1 kernel and EL2 kernel. When the + kernel is running at EL1, "contextid1" enables the PID + tracing; when the kernel is running at EL2, this enables + tracing the PID of guest applications. + + "contextid2": Only usable when the kernel is running at EL2. When + selected, enables PID tracing on EL2 kernel. + + "contextid": Will be an alias for the option that enables PID + tracing. I.e, + contextid == contextid1, on EL1 kernel. + contextid == contextid2, on EL2 kernel. + +perf will always enable PID tracing at the relevant EL, this is accomplished by +automatically enable the "contextid" config - but for EL2 it is possible to make +specific adjustments using configs "contextid1" and "contextid2", E.g. if a user +wants to trace PIDs for both host and guest, the two configs "contextid1" and +"contextid2" can be set at the same time: + + perf record -e cs_etm/contextid1,contextid2/u -- vm + Generating coverage files for Feedback Directed Optimization: AutoFDO --------------------------------------------------------------------- diff --git a/Documentation/trace/ftrace.rst b/Documentation/trace/ftrace.rst index 87cf5c010d5dda9f54d80d2f4da2eb26762d4e1b..62c98e9bbdd9d86e9141c12b6f86082e5e1a7ab6 100644 --- a/Documentation/trace/ftrace.rst +++ b/Documentation/trace/ftrace.rst @@ -1159,6 +1159,12 @@ Here are the available options: This simulates the original behavior of the trace file. When the file is closed, tracing will be enabled again. + hash-ptr + When set, "%p" in the event printk format displays the + hashed pointer value instead of real address. + This will be useful if you want to find out which hashed + value is corresponding to the real value in trace log. + record-cmd When any event or tracer is enabled, a hook is enabled in the sched_switch trace point to fill comm cache diff --git a/Documentation/translations/it_IT/process/4.Coding.rst b/Documentation/translations/it_IT/process/4.Coding.rst index a5e36aa60448c17ec3502a738b59a73fbaae59ca..8012fe9497aea50b94978a8d9a7678f7684d608f 100644 --- a/Documentation/translations/it_IT/process/4.Coding.rst +++ b/Documentation/translations/it_IT/process/4.Coding.rst @@ -256,7 +256,7 @@ e cercate di evitare le "riparazioni" che fan sparire l'avvertimento senza però averne trovato la causa. Tenete a mente che non tutti gli avvertimenti sono disabilitati di default. -Costruite il kernel con "make EXTRA_CFLAGS=-W" per ottenerli tutti. +Costruite il kernel con "make KCFLAGS=-W" per ottenerli tutti. Il kernel fornisce differenti opzioni che abilitano funzionalità di debugging; molti di queste sono trovano all'interno del sotto menu "kernel hacking". diff --git a/Documentation/translations/it_IT/process/adding-syscalls.rst b/Documentation/translations/it_IT/process/adding-syscalls.rst index bff0a82bf127c4e27fe87103ad4c354f1ba784eb..c478b6e8c292228cec1e13d9cbcf137391c682af 100644 --- a/Documentation/translations/it_IT/process/adding-syscalls.rst +++ b/Documentation/translations/it_IT/process/adding-syscalls.rst @@ -611,21 +611,21 @@ Riferimenti e fonti https://lwn.net/Articles/486306/ - Raccomandazioni da Andrew Morton circa il fatto che tutte le informazioni su una nuova chiamata di sistema dovrebbero essere contenute nello stesso - filone di discussione di email: https://lkml.org/lkml/2014/7/24/641 + filone di discussione di email: https://lore.kernel.org/r/20140724144747.3041b208832bbdf9fbce5d96@linux-foundation.org - Raccomandazioni da Michael Kerrisk circa il fatto che le nuove chiamate di - sistema dovrebbero avere una pagina man: https://lkml.org/lkml/2014/6/13/309 + sistema dovrebbero avere una pagina man: https://lore.kernel.org/r/CAKgNAkgMA39AfoSoA5Pe1r9N+ZzfYQNvNPvcRN7tOvRb8+v06Q@mail.gmail.com - Consigli da Thomas Gleixner sul fatto che il collegamento all'architettura x86 dovrebbe avvenire in un *commit* differente: - https://lkml.org/lkml/2014/11/19/254 + https://lore.kernel.org/r/alpine.DEB.2.11.1411191249560.3909@nanos - Consigli da Greg Kroah-Hartman circa la bontà d'avere una pagina man e un programma di auto-verifica per le nuove chiamate di sistema: - https://lkml.org/lkml/2014/3/19/710 + https://lore.kernel.org/r/20140320025530.GA25469@kroah.com - Discussione di Michael Kerrisk sulle nuove chiamate di sistema contro - le estensioni :manpage:`prctl(2)`: https://lkml.org/lkml/2014/6/3/411 + le estensioni :manpage:`prctl(2)`: https://lore.kernel.org/r/CAHO5Pa3F2MjfTtfNxa8LbnkeeU8=YJ+9tDqxZpw7Gz59E-4AUg@mail.gmail.com - Consigli da Ingo Molnar che le chiamate di sistema con più argomenti dovrebbero incapsularli in una struttura che includa un argomento *size* per garantire l'estensibilità futura: - https://lkml.org/lkml/2015/7/30/117 + https://lore.kernel.org/r/20150730083831.GA22182@gmail.com - Un certo numero di casi strani emersi dall'uso (riuso) dei flag O_*: - commit 75069f2b5bfb ("vfs: renumber FMODE_NONOTIFY and add to uniqueness @@ -635,9 +635,9 @@ Riferimenti e fonti - commit bb458c644a59 ("Safer ABI for O_TMPFILE") - Discussion from Matthew Wilcox about restrictions on 64-bit arguments: - https://lkml.org/lkml/2008/12/12/187 + https://lore.kernel.org/r/20081212152929.GM26095@parisc-linux.org - Raccomandazioni da Greg Kroah-Hartman sul fatto che i flag sconosciuti dovrebbero - essere controllati: https://lkml.org/lkml/2014/7/17/577 + essere controllati: https://lore.kernel.org/r/20140717193330.GB4703@kroah.com - Raccomandazioni da Linus Torvalds che le chiamate di sistema x32 dovrebbero favorire la compatibilità con le versioni a 64-bit piuttosto che quelle a 32-bit: - https://lkml.org/lkml/2011/8/31/244 + https://lore.kernel.org/r/CA+55aFxfmwfB7jbbrXxa=K7VBYPfAvmu3XOkGrLbB1UFjX1+Ew@mail.gmail.com diff --git a/Documentation/translations/it_IT/process/submit-checklist.rst b/Documentation/translations/it_IT/process/submit-checklist.rst index 3e575502690fa1f1f019e841acbf42e282a30f62..614fc17d90869a87aa3443841fc9937eea66a6ec 100644 --- a/Documentation/translations/it_IT/process/submit-checklist.rst +++ b/Documentation/translations/it_IT/process/submit-checklist.rst @@ -104,7 +104,7 @@ sottomissione delle patch, in particolare l'iniezione di fallimenti specifici per il sottosistema. 22) Il nuovo codice è stato compilato con ``gcc -W`` (usate - ``make EXTRA_CFLAGS=-W``). Questo genererà molti avvisi, ma è ottimo + ``make KCFLAGS=-W``). Questo genererà molti avvisi, ma è ottimo per scovare bachi come "warning: comparison between signed and unsigned". 23) La patch è stata verificata dopo essere stata inclusa nella serie di patch diff --git a/Documentation/translations/it_IT/process/submitting-patches.rst b/Documentation/translations/it_IT/process/submitting-patches.rst index 966cd3242a60ff50612e8b09b58af131f3d351e0..ae00352346edd76dcb2117308528895e89a45e06 100644 --- a/Documentation/translations/it_IT/process/submitting-patches.rst +++ b/Documentation/translations/it_IT/process/submitting-patches.rst @@ -731,13 +731,13 @@ Greg Kroah-Hartman, "Come scocciare un manutentore di un sottosistema" No!!!! Basta gigantesche bombe patch alle persone sulla lista linux-kernel@vger.kernel.org! - + Kernel Documentation/translations/it_IT/process/coding-style.rst: :ref:`Documentation/translations/it_IT/process/coding-style.rst ` E-mail di Linus Torvalds sul formato canonico di una patch: - + Andi Kleen, "Su come sottomettere patch del kernel" Alcune strategie su come sottomettere modifiche toste o controverse. diff --git a/Documentation/translations/ja_JP/SubmittingPatches b/Documentation/translations/ja_JP/SubmittingPatches index dd0c3280ba5ac1a10e136eed0014e6b1e305d872..6854f5add72e815854b92f2602230ca56df31aef 100644 --- a/Documentation/translations/ja_JP/SubmittingPatches +++ b/Documentation/translations/ja_JP/SubmittingPatches @@ -702,13 +702,13 @@ Greg Kroah-Hartman, "How to piss off a kernel subsystem maintainer". NO!!!! No more huge patch bombs to linux-kernel@vger.kernel.org people! - + Kernel Documentation/process/coding-style.rst: Linus Torvalds's mail on the canonical patch format: - + Andi Kleen, "On submitting kernel patches" Some strategies to get difficult or controversial changes in. diff --git a/Documentation/translations/ko_KR/howto.rst b/Documentation/translations/ko_KR/howto.rst index 240d29be38f252b2ea3a2dd0a58e69cc43d482e1..787f1e85f8a079eaab1c2d803b3624bb2d37e4a9 100644 --- a/Documentation/translations/ko_KR/howto.rst +++ b/Documentation/translations/ko_KR/howto.rst @@ -345,7 +345,7 @@ https://bugzilla.kernel.org 는 리눅스 ì»¤ë„ ê°œë°œìžë“¤ì´ 커ë„ì˜ ë²„ https://bugzilla.kernel.org/page.cgi?id=faq.html -ë©”ì¸ ì»¤ë„ ì†ŒìŠ¤ ë””ë ‰í† ë¦¬ì— ìžˆëŠ” :ref:`admin-guide/reporting-bugs.rst ` +ë©”ì¸ ì»¤ë„ ì†ŒìŠ¤ ë””ë ‰í† ë¦¬ì— ìžˆëŠ” 'Documentation/admin-guide/reporting-issues.rst' 파ì¼ì€ ì»¤ë„ ë²„ê·¸ë¼ê³  ìƒê°ë˜ëŠ” ê²ƒì„ ë³´ê³ í•˜ëŠ” ë°©ë²•ì— ê´€í•œ ì¢‹ì€ í…œí”Œë¦¿ì´ë©° 문제를 추ì í•˜ê¸° 위해서 ì»¤ë„ ê°œë°œìžë“¤ì´ 필요로 하는 ì •ë³´ê°€ 무엇들ì¸ì§€ë¥¼ ìƒì„¸ížˆ 설명하고 있다. @@ -583,7 +583,7 @@ Patì´ë¼ëŠ” ì´ë¦„ì„ ê°€ì§„ ì—¬ìžê°€ ìžˆì„ ìˆ˜ë„ ìžˆëŠ” 것ì´ë‹¤. 리눅 "The Perfect Patch" - http://www.ozlabs.org/~akpm/stuff/tpp.txt + https://www.ozlabs.org/~akpm/stuff/tpp.txt ì´ ëª¨ë“  ê²ƒì„ í•˜ëŠ” ê²ƒì€ ë§¤ìš° 어려운 ì¼ì´ë‹¤. 완벽히 소화하는 ë°ëŠ” ì ì–´ë„ ëª‡ë…„ì´ diff --git a/Documentation/translations/ko_KR/index.rst b/Documentation/translations/ko_KR/index.rst index 27995c4233de7048b5d548f3397e2babe92c4775..b9e27d20b039b523cf093ab2a1e03ee2497439d0 100644 --- a/Documentation/translations/ko_KR/index.rst +++ b/Documentation/translations/ko_KR/index.rst @@ -10,3 +10,18 @@ :maxdepth: 1 howto + + +리눅스 ì»¤ë„ ë©”ëª¨ë¦¬ 배리어 +------------------------- + +.. raw:: latex + + \footnotesize + +.. include:: ./memory-barriers.txt + :literal: + +.. raw:: latex + + \normalsize diff --git a/Documentation/translations/zh_CN/admin-guide/cpu-load.rst b/Documentation/translations/zh_CN/admin-guide/cpu-load.rst index c972731c0e5730805c1d614cf0656a937fd99ded..a73400a054ffba9a77766d340899191b8ad4179b 100644 --- a/Documentation/translations/zh_CN/admin-guide/cpu-load.rst +++ b/Documentation/translations/zh_CN/admin-guide/cpu-load.rst @@ -95,7 +95,7 @@ Linux通过``/proc/stat``å’Œ``/proc/uptime``导出å„ç§ä¿¡æ¯ï¼Œç”¨æˆ·ç©ºé—´å·¥ å‚考 --- -- http://lkml.org/lkml/2007/2/12/6 +- https://lore.kernel.org/r/loom.20070212T063225-663@post.gmane.org - Documentation/filesystems/proc.rst (1.8) diff --git a/Documentation/translations/zh_CN/arm/Booting b/Documentation/translations/zh_CN/arm/Booting index c3d26ce5f6de0e2098a15371442d0a3d77f460cd..5ecea0767893971a59a5eca3e31a1d1664713a14 100644 --- a/Documentation/translations/zh_CN/arm/Booting +++ b/Documentation/translations/zh_CN/arm/Booting @@ -124,7 +124,7 @@ bootloader 必须传递一个系统内存的ä½ç½®å’Œæœ€å°å€¼ï¼Œä»¥åŠæ ¹æ–‡ä»¶ bootloader 必须以 64bit 地å€å¯¹é½çš„å½¢å¼åŠ è½½ä¸€ä¸ªè®¾å¤‡æ ‘映åƒ(dtb)到系统 RAM 中，并用å¯åŠ¨æ•°æ®åˆå§‹åŒ–它。dtb æ ¼å¼åœ¨æ–‡æ¡£ -Documentation/devicetree/booting-without-of.rst 中。内核将会在 +https://www.devicetree.org/specifications/ 中。内核将会在 dtb 物ç†åœ°å€å¤„查找 dtb 魔数值（0xd00dfeed），以确定 dtb 是å¦å·²ç»ä»£æ›¿ 标签列表被传递进æ¥ã€‚ diff --git a/Documentation/translations/zh_CN/iio/ep93xx_adc.rst b/Documentation/translations/zh_CN/iio/ep93xx_adc.rst new file mode 100644 index 0000000000000000000000000000000000000000..7e91d2197867d08c871cc889e8166d3a2beeea65 --- /dev/null +++ b/Documentation/translations/zh_CN/iio/ep93xx_adc.rst @@ -0,0 +1,46 @@ +.. include:: ../disclaimer-zh_CN.rst + +:Original: :doc:`../../../iio/ep93xx_adc` +:Translator: Yanteng Si + +.. _cn_iio_ep93xx_adc: + + +================================== +æ€ç¿é€»è¾‘ EP93xx 模拟数字转æ¢å™¨é©±åŠ¨ +================================== + +1. 概述 +======= + +该驱动åŒæ—¶é€‚用于具有5通é“模拟数字转æ¢å™¨çš„低端 (EP9301, Ep9302) 设备和10é€šé“ +触摸å±/模拟数字转æ¢å™¨çš„高端设备(EP9307, EP9312, EP9315)。 + +2. 通é“ç¼–å· +=========== + +EP9301å’ŒEP9302æ•°æ®è¡¨å®šä¹‰äº†é€šé“0..4çš„ç¼–å·æ–¹æ¡ˆã€‚虽然EP9307, EP9312å’ŒEP9315多 +了3个通é“（一共8个），但是编å·å¹¶æ²¡æœ‰å®šä¹‰ã€‚所以说最åŽä¸‰ä¸ªé€šé“是éšæœºç¼–å·çš„。 + +如果ep93xx_adc是IIO设备0，您将在以下ä½ç½®æ‰¾åˆ°æ¡ç›® +/sys/bus/iio/devices/iio:device0/: + + +-----------------+---------------+ + | sysfs å…¥å£ | ball/pin å称 | + +=================+===============+ + | in_voltage0_raw | YM | + +-----------------+---------------+ + | in_voltage1_raw | SXP | + +-----------------+---------------+ + | in_voltage2_raw | SXM | + +-----------------+---------------+ + | in_voltage3_raw | SYP | + +-----------------+---------------+ + | in_voltage4_raw | SYM | + +-----------------+---------------+ + | in_voltage5_raw | XP | + +-----------------+---------------+ + | in_voltage6_raw | XM | + +-----------------+---------------+ + | in_voltage7_raw | YP | + +-----------------+---------------+ diff --git a/Documentation/translations/zh_CN/iio/iio_configfs.rst b/Documentation/translations/zh_CN/iio/iio_configfs.rst new file mode 100644 index 0000000000000000000000000000000000000000..274488e8dce441234cf6d6791a9fa69fda6f202a --- /dev/null +++ b/Documentation/translations/zh_CN/iio/iio_configfs.rst @@ -0,0 +1,102 @@ +.. include:: ../disclaimer-zh_CN.rst + +:Original: :doc:`../../../iio/iio_configfs` +:Translator: Yanteng Si + +.. _cn_iio_configfs: + + +===================== +工业 IIO configfsæ”¯æŒ +===================== + +1. 概述 +======= + +Configfs是一ç§å†…核对象的基于文件系统的管ç†ç³»ç»Ÿï¼ŒIIO使用一些å¯ä»¥é€šè¿‡ +configfsè½»æ¾é…置的对象（例如：设备，触å‘器）。 + +关于configfs是如何è¿è¡Œçš„，请查阅Documentation/filesystems/configfs.rst +了解更多信æ¯ã€‚ + +2. 用法 +======= +为了使configfs支æŒIIO，我们需è¦åœ¨ç¼–译时选中configçš„CONFIG_IIO_CONFIGFS +选项。 + +然åŽï¼ŒæŒ‚è½½configfs文件系统(通常在 /config directory目录下）：： + + $ mkdir/config + $ mount -t configfs none/config + +此时，将创建所有默认IIO组，并å¯ä»¥åœ¨/ config / iio下对其进行访问。 下一章 +将介ç»å¯ç”¨çš„IIOé…置对象。 + +3. 软件触å‘器 +============= + +IIO默认configfs组之一是“触å‘器â€ç»„。 挂载configfsåŽå¯ä»¥è‡ªåŠ¨è®¿é—®å®ƒï¼Œå¹¶ä¸”å¯ +以在/config/iio/triggers下找到。 + +IIO软件触å‘器为创建多ç§è§¦å‘器类型æ供了支æŒã€‚ 通常在include/linux/iio +/sw_trigger.h：中的接å£ä¸‹å°†æ–°çš„触å‘器类型实现为å•ç‹¬çš„内核模å—： +:: + + /* + * drivers/iio/trigger/iio-trig-sample.c + * 一ç§æ–°è§¦å‘器类型的内核模å—实例 + */ + #include + + + static struct iio_sw_trigger *iio_trig_sample_probe(const char *name) + { + /* + * 这将分é…并注册一个IIO触å‘器以åŠå…¶ä»–触å‘器类型特性的åˆå§‹åŒ–。 + */ + } + + static int iio_trig_sample_remove(struct iio_sw_trigger *swt) + { + /* + * 这会废弃iio_trig_sample_probe中的æ“作 + */ + } + + static const struct iio_sw_trigger_ops iio_trig_sample_ops = { + .probe = iio_trig_sample_probe, + .remove = iio_trig_sample_remove, + }; + + static struct iio_sw_trigger_type iio_trig_sample = { + .name = "trig-sample", + .owner = THIS_MODULE, + .ops = &iio_trig_sample_ops, + }; + +module_iio_sw_trigger_driver(iio_trig_sample); + +æ¯ç§è§¦å‘器类型在/config/iio/triggers下都有其自己的目录。 加载iio-trig-sample +模å—将创建“ trig-sampleâ€è§¦å‘器类型目录/config/iio/triggers/trig-sample. + +我们支æŒä»¥ä¸‹ä¸­æ–­æºï¼ˆè§¦å‘器类型） + + * hrtimer,ä½¿ç”¨é«˜åˆ†è¾¨çŽ‡å®šæ—¶å™¨ä½œä¸ºä¸­æ–­æº + +3.1 Hrtimer触å‘å™¨åˆ›å»ºä¸Žé”€æ¯ +--------------------------- + +加载iio-trig-hrtimer模å—将注册hrtimer触å‘器类型，从而å…许用户在 +/config/iio/triggers/hrtimer下创建hrtimer触å‘器。 + +例如:: + + $ mkdir /config/iio/triggers/hrtimer/instance1 + $ rmdir /config/iio/triggers/hrtimer/instance1 + +æ¯ä¸ªè§¦å‘器å¯ä»¥å…·æœ‰ä¸€ä¸ªæˆ–多个独特的触å‘器类型的属性。 + +3.2 "hrtimer" 触å‘器类型属性 +---------------------------- + +"hrtimerâ€è§¦å‘器类型没有æ¥è‡ª/config dir的任何å¯é…置属性。 diff --git a/Documentation/translations/zh_CN/iio/index.rst b/Documentation/translations/zh_CN/iio/index.rst new file mode 100644 index 0000000000000000000000000000000000000000..7087076a10f65b23ae3619df584c0bab32c57456 --- /dev/null +++ b/Documentation/translations/zh_CN/iio/index.rst @@ -0,0 +1,20 @@ +.. SPDX-License-Identifier: GPL-2.0 + +.. include:: ../disclaimer-zh_CN.rst + +:Original: :doc:`../../../iio/index` +:Translator: Yanteng Si + +.. _cn_iio_index: + + +======== +工业 I/O +======== + +.. toctree:: + :maxdepth: 1 + + iio_configfs + + ep93xx_adc diff --git a/Documentation/translations/zh_CN/mips/booting.rst b/Documentation/translations/zh_CN/mips/booting.rst new file mode 100644 index 0000000000000000000000000000000000000000..96453e1b962ee9771bd401d3e0f315115a87967e --- /dev/null +++ b/Documentation/translations/zh_CN/mips/booting.rst @@ -0,0 +1,31 @@ +.. SPDX-License-Identifier: GPL-2.0 + +.. include:: ../disclaimer-zh_CN.rst + +:Original: :doc:`../../../mips/booting` +:Translator: Yanteng Si + +.. _cn_booting: + +BMIPS设备树引导 +------------------------ + + 一些bootloadersåªæ”¯æŒåœ¨å†…核镜åƒå¼€å§‹åœ°å€å¤„çš„å•ä¸€å…¥å£ç‚¹ã€‚而其它 + bootloaders将跳转到ELF的开始地å€å¤„。两ç§æ–¹æ¡ˆéƒ½æ”¯æŒçš„；因为 + CONFIG_BOOT_RAW=y and CONFIG_NO_EXCEPT_FILL=y, 所以第一æ¡æŒ‡ä»¤ + 会立å³è·³è½¬åˆ°kernel_entry()å…¥å£å¤„执行。 + + 与arch/arm情况(b)类似，dt感知的引导加载程åºéœ€è¦è®¾ç½®ä»¥ä¸‹å¯„存器: + + a0 : 0 + + a1 : 0xffffffff + + a2 : RAM中指å‘设备树å—的物ç†æŒ‡é’ˆ(在chapterII中定义)。 + 设备树å¯ä»¥ä½äºŽå‰512MB物ç†åœ°å€ç©ºé—´(0x00000000 - + 0x1fffffff)的任何ä½ç½®ï¼Œä»¥64ä½è¾¹ç•Œå¯¹é½ã€‚ + + 传统bootloadersä¸ä¼šä½¿ç”¨è¿™æ ·çš„约定，并且它们ä¸ä¼ å…¥DTå—。 + 在这ç§æƒ…况下，Linux将通过选中CONFIG_DT_*查找DTB。 + + 以上约定åªåœ¨32ä½ç³»ç»Ÿä¸­å®šä¹‰ï¼Œå› ä¸ºç›®å‰æ²¡æœ‰ä»»ä½•64ä½çš„BMIPS实现。 diff --git a/Documentation/translations/zh_CN/mips/features.rst b/Documentation/translations/zh_CN/mips/features.rst new file mode 100644 index 0000000000000000000000000000000000000000..93d93d06b1b3e819c9f06a7c6e8c885e3e024ff7 --- /dev/null +++ b/Documentation/translations/zh_CN/mips/features.rst @@ -0,0 +1,10 @@ +.. SPDX-License-Identifier: GPL-2.0 + +.. include:: ../disclaimer-zh_CN.rst + +:Original: :doc:`../../../mips/features` +:Translator: Yanteng Si + +.. _cn_features: + +.. kernel-feat:: $srctree/Documentation/features mips diff --git a/Documentation/translations/zh_CN/mips/index.rst b/Documentation/translations/zh_CN/mips/index.rst new file mode 100644 index 0000000000000000000000000000000000000000..b85033f9d67c8f11e3d962beee3b39231fdb8b5a --- /dev/null +++ b/Documentation/translations/zh_CN/mips/index.rst @@ -0,0 +1,26 @@ +.. SPDX-License-Identifier: GPL-2.0 + +.. include:: ../disclaimer-zh_CN.rst + +:Original: :doc:`../../../mips/index` +:Translator: Yanteng Si + +=========================== +MIPS特性文档 +=========================== + +.. toctree:: + :maxdepth: 2 + :numbered: + + booting + ingenic-tcu + + features + +.. only:: subproject and html + + Indices + ======= + + * :ref:`genindex` diff --git a/Documentation/translations/zh_CN/mips/ingenic-tcu.rst b/Documentation/translations/zh_CN/mips/ingenic-tcu.rst new file mode 100644 index 0000000000000000000000000000000000000000..f04ba407384a87753487d1d60a59d13a5f9c4880 --- /dev/null +++ b/Documentation/translations/zh_CN/mips/ingenic-tcu.rst @@ -0,0 +1,69 @@ +.. SPDX-License-Identifier: GPL-2.0 + +.. include:: ../disclaimer-zh_CN.rst + +:Original: :doc:`../../../mips/ingenic-tcu` +:Translator: Yanteng Si + +.. _cn_ingenic-tcu: + +=============================================== +å›æ­£ JZ47xx SoC定时器/计数器硬件å•å…ƒ +=============================================== + +å›æ­£ JZ47xx SoC中的定时器/计数器å•å…ƒ(TCU)是一个多功能硬件å—。它有多达 +8个通é“，å¯ä»¥ç”¨ä½œè®¡æ•°å™¨ï¼Œè®¡æ—¶å™¨ï¼Œæˆ–脉冲宽度调制器。 + +- JZ4725B, JZ4750, JZ4755 åªæœ‰ï¼–个TCU通é“。其它SoC都有８个通é“。 + +- JZ4725B引入了一个独立的通é“，称为æ“作系统计时器(OST)。这是一个32ä½å¯ + 编程定时器。在JZ4760BåŠä»¥ä¸Šåž‹å·ä¸Šï¼Œå®ƒæ˜¯64ä½çš„。 + +- æ¯ä¸ªTCU通é“都有自己的时钟æºï¼Œå¯ä»¥é€šè¿‡ TCSR 寄存器设置通é“的父级时钟 + æºï¼ˆpclkã€extã€rtc）ã€å¼€å…³ä»¥åŠåˆ†é¢‘。 + + - 看门狗和OST硬件模å—在它们的寄存器空间中也有相åŒå½¢å¼çš„TCSR寄存器。 + - 用于关闭/å¼€å¯çš„ TCU 寄存器也å¯ä»¥å…³é—­/å¼€å¯çœ‹é—¨ç‹—å’Œ OST 时钟。 + +- æ¯ä¸ªTCU通é“在两ç§æ¨¡å¼çš„其中一ç§æ¨¡å¼ä¸‹è¿è¡Œï¼š + + - æ¨¡å¼ TCU1：通é“无法在ç¡çœ æ¨¡å¼ä¸‹è¿è¡Œï¼Œä½†æ›´æ˜“于æ“作。 + - æ¨¡å¼ TCU2：通é“å¯ä»¥åœ¨ç¡çœ æ¨¡å¼ä¸‹è¿è¡Œï¼Œä½†æ“作比 TCU1 通é“å¤æ‚一些。 + +- æ¯ä¸ª TCU 通é“的模å¼å–决于使用的SoC： + + - 在最è€çš„SoC（高于JZ4740），八个通é“都è¿è¡Œåœ¨TCU1模å¼ã€‚ + - 在 JZ4725B，通é“5è¿è¡Œåœ¨TCU2,其它通é“则è¿è¡Œåœ¨TCU1。 + - 在最新的SoC（JZ4750åŠä¹‹åŽï¼‰ï¼Œé€šé“1-2è¿è¡Œåœ¨TCU2，其它通é“则è¿è¡Œ + 在TCU1。 + +- æ¯ä¸ªé€šé“都å¯ä»¥ç”Ÿæˆä¸­æ–­ã€‚有些通é“共享一æ¡ä¸­æ–­çº¿ï¼Œè€Œæœ‰äº›æ²¡æœ‰ï¼Œå…¶åœ¨SoCåž‹ + å·ä¹‹é—´çš„å˜æ›´ï¼š + + - 在很è€çš„SoC（JZ4740åŠæ›´ä½Žï¼‰ï¼Œé€šé“0和通é“1有它们自己的中断线；通 + é“2-7共享最åŽä¸€æ¡ä¸­æ–­çº¿ã€‚ + - 在 JZ4725B，通é“0有它自己的中断线；通é“1-5共享一æ¡ä¸­æ–­çº¿ï¼›OST + 使用最åŽä¸€æ¡ä¸­æ–­çº¿ã€‚ + - 在比较新的SoC（JZ4750åŠä»¥åŽï¼‰ï¼Œé€šé“5有它自己的中断线；通 + é“0-4和（如果是8通é“）6-7全部共享一æ¡ä¸­æ–­çº¿ï¼›OST使用最åŽä¸€æ¡ä¸­ + 断线。 + +实现 +==== + +TCU硬件的功能分布在多个驱动程åºï¼š + +============== =================================== +时钟 drivers/clk/ingenic/tcu.c +中断 drivers/irqchip/irq-ingenic-tcu.c +定时器 drivers/clocksource/ingenic-timer.c +OST drivers/clocksource/ingenic-ost.c +脉冲宽度调制器 drivers/pwm/pwm-jz4740.c +看门狗 drivers/watchdog/jz4740_wdt.c +============== =================================== + +因为å¯ä»¥ä»Žç›¸åŒçš„寄存器控制属于ä¸åŒé©±åŠ¨ç¨‹åºå’Œæ¡†æž¶çš„TCUçš„å„ç§åŠŸèƒ½ï¼Œæ‰€ä»¥ +所有这些驱动程åºéƒ½é€šè¿‡ç›¸åŒçš„控制总线通用接å£è®¿é—®å®ƒä»¬çš„寄存器。 + +有关TCU驱动程åºçš„设备树绑定的更多信æ¯ï¼Œè¯·å‚阅: +Documentation/devicetree/bindings/timer/ingenic,tcu.yaml. diff --git a/Documentation/translations/zh_CN/process/4.Coding.rst b/Documentation/translations/zh_CN/process/4.Coding.rst index 959a06ba025c1dca461c674150b79e7b0e019126..66cd8ee076062b8d0e94db86ce915ebab6550b56 100644 --- a/Documentation/translations/zh_CN/process/4.Coding.rst +++ b/Documentation/translations/zh_CN/process/4.Coding.rst @@ -165,7 +165,7 @@ Linus对这个问题给出了最佳答案: 通常，这些警告都指å‘真正的问题。æ交以供审阅的代ç é€šå¸¸ä¸ä¼šäº§ç”Ÿä»»ä½•ç¼–译器警告。 在消除警告时，注æ„了解真正的原因，并尽é‡é¿å…“修å¤â€ï¼Œä½¿è­¦å‘Šæ¶ˆå¤±è€Œä¸è§£å†³å…¶åŽŸå› ã€‚ -请注æ„，并éžæ‰€æœ‰ç¼–译器警告都默认å¯ç”¨ã€‚使用“make EXTRA_CFLAGS=-Wâ€æž„建内核以 +请注æ„，并éžæ‰€æœ‰ç¼–译器警告都默认å¯ç”¨ã€‚使用“make KCFLAGS=-Wâ€æž„建内核以 获得完整集åˆã€‚ 内核æ供了几个é…置选项，å¯ä»¥æ‰“开调试功能；大多数é…置选项ä½äºŽâ€œkernel hacking†diff --git a/Documentation/translations/zh_CN/process/submitting-patches.rst b/Documentation/translations/zh_CN/process/submitting-patches.rst index 2e7dbaad402821956221dd2d00c6b387f526733a..4fc6d16f5196358e21c2322b2a6300388efd494b 100644 --- a/Documentation/translations/zh_CN/process/submitting-patches.rst +++ b/Documentation/translations/zh_CN/process/submitting-patches.rst @@ -668,13 +668,13 @@ Greg Kroah-Hartman, "How to piss off a kernel subsystem maintainer". NO!!!! No more huge patch bombs to linux-kernel@vger.kernel.org people! - + Kernel Documentation/process/coding-style.rst: :ref:`Documentation/translations/zh_CN/process/coding-style.rst ` Linus Torvalds's mail on the canonical patch format: - + Andi Kleen, "On submitting kernel patches" Some strategies to get difficult or controversial changes in. diff --git a/Documentation/userspace-api/ioctl/ioctl-number.rst b/Documentation/userspace-api/ioctl/ioctl-number.rst index a4c75a28c83920732d89fa3e1be66724a163a06b..599bd449394441611e726ef6f2b6c3b1b4b96481 100644 --- a/Documentation/userspace-api/ioctl/ioctl-number.rst +++ b/Documentation/userspace-api/ioctl/ioctl-number.rst @@ -157,7 +157,6 @@ Code Seq# Include File Comments 'I' all linux/isdn.h conflict! 'I' 00-0F drivers/isdn/divert/isdn_divert.h conflict! 'I' 40-4F linux/mISDNif.h conflict! -'J' 00-1F drivers/scsi/gdth_ioctl.h 'K' all linux/kd.h 'L' 00-1F linux/loop.h conflict! 'L' 10-1F drivers/scsi/mpt3sas/mpt3sas_ctl.h conflict! @@ -180,6 +179,7 @@ Code Seq# Include File Comments 'R' 00-1F linux/random.h conflict! 'R' 01 linux/rfkill.h conflict! 'R' C0-DF net/bluetooth/rfcomm.h +'R' E0 uapi/linux/fsl_mc.h 'S' all linux/cdrom.h conflict! 'S' 80-81 scsi/scsi_ioctl.h conflict! 'S' 82-FF scsi/scsi.h conflict! @@ -319,11 +319,14 @@ Code Seq# Include File Comments 0xA0 all linux/sdp/sdp.h Industrial Device Project 0xA1 0 linux/vtpm_proxy.h TPM Emulator Proxy Driver +0xA2 all uapi/linux/acrn.h ACRN hypervisor 0xA3 80-8F Port ACL in development: 0xA3 90-9F linux/dtlk.h 0xA4 00-1F uapi/linux/tee.h Generic TEE subsystem 0xA4 00-1F uapi/asm/sgx.h +0xA5 01 linux/surface_aggregator/cdev.h Microsoft Surface Platform System Aggregator + 0xAA 00-3F linux/uapi/linux/userfaultfd.h 0xAB 00-1F linux/nbd.h 0xAC 00-1F linux/raw.h @@ -352,6 +355,7 @@ Code Seq# Include File Comments 0xCC 00-0F drivers/misc/ibmvmc.h pseries VMC driver 0xCD 01 linux/reiserfs_fs.h +0xCE 01-02 uapi/linux/cxl_mem.h Compute Express Link Memory Devices 0xCF 02 fs/cifs/ioctl.c 0xDB 00-0F drivers/char/mwave/mwavepub.h 0xDD 00-3F ZFCP device driver see drivers/s390/scsi/ diff --git a/Documentation/virt/acrn/cpuid.rst b/Documentation/virt/acrn/cpuid.rst new file mode 100644 index 0000000000000000000000000000000000000000..65fa4b9c1798a8ea6f7ad9bc7df6f197d10e9b79 --- /dev/null +++ b/Documentation/virt/acrn/cpuid.rst @@ -0,0 +1,46 @@ +.. SPDX-License-Identifier: GPL-2.0 + +=============== +ACRN CPUID bits +=============== + +A guest VM running on an ACRN hypervisor can check some of its features using +CPUID. + +ACRN cpuid functions are: + +function: 0x40000000 + +returns:: + + eax = 0x40000010 + ebx = 0x4e524341 + ecx = 0x4e524341 + edx = 0x4e524341 + +Note that this value in ebx, ecx and edx corresponds to the string +"ACRNACRNACRN". The value in eax corresponds to the maximum cpuid function +present in this leaf, and will be updated if more functions are added in the +future. + +function: define ACRN_CPUID_FEATURES (0x40000001) + +returns:: + + ebx, ecx, edx + eax = an OR'ed group of (1 << flag) + +where ``flag`` is defined as below: + +================================= =========== ================================ +flag value meaning +================================= =========== ================================ +ACRN_FEATURE_PRIVILEGED_VM 0 guest VM is a privileged VM +================================= =========== ================================ + +function: 0x40000010 + +returns:: + + ebx, ecx, edx + eax = (Virtual) TSC frequency in kHz. diff --git a/Documentation/virt/acrn/index.rst b/Documentation/virt/acrn/index.rst new file mode 100644 index 0000000000000000000000000000000000000000..b5f793e73df56fbd25b95e58b13622947df28bb3 --- /dev/null +++ b/Documentation/virt/acrn/index.rst @@ -0,0 +1,12 @@ +.. SPDX-License-Identifier: GPL-2.0 + +=============== +ACRN Hypervisor +=============== + +.. toctree:: + :maxdepth: 1 + + introduction + io-request + cpuid diff --git a/Documentation/virt/acrn/introduction.rst b/Documentation/virt/acrn/introduction.rst new file mode 100644 index 0000000000000000000000000000000000000000..f8d081bc084d8f32ecf1d9ac87999f28547152cb --- /dev/null +++ b/Documentation/virt/acrn/introduction.rst @@ -0,0 +1,43 @@ +.. SPDX-License-Identifier: GPL-2.0 + +ACRN Hypervisor Introduction +============================ + +The ACRN Hypervisor is a Type 1 hypervisor, running directly on bare-metal +hardware. It has a privileged management VM, called Service VM, to manage User +VMs and do I/O emulation. + +ACRN userspace is an application running in the Service VM that emulates +devices for a User VM based on command line configurations. ACRN Hypervisor +Service Module (HSM) is a kernel module in the Service VM which provides +hypervisor services to the ACRN userspace. + +Below figure shows the architecture. + +:: + + Service VM User VM + +----------------------------+ | +------------------+ + | +--------------+ | | | | + | |ACRN userspace| | | | | + | +--------------+ | | | | + |-----------------ioctl------| | | | ... + |kernel space +----------+ | | | | + | | HSM | | | | Drivers | + | +----------+ | | | | + +--------------------|-------+ | +------------------+ + +---------------------hypercall----------------------------------------+ + | ACRN Hypervisor | + +----------------------------------------------------------------------+ + | Hardware | + +----------------------------------------------------------------------+ + +ACRN userspace allocates memory for the User VM, configures and initializes the +devices used by the User VM, loads the virtual bootloader, initializes the +virtual CPU state and handles I/O request accesses from the User VM. It uses +ioctls to communicate with the HSM. HSM implements hypervisor services by +interacting with the ACRN Hypervisor via hypercalls. HSM exports a char device +interface (/dev/acrn_hsm) to userspace. + +The ACRN hypervisor is open for contribution from anyone. The source repo is +available at https://github.com/projectacrn/acrn-hypervisor. diff --git a/Documentation/virt/acrn/io-request.rst b/Documentation/virt/acrn/io-request.rst new file mode 100644 index 0000000000000000000000000000000000000000..6cc3ea0fa1f5c5434ca76f96595ce83e5ca40323 --- /dev/null +++ b/Documentation/virt/acrn/io-request.rst @@ -0,0 +1,97 @@ +.. SPDX-License-Identifier: GPL-2.0 + +I/O request handling +==================== + +An I/O request of a User VM, which is constructed by the hypervisor, is +distributed by the ACRN Hypervisor Service Module to an I/O client +corresponding to the address range of the I/O request. Details of I/O request +handling are described in the following sections. + +1. I/O request +-------------- + +For each User VM, there is a shared 4-KByte memory region used for I/O requests +communication between the hypervisor and Service VM. An I/O request is a +256-byte structure buffer, which is 'struct acrn_io_request', that is filled by +an I/O handler of the hypervisor when a trapped I/O access happens in a User +VM. ACRN userspace in the Service VM first allocates a 4-KByte page and passes +the GPA (Guest Physical Address) of the buffer to the hypervisor. The buffer is +used as an array of 16 I/O request slots with each I/O request slot being 256 +bytes. This array is indexed by vCPU ID. + +2. I/O clients +-------------- + +An I/O client is responsible for handling User VM I/O requests whose accessed +GPA falls in a certain range. Multiple I/O clients can be associated with each +User VM. There is a special client associated with each User VM, called the +default client, that handles all I/O requests that do not fit into the range of +any other clients. The ACRN userspace acts as the default client for each User +VM. + +Below illustration shows the relationship between I/O requests shared buffer, +I/O requests and I/O clients. + +:: + + +------------------------------------------------------+ + | Service VM | + |+--------------------------------------------------+ | + || +----------------------------------------+ | | + || | shared page ACRN userspace | | | + || | +-----------------+ +------------+ | | | + || +----+->| acrn_io_request |<-+ default | | | | + || | | | +-----------------+ | I/O client | | | | + || | | | | ... | +------------+ | | | + || | | | +-----------------+ | | | + || | +-|--------------------------------------+ | | + ||---|----|-----------------------------------------| | + || | | kernel | | + || | | +----------------------+ | | + || | | | +-------------+ HSM | | | + || | +--------------+ | | | | + || | | | I/O clients | | | | + || | | | | | | | + || | | +-------------+ | | | + || | +----------------------+ | | + |+---|----------------------------------------------+ | + +----|-------------------------------------------------+ + | + +----|-------------------------------------------------+ + | +-+-----------+ | + | | I/O handler | ACRN Hypervisor | + | +-------------+ | + +------------------------------------------------------+ + +3. I/O request state transition +------------------------------- + +The state transitions of an ACRN I/O request are as follows. + +:: + + FREE -> PENDING -> PROCESSING -> COMPLETE -> FREE -> ... + +- FREE: this I/O request slot is empty +- PENDING: a valid I/O request is pending in this slot +- PROCESSING: the I/O request is being processed +- COMPLETE: the I/O request has been processed + +An I/O request in COMPLETE or FREE state is owned by the hypervisor. HSM and +ACRN userspace are in charge of processing the others. + +4. Processing flow of I/O requests +---------------------------------- + +a. The I/O handler of the hypervisor will fill an I/O request with PENDING + state when a trapped I/O access happens in a User VM. +b. The hypervisor makes an upcall, which is a notification interrupt, to + the Service VM. +c. The upcall handler schedules a worker to dispatch I/O requests. +d. The worker looks for the PENDING I/O requests, assigns them to different + registered clients based on the address of the I/O accesses, updates + their state to PROCESSING, and notifies the corresponding client to handle. +e. The notified client handles the assigned I/O requests. +f. The HSM updates I/O requests states to COMPLETE and notifies the hypervisor + of the completion via hypercalls. diff --git a/Documentation/virt/index.rst b/Documentation/virt/index.rst index 350f5c869b5642e5519b89de3350b7a30e38310a..edea7fea95a84f1a2f990bd37cd4d5a176590b5b 100644 --- a/Documentation/virt/index.rst +++ b/Documentation/virt/index.rst @@ -12,6 +12,7 @@ Linux Virtualization Support paravirt_ops guest-halt-polling ne_overview + acrn/index .. only:: html and subproject diff --git a/Documentation/virt/kvm/api.rst b/Documentation/virt/kvm/api.rst index 45fd862ac128a2ccfdfc9006c44ec6fb1998dea6..aed52b0fc16ec3ac48ba407ebfc67c4213f6c2fb 100644 --- a/Documentation/virt/kvm/api.rst +++ b/Documentation/virt/kvm/api.rst @@ -4519,6 +4519,7 @@ KVM_GET_SUPPORTED_CPUID ioctl because some of them intersect with KVM feature leaves (0x40000000, 0x40000001). Currently, the following list of CPUID leaves are returned: + - HYPERV_CPUID_VENDOR_AND_MAX_FUNCTIONS - HYPERV_CPUID_INTERFACE - HYPERV_CPUID_VERSION @@ -4543,6 +4544,7 @@ userspace should not expect to get any particular value there. Note, vcpu version of KVM_GET_SUPPORTED_HV_CPUID is currently deprecated. Unlike system ioctl which exposes all supported feature bits unconditionally, vcpu version has the following quirks: + - HYPERV_CPUID_NESTED_FEATURES leaf and HV_X64_ENLIGHTENED_VMCS_RECOMMENDED feature bit are only exposed when Enlightened VMCS was previously enabled on the corresponding vCPU (KVM_CAP_HYPERV_ENLIGHTENED_VMCS). diff --git a/Documentation/vm/arch_pgtable_helpers.rst b/Documentation/vm/arch_pgtable_helpers.rst index f3591ee3aaa89ab7274bf3cef6be6c21f9a3e1a0..552567d863b86584b814d3540c69b033e5f1b005 100644 --- a/Documentation/vm/arch_pgtable_helpers.rst +++ b/Documentation/vm/arch_pgtable_helpers.rst @@ -50,7 +50,7 @@ PTE Page Table Helpers +---------------------------+--------------------------------------------------+ | pte_mkwrite | Creates a writable PTE | +---------------------------+--------------------------------------------------+ -| pte_mkwrprotect | Creates a write protected PTE | +| pte_wrprotect | Creates a write protected PTE | +---------------------------+--------------------------------------------------+ | pte_mkspecial | Creates a special PTE | +---------------------------+--------------------------------------------------+ @@ -120,7 +120,7 @@ PMD Page Table Helpers +---------------------------+--------------------------------------------------+ | pmd_mkwrite | Creates a writable PMD | +---------------------------+--------------------------------------------------+ -| pmd_mkwrprotect | Creates a write protected PMD | +| pmd_wrprotect | Creates a write protected PMD | +---------------------------+--------------------------------------------------+ | pmd_mkspecial | Creates a special PMD | +---------------------------+--------------------------------------------------+ @@ -186,7 +186,7 @@ PUD Page Table Helpers +---------------------------+--------------------------------------------------+ | pud_mkwrite | Creates a writable PUD | +---------------------------+--------------------------------------------------+ -| pud_mkwrprotect | Creates a write protected PUD | +| pud_wrprotect | Creates a write protected PUD | +---------------------------+--------------------------------------------------+ | pud_mkdevmap | Creates a ZONE_DEVICE mapped PUD | +---------------------------+--------------------------------------------------+ @@ -224,7 +224,7 @@ HugeTLB Page Table Helpers +---------------------------+--------------------------------------------------+ | huge_pte_mkwrite | Creates a writable HugeTLB | +---------------------------+--------------------------------------------------+ -| huge_pte_mkwrprotect | Creates a write protected HugeTLB | +| huge_pte_wrprotect | Creates a write protected HugeTLB | +---------------------------+--------------------------------------------------+ | huge_ptep_get_and_clear | Clears a HugeTLB | +---------------------------+--------------------------------------------------+ diff --git a/Documentation/vm/split_page_table_lock.rst b/Documentation/vm/split_page_table_lock.rst index ff51f4a5494d70a32db3bf73319a4bc42d60ed47..c089196627045181b07b2df41ec823e4d6a60d15 100644 --- a/Documentation/vm/split_page_table_lock.rst +++ b/Documentation/vm/split_page_table_lock.rst @@ -32,7 +32,7 @@ There are helpers to lock/unlock a table and other accessor functions: Split page table lock for PTE tables is enabled compile-time if CONFIG_SPLIT_PTLOCK_CPUS (usually 4) is less or equal to NR_CPUS. -If split lock is disabled, all tables guaded by mm->page_table_lock. +If split lock is disabled, all tables are guarded by mm->page_table_lock. Split page table lock for PMD tables is enabled, if it's enabled for PTE tables and the architecture supports it (see below). diff --git a/Documentation/x86/boot.rst b/Documentation/x86/boot.rst index abb9fc164657a7caa98effa0a6ba4b0b8ebd764b..fc844913decefe47eaeb3a22d57151bb3744ddd0 100644 --- a/Documentation/x86/boot.rst +++ b/Documentation/x86/boot.rst @@ -851,7 +851,7 @@ Protocol: 2.09+ struct setup_data { __u64 next = 0 or ; __u32 type = SETUP_INDIRECT; - __u32 len = sizeof(setup_data); + __u32 len = sizeof(setup_indirect); __u8 data[sizeof(setup_indirect)] = struct setup_indirect { __u32 type = SETUP_INDIRECT | SETUP_E820_EXT; __u32 reserved = 0; diff --git a/MAINTAINERS b/MAINTAINERS index b9d55c84fc12612da4a119539b22afbe20d6b816..d92f85ca831d30d658fef768575ba538be4d4b93 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -261,6 +261,8 @@ ABI/API L: linux-api@vger.kernel.org F: include/linux/syscalls.h F: kernel/sys_ni.c +F: include/uapi/ +F: arch/*/include/uapi/ ABIT UGURU 1,2 HARDWARE MONITOR DRIVER M: Hans de Goede @@ -436,6 +438,15 @@ S: Orphan F: drivers/platform/x86/wmi.c F: include/uapi/linux/wmi.h +ACRN HYPERVISOR SERVICE MODULE +M: Shuo Liu +L: acrn-dev@lists.projectacrn.org (subscribers-only) +S: Supported +W: https://projectacrn.org +F: Documentation/virt/acrn/ +F: drivers/virt/acrn/ +F: include/uapi/linux/acrn.h + AD1889 ALSA SOUND DRIVER L: linux-parisc@vger.kernel.org S: Maintained @@ -1017,7 +1028,7 @@ F: Documentation/devicetree/bindings/mux/adi,adgs1408.txt F: drivers/mux/adgs1408.c ANALOG DEVICES INC ADIN DRIVER -M: Alexandru Ardelean +M: Michael Hennerich L: netdev@vger.kernel.org S: Supported W: http://ez.analog.com/community/linux-device-drivers @@ -1025,7 +1036,7 @@ F: Documentation/devicetree/bindings/net/adi,adin.yaml F: drivers/net/phy/adin.c ANALOG DEVICES INC ADIS DRIVER LIBRARY -M: Alexandru Ardelean +M: Nuno Sa L: linux-iio@vger.kernel.org S: Supported F: drivers/iio/imu/adis.c @@ -2126,10 +2137,13 @@ L: linux-arm-kernel@lists.infradead.org (moderated for non-subscribers) S: Maintained W: http://linux-chenxing.org/ F: Documentation/devicetree/bindings/arm/mstar/* +F: Documentation/devicetree/bindings/clock/mstar,msc313-mpll.yaml F: Documentation/devicetree/bindings/gpio/mstar,msc313-gpio.yaml F: arch/arm/boot/dts/mstar-* F: arch/arm/mach-mstar/ +F: drivers/clk/mstar/ F: drivers/gpio/gpio-msc313.c +F: include/dt-bindings/clock/mstar-* F: include/dt-bindings/gpio/msc313-gpio.h ARM/NEC MOBILEPRO 900/c MACHINE SUPPORT @@ -2566,7 +2580,7 @@ L: linux-kernel@vger.kernel.org S: Maintained F: drivers/clk/keystone/ -ARM/TEXAS INSTRUMENT KEYSTONE ClOCKSOURCE +ARM/TEXAS INSTRUMENT KEYSTONE CLOCKSOURCE M: Santosh Shilimkar L: linux-arm-kernel@lists.infradead.org (moderated for non-subscribers) L: linux-kernel@vger.kernel.org @@ -2606,10 +2620,12 @@ S: Supported T: git git://git.kernel.org/pub/scm/linux/kernel/git/iwamatsu/linux-visconti.git F: Documentation/devicetree/bindings/arm/toshiba.yaml F: Documentation/devicetree/bindings/net/toshiba,visconti-dwmac.yaml +F: Documentation/devicetree/bindings/gpio/toshiba,gpio-visconti.yaml F: Documentation/devicetree/bindings/pinctrl/toshiba,tmpv7700-pinctrl.yaml F: Documentation/devicetree/bindings/watchdog/toshiba,visconti-wdt.yaml F: arch/arm64/boot/dts/toshiba/ F: drivers/net/ethernet/stmicro/stmmac/dwmac-visconti.c +F: drivers/gpio/gpio-visconti.c F: drivers/pinctrl/visconti/ F: drivers/watchdog/visconti_wdt.c N: visconti @@ -2775,6 +2791,15 @@ F: Documentation/devicetree/bindings/interrupt-controller/aspeed,ast2xxx-scu-ic. F: drivers/irqchip/irq-aspeed-scu-ic.c F: include/dt-bindings/interrupt-controller/aspeed-scu-ic.h +ASPEED SD/MMC DRIVER +M: Andrew Jeffery +L: linux-aspeed@lists.ozlabs.org (moderated for non-subscribers) +L: openbmc@lists.ozlabs.org (moderated for non-subscribers) +L: linux-mmc@vger.kernel.org +S: Maintained +F: Documentation/devicetree/bindings/mmc/aspeed,sdhci.yaml +F: drivers/mmc/host/sdhci-of-aspeed* + ASPEED VIDEO ENGINE DRIVER M: Eddie James L: linux-media@vger.kernel.org @@ -2814,9 +2839,7 @@ S: Odd fixes W: http://sourceforge.net/projects/xscaleiop F: Documentation/crypto/async-tx-api.rst F: crypto/async_tx/ -F: drivers/dma/ F: include/linux/async_tx.h -F: include/linux/dmaengine.h AT24 EEPROM DRIVER M: Bartosz Golaszewski @@ -2961,7 +2984,7 @@ F: include/uapi/linux/audit.h F: kernel/audit* AUXILIARY DISPLAY DRIVERS -M: Miguel Ojeda Sandonis +M: Miguel Ojeda S: Maintained F: drivers/auxdisplay/ F: include/linux/cfag12864b.h @@ -3719,6 +3742,13 @@ L: netdev@vger.kernel.org S: Supported F: drivers/net/ethernet/broadcom/tg3.* +BROADCOM VK DRIVER +M: Scott Branden +L: bcm-kernel-feedback-list@broadcom.com +S: Supported +F: drivers/misc/bcm-vk/ +F: include/uapi/linux/misc/bcm_vk.h + BROCADE BFA FC SCSI DRIVER M: Anil Gurumurthy M: Sudarsana Kalluru @@ -3825,6 +3855,29 @@ W: https://github.com/Cascoda/ca8210-linux.git F: Documentation/devicetree/bindings/net/ieee802154/ca8210.txt F: drivers/net/ieee802154/ca8210.c +CANAAN/KENDRYTE K210 SOC FPIOA DRIVER +M: Damien Le Moal +L: linux-riscv@lists.infradead.org +L: linux-gpio@vger.kernel.org (pinctrl driver) +F: Documentation/devicetree/bindings/pinctrl/canaan,k210-fpioa.yaml +F: drivers/pinctrl/pinctrl-k210.c + +CANAAN/KENDRYTE K210 SOC RESET CONTROLLER DRIVER +M: Damien Le Moal +L: linux-kernel@vger.kernel.org +L: linux-riscv@lists.infradead.org +S: Maintained +F: Documentation/devicetree/bindings/reset/canaan,k210-rst.yaml +F: drivers/reset/reset-k210.c + +CANAAN/KENDRYTE K210 SOC SYSTEM CONTROLLER DRIVER +M: Damien Le Moal +L: linux-riscv@lists.infradead.org +S: Maintained +F: Documentation/devicetree/bindings/mfd/canaan,k210-sysctl.yaml +F: drivers/soc/canaan/ +F: include/soc/canaan/ + CACHEFILES: FS-CACHE BACKEND FOR CACHING ON MOUNTED FILESYSTEMS M: David Howells L: linux-cachefs@redhat.com (moderated for non-subscribers) @@ -4100,13 +4153,13 @@ F: scripts/extract-cert.c F: scripts/sign-file.c CFAG12864B LCD DRIVER -M: Miguel Ojeda Sandonis +M: Miguel Ojeda S: Maintained F: drivers/auxdisplay/cfag12864b.c F: include/linux/cfag12864b.h CFAG12864BFB LCD FRAMEBUFFER DRIVER -M: Miguel Ojeda Sandonis +M: Miguel Ojeda S: Maintained F: drivers/auxdisplay/cfag12864bfb.c F: include/linux/cfag12864b.h @@ -4276,7 +4329,7 @@ S: Supported F: drivers/infiniband/hw/usnic/ CLANG-FORMAT FILE -M: Miguel Ojeda +M: Miguel Ojeda S: Maintained F: .clang-format @@ -4291,8 +4344,6 @@ C: irc://chat.freenode.net/clangbuiltlinux F: Documentation/kbuild/llvm.rst F: include/linux/compiler-clang.h F: scripts/clang-tools/ -F: scripts/clang-version.sh -F: scripts/lld-version.sh K: \b(?i:clang|llvm)\b CLEANCACHE API @@ -4418,10 +4469,21 @@ S: Maintained F: drivers/platform/x86/compal-laptop.c COMPILER ATTRIBUTES -M: Miguel Ojeda +M: Miguel Ojeda S: Maintained F: include/linux/compiler_attributes.h +COMPUTE EXPRESS LINK (CXL) +M: Alison Schofield +M: Vishal Verma +M: Ira Weiny +M: Ben Widawsky +M: Dan Williams +L: linux-cxl@vger.kernel.org +S: Maintained +F: drivers/cxl/ +F: include/uapi/linux/cxl_mem.h + CONEXANT ACCESSRUNNER USB DRIVER L: accessrunner-general@lists.sourceforge.net S: Orphan @@ -4946,17 +5008,17 @@ M: Matthew Garrett M: Pali Rohár L: platform-driver-x86@vger.kernel.org S: Maintained -F: drivers/platform/x86/dell-laptop.c +F: drivers/platform/x86/dell/dell-laptop.c DELL LAPTOP FREEFALL DRIVER M: Pali Rohár S: Maintained -F: drivers/platform/x86/dell-smo8800.c +F: drivers/platform/x86/dell/dell-smo8800.c DELL LAPTOP RBTN DRIVER M: Pali Rohár S: Maintained -F: drivers/platform/x86/dell-rbtn.* +F: drivers/platform/x86/dell/dell-rbtn.* DELL LAPTOP SMM DRIVER M: Pali Rohár @@ -4968,26 +5030,26 @@ DELL REMOTE BIOS UPDATE DRIVER M: Stuart Hayes L: platform-driver-x86@vger.kernel.org S: Maintained -F: drivers/platform/x86/dell_rbu.c +F: drivers/platform/x86/dell/dell_rbu.c DELL SMBIOS DRIVER M: Pali Rohár M: Mario Limonciello L: platform-driver-x86@vger.kernel.org S: Maintained -F: drivers/platform/x86/dell-smbios.* +F: drivers/platform/x86/dell/dell-smbios.* DELL SMBIOS SMM DRIVER M: Mario Limonciello L: platform-driver-x86@vger.kernel.org S: Maintained -F: drivers/platform/x86/dell-smbios-smm.c +F: drivers/platform/x86/dell/dell-smbios-smm.c DELL SMBIOS WMI DRIVER M: Mario Limonciello L: platform-driver-x86@vger.kernel.org S: Maintained -F: drivers/platform/x86/dell-smbios-wmi.c +F: drivers/platform/x86/dell/dell-smbios-wmi.c F: tools/wmi/dell-smbios-example.c DELL SYSTEMS MANAGEMENT BASE DRIVER (dcdbas) @@ -4995,12 +5057,12 @@ M: Stuart Hayes L: platform-driver-x86@vger.kernel.org S: Maintained F: Documentation/driver-api/dcdbas.rst -F: drivers/platform/x86/dcdbas.* +F: drivers/platform/x86/dell/dcdbas.* DELL WMI DESCRIPTOR DRIVER M: Mario Limonciello S: Maintained -F: drivers/platform/x86/dell-wmi-descriptor.c +F: drivers/platform/x86/dell/dell-wmi-descriptor.c DELL WMI SYSMAN DRIVER M: Divya Bharathi @@ -5009,13 +5071,13 @@ M: Prasanth Ksr L: platform-driver-x86@vger.kernel.org S: Maintained F: Documentation/ABI/testing/sysfs-class-firmware-attributes -F: drivers/platform/x86/dell-wmi-sysman/ +F: drivers/platform/x86/dell/dell-wmi-sysman/ DELL WMI NOTIFICATIONS DRIVER M: Matthew Garrett M: Pali Rohár S: Maintained -F: drivers/platform/x86/dell-wmi.c +F: drivers/platform/x86/dell/dell-wmi.c DELTA ST MEDIA DRIVER M: Hugues Fruchet @@ -5257,6 +5319,7 @@ T: git git://git.kernel.org/pub/scm/linux/kernel/git/vkoul/dmaengine.git F: Documentation/devicetree/bindings/dma/ F: Documentation/driver-api/dmaengine/ F: drivers/dma/ +F: include/linux/dma/ F: include/linux/dmaengine.h F: include/linux/of_dma.h @@ -5760,6 +5823,7 @@ F: drivers/gpu/drm/vboxvideo/ DRM DRIVER FOR VMWARE VIRTUAL GPU M: "VMware Graphics" M: Roland Scheidegger +M: Zack Rusin L: dri-devel@lists.freedesktop.org S: Supported T: git git://people.freedesktop.org/~sroland/linux @@ -5961,8 +6025,8 @@ F: Documentation/devicetree/bindings/display/st,stm32-ltdc.yaml F: drivers/gpu/drm/stm DRM DRIVERS FOR TI KEYSTONE -M: Jyri Sarha -M: Tomi Valkeinen +M: Jyri Sarha +M: Tomi Valkeinen L: dri-devel@lists.freedesktop.org S: Maintained T: git git://anongit.freedesktop.org/drm/drm-misc @@ -5972,15 +6036,15 @@ F: Documentation/devicetree/bindings/display/ti/ti,k2g-dss.yaml F: drivers/gpu/drm/tidss/ DRM DRIVERS FOR TI LCDC -M: Jyri Sarha -R: Tomi Valkeinen +M: Jyri Sarha +R: Tomi Valkeinen L: dri-devel@lists.freedesktop.org S: Maintained F: Documentation/devicetree/bindings/display/tilcdc/ F: drivers/gpu/drm/tilcdc/ DRM DRIVERS FOR TI OMAP -M: Tomi Valkeinen +M: Tomi Valkeinen L: dri-devel@lists.freedesktop.org S: Maintained F: Documentation/devicetree/bindings/display/ti/ @@ -6925,9 +6989,10 @@ M: Wu Hao R: Tom Rix L: linux-fpga@vger.kernel.org S: Maintained -F: Documentation/ABI/testing/sysfs-bus-dfl +F: Documentation/ABI/testing/sysfs-bus-dfl* F: Documentation/fpga/dfl.rst F: drivers/fpga/dfl* +F: include/linux/dfl.h F: include/uapi/linux/fpga-dfl.h FPGA MANAGER FRAMEWORK @@ -7348,13 +7413,6 @@ M: Kieran Bingham S: Supported F: scripts/gdb/ -GDT SCSI DISK ARRAY CONTROLLER DRIVER -M: Achim Leubner -L: linux-scsi@vger.kernel.org -S: Supported -W: http://www.icp-vortex.com/ -F: drivers/scsi/gdt* - GEMTEK FM RADIO RECEIVER DRIVER M: Hans Verkuil L: linux-media@vger.kernel.org @@ -7900,6 +7958,12 @@ F: drivers/hid/ F: include/linux/hid* F: include/uapi/linux/hid* +HID PLAYSTATION DRIVER +M: Roderick Colenbrander +L: linux-input@vger.kernel.org +S: Supported +F: drivers/hid/hid-playstation.c + HID SENSOR HUB DRIVERS M: Jiri Kosina M: Jonathan Cameron @@ -8131,7 +8195,7 @@ F: net/hsr/ HT16K33 LED CONTROLLER DRIVER M: Robin van der Gracht S: Maintained -F: Documentation/devicetree/bindings/display/ht16k33.txt +F: Documentation/devicetree/bindings/auxdisplay/holtek,ht16k33.yaml F: drivers/auxdisplay/ht16k33.c HTCPEN TOUCHSCREEN DRIVER @@ -8849,7 +8913,6 @@ F: drivers/mfd/intel_pmc_bxt.c F: include/linux/mfd/intel_pmc_bxt.h INTEL C600 SERIES SAS CONTROLLER DRIVER -M: Intel SCU Linux support M: Artur Paszkiewicz L: linux-scsi@vger.kernel.org S: Supported @@ -8907,7 +8970,6 @@ L: linux-gpio@vger.kernel.org S: Maintained T: git git://git.kernel.org/pub/scm/linux/kernel/git/andy/linux-gpio-intel.git F: drivers/gpio/gpio-ich.c -F: drivers/gpio/gpio-intel-mid.c F: drivers/gpio/gpio-merrifield.c F: drivers/gpio/gpio-ml-ioh.c F: drivers/gpio/gpio-pch.c @@ -9031,6 +9093,17 @@ F: drivers/crypto/keembay/keembay-ocs-aes-core.c F: drivers/crypto/keembay/ocs-aes.c F: drivers/crypto/keembay/ocs-aes.h +INTEL KEEM BAY OCS HCU CRYPTO DRIVER +M: Daniele Alessandrelli +M: Declan Murphy +S: Maintained +F: Documentation/devicetree/bindings/crypto/intel,keembay-ocs-hcu.yaml +F: drivers/crypto/keembay/Kconfig +F: drivers/crypto/keembay/Makefile +F: drivers/crypto/keembay/keembay-ocs-hcu-core.c +F: drivers/crypto/keembay/ocs-hcu.c +F: drivers/crypto/keembay/ocs-hcu.h + INTEL MANAGEMENT ENGINE (mei) M: Tomas Winkler L: linux-kernel@vger.kernel.org @@ -9068,14 +9141,11 @@ M: Andy Shevchenko S: Maintained T: git git://git.kernel.org/pub/scm/linux/kernel/git/andy/linux-gpio-intel.git F: drivers/gpio/gpio-*cove.c -F: drivers/gpio/gpio-msic.c INTEL PMIC MULTIFUNCTION DEVICE DRIVERS M: Andy Shevchenko S: Maintained -F: drivers/mfd/intel_msic.c F: drivers/mfd/intel_soc_pmic* -F: include/linux/mfd/intel_msic.h F: include/linux/mfd/intel_soc_pmic* INTEL PMT DRIVER @@ -9822,6 +9892,18 @@ F: include/linux/keyctl.h F: include/uapi/linux/keyctl.h F: security/keys/ +KFENCE +M: Alexander Potapenko +M: Marco Elver +R: Dmitry Vyukov +L: kasan-dev@googlegroups.com +S: Maintained +F: Documentation/dev-tools/kfence.rst +F: arch/*/include/asm/kfence.h +F: include/linux/kfence.h +F: lib/Kconfig.kfence +F: mm/kfence/ + KFIFO M: Stefani Seibold S: Maintained @@ -9882,7 +9964,7 @@ F: include/linux/kprobes.h F: kernel/kprobes.c KS0108 LCD CONTROLLER DRIVER -M: Miguel Ojeda Sandonis +M: Miguel Ojeda S: Maintained F: Documentation/admin-guide/auxdisplay/ks0108.rst F: drivers/auxdisplay/ks0108.c @@ -11159,6 +11241,15 @@ S: Maintained F: Documentation/devicetree/bindings/i2c/i2c-mt65xx.txt F: drivers/i2c/busses/i2c-mt65xx.c +MEDIATEK IOMMU DRIVER +M: Yong Wu +L: iommu@lists.linux-foundation.org +L: linux-mediatek@lists.infradead.org (moderated for non-subscribers) +S: Supported +F: Documentation/devicetree/bindings/iommu/mediatek* +F: drivers/iommu/mtk_iommu* +F: include/dt-bindings/memory/mt*-port.h + MEDIATEK JPEG DRIVER M: Rick Chang M: Bin Liu @@ -11583,7 +11674,6 @@ F: drivers/dma/at_hdmac.c F: drivers/dma/at_hdmac_regs.h F: drivers/dma/at_xdmac.c F: include/dt-bindings/dma/at91.h -F: include/linux/platform_data/dma-atmel.h MICROCHIP AT91 SERIAL DRIVER M: Richard Genoud @@ -11669,9 +11759,9 @@ F: drivers/video/fbdev/atmel_lcdfb.c F: include/video/atmel_lcdc.h MICROCHIP MCP16502 PMIC DRIVER -M: Andrei Stefanescu +M: Claudiu Beznea L: linux-arm-kernel@lists.infradead.org (moderated for non-subscribers) -S: Maintained +S: Supported F: Documentation/devicetree/bindings/regulator/mcp16502-regulator.txt F: drivers/regulator/mcp16502.c @@ -11786,12 +11876,31 @@ S: Maintained T: git git://git.kernel.org/pub/scm/linux/kernel/git/pdx86/platform-drivers-x86.git F: drivers/platform/surface/ +MICROSOFT SURFACE HOT-PLUG DRIVER +M: Maximilian Luz +L: platform-driver-x86@vger.kernel.org +S: Maintained +F: drivers/platform/surface/surface_hotplug.c + MICROSOFT SURFACE PRO 3 BUTTON DRIVER M: Chen Yu L: platform-driver-x86@vger.kernel.org S: Supported F: drivers/platform/surface/surfacepro3_button.c +MICROSOFT SURFACE SYSTEM AGGREGATOR SUBSYSTEM +M: Maximilian Luz +S: Maintained +W: https://github.com/linux-surface/surface-aggregator-module +C: irc://chat.freenode.net/##linux-surface +F: Documentation/driver-api/surface_aggregator/ +F: drivers/platform/surface/aggregator/ +F: drivers/platform/surface/surface_acpi_notify.c +F: drivers/platform/surface/surface_aggregator_cdev.c +F: include/linux/surface_acpi_notify.h +F: include/linux/surface_aggregator/ +F: include/uapi/linux/surface_aggregator/ + MICROTEK X6 SCANNER M: Oliver Neukum S: Maintained @@ -12736,6 +12845,13 @@ F: drivers/iio/gyro/fxas21002c_core.c F: drivers/iio/gyro/fxas21002c_i2c.c F: drivers/iio/gyro/fxas21002c_spi.c +NXP i.MX CLOCK DRIVERS +M: Abel Vesa +L: linux-clk@vger.kernel.org +L: linux-imx@nxp.com +S: Maintained +F: drivers/clk/imx/ + NXP i.MX 8MQ DCSS DRIVER M: Laurentiu Palcu R: Lucas Stach @@ -13809,6 +13925,13 @@ S: Supported F: Documentation/devicetree/bindings/pci/mediatek* F: drivers/pci/controller/*mediatek* +PCIE DRIVER FOR MICROCHIP +M: Daire McNamara +L: linux-pci@vger.kernel.org +S: Supported +F: Documentation/devicetree/bindings/pci/microchip* +F: drivers/pci/controller/*microchip* + PCIE DRIVER FOR QUALCOMM MSM M: Stanimir Varbanov L: linux-pci@vger.kernel.org @@ -14620,9 +14743,11 @@ M: Stuart Yoder M: Laurentiu Tudor L: linux-kernel@vger.kernel.org S: Maintained +F: Documentation/ABI/stable/sysfs-bus-fsl-mc F: Documentation/devicetree/bindings/misc/fsl,qoriq-mc.txt F: Documentation/networking/device_drivers/ethernet/freescale/dpaa2/overview.rst F: drivers/bus/fsl-mc/ +F: include/uapi/linux/fsl_mc.h QT1010 MEDIA DRIVER M: Antti Palosaari @@ -15682,7 +15807,6 @@ F: drivers/media/i2c/s5k5baf.c SAMSUNG S5P Security SubSystem (SSS) DRIVER M: Krzysztof Kozlowski M: Vladimir Zapolskiy -M: Kamil Konieczny L: linux-crypto@vger.kernel.org L: linux-samsung-soc@vger.kernel.org S: Maintained @@ -16230,12 +16354,13 @@ S: Maintained F: Documentation/fb/sm712fb.rst F: drivers/video/fbdev/sm712* -SIMPLE FIRMWARE INTERFACE (SFI) -S: Obsolete -W: http://simplefirmware.org/ -F: arch/x86/platform/sfi/ -F: drivers/sfi/ -F: include/linux/sfi*.h +SILVACO I3C DUAL-ROLE MASTER +M: Miquel Raynal +M: Conor Culhane +L: linux-i3c@lists.infradead.org +S: Maintained +F: Documentation/devicetree/bindings/i3c/silvaco,i3c-master.yaml +F: drivers/i3c/master/svc-i3c-master.c SIMPLEFB FB DRIVER M: Hans de Goede @@ -16658,6 +16783,7 @@ R: Pierre-Louis Bossart R: Sanyog Kale L: alsa-devel@alsa-project.org (moderated for non-subscribers) S: Supported +T: git git://git.kernel.org/pub/scm/linux/kernel/git/vkoul/soundwire.git F: Documentation/driver-api/soundwire/ F: drivers/soundwire/ F: include/linux/soundwire/ @@ -17500,6 +17626,14 @@ M: Laxman Dewangan S: Supported F: drivers/spi/spi-tegra* +TEGRA QUAD SPI DRIVER +M: Thierry Reding +M: Jonathan Hunter +M: Sowjanya Komatineni +L: linux-tegra@vger.kernel.org +S: Maintained +F: drivers/spi/spi-tegra210-quad.c + TEGRA VIDEO DRIVER M: Thierry Reding M: Jonathan Hunter @@ -17592,6 +17726,15 @@ F: include/dt-bindings/soc/ti,sci_pm_domain.h F: include/linux/soc/ti/ti_sci_inta_msi.h F: include/linux/soc/ti/ti_sci_protocol.h +TEXAS INSTRUMENTS TPS23861 PoE PSE DRIVER +M: Robert Marko +M: Luka Perkov +L: linux-hwmon@vger.kernel.org +S: Maintained +F: Documentation/devicetree/bindings/hwmon/ti,tps23861.yaml +F: Documentation/hwmon/tps23861.rst +F: drivers/hwmon/tps23861.c + THANKO'S RAREMONO AM/FM/SW RADIO RECEIVER USB DRIVER M: Hans Verkuil L: linux-media@vger.kernel.org @@ -17645,7 +17788,7 @@ F: drivers/thermal/gov_power_allocator.c F: include/trace/events/thermal_power_allocator.h THINKPAD ACPI EXTRAS DRIVER -M: Henrique de Moraes Holschuh +M: Henrique de Moraes Holschuh L: ibm-acpi-devel@lists.sourceforge.net L: platform-driver-x86@vger.kernel.org S: Maintained diff --git a/Makefile b/Makefile index 95caa81be8fbda68a4dd57892c76bccde6bdcd34..f9b54da2fca054b8726ce99716b2182f0ae227c6 100644 --- a/Makefile +++ b/Makefile @@ -1,9 +1,9 @@ # SPDX-License-Identifier: GPL-2.0 VERSION = 5 -PATCHLEVEL = 11 +PATCHLEVEL = 12 SUBLEVEL = 0 -EXTRAVERSION = -NAME = 💕 Valentine's Day Edition 💕 +EXTRAVERSION = -rc1 +NAME = Frozen Wasteland # *DOCUMENTATION* # To see a list of typical targets execute "make help" @@ -96,10 +96,41 @@ endif ifneq ($(findstring s,$(filter-out --%,$(MAKEFLAGS))),) quiet=silent_ + KBUILD_VERBOSE = 0 endif export quiet Q KBUILD_VERBOSE +# Call a source code checker (by default, "sparse") as part of the +# C compilation. +# +# Use 'make C=1' to enable checking of only re-compiled files. +# Use 'make C=2' to enable checking of *all* source files, regardless +# of whether they are re-compiled or not. +# +# See the file "Documentation/dev-tools/sparse.rst" for more details, +# including where to get the "sparse" utility. + +ifeq ("$(origin C)", "command line") + KBUILD_CHECKSRC = $(C) +endif +ifndef KBUILD_CHECKSRC + KBUILD_CHECKSRC = 0 +endif + +export KBUILD_CHECKSRC + +# Use make M=dir or set the environment variable KBUILD_EXTMOD to specify the +# directory of external module to build. Setting M= takes precedence. +ifeq ("$(origin M)", "command line") + KBUILD_EXTMOD := $(M) +endif + +$(if $(word 2, $(KBUILD_EXTMOD)), \ + $(error building multiple external modules is not supported)) + +export KBUILD_EXTMOD + # Kbuild will save output files in the current working directory. # This does not need to match to the root of the kernel source tree. # @@ -145,7 +176,8 @@ else need-sub-make := 1 endif -abs_srctree := $(realpath $(dir $(lastword $(MAKEFILE_LIST)))) +this-makefile := $(lastword $(MAKEFILE_LIST)) +abs_srctree := $(realpath $(dir $(this-makefile))) ifneq ($(words $(subst :, ,$(abs_srctree))), 1) $(error source directory cannot contain spaces or colons) @@ -160,8 +192,6 @@ MAKEFLAGS += --include-dir=$(abs_srctree) need-sub-make := 1 endif -this-makefile := $(lastword $(MAKEFILE_LIST)) - ifneq ($(filter 3.%,$(MAKE_VERSION)),) # 'MAKEFLAGS += -rR' does not immediately become effective for GNU Make 3.x # We need to invoke sub-make to avoid implicit rules in the top Makefile. @@ -195,36 +225,6 @@ ifeq ($(need-sub-make),) # so that IDEs/editors are able to understand relative filenames. MAKEFLAGS += --no-print-directory -# Call a source code checker (by default, "sparse") as part of the -# C compilation. -# -# Use 'make C=1' to enable checking of only re-compiled files. -# Use 'make C=2' to enable checking of *all* source files, regardless -# of whether they are re-compiled or not. -# -# See the file "Documentation/dev-tools/sparse.rst" for more details, -# including where to get the "sparse" utility. - -ifeq ("$(origin C)", "command line") - KBUILD_CHECKSRC = $(C) -endif -ifndef KBUILD_CHECKSRC - KBUILD_CHECKSRC = 0 -endif - -# Use make M=dir or set the environment variable KBUILD_EXTMOD to specify the -# directory of external module to build. Setting M= takes precedence. -ifeq ("$(origin M)", "command line") - KBUILD_EXTMOD := $(M) -endif - -$(if $(word 2, $(KBUILD_EXTMOD)), \ - $(error building multiple external modules is not supported)) - -export KBUILD_CHECKSRC KBUILD_EXTMOD - -extmod-prefix = $(if $(KBUILD_EXTMOD),$(KBUILD_EXTMOD)/) - ifeq ($(abs_srctree),$(abs_objtree)) # building in the source tree srctree := . @@ -257,7 +257,6 @@ export building_out_of_srctree srctree objtree VPATH # of make so .config is not included in this case either (for *config). version_h := include/generated/uapi/linux/version.h -old_version_h := include/linux/version.h clean-targets := %clean mrproper cleandocs no-dot-config-targets := $(clean-targets) \ @@ -558,7 +557,13 @@ ifdef building_out_of_srctree { echo "# this is build directory, ignore it"; echo "*"; } > .gitignore endif -ifneq ($(shell $(CC) --version 2>&1 | head -n 1 | grep clang),) +# The expansion should be delayed until arch/$(SRCARCH)/Makefile is included. +# Some architectures define CROSS_COMPILE in arch/$(SRCARCH)/Makefile. +# CC_VERSION_TEXT is referenced from Kconfig (so it needs export), +# and from include/config/auto.conf.cmd to detect the compiler upgrade. +CC_VERSION_TEXT = $(shell $(CC) --version 2>/dev/null | head -n 1 | sed 's/\#//g') + +ifneq ($(findstring clang,$(CC_VERSION_TEXT)),) ifneq ($(CROSS_COMPILE),) CLANG_FLAGS += --target=$(notdir $(CROSS_COMPILE:%-=%)) GCC_TOOLCHAIN_DIR := $(dir $(shell which $(CROSS_COMPILE)elfedit)) @@ -577,12 +582,6 @@ KBUILD_AFLAGS += $(CLANG_FLAGS) export CLANG_FLAGS endif -# The expansion should be delayed until arch/$(SRCARCH)/Makefile is included. -# Some architectures define CROSS_COMPILE in arch/$(SRCARCH)/Makefile. -# CC_VERSION_TEXT is referenced from Kconfig (so it needs export), -# and from include/config/auto.conf.cmd to detect the compiler upgrade. -CC_VERSION_TEXT = $(shell $(CC) --version 2>/dev/null | head -n 1) - ifdef config-build # =========================================================================== # *config targets only - make sure prerequisites are updated, and descend @@ -832,8 +831,10 @@ ifneq ($(LLVM_IAS),1) KBUILD_AFLAGS += -Wa,-gdwarf-2 endif -ifdef CONFIG_DEBUG_INFO_DWARF4 -DEBUG_CFLAGS += -gdwarf-4 +ifndef CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT +dwarf-version-$(CONFIG_DEBUG_INFO_DWARF4) := 4 +dwarf-version-$(CONFIG_DEBUG_INFO_DWARF5) := 5 +DEBUG_CFLAGS += -gdwarf-$(dwarf-version-y) endif ifdef CONFIG_DEBUG_INFO_REDUCED @@ -853,12 +854,8 @@ KBUILD_CFLAGS += $(DEBUG_CFLAGS) export DEBUG_CFLAGS ifdef CONFIG_FUNCTION_TRACER -ifdef CONFIG_FTRACE_MCOUNT_RECORD - # gcc 5 supports generating the mcount tables directly - ifeq ($(call cc-option-yn,-mrecord-mcount),y) - CC_FLAGS_FTRACE += -mrecord-mcount - export CC_USING_RECORD_MCOUNT := 1 - endif +ifdef CONFIG_FTRACE_MCOUNT_USE_CC + CC_FLAGS_FTRACE += -mrecord-mcount ifdef CONFIG_HAVE_NOP_MCOUNT ifeq ($(call cc-option-yn, -mnop-mcount),y) CC_FLAGS_FTRACE += -mnop-mcount @@ -866,6 +863,15 @@ ifdef CONFIG_FTRACE_MCOUNT_RECORD endif endif endif +ifdef CONFIG_FTRACE_MCOUNT_USE_OBJTOOL + CC_FLAGS_USING += -DCC_USING_NOP_MCOUNT +endif +ifdef CONFIG_FTRACE_MCOUNT_USE_RECORDMCOUNT + ifdef CONFIG_HAVE_C_RECORDMCOUNT + BUILD_C_RECORDMCOUNT := y + export BUILD_C_RECORDMCOUNT + endif +endif ifdef CONFIG_HAVE_FENTRY ifeq ($(call cc-option-yn, -mfentry),y) CC_FLAGS_FTRACE += -mfentry @@ -875,12 +881,6 @@ endif export CC_FLAGS_FTRACE KBUILD_CFLAGS += $(CC_FLAGS_FTRACE) $(CC_FLAGS_USING) KBUILD_AFLAGS += $(CC_FLAGS_USING) -ifdef CONFIG_DYNAMIC_FTRACE - ifdef CONFIG_HAVE_C_RECORDMCOUNT - BUILD_C_RECORDMCOUNT := y - export BUILD_C_RECORDMCOUNT - endif -endif endif # We trigger additional mismatches with less inlining @@ -899,6 +899,25 @@ KBUILD_CFLAGS += $(CC_FLAGS_SCS) export CC_FLAGS_SCS endif +ifdef CONFIG_LTO_CLANG +ifdef CONFIG_LTO_CLANG_THIN +CC_FLAGS_LTO := -flto=thin -fsplit-lto-unit +KBUILD_LDFLAGS += --thinlto-cache-dir=$(extmod-prefix).thinlto-cache +else +CC_FLAGS_LTO := -flto +endif +CC_FLAGS_LTO += -fvisibility=hidden + +# Limit inlining across translation units to reduce binary size +KBUILD_LDFLAGS += -mllvm -import-instr-limit=5 +endif + +ifdef CONFIG_LTO +KBUILD_CFLAGS += -fno-lto $(CC_FLAGS_LTO) +KBUILD_AFLAGS += -fno-lto +export CC_FLAGS_LTO +endif + ifdef CONFIG_DEBUG_FORCE_FUNCTION_ALIGN_32B KBUILD_CFLAGS += -falign-functions=32 endif @@ -1054,7 +1073,7 @@ ifdef CONFIG_MODULE_COMPRESS mod_compress_cmd = $(KGZIP) -n -f endif # CONFIG_MODULE_COMPRESS_GZIP ifdef CONFIG_MODULE_COMPRESS_XZ - mod_compress_cmd = $(XZ) -f + mod_compress_cmd = $(XZ) --lzma2=dict=2MiB -f endif # CONFIG_MODULE_COMPRESS_XZ endif # CONFIG_MODULE_COMPRESS export mod_compress_cmd @@ -1105,6 +1124,7 @@ endif # CONFIG_BPF PHONY += prepare0 +extmod-prefix = $(if $(KBUILD_EXTMOD),$(KBUILD_EXTMOD)/) export MODORDER := $(extmod-prefix)modules.order export MODULES_NSDEPS := $(extmod-prefix)modules.nsdeps @@ -1229,6 +1249,10 @@ uapi-asm-generic: PHONY += prepare-objtool prepare-resolve_btfids prepare-objtool: $(objtool_target) ifeq ($(SKIP_STACK_VALIDATION),1) +ifdef CONFIG_FTRACE_MCOUNT_USE_OBJTOOL + @echo "error: Cannot generate __mcount_loc for CONFIG_DYNAMIC_FTRACE=y, please install libelf-dev, libelf-devel or elfutils-libelf-devel" >&2 + @false +endif ifdef CONFIG_UNWINDER_ORC @echo "error: Cannot generate ORC metadata for CONFIG_UNWINDER_ORC=y, please install libelf-dev, libelf-devel or elfutils-libelf-devel" >&2 @false @@ -1258,14 +1282,24 @@ define filechk_utsrelease.h endef define filechk_version.h - echo \#define LINUX_VERSION_CODE $(shell \ - expr $(VERSION) \* 65536 + 0$(PATCHLEVEL) \* 256 + 0$(SUBLEVEL)); \ - echo '#define KERNEL_VERSION(a,b,c) (((a) << 16) + ((b) << 8) + (c))' + if [ $(SUBLEVEL) -gt 255 ]; then \ + echo \#define LINUX_VERSION_CODE $(shell \ + expr $(VERSION) \* 65536 + $(PATCHLEVEL) \* 256 + 255); \ + else \ + echo \#define LINUX_VERSION_CODE $(shell \ + expr $(VERSION) \* 65536 + $(PATCHLEVEL) \* 256 + $(SUBLEVEL)); \ + fi; \ + echo '#define KERNEL_VERSION(a,b,c) (((a) << 16) + ((b) << 8) + \ + ((c) > 255 ? 255 : (c)))'; \ + echo \#define LINUX_VERSION_MAJOR $(VERSION); \ + echo \#define LINUX_VERSION_PATCHLEVEL $(PATCHLEVEL); \ + echo \#define LINUX_VERSION_SUBLEVEL $(SUBLEVEL) endef +$(version_h): PATCHLEVEL := $(if $(PATCHLEVEL), $(PATCHLEVEL), 0) +$(version_h): SUBLEVEL := $(if $(SUBLEVEL), $(SUBLEVEL), 0) $(version_h): FORCE $(call filechk,version.h) - $(Q)rm -f $(old_version_h) include/generated/utsrelease.h: include/config/kernel.release FORCE $(call filechk,utsrelease.h) @@ -1344,6 +1378,9 @@ ifneq ($(dtstree),) %.dtb: include/config/kernel.release scripts_dtc $(Q)$(MAKE) $(build)=$(dtstree) $(dtstree)/$@ +%.dtbo: include/config/kernel.release scripts_dtc + $(Q)$(MAKE) $(build)=$(dtstree) $(dtstree)/$@ + PHONY += dtbs dtbs_install dtbs_check dtbs: include/config/kernel.release scripts_dtc $(Q)$(MAKE) $(build)=$(dtstree) @@ -1476,7 +1513,7 @@ endif # CONFIG_MODULES # Directories & files removed with 'make clean' CLEAN_FILES += include/ksym vmlinux.symvers \ modules.builtin modules.builtin.modinfo modules.nsdeps \ - compile_commands.json + compile_commands.json .thinlto-cache # Directories & files removed with 'make mrproper' MRPROPER_FILES += include/config include/generated \ @@ -1736,7 +1773,7 @@ PHONY += compile_commands.json clean-dirs := $(KBUILD_EXTMOD) clean: rm-files := $(KBUILD_EXTMOD)/Module.symvers $(KBUILD_EXTMOD)/modules.nsdeps \ - $(KBUILD_EXTMOD)/compile_commands.json + $(KBUILD_EXTMOD)/compile_commands.json $(KBUILD_EXTMOD)/.thinlto-cache PHONY += help help: @@ -1823,7 +1860,7 @@ clean: $(clean-dirs) @find $(if $(KBUILD_EXTMOD), $(KBUILD_EXTMOD), .) $(RCS_FIND_IGNORE) \ \( -name '*.[aios]' -o -name '*.ko' -o -name '.*.cmd' \ -o -name '*.ko.*' \ - -o -name '*.dtb' -o -name '*.dtb.S' -o -name '*.dt.yaml' \ + -o -name '*.dtb' -o -name '*.dtbo' -o -name '*.dtb.S' -o -name '*.dt.yaml' \ -o -name '*.dwo' -o -name '*.lst' \ -o -name '*.su' -o -name '*.mod' \ -o -name '.*.d' -o -name '.*.tmp' -o -name '*.mod.c' \ @@ -1833,7 +1870,8 @@ clean: $(clean-dirs) -o -name '.tmp_*.o.*' \ -o -name '*.c.[012]*.*' \ -o -name '*.ll' \ - -o -name '*.gcno' \) -type f -print | xargs rm -f + -o -name '*.gcno' \ + -o -name '*.*.symversions' \) -type f -print | xargs rm -f # Generate tags for editors # --------------------------------------------------------------------------- diff --git a/arch/Kconfig b/arch/Kconfig index baf67e7f059a889f931ae1887b8b114b4facc414..2bb30673d8e6fbd5e7c3d41e8fe0800020c0f779 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -124,8 +124,8 @@ config HAVE_64BIT_ALIGNED_ACCESS accesses are required to be 64 bit aligned in this way even though it is not a 64 bit architecture. - See Documentation/unaligned-memory-access.txt for more - information on the topic of unaligned memory accesses. + See Documentation/core-api/unaligned-memory-access.rst for + more information on the topic of unaligned memory accesses. config HAVE_EFFICIENT_UNALIGNED_ACCESS bool @@ -603,6 +603,96 @@ config SHADOW_CALL_STACK reading and writing arbitrary memory may be able to locate them and hijack control flow by modifying the stacks. +config LTO + bool + help + Selected if the kernel will be built using the compiler's LTO feature. + +config LTO_CLANG + bool + select LTO + help + Selected if the kernel will be built using Clang's LTO feature. + +config ARCH_SUPPORTS_LTO_CLANG + bool + help + An architecture should select this option if it supports: + - compiling with Clang, + - compiling inline assembly with Clang's integrated assembler, + - and linking with LLD. + +config ARCH_SUPPORTS_LTO_CLANG_THIN + bool + help + An architecture should select this option if it can support Clang's + ThinLTO mode. + +config HAS_LTO_CLANG + def_bool y + # Clang >= 11: https://github.com/ClangBuiltLinux/linux/issues/510 + depends on CC_IS_CLANG && CLANG_VERSION >= 110000 && LD_IS_LLD + depends on $(success,test $(LLVM) -eq 1) + depends on $(success,test $(LLVM_IAS) -eq 1) + depends on $(success,$(NM) --help | head -n 1 | grep -qi llvm) + depends on $(success,$(AR) --help | head -n 1 | grep -qi llvm) + depends on ARCH_SUPPORTS_LTO_CLANG + depends on !FTRACE_MCOUNT_USE_RECORDMCOUNT + depends on !KASAN + depends on !GCOV_KERNEL + help + The compiler and Kconfig options support building with Clang's + LTO. + +choice + prompt "Link Time Optimization (LTO)" + default LTO_NONE + help + This option enables Link Time Optimization (LTO), which allows the + compiler to optimize binaries globally. + + If unsure, select LTO_NONE. Note that LTO is very resource-intensive + so it's disabled by default. + +config LTO_NONE + bool "None" + help + Build the kernel normally, without Link Time Optimization (LTO). + +config LTO_CLANG_FULL + bool "Clang Full LTO (EXPERIMENTAL)" + depends on HAS_LTO_CLANG + depends on !COMPILE_TEST + select LTO_CLANG + help + This option enables Clang's full Link Time Optimization (LTO), which + allows the compiler to optimize the kernel globally. If you enable + this option, the compiler generates LLVM bitcode instead of ELF + object files, and the actual compilation from bitcode happens at + the LTO link step, which may take several minutes depending on the + kernel configuration. More information can be found from LLVM's + documentation: + + https://llvm.org/docs/LinkTimeOptimization.html + + During link time, this option can use a large amount of RAM, and + may take much longer than the ThinLTO option. + +config LTO_CLANG_THIN + bool "Clang ThinLTO (EXPERIMENTAL)" + depends on HAS_LTO_CLANG && ARCH_SUPPORTS_LTO_CLANG_THIN + select LTO_CLANG + help + This option enables Clang's ThinLTO, which allows for parallel + optimization and faster incremental compiles compared to the + CONFIG_LTO_CLANG_FULL option. More information can be found + from Clang's documentation: + + https://clang.llvm.org/docs/ThinLTO.html + + If unsure, say Y. +endchoice + config HAVE_ARCH_WITHIN_STACK_FRAMES bool help @@ -731,6 +821,12 @@ config HAVE_IRQ_EXIT_ON_IRQ_STACK This spares a stack switch and improves cache usage on softirq processing. +config HAVE_SOFTIRQ_ON_OWN_STACK + bool + help + Architecture provides a function to run __do_softirq() on a + seperate stack. + config PGTABLE_LEVELS int default 2 diff --git a/arch/alpha/configs/defconfig b/arch/alpha/configs/defconfig index 6293675db1644a231fe2f4c22d3b9da94152efde..724c4075df408eefe8410e1e4ee4ea56b7b16825 100644 --- a/arch/alpha/configs/defconfig +++ b/arch/alpha/configs/defconfig @@ -1,4 +1,3 @@ -CONFIG_EXPERIMENTAL=y CONFIG_SYSVIPC=y CONFIG_POSIX_MQUEUE=y CONFIG_LOG_BUF_SHIFT=14 diff --git a/arch/alpha/kernel/process.c b/arch/alpha/kernel/process.c index 6c71554206ccb238aeb348bb3041f988710eaf39..5112ab996394773724ddf0103a9601ad0242cb07 100644 --- a/arch/alpha/kernel/process.c +++ b/arch/alpha/kernel/process.c @@ -249,7 +249,7 @@ int copy_thread(unsigned long clone_flags, unsigned long usp, childti->pcb.ksp = (unsigned long) childstack; childti->pcb.flags = 1; /* set FEN, clear everything else */ - if (unlikely(p->flags & PF_KTHREAD)) { + if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { /* kernel thread */ memset(childstack, 0, sizeof(struct switch_stack) + sizeof(struct pt_regs)); diff --git a/arch/alpha/kernel/syscalls/Makefile b/arch/alpha/kernel/syscalls/Makefile index 659faefdcb1da5f0ffbafc40fd598dd98d9a5fde..285aaba832d93936c3f2b7edb2938dbfe13fc21f 100644 --- a/arch/alpha/kernel/syscalls/Makefile +++ b/arch/alpha/kernel/syscalls/Makefile @@ -5,7 +5,7 @@ uapi := arch/$(SRCARCH)/include/generated/uapi/asm _dummy := $(shell [ -d '$(uapi)' ] || mkdir -p '$(uapi)') \ $(shell [ -d '$(kapi)' ] || mkdir -p '$(kapi)') -syscall := $(srctree)/$(src)/syscall.tbl +syscall := $(src)/syscall.tbl syshdr := $(srctree)/$(src)/syscallhdr.sh systbl := $(srctree)/$(src)/syscalltbl.sh @@ -21,18 +21,19 @@ quiet_cmd_systbl = SYSTBL $@ '$(systbl_abi_$(basetarget))' \ '$(systbl_offset_$(basetarget))' -$(uapi)/unistd_32.h: $(syscall) $(syshdr) +$(uapi)/unistd_32.h: $(syscall) $(syshdr) FORCE $(call if_changed,syshdr) -$(kapi)/syscall_table.h: $(syscall) $(systbl) +$(kapi)/syscall_table.h: $(syscall) $(systbl) FORCE $(call if_changed,systbl) uapisyshdr-y += unistd_32.h kapisyshdr-y += syscall_table.h -targets += $(uapisyshdr-y) $(kapisyshdr-y) +uapisyshdr-y := $(addprefix $(uapi)/, $(uapisyshdr-y)) +kapisyshdr-y := $(addprefix $(kapi)/, $(kapisyshdr-y)) +targets += $(addprefix ../../../../, $(uapisyshdr-y) $(kapisyshdr-y)) PHONY += all -all: $(addprefix $(uapi)/,$(uapisyshdr-y)) -all: $(addprefix $(kapi)/,$(kapisyshdr-y)) +all: $(uapisyshdr-y) $(kapisyshdr-y) @: diff --git a/arch/alpha/kernel/syscalls/syscall.tbl b/arch/alpha/kernel/syscalls/syscall.tbl index a6617067dbe6457a9d8cae8f60823e10813344ce..02f0244e005c67f5544dc09e2a7a71d67fc916d0 100644 --- a/arch/alpha/kernel/syscalls/syscall.tbl +++ b/arch/alpha/kernel/syscalls/syscall.tbl @@ -481,3 +481,4 @@ 549 common faccessat2 sys_faccessat2 550 common process_madvise sys_process_madvise 551 common epoll_pwait2 sys_epoll_pwait2 +552 common mount_setattr sys_mount_setattr diff --git a/arch/arc/kernel/process.c b/arch/arc/kernel/process.c index 37f724ad5e3992297b4513bc7498bd4e981cdac7..d838d0d576964c7d1a387488e8dd252bdb460133 100644 --- a/arch/arc/kernel/process.c +++ b/arch/arc/kernel/process.c @@ -191,7 +191,7 @@ int copy_thread(unsigned long clone_flags, unsigned long usp, childksp[0] = 0; /* fp */ childksp[1] = (unsigned long)ret_from_fork; /* blink */ - if (unlikely(p->flags & PF_KTHREAD)) { + if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { memset(c_regs, 0, sizeof(struct pt_regs)); c_callee->r13 = kthread_arg; diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index 6d0ed288893524bd7a39e65f3edc67005089ba95..853aab5ab327a430c250d4d77552e1d99ebe6071 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -23,6 +23,7 @@ config ARM select ARCH_HAS_TEARDOWN_DMA_OPS if MMU select ARCH_HAS_TICK_BROADCAST if GENERIC_CLOCKEVENTS_BROADCAST select ARCH_HAVE_CUSTOM_GPIO_H + select ARCH_HAVE_NMI_SAFE_CMPXCHG if CPU_V7 || CPU_V7M || CPU_V6K select ARCH_HAS_GCOV_PROFILE_ALL select ARCH_KEEP_MEMBLOCK select ARCH_MIGHT_HAVE_PC_PARPORT @@ -1856,9 +1857,10 @@ config AUTO_ZRELADDR help ZRELADDR is the physical address where the decompressed kernel image will be placed. If AUTO_ZRELADDR is selected, the address - will be determined at run-time by masking the current IP with - 0xf8000000. This assumes the zImage being placed in the first 128MB - from start of memory. + will be determined at run-time, either by masking the current IP + with 0xf8000000, or, if invalid, from the DTB passed in r2. + This assumes the zImage being placed in the first 128MB from + start of memory. config EFI_STUB bool diff --git a/arch/arm/Kconfig.debug b/arch/arm/Kconfig.debug index c36c5d4c6e9cb4bae396a258e2f04630baa39bd3..9e0b5e7f12af69dca9c9dd32ce6c750574e71ccd 100644 --- a/arch/arm/Kconfig.debug +++ b/arch/arm/Kconfig.debug @@ -1158,10 +1158,9 @@ choice Say Y here if you want kernel low-level debugging support on ST SPEAr13xx based platforms. - config STIH41X_DEBUG_ASC2 + config DEBUG_STIH41X_ASC2 bool "Use StiH415/416 ASC2 UART for low-level debug" depends on ARCH_STI - select DEBUG_STI_UART help Say Y here if you want kernel low-level debugging support on STiH415/416 based platforms like b2000, which has @@ -1169,10 +1168,9 @@ choice If unsure, say N. - config STIH41X_DEBUG_SBC_ASC1 + config DEBUG_STIH41X_SBC_ASC1 bool "Use StiH415/416 SBC ASC1 UART for low-level debug" depends on ARCH_STI - select DEBUG_STI_UART help Say Y here if you want kernel low-level debugging support on STiH415/416 based platforms like b2020. which has @@ -1180,6 +1178,16 @@ choice If unsure, say N. + config DEBUG_STIH418_SBC_ASC0 + bool "Use StiH418 SBC ASC0 UART for low-level debug" + depends on ARCH_STI + help + Say Y here if you want kernel low-level debugging support + on STiH418 based platforms which has default UART wired + up to SBC ASC0. + + If unsure, say N. + config STM32F4_DEBUG_UART bool "Use STM32F4 UART for low-level debug" depends on MACH_STM32F429 || MACH_STM32F469 @@ -1484,10 +1492,6 @@ config DEBUG_TEGRA_UART bool depends on ARCH_TEGRA -config DEBUG_STI_UART - bool - depends on ARCH_STI - config DEBUG_STM32_UART bool depends on ARCH_STM32 @@ -1546,7 +1550,9 @@ config DEBUG_LL_INCLUDE default "debug/renesas-scif.S" if DEBUG_RMOBILE_SCIFA4 default "debug/s3c24xx.S" if DEBUG_S3C24XX_UART || DEBUG_S3C64XX_UART default "debug/s5pv210.S" if DEBUG_S5PV210_UART - default "debug/sti.S" if DEBUG_STI_UART + default "debug/sti.S" if DEBUG_STIH41X_ASC2 + default "debug/sti.S" if DEBUG_STIH41X_SBC_ASC1 + default "debug/sti.S" if DEBUG_STIH418_SBC_ASC0 default "debug/stm32.S" if DEBUG_STM32_UART default "debug/tegra.S" if DEBUG_TEGRA_UART default "debug/ux500.S" if DEBUG_UX500_UART @@ -1579,6 +1585,7 @@ config DEBUG_UART_PHYS default 0x02531000 if DEBUG_KEYSTONE_UART1 default 0x03010fe0 if ARCH_RPC default 0x07000000 if DEBUG_SUN9I_UART0 + default 0x09530000 if DEBUG_STIH418_SBC_ASC0 default 0x10009000 if DEBUG_REALVIEW_STD_PORT || \ DEBUG_VEXPRESS_UART0_CA9 default 0x1010c000 if DEBUG_REALVIEW_PB1176_PORT @@ -1671,7 +1678,9 @@ config DEBUG_UART_PHYS default 0xfc00c000 if DEBUG_AT91_SAMA5D4_USART3 default 0xfcb00000 if DEBUG_HI3620_UART default 0xfd883000 if DEBUG_ALPINE_UART0 + default 0xfe531000 if DEBUG_STIH41X_SBC_ASC1 default 0xfe800000 if ARCH_IOP32X + default 0xfed32000 if DEBUG_STIH41X_ASC2 default 0xff690000 if DEBUG_RK32_UART2 default 0xffc02000 if DEBUG_SOCFPGA_UART0 default 0xffc02100 if DEBUG_SOCFPGA_ARRIA10_UART1 @@ -1699,7 +1708,9 @@ config DEBUG_UART_PHYS DEBUG_S3C64XX_UART || \ DEBUG_BCM63XX_UART || DEBUG_ASM9260_UART || \ DEBUG_DIGICOLOR_UA0 || \ - DEBUG_AT91_UART || DEBUG_STM32_UART + DEBUG_AT91_UART || DEBUG_STM32_UART || \ + DEBUG_STIH41X_ASC2 || DEBUG_STIH41X_SBC_ASC1 || \ + DEBUG_STIH418_SBC_ASC0 config DEBUG_UART_VIRT hex "Virtual base address of debug UART" @@ -1744,6 +1755,7 @@ config DEBUG_UART_VIRT default 0xf8090000 if DEBUG_VEXPRESS_UART0_RS1 default 0xf8ffee00 if DEBUG_AT91_SAM9263_DBGU default 0xf8fff200 if DEBUG_AT91_RM9200_DBGU + default 0xf9530000 if DEBUG_STIH418_SBC_ASC0 default 0xf9e09000 if DEBUG_AM33XXUART1 default 0xfa020000 if DEBUG_OMAP4UART3 || DEBUG_TI81XXUART1 default 0xfa022000 if DEBUG_TI81XXUART2 @@ -1762,7 +1774,9 @@ config DEBUG_UART_VIRT default 0xfb10c000 if DEBUG_REALVIEW_PB1176_PORT default 0xfcfe8600 if DEBUG_BCM63XX_UART default 0xfd000000 if DEBUG_SPEAR3XX || DEBUG_SPEAR13XX + default 0xfd531000 if DEBUG_STIH41X_SBC_ASC1 default 0xfd883000 if DEBUG_ALPINE_UART0 + default 0xfdd32000 if DEBUG_STIH41X_ASC2 default 0xfe010000 if STM32MP1_DEBUG_UART default 0xfe017000 if DEBUG_MMP_UART2 default 0xfe018000 if DEBUG_MMP_UART3 @@ -1803,7 +1817,9 @@ config DEBUG_UART_VIRT DEBUG_S3C64XX_UART || \ DEBUG_BCM63XX_UART || DEBUG_ASM9260_UART || \ DEBUG_DIGICOLOR_UA0 || \ - DEBUG_AT91_UART || DEBUG_STM32_UART + DEBUG_AT91_UART || DEBUG_STM32_UART || \ + DEBUG_STIH41X_ASC2 || DEBUG_STIH41X_SBC_ASC1 || \ + DEBUG_STIH418_SBC_ASC0 config DEBUG_UART_8250_SHIFT int "Register offset shift for the 8250 debug UART" @@ -1837,7 +1853,7 @@ config DEBUG_UNCOMPRESS depends on ARCH_MULTIPLATFORM || PLAT_SAMSUNG || ARM_SINGLE_ARMV7M depends on DEBUG_LL && !DEBUG_OMAP2PLUS_UART && \ (!DEBUG_TEGRA_UART || !ZBOOT_ROM) && \ - !DEBUG_BRCMSTB_UART + !DEBUG_BRCMSTB_UART && !DEBUG_SEMIHOSTING help This option influences the normal decompressor output for multiplatform kernels. Normally, multiplatform kernels disable diff --git a/arch/arm/boot/compressed/Makefile b/arch/arm/boot/compressed/Makefile index fb521efcc6c20a4f482df6d976049eeb3ce18d10..fd94e27ba4fa3d12f5ef16f4f0eef614e7c45ab3 100644 --- a/arch/arm/boot/compressed/Makefile +++ b/arch/arm/boot/compressed/Makefile @@ -87,10 +87,13 @@ libfdt_objs := fdt_rw.o fdt_ro.o fdt_wip.o fdt.o ifeq ($(CONFIG_ARM_ATAG_DTB_COMPAT),y) OBJS += $(libfdt_objs) atags_to_fdt.o endif +ifeq ($(CONFIG_USE_OF),y) +OBJS += $(libfdt_objs) fdt_check_mem_start.o +endif # -fstack-protector-strong triggers protection checks in this code, # but it is being used too early to link to meaningful stack_chk logic. -$(foreach o, $(libfdt_objs) atags_to_fdt.o, \ +$(foreach o, $(libfdt_objs) atags_to_fdt.o fdt_check_mem_start.o, \ $(eval CFLAGS_$(o) := -I $(srctree)/scripts/dtc/libfdt -fno-stack-protector)) # These were previously generated C files. When you are building the kernel diff --git a/arch/arm/boot/compressed/fdt_check_mem_start.c b/arch/arm/boot/compressed/fdt_check_mem_start.c new file mode 100644 index 0000000000000000000000000000000000000000..62450d824c3ca1806f56ba0996eba30ff06c676f --- /dev/null +++ b/arch/arm/boot/compressed/fdt_check_mem_start.c @@ -0,0 +1,131 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include +#include +#include + +static const void *get_prop(const void *fdt, const char *node_path, + const char *property, int minlen) +{ + const void *prop; + int offset, len; + + offset = fdt_path_offset(fdt, node_path); + if (offset < 0) + return NULL; + + prop = fdt_getprop(fdt, offset, property, &len); + if (!prop || len < minlen) + return NULL; + + return prop; +} + +static uint32_t get_cells(const void *fdt, const char *name) +{ + const fdt32_t *prop = get_prop(fdt, "/", name, sizeof(fdt32_t)); + + if (!prop) { + /* default */ + return 1; + } + + return fdt32_ld(prop); +} + +static uint64_t get_val(const fdt32_t *cells, uint32_t ncells) +{ + uint64_t r; + + r = fdt32_ld(cells); + if (ncells > 1) + r = (r << 32) | fdt32_ld(cells + 1); + + return r; +} + +/* + * Check the start of physical memory + * + * Traditionally, the start address of physical memory is obtained by masking + * the program counter. However, this does require that this address is a + * multiple of 128 MiB, precluding booting Linux on platforms where this + * requirement is not fulfilled. + * Hence validate the calculated address against the memory information in the + * DTB, and, if out-of-range, replace it by the real start address. + * To preserve backwards compatibility (systems reserving a block of memory + * at the start of physical memory, kdump, ...), the traditional method is + * always used if it yields a valid address. + * + * Return value: start address of physical memory to use + */ +uint32_t fdt_check_mem_start(uint32_t mem_start, const void *fdt) +{ + uint32_t addr_cells, size_cells, base; + uint32_t fdt_mem_start = 0xffffffff; + const fdt32_t *reg, *endp; + uint64_t size, end; + const char *type; + int offset, len; + + if (!fdt) + return mem_start; + + if (fdt_magic(fdt) != FDT_MAGIC) + return mem_start; + + /* There may be multiple cells on LPAE platforms */ + addr_cells = get_cells(fdt, "#address-cells"); + size_cells = get_cells(fdt, "#size-cells"); + if (addr_cells > 2 || size_cells > 2) + return mem_start; + + /* Walk all memory nodes and regions */ + for (offset = fdt_next_node(fdt, -1, NULL); offset >= 0; + offset = fdt_next_node(fdt, offset, NULL)) { + type = fdt_getprop(fdt, offset, "device_type", NULL); + if (!type || strcmp(type, "memory")) + continue; + + reg = fdt_getprop(fdt, offset, "linux,usable-memory", &len); + if (!reg) + reg = fdt_getprop(fdt, offset, "reg", &len); + if (!reg) + continue; + + for (endp = reg + (len / sizeof(fdt32_t)); + endp - reg >= addr_cells + size_cells; + reg += addr_cells + size_cells) { + size = get_val(reg + addr_cells, size_cells); + if (!size) + continue; + + if (addr_cells > 1 && fdt32_ld(reg)) { + /* Outside 32-bit address space, skipping */ + continue; + } + + base = fdt32_ld(reg + addr_cells - 1); + end = base + size; + if (mem_start >= base && mem_start < end) { + /* Calculated address is valid, use it */ + return mem_start; + } + + if (base < fdt_mem_start) + fdt_mem_start = base; + } + } + + if (fdt_mem_start == 0xffffffff) { + /* No usable memory found, falling back to default */ + return mem_start; + } + + /* + * The calculated address is not usable. + * Use the lowest usable physical memory address from the DTB instead, + * and make sure this is a multiple of 2 MiB for phys/virt patching. + */ + return round_up(fdt_mem_start, SZ_2M); +} diff --git a/arch/arm/boot/compressed/head.S b/arch/arm/boot/compressed/head.S index d9cce7238a3650811514396a302237b8b6ca0d0e..b1cb1972361b84e078e6a705a0f5b93c2943c3bd 100644 --- a/arch/arm/boot/compressed/head.S +++ b/arch/arm/boot/compressed/head.S @@ -174,10 +174,7 @@ .macro be32tocpu, val, tmp #ifndef __ARMEB__ /* convert to little endian */ - eor \tmp, \val, \val, ror #16 - bic \tmp, \tmp, #0x00ff0000 - mov \val, \val, ror #8 - eor \val, \val, \tmp, lsr #8 + rev_l \val, \tmp #endif .endm @@ -282,10 +279,40 @@ not_angel: * are already placing their zImage in (eg) the top 64MB * of this range. */ - mov r4, pc - and r4, r4, #0xf8000000 + mov r0, pc + and r0, r0, #0xf8000000 +#ifdef CONFIG_USE_OF + adr r1, LC1 +#ifdef CONFIG_ARM_APPENDED_DTB + /* + * Look for an appended DTB. If found, we cannot use it to + * validate the calculated start of physical memory, as its + * memory nodes may need to be augmented by ATAGS stored at + * an offset from the same start of physical memory. + */ + ldr r2, [r1, #4] @ get &_edata + add r2, r2, r1 @ relocate it + ldr r2, [r2] @ get DTB signature + ldr r3, =OF_DT_MAGIC + cmp r2, r3 @ do we have a DTB there? + beq 1f @ if yes, skip validation +#endif /* CONFIG_ARM_APPENDED_DTB */ + + /* + * Make sure we have some stack before calling C code. + * No GOT fixup has occurred yet, but none of the code we're + * about to call uses any global variables. + */ + ldr sp, [r1] @ get stack location + add sp, sp, r1 @ apply relocation + + /* Validate calculated start against passed DTB */ + mov r1, r8 + bl fdt_check_mem_start +1: +#endif /* CONFIG_USE_OF */ /* Determine final kernel image address. */ - add r4, r4, #TEXT_OFFSET + add r4, r0, #TEXT_OFFSET #else ldr r4, =zreladdr #endif @@ -1164,9 +1191,9 @@ __armv4_mmu_cache_off: __armv7_mmu_cache_off: mrc p15, 0, r0, c1, c0 #ifdef CONFIG_MMU - bic r0, r0, #0x000d + bic r0, r0, #0x0005 #else - bic r0, r0, #0x000c + bic r0, r0, #0x0004 #endif mcr p15, 0, r0, c1, c0 @ turn MMU and cache off mov r0, #0 diff --git a/arch/arm/boot/dts/cros-ec-keyboard.dtsi b/arch/arm/boot/dts/cros-ec-keyboard.dtsi index 165c5bcd510e5f41186279b501cd8cc78331d9ff..55c4744fa7e7a136bfc32af8d69015ed39036012 100644 --- a/arch/arm/boot/dts/cros-ec-keyboard.dtsi +++ b/arch/arm/boot/dts/cros-ec-keyboard.dtsi @@ -6,103 +6,18 @@ */ #include +#include &cros_ec { - keyboard-controller { + keyboard_controller: keyboard-controller { compatible = "google,cros-ec-keyb"; keypad,num-rows = <8>; keypad,num-columns = <13>; google,needs-ghost-filter; linux,keymap = < - MATRIX_KEY(0x00, 0x01, KEY_LEFTMETA) - MATRIX_KEY(0x00, 0x02, KEY_F1) - MATRIX_KEY(0x00, 0x03, KEY_B) - MATRIX_KEY(0x00, 0x04, KEY_F10) - MATRIX_KEY(0x00, 0x05, KEY_RO) - MATRIX_KEY(0x00, 0x06, KEY_N) - MATRIX_KEY(0x00, 0x08, KEY_EQUAL) - MATRIX_KEY(0x00, 0x0a, KEY_RIGHTALT) - - MATRIX_KEY(0x01, 0x01, KEY_ESC) - MATRIX_KEY(0x01, 0x02, KEY_F4) - MATRIX_KEY(0x01, 0x03, KEY_G) - MATRIX_KEY(0x01, 0x04, KEY_F7) - MATRIX_KEY(0x01, 0x06, KEY_H) - MATRIX_KEY(0x01, 0x08, KEY_APOSTROPHE) - MATRIX_KEY(0x01, 0x09, KEY_F9) - MATRIX_KEY(0x01, 0x0b, KEY_BACKSPACE) - MATRIX_KEY(0x01, 0x0c, KEY_HENKAN) - - MATRIX_KEY(0x02, 0x00, KEY_LEFTCTRL) - MATRIX_KEY(0x02, 0x01, KEY_TAB) - MATRIX_KEY(0x02, 0x02, KEY_F3) - MATRIX_KEY(0x02, 0x03, KEY_T) - MATRIX_KEY(0x02, 0x04, KEY_F6) - MATRIX_KEY(0x02, 0x05, KEY_RIGHTBRACE) - MATRIX_KEY(0x02, 0x06, KEY_Y) - MATRIX_KEY(0x02, 0x07, KEY_102ND) - MATRIX_KEY(0x02, 0x08, KEY_LEFTBRACE) - MATRIX_KEY(0x02, 0x09, KEY_F8) - MATRIX_KEY(0x02, 0x0a, KEY_YEN) - - MATRIX_KEY(0x03, 0x00, KEY_LEFTMETA) - MATRIX_KEY(0x03, 0x01, KEY_GRAVE) - MATRIX_KEY(0x03, 0x02, KEY_F2) - MATRIX_KEY(0x03, 0x03, KEY_5) - MATRIX_KEY(0x03, 0x04, KEY_F5) - MATRIX_KEY(0x03, 0x06, KEY_6) - MATRIX_KEY(0x03, 0x08, KEY_MINUS) - MATRIX_KEY(0x03, 0x09, KEY_F13) - MATRIX_KEY(0x03, 0x0b, KEY_BACKSLASH) - MATRIX_KEY(0x03, 0x0c, KEY_MUHENKAN) - - MATRIX_KEY(0x04, 0x00, KEY_RIGHTCTRL) - MATRIX_KEY(0x04, 0x01, KEY_A) - MATRIX_KEY(0x04, 0x02, KEY_D) - MATRIX_KEY(0x04, 0x03, KEY_F) - MATRIX_KEY(0x04, 0x04, KEY_S) - MATRIX_KEY(0x04, 0x05, KEY_K) - MATRIX_KEY(0x04, 0x06, KEY_J) - MATRIX_KEY(0x04, 0x08, KEY_SEMICOLON) - MATRIX_KEY(0x04, 0x09, KEY_L) - MATRIX_KEY(0x04, 0x0a, KEY_BACKSLASH) - MATRIX_KEY(0x04, 0x0b, KEY_ENTER) - - MATRIX_KEY(0x05, 0x01, KEY_Z) - MATRIX_KEY(0x05, 0x02, KEY_C) - MATRIX_KEY(0x05, 0x03, KEY_V) - MATRIX_KEY(0x05, 0x04, KEY_X) - MATRIX_KEY(0x05, 0x05, KEY_COMMA) - MATRIX_KEY(0x05, 0x06, KEY_M) - MATRIX_KEY(0x05, 0x07, KEY_LEFTSHIFT) - MATRIX_KEY(0x05, 0x08, KEY_SLASH) - MATRIX_KEY(0x05, 0x09, KEY_DOT) - MATRIX_KEY(0x05, 0x0b, KEY_SPACE) - - MATRIX_KEY(0x06, 0x01, KEY_1) - MATRIX_KEY(0x06, 0x02, KEY_3) - MATRIX_KEY(0x06, 0x03, KEY_4) - MATRIX_KEY(0x06, 0x04, KEY_2) - MATRIX_KEY(0x06, 0x05, KEY_8) - MATRIX_KEY(0x06, 0x06, KEY_7) - MATRIX_KEY(0x06, 0x08, KEY_0) - MATRIX_KEY(0x06, 0x09, KEY_9) - MATRIX_KEY(0x06, 0x0a, KEY_LEFTALT) - MATRIX_KEY(0x06, 0x0b, KEY_DOWN) - MATRIX_KEY(0x06, 0x0c, KEY_RIGHT) - - MATRIX_KEY(0x07, 0x01, KEY_Q) - MATRIX_KEY(0x07, 0x02, KEY_E) - MATRIX_KEY(0x07, 0x03, KEY_R) - MATRIX_KEY(0x07, 0x04, KEY_W) - MATRIX_KEY(0x07, 0x05, KEY_I) - MATRIX_KEY(0x07, 0x06, KEY_U) - MATRIX_KEY(0x07, 0x07, KEY_RIGHTSHIFT) - MATRIX_KEY(0x07, 0x08, KEY_P) - MATRIX_KEY(0x07, 0x09, KEY_O) - MATRIX_KEY(0x07, 0x0b, KEY_UP) - MATRIX_KEY(0x07, 0x0c, KEY_LEFT) + CROS_STD_TOP_ROW_KEYMAP + CROS_STD_MAIN_KEYMAP >; }; }; diff --git a/arch/arm/boot/dts/omap5.dtsi b/arch/arm/boot/dts/omap5.dtsi index 5f1a8bd1388049edd7413cec78d62703786dcdbc..e025b7c9a3572e45c8bb6187edf4816357f72021 100644 --- a/arch/arm/boot/dts/omap5.dtsi +++ b/arch/arm/boot/dts/omap5.dtsi @@ -518,6 +518,9 @@ dsi1: encoder@0 { clocks = <&dss_clkctrl OMAP5_DSS_CORE_CLKCTRL 8>, <&dss_clkctrl OMAP5_DSS_CORE_CLKCTRL 10>; clock-names = "fck", "sys_clk"; + + #address-cells = <1>; + #size-cells = <0>; }; }; @@ -550,6 +553,9 @@ dsi2: encoder@0 { clocks = <&dss_clkctrl OMAP5_DSS_CORE_CLKCTRL 8>, <&dss_clkctrl OMAP5_DSS_CORE_CLKCTRL 10>; clock-names = "fck", "sys_clk"; + + #address-cells = <1>; + #size-cells = <0>; }; }; diff --git a/arch/arm/common/locomo.c b/arch/arm/common/locomo.c index 62f241b09fe33008581d98d148363b9b870b6810..e45f4e4e06b698bd8ab142434a0f6df81d343ae1 100644 --- a/arch/arm/common/locomo.c +++ b/arch/arm/common/locomo.c @@ -838,11 +838,10 @@ static int locomo_bus_remove(struct device *dev) { struct locomo_dev *ldev = LOCOMO_DEV(dev); struct locomo_driver *drv = LOCOMO_DRV(dev->driver); - int ret = 0; if (drv->remove) - ret = drv->remove(ldev); - return ret; + drv->remove(ldev); + return 0; } struct bus_type locomo_bus_type = { diff --git a/arch/arm/common/sa1111.c b/arch/arm/common/sa1111.c index f89c1ea327a2541497e82abfb4099c0f995dcd81..ff5e0d04cb89238933bd20dccb778e6a53c007c1 100644 --- a/arch/arm/common/sa1111.c +++ b/arch/arm/common/sa1111.c @@ -1368,11 +1368,11 @@ static int sa1111_bus_remove(struct device *dev) { struct sa1111_dev *sadev = to_sa1111_device(dev); struct sa1111_driver *drv = SA1111_DRV(dev->driver); - int ret = 0; if (drv->remove) - ret = drv->remove(sadev); - return ret; + drv->remove(sadev); + + return 0; } struct bus_type sa1111_bus_type = { diff --git a/arch/arm/configs/bcm2835_defconfig b/arch/arm/configs/bcm2835_defconfig index 1ef2bc4c7f69d421d37b39da3dfcbdfe44403db4..383c632eba7bd7b17597fd680fa1cb7a9805e9ef 100644 --- a/arch/arm/configs/bcm2835_defconfig +++ b/arch/arm/configs/bcm2835_defconfig @@ -176,7 +176,6 @@ CONFIG_BOOT_PRINTK_DELAY=y CONFIG_DYNAMIC_DEBUG=y CONFIG_DEBUG_INFO=y # CONFIG_ENABLE_MUST_CHECK is not set -CONFIG_UNUSED_SYMBOLS=y CONFIG_DEBUG_MEMORY_INIT=y CONFIG_LOCKUP_DETECTOR=y CONFIG_SCHED_TRACER=y diff --git a/arch/arm/configs/mxs_defconfig b/arch/arm/configs/mxs_defconfig index a9c6f32a9b1c9d04ddca4ea9d7690aadcb143a40..ca32446b187f5d61da868e8a6a0d9deecc611757 100644 --- a/arch/arm/configs/mxs_defconfig +++ b/arch/arm/configs/mxs_defconfig @@ -164,7 +164,6 @@ CONFIG_FONTS=y CONFIG_PRINTK_TIME=y CONFIG_DEBUG_INFO=y CONFIG_FRAME_WARN=2048 -CONFIG_UNUSED_SYMBOLS=y CONFIG_MAGIC_SYSRQ=y CONFIG_DEBUG_KERNEL=y CONFIG_SOFTLOCKUP_DETECTOR=y diff --git a/arch/arm/configs/omap2plus_defconfig b/arch/arm/configs/omap2plus_defconfig index b26ef4866a35e63ad1ebb9e9d15282b3d40703f7..f250bf1cc022c90d5b5875b47c6dd8721be409aa 100644 --- a/arch/arm/configs/omap2plus_defconfig +++ b/arch/arm/configs/omap2plus_defconfig @@ -356,8 +356,8 @@ CONFIG_DRM_OMAP=m CONFIG_OMAP5_DSS_HDMI=y CONFIG_OMAP2_DSS_SDI=y CONFIG_OMAP2_DSS_DSI=y -CONFIG_DRM_OMAP_PANEL_DSI_CM=m CONFIG_DRM_TILCDC=m +CONFIG_DRM_PANEL_DSI_CM=m CONFIG_DRM_PANEL_SIMPLE=m CONFIG_DRM_PANEL_LG_LB035Q02=m CONFIG_DRM_PANEL_NEC_NL8048HL11=m diff --git a/arch/arm/crypto/Kconfig b/arch/arm/crypto/Kconfig index c9bf2df85cb904d735bac933817041af206b8479..2b575792363e55ea8f5fc1c32d92da2428be26dd 100644 --- a/arch/arm/crypto/Kconfig +++ b/arch/arm/crypto/Kconfig @@ -62,6 +62,25 @@ config CRYPTO_SHA512_ARM SHA-512 secure hash standard (DFIPS 180-2) implemented using optimized ARM assembler and NEON, when available. +config CRYPTO_BLAKE2S_ARM + tristate "BLAKE2s digest algorithm (ARM)" + select CRYPTO_ARCH_HAVE_LIB_BLAKE2S + help + BLAKE2s digest algorithm optimized with ARM scalar instructions. This + is faster than the generic implementations of BLAKE2s and BLAKE2b, but + slower than the NEON implementation of BLAKE2b. (There is no NEON + implementation of BLAKE2s, since NEON doesn't really help with it.) + +config CRYPTO_BLAKE2B_NEON + tristate "BLAKE2b digest algorithm (ARM NEON)" + depends on KERNEL_MODE_NEON + select CRYPTO_BLAKE2B + help + BLAKE2b digest algorithm optimized with ARM NEON instructions. + On ARM processors that have NEON support but not the ARMv8 + Crypto Extensions, typically this BLAKE2b implementation is + much faster than SHA-2 and slightly faster than SHA-1. + config CRYPTO_AES_ARM tristate "Scalar AES cipher for ARM" select CRYPTO_ALGAPI diff --git a/arch/arm/crypto/Makefile b/arch/arm/crypto/Makefile index b745c17d356fe6ea83989f1dd647427395a71304..8f26c454ea12eaf7d8e160d65b76d76e382b29eb 100644 --- a/arch/arm/crypto/Makefile +++ b/arch/arm/crypto/Makefile @@ -9,6 +9,8 @@ obj-$(CONFIG_CRYPTO_SHA1_ARM) += sha1-arm.o obj-$(CONFIG_CRYPTO_SHA1_ARM_NEON) += sha1-arm-neon.o obj-$(CONFIG_CRYPTO_SHA256_ARM) += sha256-arm.o obj-$(CONFIG_CRYPTO_SHA512_ARM) += sha512-arm.o +obj-$(CONFIG_CRYPTO_BLAKE2S_ARM) += blake2s-arm.o +obj-$(CONFIG_CRYPTO_BLAKE2B_NEON) += blake2b-neon.o obj-$(CONFIG_CRYPTO_CHACHA20_NEON) += chacha-neon.o obj-$(CONFIG_CRYPTO_POLY1305_ARM) += poly1305-arm.o obj-$(CONFIG_CRYPTO_NHPOLY1305_NEON) += nhpoly1305-neon.o @@ -29,6 +31,8 @@ sha256-arm-neon-$(CONFIG_KERNEL_MODE_NEON) := sha256_neon_glue.o sha256-arm-y := sha256-core.o sha256_glue.o $(sha256-arm-neon-y) sha512-arm-neon-$(CONFIG_KERNEL_MODE_NEON) := sha512-neon-glue.o sha512-arm-y := sha512-core.o sha512-glue.o $(sha512-arm-neon-y) +blake2s-arm-y := blake2s-core.o blake2s-glue.o +blake2b-neon-y := blake2b-neon-core.o blake2b-neon-glue.o sha1-arm-ce-y := sha1-ce-core.o sha1-ce-glue.o sha2-arm-ce-y := sha2-ce-core.o sha2-ce-glue.o aes-arm-ce-y := aes-ce-core.o aes-ce-glue.o diff --git a/arch/arm/crypto/aes-neonbs-glue.c b/arch/arm/crypto/aes-neonbs-glue.c index f70af1d0514b9ad81c3498f41a72ca21bc78092c..5c6cd3c63cbc1527bed79a4309db507c2743b289 100644 --- a/arch/arm/crypto/aes-neonbs-glue.c +++ b/arch/arm/crypto/aes-neonbs-glue.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -23,6 +24,8 @@ MODULE_ALIAS_CRYPTO("cbc(aes)-all"); MODULE_ALIAS_CRYPTO("ctr(aes)"); MODULE_ALIAS_CRYPTO("xts(aes)"); +MODULE_IMPORT_NS(CRYPTO_INTERNAL); + asmlinkage void aesbs_convert_key(u8 out[], u32 const rk[], int rounds); asmlinkage void aesbs_ecb_encrypt(u8 out[], u8 const in[], u8 const rk[], diff --git a/arch/arm/crypto/blake2b-neon-core.S b/arch/arm/crypto/blake2b-neon-core.S new file mode 100644 index 0000000000000000000000000000000000000000..0406a186377fb4a93f04568101b138050bece0c8 --- /dev/null +++ b/arch/arm/crypto/blake2b-neon-core.S @@ -0,0 +1,347 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * BLAKE2b digest algorithm, NEON accelerated + * + * Copyright 2020 Google LLC + * + * Author: Eric Biggers + */ + +#include + + .text + .fpu neon + + // The arguments to blake2b_compress_neon() + STATE .req r0 + BLOCK .req r1 + NBLOCKS .req r2 + INC .req r3 + + // Pointers to the rotation tables + ROR24_TABLE .req r4 + ROR16_TABLE .req r5 + + // The original stack pointer + ORIG_SP .req r6 + + // NEON registers which contain the message words of the current block. + // M_0-M_3 are occasionally used for other purposes too. + M_0 .req d16 + M_1 .req d17 + M_2 .req d18 + M_3 .req d19 + M_4 .req d20 + M_5 .req d21 + M_6 .req d22 + M_7 .req d23 + M_8 .req d24 + M_9 .req d25 + M_10 .req d26 + M_11 .req d27 + M_12 .req d28 + M_13 .req d29 + M_14 .req d30 + M_15 .req d31 + + .align 4 + // Tables for computing ror64(x, 24) and ror64(x, 16) using the vtbl.8 + // instruction. This is the most efficient way to implement these + // rotation amounts with NEON. (On Cortex-A53 it's the same speed as + // vshr.u64 + vsli.u64, while on Cortex-A7 it's faster.) +.Lror24_table: + .byte 3, 4, 5, 6, 7, 0, 1, 2 +.Lror16_table: + .byte 2, 3, 4, 5, 6, 7, 0, 1 + // The BLAKE2b initialization vector +.Lblake2b_IV: + .quad 0x6a09e667f3bcc908, 0xbb67ae8584caa73b + .quad 0x3c6ef372fe94f82b, 0xa54ff53a5f1d36f1 + .quad 0x510e527fade682d1, 0x9b05688c2b3e6c1f + .quad 0x1f83d9abfb41bd6b, 0x5be0cd19137e2179 + +// Execute one round of BLAKE2b by updating the state matrix v[0..15] in the +// NEON registers q0-q7. The message block is in q8..q15 (M_0-M_15). The stack +// pointer points to a 32-byte aligned buffer containing a copy of q8 and q9 +// (M_0-M_3), so that they can be reloaded if they are used as temporary +// registers. The macro arguments s0-s15 give the order in which the message +// words are used in this round. 'final' is 1 if this is the final round. +.macro _blake2b_round s0, s1, s2, s3, s4, s5, s6, s7, \ + s8, s9, s10, s11, s12, s13, s14, s15, final=0 + + // Mix the columns: + // (v[0], v[4], v[8], v[12]), (v[1], v[5], v[9], v[13]), + // (v[2], v[6], v[10], v[14]), and (v[3], v[7], v[11], v[15]). + + // a += b + m[blake2b_sigma[r][2*i + 0]]; + vadd.u64 q0, q0, q2 + vadd.u64 q1, q1, q3 + vadd.u64 d0, d0, M_\s0 + vadd.u64 d1, d1, M_\s2 + vadd.u64 d2, d2, M_\s4 + vadd.u64 d3, d3, M_\s6 + + // d = ror64(d ^ a, 32); + veor q6, q6, q0 + veor q7, q7, q1 + vrev64.32 q6, q6 + vrev64.32 q7, q7 + + // c += d; + vadd.u64 q4, q4, q6 + vadd.u64 q5, q5, q7 + + // b = ror64(b ^ c, 24); + vld1.8 {M_0}, [ROR24_TABLE, :64] + veor q2, q2, q4 + veor q3, q3, q5 + vtbl.8 d4, {d4}, M_0 + vtbl.8 d5, {d5}, M_0 + vtbl.8 d6, {d6}, M_0 + vtbl.8 d7, {d7}, M_0 + + // a += b + m[blake2b_sigma[r][2*i + 1]]; + // + // M_0 got clobbered above, so we have to reload it if any of the four + // message words this step needs happens to be M_0. Otherwise we don't + // need to reload it here, as it will just get clobbered again below. +.if \s1 == 0 || \s3 == 0 || \s5 == 0 || \s7 == 0 + vld1.8 {M_0}, [sp, :64] +.endif + vadd.u64 q0, q0, q2 + vadd.u64 q1, q1, q3 + vadd.u64 d0, d0, M_\s1 + vadd.u64 d1, d1, M_\s3 + vadd.u64 d2, d2, M_\s5 + vadd.u64 d3, d3, M_\s7 + + // d = ror64(d ^ a, 16); + vld1.8 {M_0}, [ROR16_TABLE, :64] + veor q6, q6, q0 + veor q7, q7, q1 + vtbl.8 d12, {d12}, M_0 + vtbl.8 d13, {d13}, M_0 + vtbl.8 d14, {d14}, M_0 + vtbl.8 d15, {d15}, M_0 + + // c += d; + vadd.u64 q4, q4, q6 + vadd.u64 q5, q5, q7 + + // b = ror64(b ^ c, 63); + // + // This rotation amount isn't a multiple of 8, so it has to be + // implemented using a pair of shifts, which requires temporary + // registers. Use q8-q9 (M_0-M_3) for this, and reload them afterwards. + veor q8, q2, q4 + veor q9, q3, q5 + vshr.u64 q2, q8, #63 + vshr.u64 q3, q9, #63 + vsli.u64 q2, q8, #1 + vsli.u64 q3, q9, #1 + vld1.8 {q8-q9}, [sp, :256] + + // Mix the diagonals: + // (v[0], v[5], v[10], v[15]), (v[1], v[6], v[11], v[12]), + // (v[2], v[7], v[8], v[13]), and (v[3], v[4], v[9], v[14]). + // + // There are two possible ways to do this: use 'vext' instructions to + // shift the rows of the matrix so that the diagonals become columns, + // and undo it afterwards; or just use 64-bit operations on 'd' + // registers instead of 128-bit operations on 'q' registers. We use the + // latter approach, as it performs much better on Cortex-A7. + + // a += b + m[blake2b_sigma[r][2*i + 0]]; + vadd.u64 d0, d0, d5 + vadd.u64 d1, d1, d6 + vadd.u64 d2, d2, d7 + vadd.u64 d3, d3, d4 + vadd.u64 d0, d0, M_\s8 + vadd.u64 d1, d1, M_\s10 + vadd.u64 d2, d2, M_\s12 + vadd.u64 d3, d3, M_\s14 + + // d = ror64(d ^ a, 32); + veor d15, d15, d0 + veor d12, d12, d1 + veor d13, d13, d2 + veor d14, d14, d3 + vrev64.32 d15, d15 + vrev64.32 d12, d12 + vrev64.32 d13, d13 + vrev64.32 d14, d14 + + // c += d; + vadd.u64 d10, d10, d15 + vadd.u64 d11, d11, d12 + vadd.u64 d8, d8, d13 + vadd.u64 d9, d9, d14 + + // b = ror64(b ^ c, 24); + vld1.8 {M_0}, [ROR24_TABLE, :64] + veor d5, d5, d10 + veor d6, d6, d11 + veor d7, d7, d8 + veor d4, d4, d9 + vtbl.8 d5, {d5}, M_0 + vtbl.8 d6, {d6}, M_0 + vtbl.8 d7, {d7}, M_0 + vtbl.8 d4, {d4}, M_0 + + // a += b + m[blake2b_sigma[r][2*i + 1]]; +.if \s9 == 0 || \s11 == 0 || \s13 == 0 || \s15 == 0 + vld1.8 {M_0}, [sp, :64] +.endif + vadd.u64 d0, d0, d5 + vadd.u64 d1, d1, d6 + vadd.u64 d2, d2, d7 + vadd.u64 d3, d3, d4 + vadd.u64 d0, d0, M_\s9 + vadd.u64 d1, d1, M_\s11 + vadd.u64 d2, d2, M_\s13 + vadd.u64 d3, d3, M_\s15 + + // d = ror64(d ^ a, 16); + vld1.8 {M_0}, [ROR16_TABLE, :64] + veor d15, d15, d0 + veor d12, d12, d1 + veor d13, d13, d2 + veor d14, d14, d3 + vtbl.8 d12, {d12}, M_0 + vtbl.8 d13, {d13}, M_0 + vtbl.8 d14, {d14}, M_0 + vtbl.8 d15, {d15}, M_0 + + // c += d; + vadd.u64 d10, d10, d15 + vadd.u64 d11, d11, d12 + vadd.u64 d8, d8, d13 + vadd.u64 d9, d9, d14 + + // b = ror64(b ^ c, 63); + veor d16, d4, d9 + veor d17, d5, d10 + veor d18, d6, d11 + veor d19, d7, d8 + vshr.u64 q2, q8, #63 + vshr.u64 q3, q9, #63 + vsli.u64 q2, q8, #1 + vsli.u64 q3, q9, #1 + // Reloading q8-q9 can be skipped on the final round. +.if ! \final + vld1.8 {q8-q9}, [sp, :256] +.endif +.endm + +// +// void blake2b_compress_neon(struct blake2b_state *state, +// const u8 *block, size_t nblocks, u32 inc); +// +// Only the first three fields of struct blake2b_state are used: +// u64 h[8]; (inout) +// u64 t[2]; (inout) +// u64 f[2]; (in) +// + .align 5 +ENTRY(blake2b_compress_neon) + push {r4-r10} + + // Allocate a 32-byte stack buffer that is 32-byte aligned. + mov ORIG_SP, sp + sub ip, sp, #32 + bic ip, ip, #31 + mov sp, ip + + adr ROR24_TABLE, .Lror24_table + adr ROR16_TABLE, .Lror16_table + + mov ip, STATE + vld1.64 {q0-q1}, [ip]! // Load h[0..3] + vld1.64 {q2-q3}, [ip]! // Load h[4..7] +.Lnext_block: + adr r10, .Lblake2b_IV + vld1.64 {q14-q15}, [ip] // Load t[0..1] and f[0..1] + vld1.64 {q4-q5}, [r10]! // Load IV[0..3] + vmov r7, r8, d28 // Copy t[0] to (r7, r8) + vld1.64 {q6-q7}, [r10] // Load IV[4..7] + adds r7, r7, INC // Increment counter + bcs .Lslow_inc_ctr + vmov.i32 d28[0], r7 + vst1.64 {d28}, [ip] // Update t[0] +.Linc_ctr_done: + + // Load the next message block and finish initializing the state matrix + // 'v'. Fortunately, there are exactly enough NEON registers to fit the + // entire state matrix in q0-q7 and the entire message block in q8-15. + // + // However, _blake2b_round also needs some extra registers for rotates, + // so we have to spill some registers. It's better to spill the message + // registers than the state registers, as the message doesn't change. + // Therefore we store a copy of the first 32 bytes of the message block + // (q8-q9) in an aligned buffer on the stack so that they can be + // reloaded when needed. (We could just reload directly from the + // message buffer, but it's faster to use aligned loads.) + vld1.8 {q8-q9}, [BLOCK]! + veor q6, q6, q14 // v[12..13] = IV[4..5] ^ t[0..1] + vld1.8 {q10-q11}, [BLOCK]! + veor q7, q7, q15 // v[14..15] = IV[6..7] ^ f[0..1] + vld1.8 {q12-q13}, [BLOCK]! + vst1.8 {q8-q9}, [sp, :256] + mov ip, STATE + vld1.8 {q14-q15}, [BLOCK]! + + // Execute the rounds. Each round is provided the order in which it + // needs to use the message words. + _blake2b_round 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 + _blake2b_round 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 + _blake2b_round 11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4 + _blake2b_round 7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8 + _blake2b_round 9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13 + _blake2b_round 2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9 + _blake2b_round 12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11 + _blake2b_round 13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10 + _blake2b_round 6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5 + _blake2b_round 10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13, 0 + _blake2b_round 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 + _blake2b_round 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 \ + final=1 + + // Fold the final state matrix into the hash chaining value: + // + // for (i = 0; i < 8; i++) + // h[i] ^= v[i] ^ v[i + 8]; + // + vld1.64 {q8-q9}, [ip]! // Load old h[0..3] + veor q0, q0, q4 // v[0..1] ^= v[8..9] + veor q1, q1, q5 // v[2..3] ^= v[10..11] + vld1.64 {q10-q11}, [ip] // Load old h[4..7] + veor q2, q2, q6 // v[4..5] ^= v[12..13] + veor q3, q3, q7 // v[6..7] ^= v[14..15] + veor q0, q0, q8 // v[0..1] ^= h[0..1] + veor q1, q1, q9 // v[2..3] ^= h[2..3] + mov ip, STATE + subs NBLOCKS, NBLOCKS, #1 // nblocks-- + vst1.64 {q0-q1}, [ip]! // Store new h[0..3] + veor q2, q2, q10 // v[4..5] ^= h[4..5] + veor q3, q3, q11 // v[6..7] ^= h[6..7] + vst1.64 {q2-q3}, [ip]! // Store new h[4..7] + + // Advance to the next block, if there is one. + bne .Lnext_block // nblocks != 0? + + mov sp, ORIG_SP + pop {r4-r10} + mov pc, lr + +.Lslow_inc_ctr: + // Handle the case where the counter overflowed its low 32 bits, by + // carrying the overflow bit into the full 128-bit counter. + vmov r9, r10, d29 + adcs r8, r8, #0 + adcs r9, r9, #0 + adc r10, r10, #0 + vmov d28, r7, r8 + vmov d29, r9, r10 + vst1.64 {q14}, [ip] // Update t[0] and t[1] + b .Linc_ctr_done +ENDPROC(blake2b_compress_neon) diff --git a/arch/arm/crypto/blake2b-neon-glue.c b/arch/arm/crypto/blake2b-neon-glue.c new file mode 100644 index 0000000000000000000000000000000000000000..34d73200e7fa6eff750bb2ae5fe9a9017c24af0f --- /dev/null +++ b/arch/arm/crypto/blake2b-neon-glue.c @@ -0,0 +1,105 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * BLAKE2b digest algorithm, NEON accelerated + * + * Copyright 2020 Google LLC + */ + +#include +#include +#include + +#include +#include + +#include +#include + +asmlinkage void blake2b_compress_neon(struct blake2b_state *state, + const u8 *block, size_t nblocks, u32 inc); + +static void blake2b_compress_arch(struct blake2b_state *state, + const u8 *block, size_t nblocks, u32 inc) +{ + if (!crypto_simd_usable()) { + blake2b_compress_generic(state, block, nblocks, inc); + return; + } + + do { + const size_t blocks = min_t(size_t, nblocks, + SZ_4K / BLAKE2B_BLOCK_SIZE); + + kernel_neon_begin(); + blake2b_compress_neon(state, block, blocks, inc); + kernel_neon_end(); + + nblocks -= blocks; + block += blocks * BLAKE2B_BLOCK_SIZE; + } while (nblocks); +} + +static int crypto_blake2b_update_neon(struct shash_desc *desc, + const u8 *in, unsigned int inlen) +{ + return crypto_blake2b_update(desc, in, inlen, blake2b_compress_arch); +} + +static int crypto_blake2b_final_neon(struct shash_desc *desc, u8 *out) +{ + return crypto_blake2b_final(desc, out, blake2b_compress_arch); +} + +#define BLAKE2B_ALG(name, driver_name, digest_size) \ + { \ + .base.cra_name = name, \ + .base.cra_driver_name = driver_name, \ + .base.cra_priority = 200, \ + .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY, \ + .base.cra_blocksize = BLAKE2B_BLOCK_SIZE, \ + .base.cra_ctxsize = sizeof(struct blake2b_tfm_ctx), \ + .base.cra_module = THIS_MODULE, \ + .digestsize = digest_size, \ + .setkey = crypto_blake2b_setkey, \ + .init = crypto_blake2b_init, \ + .update = crypto_blake2b_update_neon, \ + .final = crypto_blake2b_final_neon, \ + .descsize = sizeof(struct blake2b_state), \ + } + +static struct shash_alg blake2b_neon_algs[] = { + BLAKE2B_ALG("blake2b-160", "blake2b-160-neon", BLAKE2B_160_HASH_SIZE), + BLAKE2B_ALG("blake2b-256", "blake2b-256-neon", BLAKE2B_256_HASH_SIZE), + BLAKE2B_ALG("blake2b-384", "blake2b-384-neon", BLAKE2B_384_HASH_SIZE), + BLAKE2B_ALG("blake2b-512", "blake2b-512-neon", BLAKE2B_512_HASH_SIZE), +}; + +static int __init blake2b_neon_mod_init(void) +{ + if (!(elf_hwcap & HWCAP_NEON)) + return -ENODEV; + + return crypto_register_shashes(blake2b_neon_algs, + ARRAY_SIZE(blake2b_neon_algs)); +} + +static void __exit blake2b_neon_mod_exit(void) +{ + return crypto_unregister_shashes(blake2b_neon_algs, + ARRAY_SIZE(blake2b_neon_algs)); +} + +module_init(blake2b_neon_mod_init); +module_exit(blake2b_neon_mod_exit); + +MODULE_DESCRIPTION("BLAKE2b digest algorithm, NEON accelerated"); +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Eric Biggers "); +MODULE_ALIAS_CRYPTO("blake2b-160"); +MODULE_ALIAS_CRYPTO("blake2b-160-neon"); +MODULE_ALIAS_CRYPTO("blake2b-256"); +MODULE_ALIAS_CRYPTO("blake2b-256-neon"); +MODULE_ALIAS_CRYPTO("blake2b-384"); +MODULE_ALIAS_CRYPTO("blake2b-384-neon"); +MODULE_ALIAS_CRYPTO("blake2b-512"); +MODULE_ALIAS_CRYPTO("blake2b-512-neon"); diff --git a/arch/arm/crypto/blake2s-core.S b/arch/arm/crypto/blake2s-core.S new file mode 100644 index 0000000000000000000000000000000000000000..bed897e9a181a1ae0c7104fdd7b73e34e4f74975 --- /dev/null +++ b/arch/arm/crypto/blake2s-core.S @@ -0,0 +1,285 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * BLAKE2s digest algorithm, ARM scalar implementation + * + * Copyright 2020 Google LLC + * + * Author: Eric Biggers + */ + +#include + + // Registers used to hold message words temporarily. There aren't + // enough ARM registers to hold the whole message block, so we have to + // load the words on-demand. + M_0 .req r12 + M_1 .req r14 + +// The BLAKE2s initialization vector +.Lblake2s_IV: + .word 0x6A09E667, 0xBB67AE85, 0x3C6EF372, 0xA54FF53A + .word 0x510E527F, 0x9B05688C, 0x1F83D9AB, 0x5BE0CD19 + +.macro __ldrd a, b, src, offset +#if __LINUX_ARM_ARCH__ >= 6 + ldrd \a, \b, [\src, #\offset] +#else + ldr \a, [\src, #\offset] + ldr \b, [\src, #\offset + 4] +#endif +.endm + +.macro __strd a, b, dst, offset +#if __LINUX_ARM_ARCH__ >= 6 + strd \a, \b, [\dst, #\offset] +#else + str \a, [\dst, #\offset] + str \b, [\dst, #\offset + 4] +#endif +.endm + +// Execute a quarter-round of BLAKE2s by mixing two columns or two diagonals. +// (a0, b0, c0, d0) and (a1, b1, c1, d1) give the registers containing the two +// columns/diagonals. s0-s1 are the word offsets to the message words the first +// column/diagonal needs, and likewise s2-s3 for the second column/diagonal. +// M_0 and M_1 are free to use, and the message block can be found at sp + 32. +// +// Note that to save instructions, the rotations don't happen when the +// pseudocode says they should, but rather they are delayed until the values are +// used. See the comment above _blake2s_round(). +.macro _blake2s_quarterround a0, b0, c0, d0, a1, b1, c1, d1, s0, s1, s2, s3 + + ldr M_0, [sp, #32 + 4 * \s0] + ldr M_1, [sp, #32 + 4 * \s2] + + // a += b + m[blake2s_sigma[r][2*i + 0]]; + add \a0, \a0, \b0, ror #brot + add \a1, \a1, \b1, ror #brot + add \a0, \a0, M_0 + add \a1, \a1, M_1 + + // d = ror32(d ^ a, 16); + eor \d0, \a0, \d0, ror #drot + eor \d1, \a1, \d1, ror #drot + + // c += d; + add \c0, \c0, \d0, ror #16 + add \c1, \c1, \d1, ror #16 + + // b = ror32(b ^ c, 12); + eor \b0, \c0, \b0, ror #brot + eor \b1, \c1, \b1, ror #brot + + ldr M_0, [sp, #32 + 4 * \s1] + ldr M_1, [sp, #32 + 4 * \s3] + + // a += b + m[blake2s_sigma[r][2*i + 1]]; + add \a0, \a0, \b0, ror #12 + add \a1, \a1, \b1, ror #12 + add \a0, \a0, M_0 + add \a1, \a1, M_1 + + // d = ror32(d ^ a, 8); + eor \d0, \a0, \d0, ror#16 + eor \d1, \a1, \d1, ror#16 + + // c += d; + add \c0, \c0, \d0, ror#8 + add \c1, \c1, \d1, ror#8 + + // b = ror32(b ^ c, 7); + eor \b0, \c0, \b0, ror#12 + eor \b1, \c1, \b1, ror#12 +.endm + +// Execute one round of BLAKE2s by updating the state matrix v[0..15]. v[0..9] +// are in r0..r9. The stack pointer points to 8 bytes of scratch space for +// spilling v[8..9], then to v[9..15], then to the message block. r10-r12 and +// r14 are free to use. The macro arguments s0-s15 give the order in which the +// message words are used in this round. +// +// All rotates are performed using the implicit rotate operand accepted by the +// 'add' and 'eor' instructions. This is faster than using explicit rotate +// instructions. To make this work, we allow the values in the second and last +// rows of the BLAKE2s state matrix (rows 'b' and 'd') to temporarily have the +// wrong rotation amount. The rotation amount is then fixed up just in time +// when the values are used. 'brot' is the number of bits the values in row 'b' +// need to be rotated right to arrive at the correct values, and 'drot' +// similarly for row 'd'. (brot, drot) start out as (0, 0) but we make it such +// that they end up as (7, 8) after every round. +.macro _blake2s_round s0, s1, s2, s3, s4, s5, s6, s7, \ + s8, s9, s10, s11, s12, s13, s14, s15 + + // Mix first two columns: + // (v[0], v[4], v[8], v[12]) and (v[1], v[5], v[9], v[13]). + __ldrd r10, r11, sp, 16 // load v[12] and v[13] + _blake2s_quarterround r0, r4, r8, r10, r1, r5, r9, r11, \ + \s0, \s1, \s2, \s3 + __strd r8, r9, sp, 0 + __strd r10, r11, sp, 16 + + // Mix second two columns: + // (v[2], v[6], v[10], v[14]) and (v[3], v[7], v[11], v[15]). + __ldrd r8, r9, sp, 8 // load v[10] and v[11] + __ldrd r10, r11, sp, 24 // load v[14] and v[15] + _blake2s_quarterround r2, r6, r8, r10, r3, r7, r9, r11, \ + \s4, \s5, \s6, \s7 + str r10, [sp, #24] // store v[14] + // v[10], v[11], and v[15] are used below, so no need to store them yet. + + .set brot, 7 + .set drot, 8 + + // Mix first two diagonals: + // (v[0], v[5], v[10], v[15]) and (v[1], v[6], v[11], v[12]). + ldr r10, [sp, #16] // load v[12] + _blake2s_quarterround r0, r5, r8, r11, r1, r6, r9, r10, \ + \s8, \s9, \s10, \s11 + __strd r8, r9, sp, 8 + str r11, [sp, #28] + str r10, [sp, #16] + + // Mix second two diagonals: + // (v[2], v[7], v[8], v[13]) and (v[3], v[4], v[9], v[14]). + __ldrd r8, r9, sp, 0 // load v[8] and v[9] + __ldrd r10, r11, sp, 20 // load v[13] and v[14] + _blake2s_quarterround r2, r7, r8, r10, r3, r4, r9, r11, \ + \s12, \s13, \s14, \s15 + __strd r10, r11, sp, 20 +.endm + +// +// void blake2s_compress_arch(struct blake2s_state *state, +// const u8 *block, size_t nblocks, u32 inc); +// +// Only the first three fields of struct blake2s_state are used: +// u32 h[8]; (inout) +// u32 t[2]; (inout) +// u32 f[2]; (in) +// + .align 5 +ENTRY(blake2s_compress_arch) + push {r0-r2,r4-r11,lr} // keep this an even number + +.Lnext_block: + // r0 is 'state' + // r1 is 'block' + // r3 is 'inc' + + // Load and increment the counter t[0..1]. + __ldrd r10, r11, r0, 32 + adds r10, r10, r3 + adc r11, r11, #0 + __strd r10, r11, r0, 32 + + // _blake2s_round is very short on registers, so copy the message block + // to the stack to save a register during the rounds. This also has the + // advantage that misalignment only needs to be dealt with in one place. + sub sp, sp, #64 + mov r12, sp + tst r1, #3 + bne .Lcopy_block_misaligned + ldmia r1!, {r2-r9} + stmia r12!, {r2-r9} + ldmia r1!, {r2-r9} + stmia r12, {r2-r9} +.Lcopy_block_done: + str r1, [sp, #68] // Update message pointer + + // Calculate v[8..15]. Push v[9..15] onto the stack, and leave space + // for spilling v[8..9]. Leave v[8..9] in r8-r9. + mov r14, r0 // r14 = state + adr r12, .Lblake2s_IV + ldmia r12!, {r8-r9} // load IV[0..1] + __ldrd r0, r1, r14, 40 // load f[0..1] + ldm r12, {r2-r7} // load IV[3..7] + eor r4, r4, r10 // v[12] = IV[4] ^ t[0] + eor r5, r5, r11 // v[13] = IV[5] ^ t[1] + eor r6, r6, r0 // v[14] = IV[6] ^ f[0] + eor r7, r7, r1 // v[15] = IV[7] ^ f[1] + push {r2-r7} // push v[9..15] + sub sp, sp, #8 // leave space for v[8..9] + + // Load h[0..7] == v[0..7]. + ldm r14, {r0-r7} + + // Execute the rounds. Each round is provided the order in which it + // needs to use the message words. + .set brot, 0 + .set drot, 0 + _blake2s_round 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 + _blake2s_round 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 + _blake2s_round 11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4 + _blake2s_round 7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8 + _blake2s_round 9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13 + _blake2s_round 2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9 + _blake2s_round 12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11 + _blake2s_round 13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10 + _blake2s_round 6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5 + _blake2s_round 10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13, 0 + + // Fold the final state matrix into the hash chaining value: + // + // for (i = 0; i < 8; i++) + // h[i] ^= v[i] ^ v[i + 8]; + // + ldr r14, [sp, #96] // r14 = &h[0] + add sp, sp, #8 // v[8..9] are already loaded. + pop {r10-r11} // load v[10..11] + eor r0, r0, r8 + eor r1, r1, r9 + eor r2, r2, r10 + eor r3, r3, r11 + ldm r14, {r8-r11} // load h[0..3] + eor r0, r0, r8 + eor r1, r1, r9 + eor r2, r2, r10 + eor r3, r3, r11 + stmia r14!, {r0-r3} // store new h[0..3] + ldm r14, {r0-r3} // load old h[4..7] + pop {r8-r11} // load v[12..15] + eor r0, r0, r4, ror #brot + eor r1, r1, r5, ror #brot + eor r2, r2, r6, ror #brot + eor r3, r3, r7, ror #brot + eor r0, r0, r8, ror #drot + eor r1, r1, r9, ror #drot + eor r2, r2, r10, ror #drot + eor r3, r3, r11, ror #drot + add sp, sp, #64 // skip copy of message block + stm r14, {r0-r3} // store new h[4..7] + + // Advance to the next block, if there is one. Note that if there are + // multiple blocks, then 'inc' (the counter increment amount) must be + // 64. So we can simply set it to 64 without re-loading it. + ldm sp, {r0, r1, r2} // load (state, block, nblocks) + mov r3, #64 // set 'inc' + subs r2, r2, #1 // nblocks-- + str r2, [sp, #8] + bne .Lnext_block // nblocks != 0? + + pop {r0-r2,r4-r11,pc} + + // The next message block (pointed to by r1) isn't 4-byte aligned, so it + // can't be loaded using ldmia. Copy it to the stack buffer (pointed to + // by r12) using an alternative method. r2-r9 are free to use. +.Lcopy_block_misaligned: + mov r2, #64 +1: +#ifdef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS + ldr r3, [r1], #4 +#else + ldrb r3, [r1, #0] + ldrb r4, [r1, #1] + ldrb r5, [r1, #2] + ldrb r6, [r1, #3] + add r1, r1, #4 + orr r3, r3, r4, lsl #8 + orr r3, r3, r5, lsl #16 + orr r3, r3, r6, lsl #24 +#endif + subs r2, r2, #4 + str r3, [r12], #4 + bne 1b + b .Lcopy_block_done +ENDPROC(blake2s_compress_arch) diff --git a/arch/arm/crypto/blake2s-glue.c b/arch/arm/crypto/blake2s-glue.c new file mode 100644 index 0000000000000000000000000000000000000000..f2cc1e5fc9ec1830f4e764fe32d4de0083cae3ec --- /dev/null +++ b/arch/arm/crypto/blake2s-glue.c @@ -0,0 +1,78 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * BLAKE2s digest algorithm, ARM scalar implementation + * + * Copyright 2020 Google LLC + */ + +#include +#include + +#include + +/* defined in blake2s-core.S */ +EXPORT_SYMBOL(blake2s_compress_arch); + +static int crypto_blake2s_update_arm(struct shash_desc *desc, + const u8 *in, unsigned int inlen) +{ + return crypto_blake2s_update(desc, in, inlen, blake2s_compress_arch); +} + +static int crypto_blake2s_final_arm(struct shash_desc *desc, u8 *out) +{ + return crypto_blake2s_final(desc, out, blake2s_compress_arch); +} + +#define BLAKE2S_ALG(name, driver_name, digest_size) \ + { \ + .base.cra_name = name, \ + .base.cra_driver_name = driver_name, \ + .base.cra_priority = 200, \ + .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY, \ + .base.cra_blocksize = BLAKE2S_BLOCK_SIZE, \ + .base.cra_ctxsize = sizeof(struct blake2s_tfm_ctx), \ + .base.cra_module = THIS_MODULE, \ + .digestsize = digest_size, \ + .setkey = crypto_blake2s_setkey, \ + .init = crypto_blake2s_init, \ + .update = crypto_blake2s_update_arm, \ + .final = crypto_blake2s_final_arm, \ + .descsize = sizeof(struct blake2s_state), \ + } + +static struct shash_alg blake2s_arm_algs[] = { + BLAKE2S_ALG("blake2s-128", "blake2s-128-arm", BLAKE2S_128_HASH_SIZE), + BLAKE2S_ALG("blake2s-160", "blake2s-160-arm", BLAKE2S_160_HASH_SIZE), + BLAKE2S_ALG("blake2s-224", "blake2s-224-arm", BLAKE2S_224_HASH_SIZE), + BLAKE2S_ALG("blake2s-256", "blake2s-256-arm", BLAKE2S_256_HASH_SIZE), +}; + +static int __init blake2s_arm_mod_init(void) +{ + return IS_REACHABLE(CONFIG_CRYPTO_HASH) ? + crypto_register_shashes(blake2s_arm_algs, + ARRAY_SIZE(blake2s_arm_algs)) : 0; +} + +static void __exit blake2s_arm_mod_exit(void) +{ + if (IS_REACHABLE(CONFIG_CRYPTO_HASH)) + crypto_unregister_shashes(blake2s_arm_algs, + ARRAY_SIZE(blake2s_arm_algs)); +} + +module_init(blake2s_arm_mod_init); +module_exit(blake2s_arm_mod_exit); + +MODULE_DESCRIPTION("BLAKE2s digest algorithm, ARM scalar implementation"); +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Eric Biggers "); +MODULE_ALIAS_CRYPTO("blake2s-128"); +MODULE_ALIAS_CRYPTO("blake2s-128-arm"); +MODULE_ALIAS_CRYPTO("blake2s-160"); +MODULE_ALIAS_CRYPTO("blake2s-160-arm"); +MODULE_ALIAS_CRYPTO("blake2s-224"); +MODULE_ALIAS_CRYPTO("blake2s-224-arm"); +MODULE_ALIAS_CRYPTO("blake2s-256"); +MODULE_ALIAS_CRYPTO("blake2s-256-arm"); diff --git a/arch/arm/include/asm/assembler.h b/arch/arm/include/asm/assembler.h index 6ed30421f697982f6cb58dfc5f2f73673a790d72..e2b1fd558bf3c4d1d6ccc55388e7f3e7b3b090dc 100644 --- a/arch/arm/include/asm/assembler.h +++ b/arch/arm/include/asm/assembler.h @@ -578,4 +578,21 @@ THUMB( orr \reg , \reg , #PSR_T_BIT ) __adldst_l str, \src, \sym, \tmp, \cond .endm + /* + * rev_l - byte-swap a 32-bit value + * + * @val: source/destination register + * @tmp: scratch register + */ + .macro rev_l, val:req, tmp:req + .if __LINUX_ARM_ARCH__ < 6 + eor \tmp, \val, \val, ror #16 + bic \tmp, \tmp, #0x00ff0000 + mov \val, \val, ror #8 + eor \val, \val, \tmp, lsr #8 + .else + rev \val, \val + .endif + .endm + #endif /* __ASM_ASSEMBLER_H__ */ diff --git a/arch/arm/include/asm/hardware/locomo.h b/arch/arm/include/asm/hardware/locomo.h index f8712e3c29cfec59749576e56bbda43a5a52d5fa..246a3de25931db6d65259933fee7d50169a81f2f 100644 --- a/arch/arm/include/asm/hardware/locomo.h +++ b/arch/arm/include/asm/hardware/locomo.h @@ -188,7 +188,7 @@ struct locomo_driver { struct device_driver drv; unsigned int devid; int (*probe)(struct locomo_dev *); - int (*remove)(struct locomo_dev *); + void (*remove)(struct locomo_dev *); }; #define LOCOMO_DRV(_d) container_of((_d), struct locomo_driver, drv) diff --git a/arch/arm/include/asm/hardware/sa1111.h b/arch/arm/include/asm/hardware/sa1111.h index d134b9a5ff94794b6f1b251cf97f66b28d531dd2..2e70db6f22ea374b25e1684d25501d36b799168d 100644 --- a/arch/arm/include/asm/hardware/sa1111.h +++ b/arch/arm/include/asm/hardware/sa1111.h @@ -403,7 +403,7 @@ struct sa1111_driver { struct device_driver drv; unsigned int devid; int (*probe)(struct sa1111_dev *); - int (*remove)(struct sa1111_dev *); + void (*remove)(struct sa1111_dev *); }; #define SA1111_DRV(_d) container_of((_d), struct sa1111_driver, drv) diff --git a/arch/arm/include/debug/sti.S b/arch/arm/include/debug/sti.S index 72d0525118908804f9d24b751211ae191dbe9100..dc796ac2ac57e290f853638664710f976e89fd0f 100644 --- a/arch/arm/include/debug/sti.S +++ b/arch/arm/include/debug/sti.S @@ -6,28 +6,6 @@ * Copyright (C) 2013 STMicroelectronics (R&D) Limited. */ -#define STIH41X_COMMS_BASE 0xfed00000 -#define STIH41X_ASC2_BASE (STIH41X_COMMS_BASE+0x32000) - -#define STIH41X_SBC_LPM_BASE 0xfe400000 -#define STIH41X_SBC_COMMS_BASE (STIH41X_SBC_LPM_BASE + 0x100000) -#define STIH41X_SBC_ASC1_BASE (STIH41X_SBC_COMMS_BASE + 0x31000) - - -#define VIRT_ADDRESS(x) (x - 0x1000000) - -#if IS_ENABLED(CONFIG_STIH41X_DEBUG_ASC2) -#define DEBUG_LL_UART_BASE STIH41X_ASC2_BASE -#endif - -#if IS_ENABLED(CONFIG_STIH41X_DEBUG_SBC_ASC1) -#define DEBUG_LL_UART_BASE STIH41X_SBC_ASC1_BASE -#endif - -#ifndef DEBUG_LL_UART_BASE -#error "DEBUG UART is not Configured" -#endif - #define ASC_TX_BUF_OFF 0x04 #define ASC_CTRL_OFF 0x0c #define ASC_STA_OFF 0x14 @@ -37,8 +15,8 @@ .macro addruart, rp, rv, tmp - ldr \rp, =DEBUG_LL_UART_BASE @ physical base - ldr \rv, =VIRT_ADDRESS(DEBUG_LL_UART_BASE) @ virt base + ldr \rp, =CONFIG_DEBUG_UART_PHYS @ physical base + ldr \rv, =CONFIG_DEBUG_UART_VIRT @ virt base .endm .macro senduart,rd,rx diff --git a/arch/arm/kernel/process.c b/arch/arm/kernel/process.c index ee3aee69e444c21093ef02c2662fdafed8307305..5199a2bb4111b7bff8c04b1eb4efa2bcdd8bf090 100644 --- a/arch/arm/kernel/process.c +++ b/arch/arm/kernel/process.c @@ -243,7 +243,7 @@ int copy_thread(unsigned long clone_flags, unsigned long stack_start, thread->cpu_domain = get_domain(); #endif - if (likely(!(p->flags & PF_KTHREAD))) { + if (likely(!(p->flags & (PF_KTHREAD | PF_IO_WORKER)))) { *childregs = *current_pt_regs(); childregs->ARM_r0 = 0; if (stack_start) diff --git a/arch/arm/kernel/smp.c b/arch/arm/kernel/smp.c index 5c48eb4fd0e5ea7d51470bfff94c2989ed8644a9..74679240a9d8e78e3f738d3eb6d79a45b2f491f9 100644 --- a/arch/arm/kernel/smp.c +++ b/arch/arm/kernel/smp.c @@ -540,12 +540,9 @@ void show_ipi_list(struct seq_file *p, int prec) unsigned int cpu, i; for (i = 0; i < NR_IPI; i++) { - unsigned int irq; - if (!ipi_desc[i]) continue; - irq = irq_desc_get_irq(ipi_desc[i]); seq_printf(p, "%*s%u: ", prec - 1, "IPI", i); for_each_online_cpu(cpu) diff --git a/arch/arm/kernel/sys_oabi-compat.c b/arch/arm/kernel/sys_oabi-compat.c index 0203e545bbc8df4cd7cc3f5716d78ace12c27bd0..075a2e0ed2c157d1cbf2167a872aad603b49a9ca 100644 --- a/arch/arm/kernel/sys_oabi-compat.c +++ b/arch/arm/kernel/sys_oabi-compat.c @@ -248,6 +248,7 @@ struct oabi_epoll_event { __u64 data; } __attribute__ ((packed,aligned(4))); +#ifdef CONFIG_EPOLL asmlinkage long sys_oabi_epoll_ctl(int epfd, int op, int fd, struct oabi_epoll_event __user *event) { @@ -298,6 +299,20 @@ asmlinkage long sys_oabi_epoll_wait(int epfd, kfree(kbuf); return err ? -EFAULT : ret; } +#else +asmlinkage long sys_oabi_epoll_ctl(int epfd, int op, int fd, + struct oabi_epoll_event __user *event) +{ + return -EINVAL; +} + +asmlinkage long sys_oabi_epoll_wait(int epfd, + struct oabi_epoll_event __user *events, + int maxevents, int timeout) +{ + return -EINVAL; +} +#endif struct oabi_sembuf { unsigned short sem_num; diff --git a/arch/arm/mach-imx/common.h b/arch/arm/mach-imx/common.h index 2d76e2c6c99e33f0de6e2db2136a9ccf8a25a6ec..2b004cc4f95e5637fef01a8dc6d25581ea7d5502 100644 --- a/arch/arm/mach-imx/common.h +++ b/arch/arm/mach-imx/common.h @@ -85,7 +85,6 @@ void imx_anatop_pre_suspend(void); void imx_anatop_post_resume(void); int imx6_set_lpm(enum mxc_cpu_pwr_mode mode); void imx6_set_int_mem_clk_lpm(bool enable); -void imx6sl_set_wait_clk(bool enter); int imx_mmdc_get_ddr_type(void); int imx7ulp_set_lpm(enum ulp_cpu_pwr_mode mode); diff --git a/arch/arm/mach-imx/cpuidle-imx6sl.c b/arch/arm/mach-imx/cpuidle-imx6sl.c index 4521e5352bf64825043254da51664e1d8970351e..b86ffbeb28e42f8693a2f3cf07957c4c3dc34d17 100644 --- a/arch/arm/mach-imx/cpuidle-imx6sl.c +++ b/arch/arm/mach-imx/cpuidle-imx6sl.c @@ -3,6 +3,7 @@ * Copyright (C) 2014 Freescale Semiconductor, Inc. */ +#include #include #include #include diff --git a/arch/arm/mach-imx/pm-imx6.c b/arch/arm/mach-imx/pm-imx6.c index 40c74b4c4d730151ea8a805344961acc90470dcb..9244437cb1b9b5345e248ee542af658b4bea8a72 100644 --- a/arch/arm/mach-imx/pm-imx6.c +++ b/arch/arm/mach-imx/pm-imx6.c @@ -4,6 +4,7 @@ * Copyright 2011 Linaro Ltd. */ +#include #include #include #include diff --git a/arch/arm/mach-ixp4xx/Kconfig b/arch/arm/mach-ixp4xx/Kconfig index f7211b57b1e7885a7ce5f15f6080c99e856aabb4..165c184801e19f4699035a46dd5c1d713cd262ce 100644 --- a/arch/arm/mach-ixp4xx/Kconfig +++ b/arch/arm/mach-ixp4xx/Kconfig @@ -13,7 +13,6 @@ config MACH_IXP4XX_OF select I2C select I2C_IOP3XX select PCI - select TIMER_OF select USE_OF help Say 'Y' here to support Device Tree-based IXP4xx platforms. diff --git a/arch/arm/mach-pxa/mioa701.c b/arch/arm/mach-pxa/mioa701.c index d3af80317f2dab5b80e5f153097148b5a238472e..a79f296e81e029b0c0ad1a4c8280dc80779587db 100644 --- a/arch/arm/mach-pxa/mioa701.c +++ b/arch/arm/mach-pxa/mioa701.c @@ -577,7 +577,6 @@ static struct platform_device power_dev = { static struct wm97xx_batt_pdata mioa701_battery_data = { .batt_aux = WM97XX_AUX_ID1, .temp_aux = -1, - .charge_gpio = -1, .min_voltage = 0xc00, .max_voltage = 0xfc0, .batt_tech = POWER_SUPPLY_TECHNOLOGY_LION, diff --git a/arch/arm/mach-pxa/palm27x.c b/arch/arm/mach-pxa/palm27x.c index 0d246a1aebbcd624d77183efac8b3833b1f63a50..6230381a7ca0ca4f1c13f3d8839b601160602fbc 100644 --- a/arch/arm/mach-pxa/palm27x.c +++ b/arch/arm/mach-pxa/palm27x.c @@ -212,7 +212,6 @@ void __init palm27x_irda_init(int pwdn) static struct wm97xx_batt_pdata palm27x_batt_pdata = { .batt_aux = WM97XX_AUX_ID3, .temp_aux = WM97XX_AUX_ID2, - .charge_gpio = -1, .batt_mult = 1000, .batt_div = 414, .temp_mult = 1, diff --git a/arch/arm/mach-pxa/palmte2.c b/arch/arm/mach-pxa/palmte2.c index e3bcf58b4e6399325d3d6dd8eea3902d4ef23d19..a2b10db4aacc4605f25d8cdd31516a64cb49ae53 100644 --- a/arch/arm/mach-pxa/palmte2.c +++ b/arch/arm/mach-pxa/palmte2.c @@ -273,7 +273,6 @@ static struct platform_device power_supply = { static struct wm97xx_batt_pdata palmte2_batt_pdata = { .batt_aux = WM97XX_AUX_ID3, .temp_aux = WM97XX_AUX_ID2, - .charge_gpio = -1, .max_voltage = PALMTE2_BAT_MAX_VOLTAGE, .min_voltage = PALMTE2_BAT_MIN_VOLTAGE, .batt_mult = 1000, diff --git a/arch/arm/mach-pxa/z2.c b/arch/arm/mach-pxa/z2.c index 21fd76bb09cdf2946e8177202467edd8fd8de84c..8e74fbb0a96e282294f4332e69a612e92c35a25a 100644 --- a/arch/arm/mach-pxa/z2.c +++ b/arch/arm/mach-pxa/z2.c @@ -20,7 +20,6 @@ #include #include #include -#include #include #include #include @@ -488,7 +487,6 @@ static struct z2_battery_info batt_chip_info = { .batt_I2C_bus = 0, .batt_I2C_addr = 0x55, .batt_I2C_reg = 2, - .charge_gpio = GPIO0_ZIPITZ2_AC_DETECT, .min_voltage = 3475000, .max_voltage = 4190000, .batt_div = 59, @@ -497,9 +495,19 @@ static struct z2_battery_info batt_chip_info = { .batt_name = "Z2", }; +static struct gpiod_lookup_table z2_battery_gpio_table = { + .dev_id = "aer915", + .table = { + GPIO_LOOKUP("gpio-pxa", GPIO0_ZIPITZ2_AC_DETECT, + NULL, GPIO_ACTIVE_HIGH), + { }, + }, +}; + static struct i2c_board_info __initdata z2_i2c_board_info[] = { { I2C_BOARD_INFO("aer915", 0x55), + .dev_name = "aer915", .platform_data = &batt_chip_info, }, { I2C_BOARD_INFO("wm8750", 0x1b), @@ -510,6 +518,7 @@ static struct i2c_board_info __initdata z2_i2c_board_info[] = { static void __init z2_i2c_init(void) { pxa_set_i2c_info(NULL); + gpiod_add_lookup_table(&z2_battery_gpio_table); i2c_register_board_info(0, ARRAY_AND_SIZE(z2_i2c_board_info)); } #else @@ -578,8 +587,13 @@ static struct pxa2xx_spi_chip lms283_chip_info = { .gpio_cs = GPIO88_ZIPITZ2_LCD_CS, }; -static const struct lms283gf05_pdata lms283_pdata = { - .reset_gpio = GPIO19_ZIPITZ2_LCD_RESET, +static struct gpiod_lookup_table lms283_gpio_table = { + .dev_id = "spi2.0", /* SPI bus 2 chip select 0 */ + .table = { + GPIO_LOOKUP("gpio-pxa", GPIO19_ZIPITZ2_LCD_RESET, + "reset", GPIO_ACTIVE_LOW), + { }, + }, }; static struct spi_board_info spi_board_info[] __initdata = { @@ -595,7 +609,6 @@ static struct spi_board_info spi_board_info[] __initdata = { { .modalias = "lms283gf05", .controller_data = &lms283_chip_info, - .platform_data = &lms283_pdata, .max_speed_hz = 400000, .bus_num = 2, .chip_select = 0, @@ -615,6 +628,7 @@ static void __init z2_spi_init(void) { pxa2xx_set_spi_info(1, &pxa_ssp1_master_info); pxa2xx_set_spi_info(2, &pxa_ssp2_master_info); + gpiod_add_lookup_table(&lms283_gpio_table); spi_register_board_info(spi_board_info, ARRAY_SIZE(spi_board_info)); } #else diff --git a/arch/arm/mach-sa1100/collie.c b/arch/arm/mach-sa1100/collie.c index d4e89a02c8c8867a1762d6856d3844ffe62653ea..14c33ed053180bbaa037761dda25f85b31471727 100644 --- a/arch/arm/mach-sa1100/collie.c +++ b/arch/arm/mach-sa1100/collie.c @@ -224,18 +224,12 @@ static int collie_uart_probe(struct locomo_dev *dev) return 0; } -static int collie_uart_remove(struct locomo_dev *dev) -{ - return 0; -} - static struct locomo_driver collie_uart_driver = { .drv = { .name = "collie_uart", }, .devid = LOCOMO_DEVID_UART, .probe = collie_uart_probe, - .remove = collie_uart_remove, }; static int __init collie_uart_init(void) diff --git a/arch/arm/mach-spear/generic.h b/arch/arm/mach-spear/generic.h index 25b4c5e66e39610e34de8ba0128e1b43ac18ebb0..8ec2b92dca192ff35f8f7bfca6f44bda42401179 100644 --- a/arch/arm/mach-spear/generic.h +++ b/arch/arm/mach-spear/generic.h @@ -43,16 +43,4 @@ void spear13xx_cpu_die(unsigned int cpu); extern const struct smp_operations spear13xx_smp_ops; -#ifdef CONFIG_MACH_SPEAR1310 -void __init spear1310_clk_init(void __iomem *misc_base, void __iomem *ras_base); -#else -static inline void spear1310_clk_init(void __iomem *misc_base, void __iomem *ras_base) {} -#endif - -#ifdef CONFIG_MACH_SPEAR1340 -void __init spear1340_clk_init(void __iomem *misc_base); -#else -static inline void spear1340_clk_init(void __iomem *misc_base) {} -#endif - #endif /* __MACH_GENERIC_H */ diff --git a/arch/arm/mach-spear/spear13xx.c b/arch/arm/mach-spear/spear13xx.c index 31c43cabf362399a9af40abdaf75e50b12f0f8f9..74d1ca2a529a7bd2339b5943b5e2008658f26a03 100644 --- a/arch/arm/mach-spear/spear13xx.c +++ b/arch/arm/mach-spear/spear13xx.c @@ -15,6 +15,7 @@ #include #include +#include #include #include #include diff --git a/arch/arm/mm/dump.c b/arch/arm/mm/dump.c index c18d23a5e5f1234828cfbb490d67915762fea16e..93ff0097f00bb878698f389ae0af0511099dc36d 100644 --- a/arch/arm/mm/dump.c +++ b/arch/arm/mm/dump.c @@ -19,6 +19,10 @@ #include static struct addr_marker address_markers[] = { +#ifdef CONFIG_KASAN + { KASAN_SHADOW_START, "Kasan shadow start"}, + { KASAN_SHADOW_END, "Kasan shadow end"}, +#endif { MODULES_VADDR, "Modules" }, { PAGE_OFFSET, "Kernel Mapping" }, { 0, "vmalloc() Area" }, @@ -429,8 +433,11 @@ static void ptdump_initialize(void) if (pg_level[i].bits[j].nx_bit) pg_level[i].nx_bit = &pg_level[i].bits[j]; } - +#ifdef CONFIG_KASAN + address_markers[4].start_address = VMALLOC_START; +#else address_markers[2].start_address = VMALLOC_START; +#endif } static struct ptdump_info kernel_ptdump_info = { diff --git a/arch/arm/mm/mmu.c b/arch/arm/mm/mmu.c index c06ebfbc48c4ae184ad21e601dacb7c77daeb77d..a25b660c3017263a3ec403368f35ac7a26498b5e 100644 --- a/arch/arm/mm/mmu.c +++ b/arch/arm/mm/mmu.c @@ -18,7 +18,6 @@ #include #include #include -#include #include #include #include diff --git a/arch/arm/tools/Makefile b/arch/arm/tools/Makefile index 27d8beb7c9410bedc6010ccbacd77ef3b8526558..3654f979851b71efac6dec5d46e461e2d8571b32 100644 --- a/arch/arm/tools/Makefile +++ b/arch/arm/tools/Makefile @@ -11,7 +11,7 @@ uapi := $(gen)/uapi/asm syshdr := $(srctree)/$(src)/syscallhdr.sh sysnr := $(srctree)/$(src)/syscallnr.sh systbl := $(srctree)/$(src)/syscalltbl.sh -syscall := $(srctree)/$(src)/syscall.tbl +syscall := $(src)/syscall.tbl gen-y := $(gen)/calls-oabi.S gen-y += $(gen)/calls-eabi.S diff --git a/arch/arm/tools/syscall.tbl b/arch/arm/tools/syscall.tbl index 20e1170e2e0aaeb52ea10f1802c637164ea79b5e..dcc1191291a225509693b769ecee3be417403a34 100644 --- a/arch/arm/tools/syscall.tbl +++ b/arch/arm/tools/syscall.tbl @@ -455,3 +455,4 @@ 439 common faccessat2 sys_faccessat2 440 common process_madvise sys_process_madvise 441 common epoll_pwait2 sys_epoll_pwait2 +442 common mount_setattr sys_mount_setattr diff --git a/arch/arm/vdso/Makefile b/arch/arm/vdso/Makefile index b558bee0e1f6b9e4fa89700e26698ceecf7455eb..7c9e395b77f7cbbab8f72960c79add9f50257b29 100644 --- a/arch/arm/vdso/Makefile +++ b/arch/arm/vdso/Makefile @@ -23,7 +23,6 @@ ldflags-y := -Bsymbolic --no-undefined -soname=linux-vdso.so.1 \ -T obj-$(CONFIG_VDSO) += vdso.o -extra-$(CONFIG_VDSO) += vdso.lds CPPFLAGS_vdso.lds += -P -C -U$(ARCH) CFLAGS_REMOVE_vdso.o = -pg diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 1b8cecdc028e92e8a15594434c86fb0d4626b436..1f212b47a48a1033d1799c946f0bb0b9dca96fd2 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -73,6 +73,8 @@ config ARM64 select ARCH_SUPPORTS_DEBUG_PAGEALLOC select ARCH_SUPPORTS_MEMORY_FAILURE select ARCH_SUPPORTS_SHADOW_CALL_STACK if CC_HAVE_SHADOW_CALL_STACK + select ARCH_SUPPORTS_LTO_CLANG if CPU_LITTLE_ENDIAN + select ARCH_SUPPORTS_LTO_CLANG_THIN select ARCH_SUPPORTS_ATOMIC_RMW select ARCH_SUPPORTS_INT128 if CC_HAS_INT128 && (GCC_VERSION >= 50000 || CC_IS_CLANG) select ARCH_SUPPORTS_NUMA_BALANCING @@ -138,6 +140,7 @@ config ARM64 select HAVE_ARCH_KASAN if !(ARM64_16K_PAGES && ARM64_VA_BITS_48) select HAVE_ARCH_KASAN_SW_TAGS if HAVE_ARCH_KASAN select HAVE_ARCH_KASAN_HW_TAGS if (HAVE_ARCH_KASAN && ARM64_MTE) + select HAVE_ARCH_KFENCE select HAVE_ARCH_KGDB select HAVE_ARCH_MMAP_RND_BITS select HAVE_ARCH_MMAP_RND_COMPAT_BITS if COMPAT @@ -162,6 +165,8 @@ config ARM64 select HAVE_DYNAMIC_FTRACE select HAVE_DYNAMIC_FTRACE_WITH_REGS \ if $(cc-option,-fpatchable-function-entry=2) + select FTRACE_MCOUNT_USE_PATCHABLE_FUNCTION_ENTRY \ + if DYNAMIC_FTRACE_WITH_REGS select HAVE_EFFICIENT_UNALIGNED_ACCESS select HAVE_FAST_GUP select HAVE_FTRACE_MCOUNT_RECORD @@ -994,6 +999,7 @@ config HOTPLUG_CPU # Common NUMA Features config NUMA bool "NUMA Memory Allocation and Scheduler Support" + select GENERIC_ARCH_NUMA select ACPI_NUMA if ACPI select OF_NUMA help @@ -1481,7 +1487,7 @@ config ARM64_PTR_AUTH depends on (CC_HAS_SIGN_RETURN_ADDRESS || CC_HAS_BRANCH_PROT_PAC_RET) && AS_HAS_PAC # Modern compilers insert a .note.gnu.property section note for PAC # which is only understood by binutils starting with version 2.33.1. - depends on LD_IS_LLD || LD_VERSION >= 233010000 || (CC_IS_GCC && GCC_VERSION < 90100) + depends on LD_IS_LLD || LD_VERSION >= 23301 || (CC_IS_GCC && GCC_VERSION < 90100) depends on !CC_IS_CLANG || AS_HAS_CFI_NEGATE_RA_STATE depends on (!FUNCTION_GRAPH_TRACER || DYNAMIC_FTRACE_WITH_REGS) help diff --git a/arch/arm64/boot/dts/qcom/qrb5165-rb5.dts b/arch/arm64/boot/dts/qcom/qrb5165-rb5.dts index 26a4a5a6871e76d47b5dc81a0a59bd94da498393..2f0528d012993571967d5ab9465ab5405321f6f4 100644 --- a/arch/arm64/boot/dts/qcom/qrb5165-rb5.dts +++ b/arch/arm64/boot/dts/qcom/qrb5165-rb5.dts @@ -232,7 +232,7 @@ &adsp { &apps_rsc { pm8009-rpmh-regulators { - compatible = "qcom,pm8009-rpmh-regulators"; + compatible = "qcom,pm8009-1-rpmh-regulators"; qcom,pmic-id = "f"; vdd-s1-supply = <&vph_pwr>; @@ -241,6 +241,13 @@ pm8009-rpmh-regulators { vdd-l5-l6-supply = <&vreg_bob>; vdd-l7-supply = <&vreg_s4a_1p8>; + vreg_s2f_0p95: smps2 { + regulator-name = "vreg_s2f_0p95"; + regulator-min-microvolt = <900000>; + regulator-max-microvolt = <952000>; + regulator-initial-mode = ; + }; + vreg_l1f_1p1: ldo1 { regulator-name = "vreg_l1f_1p1"; regulator-min-microvolt = <1104000>; diff --git a/arch/arm64/configs/defconfig b/arch/arm64/configs/defconfig index 43c4280bfc4ffea119962c4030d986729e36b7d5..d612f633b7719e6f79e3c66da4a92331d2c5fdb2 100644 --- a/arch/arm64/configs/defconfig +++ b/arch/arm64/configs/defconfig @@ -770,7 +770,8 @@ CONFIG_SND_SOC_LPASS_VA_MACRO=m CONFIG_SND_SIMPLE_CARD=m CONFIG_SND_AUDIO_GRAPH_CARD=m CONFIG_HID_MULTITOUCH=m -CONFIG_I2C_HID=m +CONFIG_I2C_HID_ACPI=m +CONFIG_I2C_HID_OF=m CONFIG_USB_CONN_GPIO=m CONFIG_USB=y CONFIG_USB_OTG=y diff --git a/arch/arm64/crypto/aes-glue.c b/arch/arm64/crypto/aes-glue.c index 34b8a89197be33f3f2863e257af7614254b02e43..17e735931a0c9ab9f7afdfe50f4f55ed95eb2a7d 100644 --- a/arch/arm64/crypto/aes-glue.c +++ b/arch/arm64/crypto/aes-glue.c @@ -24,6 +24,7 @@ #ifdef USE_V8_CRYPTO_EXTENSIONS #define MODE "ce" #define PRIO 300 +#define STRIDE 5 #define aes_expandkey ce_aes_expandkey #define aes_ecb_encrypt ce_aes_ecb_encrypt #define aes_ecb_decrypt ce_aes_ecb_decrypt @@ -41,6 +42,7 @@ MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS using ARMv8 Crypto Extensions"); #else #define MODE "neon" #define PRIO 200 +#define STRIDE 4 #define aes_ecb_encrypt neon_aes_ecb_encrypt #define aes_ecb_decrypt neon_aes_ecb_decrypt #define aes_cbc_encrypt neon_aes_cbc_encrypt @@ -55,7 +57,7 @@ MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS using ARMv8 Crypto Extensions"); #define aes_mac_update neon_aes_mac_update MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS using ARMv8 NEON"); #endif -#if defined(USE_V8_CRYPTO_EXTENSIONS) || !defined(CONFIG_CRYPTO_AES_ARM64_BS) +#if defined(USE_V8_CRYPTO_EXTENSIONS) || !IS_ENABLED(CONFIG_CRYPTO_AES_ARM64_BS) MODULE_ALIAS_CRYPTO("ecb(aes)"); MODULE_ALIAS_CRYPTO("cbc(aes)"); MODULE_ALIAS_CRYPTO("ctr(aes)"); @@ -87,7 +89,7 @@ asmlinkage void aes_cbc_cts_decrypt(u8 out[], u8 const in[], u32 const rk[], int rounds, int bytes, u8 const iv[]); asmlinkage void aes_ctr_encrypt(u8 out[], u8 const in[], u32 const rk[], - int rounds, int blocks, u8 ctr[]); + int rounds, int bytes, u8 ctr[], u8 finalbuf[]); asmlinkage void aes_xts_encrypt(u8 out[], u8 const in[], u32 const rk1[], int rounds, int bytes, u32 const rk2[], u8 iv[], @@ -103,9 +105,9 @@ asmlinkage void aes_essiv_cbc_decrypt(u8 out[], u8 const in[], u32 const rk1[], int rounds, int blocks, u8 iv[], u32 const rk2[]); -asmlinkage void aes_mac_update(u8 const in[], u32 const rk[], int rounds, - int blocks, u8 dg[], int enc_before, - int enc_after); +asmlinkage int aes_mac_update(u8 const in[], u32 const rk[], int rounds, + int blocks, u8 dg[], int enc_before, + int enc_after); struct crypto_aes_xts_ctx { struct crypto_aes_ctx key1; @@ -448,34 +450,36 @@ static int ctr_encrypt(struct skcipher_request *req) struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm); int err, rounds = 6 + ctx->key_length / 4; struct skcipher_walk walk; - int blocks; err = skcipher_walk_virt(&walk, req, false); - while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) { - kernel_neon_begin(); - aes_ctr_encrypt(walk.dst.virt.addr, walk.src.virt.addr, - ctx->key_enc, rounds, blocks, walk.iv); - kernel_neon_end(); - err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE); - } - if (walk.nbytes) { - u8 __aligned(8) tail[AES_BLOCK_SIZE]; + while (walk.nbytes > 0) { + const u8 *src = walk.src.virt.addr; unsigned int nbytes = walk.nbytes; - u8 *tdst = walk.dst.virt.addr; - u8 *tsrc = walk.src.virt.addr; + u8 *dst = walk.dst.virt.addr; + u8 buf[AES_BLOCK_SIZE]; + unsigned int tail; - /* - * Tell aes_ctr_encrypt() to process a tail block. - */ - blocks = -1; + if (unlikely(nbytes < AES_BLOCK_SIZE)) + src = memcpy(buf, src, nbytes); + else if (nbytes < walk.total) + nbytes &= ~(AES_BLOCK_SIZE - 1); kernel_neon_begin(); - aes_ctr_encrypt(tail, NULL, ctx->key_enc, rounds, - blocks, walk.iv); + aes_ctr_encrypt(dst, src, ctx->key_enc, rounds, nbytes, + walk.iv, buf); kernel_neon_end(); - crypto_xor_cpy(tdst, tsrc, tail, nbytes); - err = skcipher_walk_done(&walk, 0); + + tail = nbytes % (STRIDE * AES_BLOCK_SIZE); + if (tail > 0 && tail < AES_BLOCK_SIZE) + /* + * The final partial block could not be returned using + * an overlapping store, so it was passed via buf[] + * instead. + */ + memcpy(dst + nbytes - tail, buf, tail); + + err = skcipher_walk_done(&walk, walk.nbytes - nbytes); } return err; @@ -650,7 +654,7 @@ static int __maybe_unused xts_decrypt(struct skcipher_request *req) } static struct skcipher_alg aes_algs[] = { { -#if defined(USE_V8_CRYPTO_EXTENSIONS) || !defined(CONFIG_CRYPTO_AES_ARM64_BS) +#if defined(USE_V8_CRYPTO_EXTENSIONS) || !IS_ENABLED(CONFIG_CRYPTO_AES_ARM64_BS) .base = { .cra_name = "__ecb(aes)", .cra_driver_name = "__ecb-aes-" MODE, @@ -852,10 +856,17 @@ static void mac_do_update(struct crypto_aes_ctx *ctx, u8 const in[], int blocks, int rounds = 6 + ctx->key_length / 4; if (crypto_simd_usable()) { - kernel_neon_begin(); - aes_mac_update(in, ctx->key_enc, rounds, blocks, dg, enc_before, - enc_after); - kernel_neon_end(); + int rem; + + do { + kernel_neon_begin(); + rem = aes_mac_update(in, ctx->key_enc, rounds, blocks, + dg, enc_before, enc_after); + kernel_neon_end(); + in += (blocks - rem) * AES_BLOCK_SIZE; + blocks = rem; + enc_before = 0; + } while (blocks); } else { if (enc_before) aes_encrypt(ctx, dg, dg); diff --git a/arch/arm64/crypto/aes-modes.S b/arch/arm64/crypto/aes-modes.S index cf618d8f6cec420753cc0a0160e38516f04b3103..bbdb54702aa7a45d40fd26b7c2db5e2c98ffbca9 100644 --- a/arch/arm64/crypto/aes-modes.S +++ b/arch/arm64/crypto/aes-modes.S @@ -321,42 +321,76 @@ AES_FUNC_END(aes_cbc_cts_decrypt) /* * aes_ctr_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds, - * int blocks, u8 ctr[]) + * int bytes, u8 ctr[], u8 finalbuf[]) */ AES_FUNC_START(aes_ctr_encrypt) stp x29, x30, [sp, #-16]! mov x29, sp - enc_prepare w3, x2, x6 + enc_prepare w3, x2, x12 ld1 {vctr.16b}, [x5] - umov x6, vctr.d[1] /* keep swabbed ctr in reg */ - rev x6, x6 - cmn w6, w4 /* 32 bit overflow? */ - bcs .Lctrloop + umov x12, vctr.d[1] /* keep swabbed ctr in reg */ + rev x12, x12 + .LctrloopNx: - subs w4, w4, #MAX_STRIDE - bmi .Lctr1x - add w7, w6, #1 + add w7, w4, #15 + sub w4, w4, #MAX_STRIDE << 4 + lsr w7, w7, #4 + mov w8, #MAX_STRIDE + cmp w7, w8 + csel w7, w7, w8, lt + adds x12, x12, x7 + mov v0.16b, vctr.16b - add w8, w6, #2 mov v1.16b, vctr.16b - add w9, w6, #3 mov v2.16b, vctr.16b - add w9, w6, #3 - rev w7, w7 mov v3.16b, vctr.16b - rev w8, w8 ST5( mov v4.16b, vctr.16b ) - mov v1.s[3], w7 - rev w9, w9 -ST5( add w10, w6, #4 ) - mov v2.s[3], w8 -ST5( rev w10, w10 ) - mov v3.s[3], w9 -ST5( mov v4.s[3], w10 ) - ld1 {v5.16b-v7.16b}, [x1], #48 /* get 3 input blocks */ + bcs 0f + + .subsection 1 + /* apply carry to outgoing counter */ +0: umov x8, vctr.d[0] + rev x8, x8 + add x8, x8, #1 + rev x8, x8 + ins vctr.d[0], x8 + + /* apply carry to N counter blocks for N := x12 */ + adr x16, 1f + sub x16, x16, x12, lsl #3 + br x16 + hint 34 // bti c + mov v0.d[0], vctr.d[0] + hint 34 // bti c + mov v1.d[0], vctr.d[0] + hint 34 // bti c + mov v2.d[0], vctr.d[0] + hint 34 // bti c + mov v3.d[0], vctr.d[0] +ST5( hint 34 ) +ST5( mov v4.d[0], vctr.d[0] ) +1: b 2f + .previous + +2: rev x7, x12 + ins vctr.d[1], x7 + sub x7, x12, #MAX_STRIDE - 1 + sub x8, x12, #MAX_STRIDE - 2 + sub x9, x12, #MAX_STRIDE - 3 + rev x7, x7 + rev x8, x8 + mov v1.d[1], x7 + rev x9, x9 +ST5( sub x10, x12, #MAX_STRIDE - 4 ) + mov v2.d[1], x8 +ST5( rev x10, x10 ) + mov v3.d[1], x9 +ST5( mov v4.d[1], x10 ) + tbnz w4, #31, .Lctrtail + ld1 {v5.16b-v7.16b}, [x1], #48 ST4( bl aes_encrypt_block4x ) ST5( bl aes_encrypt_block5x ) eor v0.16b, v5.16b, v0.16b @@ -368,47 +402,72 @@ ST5( ld1 {v5.16b-v6.16b}, [x1], #32 ) ST5( eor v4.16b, v6.16b, v4.16b ) st1 {v0.16b-v3.16b}, [x0], #64 ST5( st1 {v4.16b}, [x0], #16 ) - add x6, x6, #MAX_STRIDE - rev x7, x6 - ins vctr.d[1], x7 cbz w4, .Lctrout b .LctrloopNx -.Lctr1x: - adds w4, w4, #MAX_STRIDE - beq .Lctrout -.Lctrloop: - mov v0.16b, vctr.16b - encrypt_block v0, w3, x2, x8, w7 - - adds x6, x6, #1 /* increment BE ctr */ - rev x7, x6 - ins vctr.d[1], x7 - bcs .Lctrcarry /* overflow? */ - -.Lctrcarrydone: - subs w4, w4, #1 - bmi .Lctrtailblock /* blocks <0 means tail block */ - ld1 {v3.16b}, [x1], #16 - eor v3.16b, v0.16b, v3.16b - st1 {v3.16b}, [x0], #16 - bne .Lctrloop .Lctrout: st1 {vctr.16b}, [x5] /* return next CTR value */ ldp x29, x30, [sp], #16 ret -.Lctrtailblock: - st1 {v0.16b}, [x0] +.Lctrtail: + /* XOR up to MAX_STRIDE * 16 - 1 bytes of in/output with v0 ... v3/v4 */ + mov x16, #16 + ands x13, x4, #0xf + csel x13, x13, x16, ne + +ST5( cmp w4, #64 - (MAX_STRIDE << 4) ) +ST5( csel x14, x16, xzr, gt ) + cmp w4, #48 - (MAX_STRIDE << 4) + csel x15, x16, xzr, gt + cmp w4, #32 - (MAX_STRIDE << 4) + csel x16, x16, xzr, gt + cmp w4, #16 - (MAX_STRIDE << 4) + ble .Lctrtail1x + + adr_l x12, .Lcts_permute_table + add x12, x12, x13 + +ST5( ld1 {v5.16b}, [x1], x14 ) + ld1 {v6.16b}, [x1], x15 + ld1 {v7.16b}, [x1], x16 + +ST4( bl aes_encrypt_block4x ) +ST5( bl aes_encrypt_block5x ) + + ld1 {v8.16b}, [x1], x13 + ld1 {v9.16b}, [x1] + ld1 {v10.16b}, [x12] + +ST4( eor v6.16b, v6.16b, v0.16b ) +ST4( eor v7.16b, v7.16b, v1.16b ) +ST4( tbl v3.16b, {v3.16b}, v10.16b ) +ST4( eor v8.16b, v8.16b, v2.16b ) +ST4( eor v9.16b, v9.16b, v3.16b ) + +ST5( eor v5.16b, v5.16b, v0.16b ) +ST5( eor v6.16b, v6.16b, v1.16b ) +ST5( tbl v4.16b, {v4.16b}, v10.16b ) +ST5( eor v7.16b, v7.16b, v2.16b ) +ST5( eor v8.16b, v8.16b, v3.16b ) +ST5( eor v9.16b, v9.16b, v4.16b ) + +ST5( st1 {v5.16b}, [x0], x14 ) + st1 {v6.16b}, [x0], x15 + st1 {v7.16b}, [x0], x16 + add x13, x13, x0 + st1 {v9.16b}, [x13] // overlapping stores + st1 {v8.16b}, [x0] b .Lctrout -.Lctrcarry: - umov x7, vctr.d[0] /* load upper word of ctr */ - rev x7, x7 /* ... to handle the carry */ - add x7, x7, #1 - rev x7, x7 - ins vctr.d[0], x7 - b .Lctrcarrydone +.Lctrtail1x: + csel x0, x0, x6, eq // use finalbuf if less than a full block + ld1 {v5.16b}, [x1] +ST5( mov v3.16b, v4.16b ) + encrypt_block v3, w3, x2, x8, w7 + eor v5.16b, v5.16b, v3.16b + st1 {v5.16b}, [x0] + b .Lctrout AES_FUNC_END(aes_ctr_encrypt) @@ -619,61 +678,47 @@ AES_FUNC_END(aes_xts_decrypt) * int blocks, u8 dg[], int enc_before, int enc_after) */ AES_FUNC_START(aes_mac_update) - frame_push 6 - - mov x19, x0 - mov x20, x1 - mov x21, x2 - mov x22, x3 - mov x23, x4 - mov x24, x6 - - ld1 {v0.16b}, [x23] /* get dg */ + ld1 {v0.16b}, [x4] /* get dg */ enc_prepare w2, x1, x7 cbz w5, .Lmacloop4x encrypt_block v0, w2, x1, x7, w8 .Lmacloop4x: - subs w22, w22, #4 + subs w3, w3, #4 bmi .Lmac1x - ld1 {v1.16b-v4.16b}, [x19], #64 /* get next pt block */ + ld1 {v1.16b-v4.16b}, [x0], #64 /* get next pt block */ eor v0.16b, v0.16b, v1.16b /* ..and xor with dg */ - encrypt_block v0, w21, x20, x7, w8 + encrypt_block v0, w2, x1, x7, w8 eor v0.16b, v0.16b, v2.16b - encrypt_block v0, w21, x20, x7, w8 + encrypt_block v0, w2, x1, x7, w8 eor v0.16b, v0.16b, v3.16b - encrypt_block v0, w21, x20, x7, w8 + encrypt_block v0, w2, x1, x7, w8 eor v0.16b, v0.16b, v4.16b - cmp w22, wzr - csinv x5, x24, xzr, eq + cmp w3, wzr + csinv x5, x6, xzr, eq cbz w5, .Lmacout - encrypt_block v0, w21, x20, x7, w8 - st1 {v0.16b}, [x23] /* return dg */ - cond_yield_neon .Lmacrestart + encrypt_block v0, w2, x1, x7, w8 + st1 {v0.16b}, [x4] /* return dg */ + cond_yield .Lmacout, x7 b .Lmacloop4x .Lmac1x: - add w22, w22, #4 + add w3, w3, #4 .Lmacloop: - cbz w22, .Lmacout - ld1 {v1.16b}, [x19], #16 /* get next pt block */ + cbz w3, .Lmacout + ld1 {v1.16b}, [x0], #16 /* get next pt block */ eor v0.16b, v0.16b, v1.16b /* ..and xor with dg */ - subs w22, w22, #1 - csinv x5, x24, xzr, eq + subs w3, w3, #1 + csinv x5, x6, xzr, eq cbz w5, .Lmacout .Lmacenc: - encrypt_block v0, w21, x20, x7, w8 + encrypt_block v0, w2, x1, x7, w8 b .Lmacloop .Lmacout: - st1 {v0.16b}, [x23] /* return dg */ - frame_pop + st1 {v0.16b}, [x4] /* return dg */ + mov w0, w3 ret - -.Lmacrestart: - ld1 {v0.16b}, [x23] /* get dg */ - enc_prepare w21, x20, x0 - b .Lmacloop4x AES_FUNC_END(aes_mac_update) diff --git a/arch/arm64/crypto/aes-neonbs-core.S b/arch/arm64/crypto/aes-neonbs-core.S index 63a52ad9a75c03deab976e09e5a709135f71102f..a3405b8c344b5b24c71c831931cb69921f39990a 100644 --- a/arch/arm64/crypto/aes-neonbs-core.S +++ b/arch/arm64/crypto/aes-neonbs-core.S @@ -613,7 +613,6 @@ SYM_FUNC_END(aesbs_decrypt8) st1 {\o7\().16b}, [x19], #16 cbz x23, 1f - cond_yield_neon b 99b 1: frame_pop @@ -715,7 +714,6 @@ SYM_FUNC_START(aesbs_cbc_decrypt) 1: st1 {v24.16b}, [x24] // store IV cbz x23, 2f - cond_yield_neon b 99b 2: frame_pop @@ -801,7 +799,7 @@ SYM_FUNC_END(__xts_crypt8) mov x23, x4 mov x24, x5 -0: movi v30.2s, #0x1 + movi v30.2s, #0x1 movi v25.2s, #0x87 uzp1 v30.4s, v30.4s, v25.4s ld1 {v25.16b}, [x24] @@ -846,7 +844,6 @@ SYM_FUNC_END(__xts_crypt8) cbz x23, 1f st1 {v25.16b}, [x24] - cond_yield_neon 0b b 99b 1: st1 {v25.16b}, [x24] @@ -889,7 +886,7 @@ SYM_FUNC_START(aesbs_ctr_encrypt) cset x26, ne add x23, x23, x26 // do one extra block if final -98: ldp x7, x8, [x24] + ldp x7, x8, [x24] ld1 {v0.16b}, [x24] CPU_LE( rev x7, x7 ) CPU_LE( rev x8, x8 ) @@ -967,7 +964,6 @@ CPU_LE( rev x8, x8 ) st1 {v0.16b}, [x24] cbz x23, .Lctr_done - cond_yield_neon 98b b 99b .Lctr_done: diff --git a/arch/arm64/crypto/crct10dif-ce-core.S b/arch/arm64/crypto/crct10dif-ce-core.S index 111d9c9abddd1885785c135cfb4a4f07ff8ab807..dce6dcebfca189ee4f99d8c7ee03c00cc879f878 100644 --- a/arch/arm64/crypto/crct10dif-ce-core.S +++ b/arch/arm64/crypto/crct10dif-ce-core.S @@ -68,10 +68,10 @@ .text .arch armv8-a+crypto - init_crc .req w19 - buf .req x20 - len .req x21 - fold_consts_ptr .req x22 + init_crc .req w0 + buf .req x1 + len .req x2 + fold_consts_ptr .req x3 fold_consts .req v10 @@ -257,12 +257,6 @@ CPU_LE( ext v12.16b, v12.16b, v12.16b, #8 ) .endm .macro crc_t10dif_pmull, p - frame_push 4, 128 - - mov init_crc, w0 - mov buf, x1 - mov len, x2 - __pmull_init_\p // For sizes less than 256 bytes, we can't fold 128 bytes at a time. @@ -317,26 +311,7 @@ CPU_LE( ext v7.16b, v7.16b, v7.16b, #8 ) fold_32_bytes \p, v6, v7 subs len, len, #128 - b.lt .Lfold_128_bytes_loop_done_\@ - - if_will_cond_yield_neon - stp q0, q1, [sp, #.Lframe_local_offset] - stp q2, q3, [sp, #.Lframe_local_offset + 32] - stp q4, q5, [sp, #.Lframe_local_offset + 64] - stp q6, q7, [sp, #.Lframe_local_offset + 96] - do_cond_yield_neon - ldp q0, q1, [sp, #.Lframe_local_offset] - ldp q2, q3, [sp, #.Lframe_local_offset + 32] - ldp q4, q5, [sp, #.Lframe_local_offset + 64] - ldp q6, q7, [sp, #.Lframe_local_offset + 96] - ld1 {fold_consts.2d}, [fold_consts_ptr] - __pmull_init_\p - __pmull_pre_\p fold_consts - endif_yield_neon - - b .Lfold_128_bytes_loop_\@ - -.Lfold_128_bytes_loop_done_\@: + b.ge .Lfold_128_bytes_loop_\@ // Now fold the 112 bytes in v0-v6 into the 16 bytes in v7. @@ -453,7 +428,9 @@ CPU_LE( ext v0.16b, v0.16b, v0.16b, #8 ) // Final CRC value (x^16 * M(x)) mod G(x) is in low 16 bits of v0. umov w0, v0.h[0] - frame_pop + .ifc \p, p8 + ldp x29, x30, [sp], #16 + .endif ret .Lless_than_256_bytes_\@: @@ -489,7 +466,9 @@ CPU_LE( ext v7.16b, v7.16b, v7.16b, #8 ) // Assumes len >= 16. // SYM_FUNC_START(crc_t10dif_pmull_p8) - crc_t10dif_pmull p8 + stp x29, x30, [sp, #-16]! + mov x29, sp + crc_t10dif_pmull p8 SYM_FUNC_END(crc_t10dif_pmull_p8) .align 5 diff --git a/arch/arm64/crypto/crct10dif-ce-glue.c b/arch/arm64/crypto/crct10dif-ce-glue.c index ccc3f6067742433d3fe684833c4193293e360ba9..09eb1456aed4a9450b433ef7e6b10358c6a84557 100644 --- a/arch/arm64/crypto/crct10dif-ce-glue.c +++ b/arch/arm64/crypto/crct10dif-ce-glue.c @@ -37,9 +37,18 @@ static int crct10dif_update_pmull_p8(struct shash_desc *desc, const u8 *data, u16 *crc = shash_desc_ctx(desc); if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE && crypto_simd_usable()) { - kernel_neon_begin(); - *crc = crc_t10dif_pmull_p8(*crc, data, length); - kernel_neon_end(); + do { + unsigned int chunk = length; + + if (chunk > SZ_4K + CRC_T10DIF_PMULL_CHUNK_SIZE) + chunk = SZ_4K; + + kernel_neon_begin(); + *crc = crc_t10dif_pmull_p8(*crc, data, chunk); + kernel_neon_end(); + data += chunk; + length -= chunk; + } while (length); } else { *crc = crc_t10dif_generic(*crc, data, length); } @@ -53,9 +62,18 @@ static int crct10dif_update_pmull_p64(struct shash_desc *desc, const u8 *data, u16 *crc = shash_desc_ctx(desc); if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE && crypto_simd_usable()) { - kernel_neon_begin(); - *crc = crc_t10dif_pmull_p64(*crc, data, length); - kernel_neon_end(); + do { + unsigned int chunk = length; + + if (chunk > SZ_4K + CRC_T10DIF_PMULL_CHUNK_SIZE) + chunk = SZ_4K; + + kernel_neon_begin(); + *crc = crc_t10dif_pmull_p64(*crc, data, chunk); + kernel_neon_end(); + data += chunk; + length -= chunk; + } while (length); } else { *crc = crc_t10dif_generic(*crc, data, length); } diff --git a/arch/arm64/crypto/sha1-ce-core.S b/arch/arm64/crypto/sha1-ce-core.S index 92d0d2753e812fef09862c2a1b231f8113c7b555..8c02bbc2684ed7f88428bf3f4b96482a64c202a5 100644 --- a/arch/arm64/crypto/sha1-ce-core.S +++ b/arch/arm64/crypto/sha1-ce-core.S @@ -62,40 +62,34 @@ .endm /* - * void sha1_ce_transform(struct sha1_ce_state *sst, u8 const *src, - * int blocks) + * int sha1_ce_transform(struct sha1_ce_state *sst, u8 const *src, + * int blocks) */ SYM_FUNC_START(sha1_ce_transform) - frame_push 3 - - mov x19, x0 - mov x20, x1 - mov x21, x2 - /* load round constants */ -0: loadrc k0.4s, 0x5a827999, w6 + loadrc k0.4s, 0x5a827999, w6 loadrc k1.4s, 0x6ed9eba1, w6 loadrc k2.4s, 0x8f1bbcdc, w6 loadrc k3.4s, 0xca62c1d6, w6 /* load state */ - ld1 {dgav.4s}, [x19] - ldr dgb, [x19, #16] + ld1 {dgav.4s}, [x0] + ldr dgb, [x0, #16] /* load sha1_ce_state::finalize */ ldr_l w4, sha1_ce_offsetof_finalize, x4 - ldr w4, [x19, x4] + ldr w4, [x0, x4] /* load input */ -1: ld1 {v8.4s-v11.4s}, [x20], #64 - sub w21, w21, #1 +0: ld1 {v8.4s-v11.4s}, [x1], #64 + sub w2, w2, #1 CPU_LE( rev32 v8.16b, v8.16b ) CPU_LE( rev32 v9.16b, v9.16b ) CPU_LE( rev32 v10.16b, v10.16b ) CPU_LE( rev32 v11.16b, v11.16b ) -2: add t0.4s, v8.4s, k0.4s +1: add t0.4s, v8.4s, k0.4s mov dg0v.16b, dgav.16b add_update c, ev, k0, 8, 9, 10, 11, dgb @@ -126,25 +120,18 @@ CPU_LE( rev32 v11.16b, v11.16b ) add dgbv.2s, dgbv.2s, dg1v.2s add dgav.4s, dgav.4s, dg0v.4s - cbz w21, 3f - - if_will_cond_yield_neon - st1 {dgav.4s}, [x19] - str dgb, [x19, #16] - do_cond_yield_neon + cbz w2, 2f + cond_yield 3f, x5 b 0b - endif_yield_neon - - b 1b /* * Final block: add padding and total bit count. * Skip if the input size was not a round multiple of the block size, * the padding is handled by the C code in that case. */ -3: cbz x4, 4f +2: cbz x4, 3f ldr_l w4, sha1_ce_offsetof_count, x4 - ldr x4, [x19, x4] + ldr x4, [x0, x4] movi v9.2d, #0 mov x8, #0x80000000 movi v10.2d, #0 @@ -153,11 +140,11 @@ CPU_LE( rev32 v11.16b, v11.16b ) mov x4, #0 mov v11.d[0], xzr mov v11.d[1], x7 - b 2b + b 1b /* store new state */ -4: st1 {dgav.4s}, [x19] - str dgb, [x19, #16] - frame_pop +3: st1 {dgav.4s}, [x0] + str dgb, [x0, #16] + mov w0, w2 ret SYM_FUNC_END(sha1_ce_transform) diff --git a/arch/arm64/crypto/sha1-ce-glue.c b/arch/arm64/crypto/sha1-ce-glue.c index c93121bcfdeba5c020ef5b013c0fa2654476693f..71fa4f1122d747b9d67fd0b95944609a89aadad5 100644 --- a/arch/arm64/crypto/sha1-ce-glue.c +++ b/arch/arm64/crypto/sha1-ce-glue.c @@ -19,6 +19,7 @@ MODULE_DESCRIPTION("SHA1 secure hash using ARMv8 Crypto Extensions"); MODULE_AUTHOR("Ard Biesheuvel "); MODULE_LICENSE("GPL v2"); +MODULE_ALIAS_CRYPTO("sha1"); struct sha1_ce_state { struct sha1_state sst; @@ -28,14 +29,22 @@ struct sha1_ce_state { extern const u32 sha1_ce_offsetof_count; extern const u32 sha1_ce_offsetof_finalize; -asmlinkage void sha1_ce_transform(struct sha1_ce_state *sst, u8 const *src, - int blocks); +asmlinkage int sha1_ce_transform(struct sha1_ce_state *sst, u8 const *src, + int blocks); static void __sha1_ce_transform(struct sha1_state *sst, u8 const *src, int blocks) { - sha1_ce_transform(container_of(sst, struct sha1_ce_state, sst), src, - blocks); + while (blocks) { + int rem; + + kernel_neon_begin(); + rem = sha1_ce_transform(container_of(sst, struct sha1_ce_state, + sst), src, blocks); + kernel_neon_end(); + src += (blocks - rem) * SHA1_BLOCK_SIZE; + blocks = rem; + } } const u32 sha1_ce_offsetof_count = offsetof(struct sha1_ce_state, sst.count); @@ -50,9 +59,7 @@ static int sha1_ce_update(struct shash_desc *desc, const u8 *data, return crypto_sha1_update(desc, data, len); sctx->finalize = 0; - kernel_neon_begin(); sha1_base_do_update(desc, data, len, __sha1_ce_transform); - kernel_neon_end(); return 0; } @@ -72,11 +79,9 @@ static int sha1_ce_finup(struct shash_desc *desc, const u8 *data, */ sctx->finalize = finalize; - kernel_neon_begin(); sha1_base_do_update(desc, data, len, __sha1_ce_transform); if (!finalize) sha1_base_do_finalize(desc, __sha1_ce_transform); - kernel_neon_end(); return sha1_base_finish(desc, out); } @@ -88,9 +93,7 @@ static int sha1_ce_final(struct shash_desc *desc, u8 *out) return crypto_sha1_finup(desc, NULL, 0, out); sctx->finalize = 0; - kernel_neon_begin(); sha1_base_do_finalize(desc, __sha1_ce_transform); - kernel_neon_end(); return sha1_base_finish(desc, out); } diff --git a/arch/arm64/crypto/sha2-ce-core.S b/arch/arm64/crypto/sha2-ce-core.S index 3f9d0f32698757893bd0440fca0ca5ee12e0c72e..6cdea7d560593ba90adc0047958823fd8057da7b 100644 --- a/arch/arm64/crypto/sha2-ce-core.S +++ b/arch/arm64/crypto/sha2-ce-core.S @@ -76,36 +76,30 @@ */ .text SYM_FUNC_START(sha2_ce_transform) - frame_push 3 - - mov x19, x0 - mov x20, x1 - mov x21, x2 - /* load round constants */ -0: adr_l x8, .Lsha2_rcon + adr_l x8, .Lsha2_rcon ld1 { v0.4s- v3.4s}, [x8], #64 ld1 { v4.4s- v7.4s}, [x8], #64 ld1 { v8.4s-v11.4s}, [x8], #64 ld1 {v12.4s-v15.4s}, [x8] /* load state */ - ld1 {dgav.4s, dgbv.4s}, [x19] + ld1 {dgav.4s, dgbv.4s}, [x0] /* load sha256_ce_state::finalize */ ldr_l w4, sha256_ce_offsetof_finalize, x4 - ldr w4, [x19, x4] + ldr w4, [x0, x4] /* load input */ -1: ld1 {v16.4s-v19.4s}, [x20], #64 - sub w21, w21, #1 +0: ld1 {v16.4s-v19.4s}, [x1], #64 + sub w2, w2, #1 CPU_LE( rev32 v16.16b, v16.16b ) CPU_LE( rev32 v17.16b, v17.16b ) CPU_LE( rev32 v18.16b, v18.16b ) CPU_LE( rev32 v19.16b, v19.16b ) -2: add t0.4s, v16.4s, v0.4s +1: add t0.4s, v16.4s, v0.4s mov dg0v.16b, dgav.16b mov dg1v.16b, dgbv.16b @@ -134,24 +128,18 @@ CPU_LE( rev32 v19.16b, v19.16b ) add dgbv.4s, dgbv.4s, dg1v.4s /* handled all input blocks? */ - cbz w21, 3f - - if_will_cond_yield_neon - st1 {dgav.4s, dgbv.4s}, [x19] - do_cond_yield_neon + cbz w2, 2f + cond_yield 3f, x5 b 0b - endif_yield_neon - - b 1b /* * Final block: add padding and total bit count. * Skip if the input size was not a round multiple of the block size, * the padding is handled by the C code in that case. */ -3: cbz x4, 4f +2: cbz x4, 3f ldr_l w4, sha256_ce_offsetof_count, x4 - ldr x4, [x19, x4] + ldr x4, [x0, x4] movi v17.2d, #0 mov x8, #0x80000000 movi v18.2d, #0 @@ -160,10 +148,10 @@ CPU_LE( rev32 v19.16b, v19.16b ) mov x4, #0 mov v19.d[0], xzr mov v19.d[1], x7 - b 2b + b 1b /* store new state */ -4: st1 {dgav.4s, dgbv.4s}, [x19] - frame_pop +3: st1 {dgav.4s, dgbv.4s}, [x0] + mov w0, w2 ret SYM_FUNC_END(sha2_ce_transform) diff --git a/arch/arm64/crypto/sha2-ce-glue.c b/arch/arm64/crypto/sha2-ce-glue.c index 31ba3da5e61bd4265b8a06c66b061b787533cdd1..c57a6119fefc586d9ac1264efaff8159bfe6aec3 100644 --- a/arch/arm64/crypto/sha2-ce-glue.c +++ b/arch/arm64/crypto/sha2-ce-glue.c @@ -19,6 +19,8 @@ MODULE_DESCRIPTION("SHA-224/SHA-256 secure hash using ARMv8 Crypto Extensions"); MODULE_AUTHOR("Ard Biesheuvel "); MODULE_LICENSE("GPL v2"); +MODULE_ALIAS_CRYPTO("sha224"); +MODULE_ALIAS_CRYPTO("sha256"); struct sha256_ce_state { struct sha256_state sst; @@ -28,14 +30,22 @@ struct sha256_ce_state { extern const u32 sha256_ce_offsetof_count; extern const u32 sha256_ce_offsetof_finalize; -asmlinkage void sha2_ce_transform(struct sha256_ce_state *sst, u8 const *src, - int blocks); +asmlinkage int sha2_ce_transform(struct sha256_ce_state *sst, u8 const *src, + int blocks); static void __sha2_ce_transform(struct sha256_state *sst, u8 const *src, int blocks) { - sha2_ce_transform(container_of(sst, struct sha256_ce_state, sst), src, - blocks); + while (blocks) { + int rem; + + kernel_neon_begin(); + rem = sha2_ce_transform(container_of(sst, struct sha256_ce_state, + sst), src, blocks); + kernel_neon_end(); + src += (blocks - rem) * SHA256_BLOCK_SIZE; + blocks = rem; + } } const u32 sha256_ce_offsetof_count = offsetof(struct sha256_ce_state, @@ -61,9 +71,7 @@ static int sha256_ce_update(struct shash_desc *desc, const u8 *data, __sha256_block_data_order); sctx->finalize = 0; - kernel_neon_begin(); sha256_base_do_update(desc, data, len, __sha2_ce_transform); - kernel_neon_end(); return 0; } @@ -88,11 +96,9 @@ static int sha256_ce_finup(struct shash_desc *desc, const u8 *data, */ sctx->finalize = finalize; - kernel_neon_begin(); sha256_base_do_update(desc, data, len, __sha2_ce_transform); if (!finalize) sha256_base_do_finalize(desc, __sha2_ce_transform); - kernel_neon_end(); return sha256_base_finish(desc, out); } @@ -106,9 +112,7 @@ static int sha256_ce_final(struct shash_desc *desc, u8 *out) } sctx->finalize = 0; - kernel_neon_begin(); sha256_base_do_finalize(desc, __sha2_ce_transform); - kernel_neon_end(); return sha256_base_finish(desc, out); } diff --git a/arch/arm64/crypto/sha3-ce-core.S b/arch/arm64/crypto/sha3-ce-core.S index 1cfb768df350b43151e6b5f9d0224b2db96c4051..6f5208414fe3fdb03c33482a26eda8d0cddc09e4 100644 --- a/arch/arm64/crypto/sha3-ce-core.S +++ b/arch/arm64/crypto/sha3-ce-core.S @@ -37,20 +37,13 @@ .endm /* - * sha3_ce_transform(u64 *st, const u8 *data, int blocks, int dg_size) + * int sha3_ce_transform(u64 *st, const u8 *data, int blocks, int dg_size) */ .text SYM_FUNC_START(sha3_ce_transform) - frame_push 4 - - mov x19, x0 - mov x20, x1 - mov x21, x2 - mov x22, x3 - -0: /* load state */ - add x8, x19, #32 - ld1 { v0.1d- v3.1d}, [x19] + /* load state */ + add x8, x0, #32 + ld1 { v0.1d- v3.1d}, [x0] ld1 { v4.1d- v7.1d}, [x8], #32 ld1 { v8.1d-v11.1d}, [x8], #32 ld1 {v12.1d-v15.1d}, [x8], #32 @@ -58,13 +51,13 @@ SYM_FUNC_START(sha3_ce_transform) ld1 {v20.1d-v23.1d}, [x8], #32 ld1 {v24.1d}, [x8] -1: sub w21, w21, #1 +0: sub w2, w2, #1 mov w8, #24 adr_l x9, .Lsha3_rcon /* load input */ - ld1 {v25.8b-v28.8b}, [x20], #32 - ld1 {v29.8b-v31.8b}, [x20], #24 + ld1 {v25.8b-v28.8b}, [x1], #32 + ld1 {v29.8b-v31.8b}, [x1], #24 eor v0.8b, v0.8b, v25.8b eor v1.8b, v1.8b, v26.8b eor v2.8b, v2.8b, v27.8b @@ -73,10 +66,10 @@ SYM_FUNC_START(sha3_ce_transform) eor v5.8b, v5.8b, v30.8b eor v6.8b, v6.8b, v31.8b - tbnz x22, #6, 3f // SHA3-512 + tbnz x3, #6, 2f // SHA3-512 - ld1 {v25.8b-v28.8b}, [x20], #32 - ld1 {v29.8b-v30.8b}, [x20], #16 + ld1 {v25.8b-v28.8b}, [x1], #32 + ld1 {v29.8b-v30.8b}, [x1], #16 eor v7.8b, v7.8b, v25.8b eor v8.8b, v8.8b, v26.8b eor v9.8b, v9.8b, v27.8b @@ -84,34 +77,34 @@ SYM_FUNC_START(sha3_ce_transform) eor v11.8b, v11.8b, v29.8b eor v12.8b, v12.8b, v30.8b - tbnz x22, #4, 2f // SHA3-384 or SHA3-224 + tbnz x3, #4, 1f // SHA3-384 or SHA3-224 // SHA3-256 - ld1 {v25.8b-v28.8b}, [x20], #32 + ld1 {v25.8b-v28.8b}, [x1], #32 eor v13.8b, v13.8b, v25.8b eor v14.8b, v14.8b, v26.8b eor v15.8b, v15.8b, v27.8b eor v16.8b, v16.8b, v28.8b - b 4f + b 3f -2: tbz x22, #2, 4f // bit 2 cleared? SHA-384 +1: tbz x3, #2, 3f // bit 2 cleared? SHA-384 // SHA3-224 - ld1 {v25.8b-v28.8b}, [x20], #32 - ld1 {v29.8b}, [x20], #8 + ld1 {v25.8b-v28.8b}, [x1], #32 + ld1 {v29.8b}, [x1], #8 eor v13.8b, v13.8b, v25.8b eor v14.8b, v14.8b, v26.8b eor v15.8b, v15.8b, v27.8b eor v16.8b, v16.8b, v28.8b eor v17.8b, v17.8b, v29.8b - b 4f + b 3f // SHA3-512 -3: ld1 {v25.8b-v26.8b}, [x20], #16 +2: ld1 {v25.8b-v26.8b}, [x1], #16 eor v7.8b, v7.8b, v25.8b eor v8.8b, v8.8b, v26.8b -4: sub w8, w8, #1 +3: sub w8, w8, #1 eor3 v29.16b, v4.16b, v9.16b, v14.16b eor3 v26.16b, v1.16b, v6.16b, v11.16b @@ -190,33 +183,19 @@ SYM_FUNC_START(sha3_ce_transform) eor v0.16b, v0.16b, v31.16b - cbnz w8, 4b - cbz w21, 5f - - if_will_cond_yield_neon - add x8, x19, #32 - st1 { v0.1d- v3.1d}, [x19] - st1 { v4.1d- v7.1d}, [x8], #32 - st1 { v8.1d-v11.1d}, [x8], #32 - st1 {v12.1d-v15.1d}, [x8], #32 - st1 {v16.1d-v19.1d}, [x8], #32 - st1 {v20.1d-v23.1d}, [x8], #32 - st1 {v24.1d}, [x8] - do_cond_yield_neon - b 0b - endif_yield_neon - - b 1b + cbnz w8, 3b + cond_yield 3f, x8 + cbnz w2, 0b /* save state */ -5: st1 { v0.1d- v3.1d}, [x19], #32 - st1 { v4.1d- v7.1d}, [x19], #32 - st1 { v8.1d-v11.1d}, [x19], #32 - st1 {v12.1d-v15.1d}, [x19], #32 - st1 {v16.1d-v19.1d}, [x19], #32 - st1 {v20.1d-v23.1d}, [x19], #32 - st1 {v24.1d}, [x19] - frame_pop +3: st1 { v0.1d- v3.1d}, [x0], #32 + st1 { v4.1d- v7.1d}, [x0], #32 + st1 { v8.1d-v11.1d}, [x0], #32 + st1 {v12.1d-v15.1d}, [x0], #32 + st1 {v16.1d-v19.1d}, [x0], #32 + st1 {v20.1d-v23.1d}, [x0], #32 + st1 {v24.1d}, [x0] + mov w0, w2 ret SYM_FUNC_END(sha3_ce_transform) diff --git a/arch/arm64/crypto/sha3-ce-glue.c b/arch/arm64/crypto/sha3-ce-glue.c index e5a2936f08864361d76833c8bfec9257b053cfd9..8c65cecf560a9505bfd4eeb81aaeaaf487595d62 100644 --- a/arch/arm64/crypto/sha3-ce-glue.c +++ b/arch/arm64/crypto/sha3-ce-glue.c @@ -23,9 +23,13 @@ MODULE_DESCRIPTION("SHA3 secure hash using ARMv8 Crypto Extensions"); MODULE_AUTHOR("Ard Biesheuvel "); MODULE_LICENSE("GPL v2"); +MODULE_ALIAS_CRYPTO("sha3-224"); +MODULE_ALIAS_CRYPTO("sha3-256"); +MODULE_ALIAS_CRYPTO("sha3-384"); +MODULE_ALIAS_CRYPTO("sha3-512"); -asmlinkage void sha3_ce_transform(u64 *st, const u8 *data, int blocks, - int md_len); +asmlinkage int sha3_ce_transform(u64 *st, const u8 *data, int blocks, + int md_len); static int sha3_update(struct shash_desc *desc, const u8 *data, unsigned int len) @@ -55,11 +59,15 @@ static int sha3_update(struct shash_desc *desc, const u8 *data, blocks = len / sctx->rsiz; len %= sctx->rsiz; - if (blocks) { + while (blocks) { + int rem; + kernel_neon_begin(); - sha3_ce_transform(sctx->st, data, blocks, digest_size); + rem = sha3_ce_transform(sctx->st, data, blocks, + digest_size); kernel_neon_end(); - data += blocks * sctx->rsiz; + data += (blocks - rem) * sctx->rsiz; + blocks = rem; } } diff --git a/arch/arm64/crypto/sha512-ce-core.S b/arch/arm64/crypto/sha512-ce-core.S index cde606c0323ee93262e0b2d2993894f7efcde511..d6e7f6c95fa6f26f20471426ca04bc7498178c5b 100644 --- a/arch/arm64/crypto/sha512-ce-core.S +++ b/arch/arm64/crypto/sha512-ce-core.S @@ -107,23 +107,17 @@ */ .text SYM_FUNC_START(sha512_ce_transform) - frame_push 3 - - mov x19, x0 - mov x20, x1 - mov x21, x2 - /* load state */ -0: ld1 {v8.2d-v11.2d}, [x19] + ld1 {v8.2d-v11.2d}, [x0] /* load first 4 round constants */ adr_l x3, .Lsha512_rcon ld1 {v20.2d-v23.2d}, [x3], #64 /* load input */ -1: ld1 {v12.2d-v15.2d}, [x20], #64 - ld1 {v16.2d-v19.2d}, [x20], #64 - sub w21, w21, #1 +0: ld1 {v12.2d-v15.2d}, [x1], #64 + ld1 {v16.2d-v19.2d}, [x1], #64 + sub w2, w2, #1 CPU_LE( rev64 v12.16b, v12.16b ) CPU_LE( rev64 v13.16b, v13.16b ) @@ -201,19 +195,12 @@ CPU_LE( rev64 v19.16b, v19.16b ) add v10.2d, v10.2d, v2.2d add v11.2d, v11.2d, v3.2d + cond_yield 3f, x4 /* handled all input blocks? */ - cbz w21, 3f - - if_will_cond_yield_neon - st1 {v8.2d-v11.2d}, [x19] - do_cond_yield_neon - b 0b - endif_yield_neon - - b 1b + cbnz w2, 0b /* store new state */ -3: st1 {v8.2d-v11.2d}, [x19] - frame_pop +3: st1 {v8.2d-v11.2d}, [x0] + mov w0, w2 ret SYM_FUNC_END(sha512_ce_transform) diff --git a/arch/arm64/crypto/sha512-ce-glue.c b/arch/arm64/crypto/sha512-ce-glue.c index faa83f6cf376c452a3f9f80d75e2b79ca4c66229..e62a094a9d52612fcda5b972c728831d073442dc 100644 --- a/arch/arm64/crypto/sha512-ce-glue.c +++ b/arch/arm64/crypto/sha512-ce-glue.c @@ -23,12 +23,28 @@ MODULE_DESCRIPTION("SHA-384/SHA-512 secure hash using ARMv8 Crypto Extensions"); MODULE_AUTHOR("Ard Biesheuvel "); MODULE_LICENSE("GPL v2"); +MODULE_ALIAS_CRYPTO("sha384"); +MODULE_ALIAS_CRYPTO("sha512"); -asmlinkage void sha512_ce_transform(struct sha512_state *sst, u8 const *src, - int blocks); +asmlinkage int sha512_ce_transform(struct sha512_state *sst, u8 const *src, + int blocks); asmlinkage void sha512_block_data_order(u64 *digest, u8 const *src, int blocks); +static void __sha512_ce_transform(struct sha512_state *sst, u8 const *src, + int blocks) +{ + while (blocks) { + int rem; + + kernel_neon_begin(); + rem = sha512_ce_transform(sst, src, blocks); + kernel_neon_end(); + src += (blocks - rem) * SHA512_BLOCK_SIZE; + blocks = rem; + } +} + static void __sha512_block_data_order(struct sha512_state *sst, u8 const *src, int blocks) { @@ -38,45 +54,30 @@ static void __sha512_block_data_order(struct sha512_state *sst, u8 const *src, static int sha512_ce_update(struct shash_desc *desc, const u8 *data, unsigned int len) { - if (!crypto_simd_usable()) - return sha512_base_do_update(desc, data, len, - __sha512_block_data_order); - - kernel_neon_begin(); - sha512_base_do_update(desc, data, len, sha512_ce_transform); - kernel_neon_end(); + sha512_block_fn *fn = crypto_simd_usable() ? __sha512_ce_transform + : __sha512_block_data_order; + sha512_base_do_update(desc, data, len, fn); return 0; } static int sha512_ce_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) { - if (!crypto_simd_usable()) { - if (len) - sha512_base_do_update(desc, data, len, - __sha512_block_data_order); - sha512_base_do_finalize(desc, __sha512_block_data_order); - return sha512_base_finish(desc, out); - } + sha512_block_fn *fn = crypto_simd_usable() ? __sha512_ce_transform + : __sha512_block_data_order; - kernel_neon_begin(); - sha512_base_do_update(desc, data, len, sha512_ce_transform); - sha512_base_do_finalize(desc, sha512_ce_transform); - kernel_neon_end(); + sha512_base_do_update(desc, data, len, fn); + sha512_base_do_finalize(desc, fn); return sha512_base_finish(desc, out); } static int sha512_ce_final(struct shash_desc *desc, u8 *out) { - if (!crypto_simd_usable()) { - sha512_base_do_finalize(desc, __sha512_block_data_order); - return sha512_base_finish(desc, out); - } + sha512_block_fn *fn = crypto_simd_usable() ? __sha512_ce_transform + : __sha512_block_data_order; - kernel_neon_begin(); - sha512_base_do_finalize(desc, sha512_ce_transform); - kernel_neon_end(); + sha512_base_do_finalize(desc, fn); return sha512_base_finish(desc, out); } diff --git a/arch/arm64/include/asm/cache.h b/arch/arm64/include/asm/cache.h index 77cbbe3625f215d179078cf482c72733df77d5e1..a074459f8f2fb7761c046518d7ce0ac947ff0cda 100644 --- a/arch/arm64/include/asm/cache.h +++ b/arch/arm64/include/asm/cache.h @@ -6,7 +6,6 @@ #define __ASM_CACHE_H #include -#include #define CTR_L1IP_SHIFT 14 #define CTR_L1IP_MASK 3 diff --git a/arch/arm64/include/asm/kasan.h b/arch/arm64/include/asm/kasan.h index 0aaf9044cd6a5da848d18f6716d41f4009ce533f..12d5f47f7dbec62806c89f2010448fd5d083378d 100644 --- a/arch/arm64/include/asm/kasan.h +++ b/arch/arm64/include/asm/kasan.h @@ -6,6 +6,7 @@ #include #include +#include #include #define arch_kasan_set_tag(addr, tag) __tag_set(addr, tag) diff --git a/arch/arm64/include/asm/kfence.h b/arch/arm64/include/asm/kfence.h new file mode 100644 index 0000000000000000000000000000000000000000..d061176d57ea178b8bcd91f673d86142735a7524 --- /dev/null +++ b/arch/arm64/include/asm/kfence.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * arm64 KFENCE support. + * + * Copyright (C) 2020, Google LLC. + */ + +#ifndef __ASM_KFENCE_H +#define __ASM_KFENCE_H + +#include + +static inline bool arch_kfence_init_pool(void) { return true; } + +static inline bool kfence_protect_page(unsigned long addr, bool protect) +{ + set_memory_valid(addr, 1, !protect); + + return true; +} + +#endif /* __ASM_KFENCE_H */ diff --git a/arch/arm64/include/asm/memory.h b/arch/arm64/include/asm/memory.h index bc09af26c1b892484d5f3866c27d202d3b44b681..c759faf7a1ff9a59299e4aad27af246f4c228cec 100644 --- a/arch/arm64/include/asm/memory.h +++ b/arch/arm64/include/asm/memory.h @@ -244,6 +244,7 @@ static inline const void *__tag_set(const void *addr, u8 tag) #ifdef CONFIG_KASAN_HW_TAGS #define arch_enable_tagging() mte_enable_kernel() +#define arch_set_tagging_report_once(state) mte_set_report_once(state) #define arch_init_tags(max_tag) mte_init_tags(max_tag) #define arch_get_random_tag() mte_get_random_tag() #define arch_get_mem_tag(addr) mte_get_mem_tag(addr) diff --git a/arch/arm64/include/asm/module.lds.h b/arch/arm64/include/asm/module.lds.h index 691f15af788e4c64fd1efcaaf0da8097543cb35c..810045628c66e7588da28fb9208b6882e760b41d 100644 --- a/arch/arm64/include/asm/module.lds.h +++ b/arch/arm64/include/asm/module.lds.h @@ -1,7 +1,7 @@ #ifdef CONFIG_ARM64_MODULE_PLTS SECTIONS { - .plt (NOLOAD) : { BYTE(0) } - .init.plt (NOLOAD) : { BYTE(0) } - .text.ftrace_trampoline (NOLOAD) : { BYTE(0) } + .plt 0 (NOLOAD) : { BYTE(0) } + .init.plt 0 (NOLOAD) : { BYTE(0) } + .text.ftrace_trampoline 0 (NOLOAD) : { BYTE(0) } } #endif diff --git a/arch/arm64/include/asm/mte-def.h b/arch/arm64/include/asm/mte-def.h index 2d73a1612f099a6f4d1e7adb2021a3099c720f44..cf241b0f0a425a4e4a76a7d0a74edb604188ae61 100644 --- a/arch/arm64/include/asm/mte-def.h +++ b/arch/arm64/include/asm/mte-def.h @@ -11,4 +11,6 @@ #define MTE_TAG_SIZE 4 #define MTE_TAG_MASK GENMASK((MTE_TAG_SHIFT + (MTE_TAG_SIZE - 1)), MTE_TAG_SHIFT) +#define __MTE_PREAMBLE ARM64_ASM_PREAMBLE ".arch_extension memtag\n" + #endif /* __ASM_MTE_DEF_H */ diff --git a/arch/arm64/include/asm/mte-kasan.h b/arch/arm64/include/asm/mte-kasan.h index 26349a4b5e2ead7ce7c5ddfa4d2533da35f45fe9..7ab500e2ad1762c054e4f36f0840a78696ba5985 100644 --- a/arch/arm64/include/asm/mte-kasan.h +++ b/arch/arm64/include/asm/mte-kasan.h @@ -11,11 +11,14 @@ #include +#ifdef CONFIG_ARM64_MTE + /* - * The functions below are meant to be used only for the - * KASAN_HW_TAGS interface defined in asm/memory.h. + * These functions are meant to be only used from KASAN runtime through + * the arch_*() interface defined in asm/memory.h. + * These functions don't include system_supports_mte() checks, + * as KASAN only calls them when MTE is supported and enabled. */ -#ifdef CONFIG_ARM64_MTE static inline u8 mte_get_ptr_tag(void *ptr) { @@ -25,13 +28,61 @@ static inline u8 mte_get_ptr_tag(void *ptr) return tag; } -u8 mte_get_mem_tag(void *addr); -u8 mte_get_random_tag(void); -void *mte_set_mem_tag_range(void *addr, size_t size, u8 tag); +/* Get allocation tag for the address. */ +static inline u8 mte_get_mem_tag(void *addr) +{ + asm(__MTE_PREAMBLE "ldg %0, [%0]" + : "+r" (addr)); + + return mte_get_ptr_tag(addr); +} + +/* Generate a random tag. */ +static inline u8 mte_get_random_tag(void) +{ + void *addr; + + asm(__MTE_PREAMBLE "irg %0, %0" + : "=r" (addr)); + + return mte_get_ptr_tag(addr); +} + +/* + * Assign allocation tags for a region of memory based on the pointer tag. + * Note: The address must be non-NULL and MTE_GRANULE_SIZE aligned and + * size must be non-zero and MTE_GRANULE_SIZE aligned. + */ +static inline void mte_set_mem_tag_range(void *addr, size_t size, u8 tag) +{ + u64 curr, end; + + if (!size) + return; + + curr = (u64)__tag_set(addr, tag); + end = curr + size; + + do { + /* + * 'asm volatile' is required to prevent the compiler to move + * the statement outside of the loop. + */ + asm volatile(__MTE_PREAMBLE "stg %0, [%0]" + : + : "r" (curr) + : "memory"); + + curr += MTE_GRANULE_SIZE; + } while (curr != end); +} void mte_enable_kernel(void); void mte_init_tags(u64 max_tag); +void mte_set_report_once(bool state); +bool mte_report_once(void); + #else /* CONFIG_ARM64_MTE */ static inline u8 mte_get_ptr_tag(void *ptr) @@ -43,13 +94,14 @@ static inline u8 mte_get_mem_tag(void *addr) { return 0xFF; } + static inline u8 mte_get_random_tag(void) { return 0xFF; } -static inline void *mte_set_mem_tag_range(void *addr, size_t size, u8 tag) + +static inline void mte_set_mem_tag_range(void *addr, size_t size, u8 tag) { - return addr; } static inline void mte_enable_kernel(void) @@ -60,6 +112,15 @@ static inline void mte_init_tags(u64 max_tag) { } +static inline void mte_set_report_once(bool state) +{ +} + +static inline bool mte_report_once(void) +{ + return false; +} + #endif /* CONFIG_ARM64_MTE */ #endif /* __ASSEMBLY__ */ diff --git a/arch/arm64/include/asm/mte.h b/arch/arm64/include/asm/mte.h index d02aff9f493d095fae8f3c746499ccbd526ade79..9b557a457f244c7f8f79c129a916fc0fb1c12563 100644 --- a/arch/arm64/include/asm/mte.h +++ b/arch/arm64/include/asm/mte.h @@ -8,8 +8,6 @@ #include #include -#define __MTE_PREAMBLE ARM64_ASM_PREAMBLE ".arch_extension memtag\n" - #ifndef __ASSEMBLY__ #include diff --git a/arch/arm64/include/asm/numa.h b/arch/arm64/include/asm/numa.h index dd870390d639f2b7361fe1c0626f4a7dace0e26c..8c8cf4297cc321ace630d4d1a78049d75b70e524 100644 --- a/arch/arm64/include/asm/numa.h +++ b/arch/arm64/include/asm/numa.h @@ -3,52 +3,6 @@ #define __ASM_NUMA_H #include - -#ifdef CONFIG_NUMA - -#define NR_NODE_MEMBLKS (MAX_NUMNODES * 2) - -int __node_distance(int from, int to); -#define node_distance(a, b) __node_distance(a, b) - -extern nodemask_t numa_nodes_parsed __initdata; - -extern bool numa_off; - -/* Mappings between node number and cpus on that node. */ -extern cpumask_var_t node_to_cpumask_map[MAX_NUMNODES]; -void numa_clear_node(unsigned int cpu); - -#ifdef CONFIG_DEBUG_PER_CPU_MAPS -const struct cpumask *cpumask_of_node(int node); -#else -/* Returns a pointer to the cpumask of CPUs on Node 'node'. */ -static inline const struct cpumask *cpumask_of_node(int node) -{ - if (node == NUMA_NO_NODE) - return cpu_all_mask; - - return node_to_cpumask_map[node]; -} -#endif - -void __init arm64_numa_init(void); -int __init numa_add_memblk(int nodeid, u64 start, u64 end); -void __init numa_set_distance(int from, int to, int distance); -void __init numa_free_distance(void); -void __init early_map_cpu_to_node(unsigned int cpu, int nid); -void numa_store_cpu_info(unsigned int cpu); -void numa_add_cpu(unsigned int cpu); -void numa_remove_cpu(unsigned int cpu); - -#else /* CONFIG_NUMA */ - -static inline void numa_store_cpu_info(unsigned int cpu) { } -static inline void numa_add_cpu(unsigned int cpu) { } -static inline void numa_remove_cpu(unsigned int cpu) { } -static inline void arm64_numa_init(void) { } -static inline void early_map_cpu_to_node(unsigned int cpu, int nid) { } - -#endif /* CONFIG_NUMA */ +#include #endif /* __ASM_NUMA_H */ diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h index f9fbbb4734e80254609c0cbd263c02b1722cab84..dfd4edbfe36086df7c67bafe532929970c73dcb7 100644 --- a/arch/arm64/include/asm/sysreg.h +++ b/arch/arm64/include/asm/sysreg.h @@ -191,6 +191,7 @@ #define SYS_GCR_EL1 sys_reg(3, 0, 1, 0, 6) #define SYS_ZCR_EL1 sys_reg(3, 0, 1, 2, 0) +#define SYS_TRFCR_EL1 sys_reg(3, 0, 1, 2, 1) #define SYS_TTBR0_EL1 sys_reg(3, 0, 2, 0, 0) #define SYS_TTBR1_EL1 sys_reg(3, 0, 2, 0, 1) @@ -475,6 +476,7 @@ #define SYS_SCTLR_EL2 sys_reg(3, 4, 1, 0, 0) #define SYS_ZCR_EL2 sys_reg(3, 4, 1, 2, 0) +#define SYS_TRFCR_EL2 sys_reg(3, 4, 1, 2, 1) #define SYS_DACR32_EL2 sys_reg(3, 4, 3, 0, 0) #define SYS_SPSR_EL2 sys_reg(3, 4, 4, 0, 0) #define SYS_ELR_EL2 sys_reg(3, 4, 4, 0, 1) @@ -833,6 +835,7 @@ #define ID_AA64MMFR2_CNP_SHIFT 0 /* id_aa64dfr0 */ +#define ID_AA64DFR0_TRACE_FILT_SHIFT 40 #define ID_AA64DFR0_DOUBLELOCK_SHIFT 36 #define ID_AA64DFR0_PMSVER_SHIFT 32 #define ID_AA64DFR0_CTX_CMPS_SHIFT 28 @@ -1013,6 +1016,14 @@ /* Safe value for MPIDR_EL1: Bit31:RES1, Bit30:U:0, Bit24:MT:0 */ #define SYS_MPIDR_SAFE_VAL (BIT(31)) +#define TRFCR_ELx_TS_SHIFT 5 +#define TRFCR_ELx_TS_VIRTUAL ((0x1UL) << TRFCR_ELx_TS_SHIFT) +#define TRFCR_ELx_TS_GUEST_PHYSICAL ((0x2UL) << TRFCR_ELx_TS_SHIFT) +#define TRFCR_ELx_TS_PHYSICAL ((0x3UL) << TRFCR_ELx_TS_SHIFT) +#define TRFCR_EL2_CX BIT(3) +#define TRFCR_ELx_ExTRE BIT(1) +#define TRFCR_ELx_E0TRE BIT(0) + #ifdef __ASSEMBLY__ .irp num,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30 diff --git a/arch/arm64/include/asm/unistd.h b/arch/arm64/include/asm/unistd.h index 86a9d7b3eabe9ee254df4abd7f4fd008176d1cf5..949788f5ba4007049d0e92b55895ef738c1949fa 100644 --- a/arch/arm64/include/asm/unistd.h +++ b/arch/arm64/include/asm/unistd.h @@ -38,7 +38,7 @@ #define __ARM_NR_compat_set_tls (__ARM_NR_COMPAT_BASE + 5) #define __ARM_NR_COMPAT_END (__ARM_NR_COMPAT_BASE + 0x800) -#define __NR_compat_syscalls 442 +#define __NR_compat_syscalls 443 #endif #define __ARCH_WANT_SYS_CLONE diff --git a/arch/arm64/include/asm/unistd32.h b/arch/arm64/include/asm/unistd32.h index cccfbbefbf952e2c37d3cb39e046a3877df40638..3d874f624056b15b7e948d4c6821261f3907d4f5 100644 --- a/arch/arm64/include/asm/unistd32.h +++ b/arch/arm64/include/asm/unistd32.h @@ -891,6 +891,8 @@ __SYSCALL(__NR_faccessat2, sys_faccessat2) __SYSCALL(__NR_process_madvise, sys_process_madvise) #define __NR_epoll_pwait2 441 __SYSCALL(__NR_epoll_pwait2, compat_sys_epoll_pwait2) +#define __NR_mount_setattr 442 +__SYSCALL(__NR_mount_setattr, sys_mount_setattr) /* * Please add new compat syscalls above this comment and update diff --git a/arch/arm64/kernel/acpi_numa.c b/arch/arm64/kernel/acpi_numa.c index 7ff800045434602507a4a9aefaf50230cee4b289..fdfecf0991ced52e1b9216462e5618be33061bdb 100644 --- a/arch/arm64/kernel/acpi_numa.c +++ b/arch/arm64/kernel/acpi_numa.c @@ -118,15 +118,3 @@ void __init acpi_numa_gicc_affinity_init(struct acpi_srat_gicc_affinity *pa) node_set(node, numa_nodes_parsed); } -int __init arm64_acpi_numa_init(void) -{ - int ret; - - ret = acpi_numa_init(); - if (ret) { - pr_info("Failed to initialise from firmware\n"); - return ret; - } - - return srat_disabled() ? -EINVAL : 0; -} diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S index 1e30b5550d2a76696a00dfeddedaeb4b32769f6a..66b0e0b66e3122a9ba94a7ce648525b7f048bdca 100644 --- a/arch/arm64/kernel/head.S +++ b/arch/arm64/kernel/head.S @@ -837,6 +837,7 @@ SYM_FUNC_START_LOCAL(__primary_switch) tlbi vmalle1 // Remove any stale TLB entries dsb nsh + isb set_sctlr_el1 x19 // re-enable the MMU diff --git a/arch/arm64/kernel/hyp-stub.S b/arch/arm64/kernel/hyp-stub.S index 678cd2c618eec5d27b155e67abbe74b380b4c926..5eccbd62fec8a84fd35dc715a2f81e907847ea3b 100644 --- a/arch/arm64/kernel/hyp-stub.S +++ b/arch/arm64/kernel/hyp-stub.S @@ -75,9 +75,6 @@ SYM_CODE_END(el1_sync) // nVHE? No way! Give me the real thing! SYM_CODE_START_LOCAL(mutate_to_vhe) - // Be prepared to fail - mov_q x0, HVC_STUB_ERR - // Sanity check: MMU *must* be off mrs x1, sctlr_el2 tbnz x1, #0, 1f @@ -96,8 +93,11 @@ SYM_CODE_START_LOCAL(mutate_to_vhe) cmp x1, xzr and x2, x2, x1 csinv x2, x2, xzr, ne - cbz x2, 1f + cbnz x2, 2f +1: mov_q x0, HVC_STUB_ERR + eret +2: // Engage the VHE magic! mov_q x0, HCR_HOST_VHE_FLAGS msr hcr_el2, x0 @@ -131,9 +131,28 @@ SYM_CODE_START_LOCAL(mutate_to_vhe) msr mair_el1, x0 isb + // Hack the exception return to stay at EL2 + mrs x0, spsr_el1 + and x0, x0, #~PSR_MODE_MASK + mov x1, #PSR_MODE_EL2h + orr x0, x0, x1 + msr spsr_el1, x0 + + b enter_vhe +SYM_CODE_END(mutate_to_vhe) + + // At the point where we reach enter_vhe(), we run with + // the MMU off (which is enforced by mutate_to_vhe()). + // We thus need to be in the idmap, or everything will + // explode when enabling the MMU. + + .pushsection .idmap.text, "ax" + +SYM_CODE_START_LOCAL(enter_vhe) // Invalidate TLBs before enabling the MMU tlbi vmalle1 dsb nsh + isb // Enable the EL2 S1 MMU, as set up from EL1 mrs_s x0, SYS_SCTLR_EL12 @@ -143,17 +162,12 @@ SYM_CODE_START_LOCAL(mutate_to_vhe) mov_q x0, INIT_SCTLR_EL1_MMU_OFF msr_s SYS_SCTLR_EL12, x0 - // Hack the exception return to stay at EL2 - mrs x0, spsr_el1 - and x0, x0, #~PSR_MODE_MASK - mov x1, #PSR_MODE_EL2h - orr x0, x0, x1 - msr spsr_el1, x0 - mov x0, xzr -1: eret -SYM_CODE_END(mutate_to_vhe) + eret +SYM_CODE_END(enter_vhe) + + .popsection .macro invalid_vector label SYM_CODE_START_LOCAL(\label) diff --git a/arch/arm64/kernel/machine_kexec_file.c b/arch/arm64/kernel/machine_kexec_file.c index 03210f64479000152e2b2a60a25cb0914da8c64b..0cde47a63bebf42fccc433fe11b3a18cb0e703ff 100644 --- a/arch/arm64/kernel/machine_kexec_file.c +++ b/arch/arm64/kernel/machine_kexec_file.c @@ -182,8 +182,10 @@ static int create_dtb(struct kimage *image, /* duplicate a device tree blob */ ret = fdt_open_into(initial_boot_params, buf, buf_size); - if (ret) + if (ret) { + vfree(buf); return -EINVAL; + } ret = setup_dtb(image, initrd_load_addr, initrd_len, cmdline, buf); diff --git a/arch/arm64/kernel/mte.c b/arch/arm64/kernel/mte.c index 80b62fe49dcfd329971065afb08adc59b61647d2..b3c70a612c7a93324b735559853069320519dd94 100644 --- a/arch/arm64/kernel/mte.c +++ b/arch/arm64/kernel/mte.c @@ -19,12 +19,13 @@ #include #include #include -#include #include #include u64 gcr_kernel_excl __ro_after_init; +static bool report_fault_once = true; + static void mte_sync_page_tags(struct page *page, pte_t *ptep, bool check_swap) { pte_t old_pte = READ_ONCE(*ptep); @@ -86,51 +87,6 @@ int memcmp_pages(struct page *page1, struct page *page2) return ret; } -u8 mte_get_mem_tag(void *addr) -{ - if (!system_supports_mte()) - return 0xFF; - - asm(__MTE_PREAMBLE "ldg %0, [%0]" - : "+r" (addr)); - - return mte_get_ptr_tag(addr); -} - -u8 mte_get_random_tag(void) -{ - void *addr; - - if (!system_supports_mte()) - return 0xFF; - - asm(__MTE_PREAMBLE "irg %0, %0" - : "+r" (addr)); - - return mte_get_ptr_tag(addr); -} - -void *mte_set_mem_tag_range(void *addr, size_t size, u8 tag) -{ - void *ptr = addr; - - if ((!system_supports_mte()) || (size == 0)) - return addr; - - /* Make sure that size is MTE granule aligned. */ - WARN_ON(size & (MTE_GRANULE_SIZE - 1)); - - /* Make sure that the address is MTE granule aligned. */ - WARN_ON((u64)addr & (MTE_GRANULE_SIZE - 1)); - - tag = 0xF0 | tag; - ptr = (void *)__tag_set(ptr, tag); - - mte_assign_mem_tag_range(ptr, size); - - return ptr; -} - void mte_init_tags(u64 max_tag) { static bool gcr_kernel_excl_initialized; @@ -158,6 +114,16 @@ void mte_enable_kernel(void) isb(); } +void mte_set_report_once(bool state) +{ + WRITE_ONCE(report_fault_once, state); +} + +bool mte_report_once(void) +{ + return READ_ONCE(report_fault_once); +} + static void update_sctlr_el1_tcf0(u64 tcf0) { /* ISB required for the kernel uaccess routines */ diff --git a/arch/arm64/kernel/probes/uprobes.c b/arch/arm64/kernel/probes/uprobes.c index a412d8edbcd24428150a16e93ae059d45817b355..2c247634552b196b59e32e096381ce4dcdb827f0 100644 --- a/arch/arm64/kernel/probes/uprobes.c +++ b/arch/arm64/kernel/probes/uprobes.c @@ -38,7 +38,7 @@ int arch_uprobe_analyze_insn(struct arch_uprobe *auprobe, struct mm_struct *mm, /* TODO: Currently we do not support AARCH32 instruction probing */ if (mm->context.flags & MMCF_AARCH32) - return -ENOTSUPP; + return -EOPNOTSUPP; else if (!IS_ALIGNED(addr, AARCH64_INSN_SIZE)) return -EINVAL; diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c index 4cc1ccc8d6ab4184d054381f190c145ee15810bb..325c83b1a24df2101e19f8d695fe3ffde1230431 100644 --- a/arch/arm64/kernel/process.c +++ b/arch/arm64/kernel/process.c @@ -398,7 +398,7 @@ int copy_thread(unsigned long clone_flags, unsigned long stack_start, ptrauth_thread_init_kernel(p); - if (likely(!(p->flags & PF_KTHREAD))) { + if (likely(!(p->flags & (PF_KTHREAD | PF_IO_WORKER)))) { *childregs = *current_pt_regs(); childregs->regs[0] = 0; diff --git a/arch/arm64/kernel/ptrace.c b/arch/arm64/kernel/ptrace.c index 3d5c8afca75b1ea9a7e2b6c525452bfadc85b648..170f42fd6101a8cb9e9781064bfe7ded6c2a0ce2 100644 --- a/arch/arm64/kernel/ptrace.c +++ b/arch/arm64/kernel/ptrace.c @@ -1797,7 +1797,7 @@ int syscall_trace_enter(struct pt_regs *regs) if (flags & (_TIF_SYSCALL_EMU | _TIF_SYSCALL_TRACE)) { tracehook_report_syscall(regs, PTRACE_SYSCALL_ENTER); - if (!in_syscall(regs) || (flags & _TIF_SYSCALL_EMU)) + if (flags & _TIF_SYSCALL_EMU) return NO_SYSCALL; } diff --git a/arch/arm64/kernel/stacktrace.c b/arch/arm64/kernel/stacktrace.c index 0fb42129b46910e2fdb84ad86f2650df6443d632..ad20981dfda474bc60d1772b3ade0ef7831dbf7a 100644 --- a/arch/arm64/kernel/stacktrace.c +++ b/arch/arm64/kernel/stacktrace.c @@ -46,7 +46,7 @@ int notrace unwind_frame(struct task_struct *tsk, struct stackframe *frame) /* Terminal record; nothing to unwind */ if (!fp) - return -EINVAL; + return -ENOENT; if (fp & 0xf) return -EINVAL; diff --git a/arch/arm64/kernel/suspend.c b/arch/arm64/kernel/suspend.c index a67b37a7a47e1fc7e38ccb88acd236a797837ca8..d7564891ffe12f656d80f76363821870886c7ad6 100644 --- a/arch/arm64/kernel/suspend.c +++ b/arch/arm64/kernel/suspend.c @@ -119,7 +119,7 @@ int cpu_suspend(unsigned long arg, int (*fn)(unsigned long)) if (!ret) ret = -EOPNOTSUPP; } else { - __cpu_suspend_exit(); + RCU_NONIDLE(__cpu_suspend_exit()); } unpause_graph_tracing(); diff --git a/arch/arm64/kernel/vdso/Makefile b/arch/arm64/kernel/vdso/Makefile index 76c0255ecc91979cd8e28761a2e197b99d82245d..945e6bb326e3ed7f1b7127c3fe03ffb9d6a00cc0 100644 --- a/arch/arm64/kernel/vdso/Makefile +++ b/arch/arm64/kernel/vdso/Makefile @@ -29,7 +29,8 @@ ldflags-y := -shared -nostdlib -soname=linux-vdso.so.1 --hash-style=sysv \ ccflags-y := -fno-common -fno-builtin -fno-stack-protector -ffixed-x18 ccflags-y += -DDISABLE_BRANCH_PROFILING -DBUILD_VDSO -CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE) -Os $(CC_FLAGS_SCS) $(GCC_PLUGINS_CFLAGS) +CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE) -Os $(CC_FLAGS_SCS) $(GCC_PLUGINS_CFLAGS) \ + $(CC_FLAGS_LTO) KASAN_SANITIZE := n UBSAN_SANITIZE := n OBJECT_FILES_NON_STANDARD := y diff --git a/arch/arm64/kvm/hyp/hyp-entry.S b/arch/arm64/kvm/hyp/hyp-entry.S index d179056e1af819fa6a60f55abed0aade12b55753..5f49df4ffdd8651d4277b42d74c514166a21cba0 100644 --- a/arch/arm64/kvm/hyp/hyp-entry.S +++ b/arch/arm64/kvm/hyp/hyp-entry.S @@ -119,7 +119,7 @@ el2_error: .macro invalid_vector label, target = __guest_exit_panic .align 2 -SYM_CODE_START(\label) +SYM_CODE_START_LOCAL(\label) b \target SYM_CODE_END(\label) .endm diff --git a/arch/arm64/lib/mte.S b/arch/arm64/lib/mte.S index 9e1a12e1005336a5d0b20ffe4971c9709ec396a7..351537c12f36eac5826adce7d74ac24942e03d36 100644 --- a/arch/arm64/lib/mte.S +++ b/arch/arm64/lib/mte.S @@ -149,19 +149,3 @@ SYM_FUNC_START(mte_restore_page_tags) ret SYM_FUNC_END(mte_restore_page_tags) - -/* - * Assign allocation tags for a region of memory based on the pointer tag - * x0 - source pointer - * x1 - size - * - * Note: The address must be non-NULL and MTE_GRANULE_SIZE aligned and - * size must be non-zero and MTE_GRANULE_SIZE aligned. - */ -SYM_FUNC_START(mte_assign_mem_tag_range) -1: stg x0, [x0] - add x0, x0, #MTE_GRANULE_SIZE - subs x1, x1, #MTE_GRANULE_SIZE - b.gt 1b - ret -SYM_FUNC_END(mte_assign_mem_tag_range) diff --git a/arch/arm64/mm/Makefile b/arch/arm64/mm/Makefile index 77222d92667a7222d3fd715d048f2adfeffd7440..f188c9092696308c9ae75ad0505af66ee0712bf8 100644 --- a/arch/arm64/mm/Makefile +++ b/arch/arm64/mm/Makefile @@ -7,7 +7,6 @@ obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o obj-$(CONFIG_PTDUMP_CORE) += ptdump.o obj-$(CONFIG_PTDUMP_DEBUGFS) += ptdump_debugfs.o obj-$(CONFIG_TRANS_TABLE) += trans_pgd.o -obj-$(CONFIG_NUMA) += numa.o obj-$(CONFIG_DEBUG_VIRTUAL) += physaddr.o obj-$(CONFIG_ARM64_MTE) += mteswap.o KASAN_SANITIZE_physaddr.o += n diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c index 2e339f0bd9588c16153a0d7cf380311a1cedc455..f37d4e3830b79a9459f1de8c4349afda05e1c347 100644 --- a/arch/arm64/mm/fault.c +++ b/arch/arm64/mm/fault.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -302,12 +303,24 @@ static void die_kernel_fault(const char *msg, unsigned long addr, static void report_tag_fault(unsigned long addr, unsigned int esr, struct pt_regs *regs) { - bool is_write = ((esr & ESR_ELx_WNR) >> ESR_ELx_WNR_SHIFT) != 0; + static bool reported; + bool is_write; + + if (READ_ONCE(reported)) + return; + + /* + * This is used for KASAN tests and assumes that no MTE faults + * happened before running the tests. + */ + if (mte_report_once()) + WRITE_ONCE(reported, true); /* * SAS bits aren't set for all faults reported in EL1, so we can't * find out access size. */ + is_write = !!(esr & ESR_ELx_WNR); kasan_report(addr, 0, is_write, regs->pc); } #else @@ -319,12 +332,8 @@ static inline void report_tag_fault(unsigned long addr, unsigned int esr, static void do_tag_recovery(unsigned long addr, unsigned int esr, struct pt_regs *regs) { - static bool reported; - if (!READ_ONCE(reported)) { - report_tag_fault(addr, esr, regs); - WRITE_ONCE(reported, true); - } + report_tag_fault(addr, esr, regs); /* * Disable MTE Tag Checking on the local CPU for the current EL. @@ -381,6 +390,9 @@ static void __do_kernel_fault(unsigned long addr, unsigned int esr, } else if (addr < PAGE_SIZE) { msg = "NULL pointer dereference"; } else { + if (kfence_handle_page_fault(addr, esr & ESR_ELx_WNR, regs)) + return; + msg = "paging request"; } diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c index 709d98fea90cc1f8b08c85864fbaa6fd6a0b7d42..0ace5e68efba0ea05b32c989ac6ae987364fc89c 100644 --- a/arch/arm64/mm/init.c +++ b/arch/arm64/mm/init.c @@ -416,10 +416,10 @@ void __init bootmem_init(void) max_pfn = max_low_pfn = max; min_low_pfn = min; - arm64_numa_init(); + arch_numa_init(); /* - * must be done after arm64_numa_init() which calls numa_init() to + * must be done after arch_numa_init() which calls numa_init() to * initialize node_online_map that gets used in hugetlb_cma_reserve() * while allocating required CMA size across online nodes. */ diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index 25af183e4bed9d19c439d708e2f2bd6ce3fc98a2..3802cfbdd20d12e93388cb8f5f2029eafdc9f60f 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -1155,7 +1155,7 @@ void vmemmap_free(unsigned long start, unsigned long end, } #endif /* CONFIG_SPARSEMEM_VMEMMAP */ -static inline pud_t * fixmap_pud(unsigned long addr) +static inline pud_t *fixmap_pud(unsigned long addr) { pgd_t *pgdp = pgd_offset_k(addr); p4d_t *p4dp = p4d_offset(pgdp, addr); @@ -1166,7 +1166,7 @@ static inline pud_t * fixmap_pud(unsigned long addr) return pud_offset_kimg(p4dp, addr); } -static inline pmd_t * fixmap_pmd(unsigned long addr) +static inline pmd_t *fixmap_pmd(unsigned long addr) { pud_t *pudp = fixmap_pud(addr); pud_t pud = READ_ONCE(*pudp); @@ -1176,7 +1176,7 @@ static inline pmd_t * fixmap_pmd(unsigned long addr) return pmd_offset_kimg(pudp, addr); } -static inline pte_t * fixmap_pte(unsigned long addr) +static inline pte_t *fixmap_pte(unsigned long addr) { return &bm_pte[pte_index(addr)]; } @@ -1444,16 +1444,19 @@ static void __remove_pgd_mapping(pgd_t *pgdir, unsigned long start, u64 size) free_empty_tables(start, end, PAGE_OFFSET, PAGE_END); } -static bool inside_linear_region(u64 start, u64 size) +struct range arch_get_mappable_range(void) { + struct range mhp_range; + /* * Linear mapping region is the range [PAGE_OFFSET..(PAGE_END - 1)] * accommodating both its ends but excluding PAGE_END. Max physical * range which can be mapped inside this linear mapping range, must * also be derived from its end points. */ - return start >= __pa(_PAGE_OFFSET(vabits_actual)) && - (start + size - 1) <= __pa(PAGE_END - 1); + mhp_range.start = __pa(_PAGE_OFFSET(vabits_actual)); + mhp_range.end = __pa(PAGE_END - 1); + return mhp_range; } int arch_add_memory(int nid, u64 start, u64 size, @@ -1461,12 +1464,14 @@ int arch_add_memory(int nid, u64 start, u64 size, { int ret, flags = 0; - if (!inside_linear_region(start, size)) { - pr_err("[%llx %llx] is outside linear mapping region\n", start, start + size); - return -EINVAL; - } + VM_BUG_ON(!mhp_range_allowed(start, size, true)); - if (rodata_full || debug_pagealloc_enabled()) + /* + * KFENCE requires linear map to be mapped at page granularity, so that + * it is possible to protect/unprotect single pages in the KFENCE pool. + */ + if (rodata_full || debug_pagealloc_enabled() || + IS_ENABLED(CONFIG_KFENCE)) flags = NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS; __create_pgd_mapping(swapper_pg_dir, start, __phys_to_virt(start), diff --git a/arch/csky/Kconfig b/arch/csky/Kconfig index 89dd2fcf38fa168c16d15f028e0442e97193463c..34e91224adc395c5c6912042f8c8fdccfc364bbc 100644 --- a/arch/csky/Kconfig +++ b/arch/csky/Kconfig @@ -7,7 +7,7 @@ config CSKY select ARCH_HAS_SYNC_DMA_FOR_CPU select ARCH_HAS_SYNC_DMA_FOR_DEVICE select ARCH_USE_BUILTIN_BSWAP - select ARCH_USE_QUEUED_RWLOCKS if NR_CPUS>2 + select ARCH_USE_QUEUED_RWLOCKS select ARCH_WANT_FRAME_POINTERS if !CPU_CK610 select ARCH_WANT_DEFAULT_TOPDOWN_MMAP_LAYOUT select COMMON_CLK @@ -35,6 +35,9 @@ config CSKY select GENERIC_IRQ_MULTI_HANDLER select GENERIC_SCHED_CLOCK select GENERIC_SMP_IDLE_THREAD + select GENERIC_TIME_VSYSCALL + select GENERIC_VDSO_32 + select GENERIC_GETTIMEOFDAY select GX6605S_TIMER if CPU_CK610 select HAVE_ARCH_TRACEHOOK select HAVE_ARCH_AUDITSYSCALL @@ -43,11 +46,14 @@ config CSKY select HAVE_CONTEXT_TRACKING select HAVE_VIRT_CPU_ACCOUNTING_GEN select HAVE_DEBUG_BUGVERBOSE + select HAVE_DEBUG_KMEMLEAK select HAVE_DYNAMIC_FTRACE select HAVE_DYNAMIC_FTRACE_WITH_REGS + select HAVE_GENERIC_VDSO select HAVE_FUNCTION_TRACER select HAVE_FUNCTION_GRAPH_TRACER select HAVE_FUNCTION_ERROR_INJECTION + select HAVE_FUTEX_CMPXCHG if FUTEX && SMP select HAVE_FTRACE_MCOUNT_RECORD select HAVE_KERNEL_GZIP select HAVE_KERNEL_LZO @@ -192,6 +198,22 @@ config CPU_CK860 endchoice choice + prompt "PAGE OFFSET" + default PAGE_OFFSET_80000000 + +config PAGE_OFFSET_80000000 + bool "PAGE OFFSET 2G (user:kernel = 2:2)" + +config PAGE_OFFSET_A0000000 + bool "PAGE OFFSET 2.5G (user:kernel = 2.5:1.5)" +endchoice + +config PAGE_OFFSET + hex + default 0x80000000 if PAGE_OFFSET_80000000 + default 0xa0000000 if PAGE_OFFSET_A0000000 +choice + prompt "C-SKY PMU type" depends on PERF_EVENTS depends on CPU_CK807 || CPU_CK810 || CPU_CK860 diff --git a/arch/csky/abiv1/inc/abi/cacheflush.h b/arch/csky/abiv1/inc/abi/cacheflush.h index d3e04208d53c23e36eee552e89e484ef81d8f2c5..6cab7afae96262e91f24ea519e23e19a2694e8f2 100644 --- a/arch/csky/abiv1/inc/abi/cacheflush.h +++ b/arch/csky/abiv1/inc/abi/cacheflush.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ABI_CSKY_CACHEFLUSH_H #define __ABI_CSKY_CACHEFLUSH_H diff --git a/arch/csky/abiv1/inc/abi/ckmmu.h b/arch/csky/abiv1/inc/abi/ckmmu.h index ba8eb58708351a8296d64268408be9a191103e0c..416b30c579831fed32ed1e70e8b21ba6070b7301 100644 --- a/arch/csky/abiv1/inc/abi/ckmmu.h +++ b/arch/csky/abiv1/inc/abi/ckmmu.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_CKMMUV1_H #define __ASM_CSKY_CKMMUV1_H @@ -89,13 +88,14 @@ static inline void tlb_invalid_indexed(void) cpwcr("cpcr8", 0x02000000); } -static inline void setup_pgd(unsigned long pgd, bool kernel) +static inline void setup_pgd(pgd_t *pgd, int asid) { - cpwcr("cpcr29", pgd | BIT(0)); + cpwcr("cpcr29", __pa(pgd) | BIT(0)); + write_mmu_entryhi(asid); } -static inline unsigned long get_pgd(void) +static inline pgd_t *get_pgd(void) { - return cprcr("cpcr29") & ~BIT(0); + return __va(cprcr("cpcr29") & ~BIT(0)); } #endif /* __ASM_CSKY_CKMMUV1_H */ diff --git a/arch/csky/abiv1/inc/abi/entry.h b/arch/csky/abiv1/inc/abi/entry.h index 13c23e2c707ccfae861fedddfb16374a527a9776..b6a2109b895eb64e1014d91f83ff687ef9cf5689 100644 --- a/arch/csky/abiv1/inc/abi/entry.h +++ b/arch/csky/abiv1/inc/abi/entry.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_ENTRY_H #define __ASM_CSKY_ENTRY_H diff --git a/arch/csky/abiv1/inc/abi/page.h b/arch/csky/abiv1/inc/abi/page.h index c864519117c71581da59bc733ef740c1df1cc9fa..2d2159933b76624e905efc92663f22c0af06f09e 100644 --- a/arch/csky/abiv1/inc/abi/page.h +++ b/arch/csky/abiv1/inc/abi/page.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #include diff --git a/arch/csky/abiv1/inc/abi/pgtable-bits.h b/arch/csky/abiv1/inc/abi/pgtable-bits.h index d605445aad9ad4362f51d3ad41f47707c1946a07..752c8b3f91945dc85b89dbb57aab4e828eaa16b4 100644 --- a/arch/csky/abiv1/inc/abi/pgtable-bits.h +++ b/arch/csky/abiv1/inc/abi/pgtable-bits.h @@ -1,37 +1,49 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_PGTABLE_BITS_H #define __ASM_CSKY_PGTABLE_BITS_H /* implemented in software */ -#define _PAGE_ACCESSED (1<<3) -#define PAGE_ACCESSED_BIT (3) - +#define _PAGE_PRESENT (1<<0) #define _PAGE_READ (1<<1) #define _PAGE_WRITE (1<<2) -#define _PAGE_PRESENT (1<<0) - +#define _PAGE_ACCESSED (1<<3) #define _PAGE_MODIFIED (1<<4) -#define PAGE_MODIFIED_BIT (4) /* implemented in hardware */ #define _PAGE_GLOBAL (1<<6) - #define _PAGE_VALID (1<<7) -#define PAGE_VALID_BIT (7) - #define _PAGE_DIRTY (1<<8) -#define PAGE_DIRTY_BIT (8) #define _PAGE_CACHE (3<<9) #define _PAGE_UNCACHE (2<<9) #define _PAGE_SO _PAGE_UNCACHE - #define _CACHE_MASK (7<<9) -#define _CACHE_CACHED (_PAGE_VALID | _PAGE_CACHE) -#define _CACHE_UNCACHED (_PAGE_VALID | _PAGE_UNCACHE) +#define _CACHE_CACHED _PAGE_CACHE +#define _CACHE_UNCACHED _PAGE_UNCACHE + +#define _PAGE_PROT_NONE _PAGE_READ + +/* + * Encode and decode a swap entry + * + * Format of swap PTE: + * bit 0: _PAGE_PRESENT (zero) + * bit 1: _PAGE_READ (zero) + * bit 2 - 5: swap type[0 - 3] + * bit 6: _PAGE_GLOBAL (zero) + * bit 7: _PAGE_VALID (zero) + * bit 8: swap type[4] + * bit 9 - 31: swap offset + */ +#define __swp_type(x) ((((x).val >> 2) & 0xf) | \ + (((x).val >> 4) & 0x10)) +#define __swp_offset(x) ((x).val >> 9) +#define __swp_entry(type, offset) ((swp_entry_t) { \ + ((type & 0xf) << 2) | \ + ((type & 0x10) << 4) | \ + ((offset) << 9)}) #define HAVE_ARCH_UNMAPPED_AREA diff --git a/arch/csky/abiv1/inc/abi/reg_ops.h b/arch/csky/abiv1/inc/abi/reg_ops.h index a153bd3918f7737cbe26f939561ecbc309d78fbc..abd01a2433887a4069de79becccd6d5804f66037 100644 --- a/arch/csky/abiv1/inc/abi/reg_ops.h +++ b/arch/csky/abiv1/inc/abi/reg_ops.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ABI_REG_OPS_H #define __ABI_REG_OPS_H diff --git a/arch/csky/abiv1/inc/abi/regdef.h b/arch/csky/abiv1/inc/abi/regdef.h index 104707fbdcc1ceaa389659714d3e376038a5351c..7b386fd6707025d3ac41e5c07ef8dea47c2d19ba 100644 --- a/arch/csky/abiv1/inc/abi/regdef.h +++ b/arch/csky/abiv1/inc/abi/regdef.h @@ -1,10 +1,14 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_REGDEF_H #define __ASM_CSKY_REGDEF_H +#ifdef __ASSEMBLY__ #define syscallid r1 +#else +#define syscallid "r1" +#endif + #define regs_syscallid(regs) regs->regs[9] #define regs_fp(regs) regs->regs[2] diff --git a/arch/csky/abiv1/inc/abi/string.h b/arch/csky/abiv1/inc/abi/string.h index 0cd43384f8d22295b2d34b6d071aa0a8f18ba165..9d95594b0febfc3aba5bd5c57c0984ffe6fcf366 100644 --- a/arch/csky/abiv1/inc/abi/string.h +++ b/arch/csky/abiv1/inc/abi/string.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ABI_CSKY_STRING_H #define __ABI_CSKY_STRING_H diff --git a/arch/csky/abiv1/inc/abi/switch_context.h b/arch/csky/abiv1/inc/abi/switch_context.h index 17c82686498e8b1cbdf3a9e7dc124f4c6572923d..ec73fd7c9f871976b8b1b8c71f54b6d2f44cd3a8 100644 --- a/arch/csky/abiv1/inc/abi/switch_context.h +++ b/arch/csky/abiv1/inc/abi/switch_context.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ABI_CSKY_PTRACE_H #define __ABI_CSKY_PTRACE_H diff --git a/arch/csky/abiv1/inc/abi/vdso.h b/arch/csky/abiv1/inc/abi/vdso.h index 14352f524f1d11a4612c9628729db8121e82107c..9e6d0a2fdd2b7613908b21231cc5ec7a9d43908a 100644 --- a/arch/csky/abiv1/inc/abi/vdso.h +++ b/arch/csky/abiv1/inc/abi/vdso.h @@ -1,17 +1,9 @@ /* SPDX-License-Identifier: GPL-2.0 */ -#include +#ifndef __ABI_CSKY_VDSO_H +#define __ABI_CSKY_VDSO_H -static inline int setup_vdso_page(unsigned short *ptr) -{ - int err = 0; +/* movi r1, 127; addi r1, (139 - 127) */ +#define SET_SYSCALL_ID .long 0x20b167f1 - /* movi r1, 127 */ - err |= __put_user(0x67f1, ptr + 0); - /* addi r1, (139 - 127) */ - err |= __put_user(0x20b1, ptr + 1); - /* trap 0 */ - err |= __put_user(0x0008, ptr + 2); - - return err; -} +#endif /* __ABI_CSKY_VDSO_H */ diff --git a/arch/csky/abiv2/cacheflush.c b/arch/csky/abiv2/cacheflush.c index 790f1ebfba44ba925ccd94f1927b61e03137dbc6..39c51399dd81da57ef4d9c580bd31bdf5e2b6f0b 100644 --- a/arch/csky/abiv2/cacheflush.c +++ b/arch/csky/abiv2/cacheflush.c @@ -12,6 +12,9 @@ void update_mmu_cache(struct vm_area_struct *vma, unsigned long address, unsigned long addr; struct page *page; + if (!pfn_valid(pte_pfn(*pte))) + return; + page = pfn_to_page(pte_pfn(*pte)); if (page == ZERO_PAGE(0)) return; diff --git a/arch/csky/abiv2/inc/abi/ckmmu.h b/arch/csky/abiv2/inc/abi/ckmmu.h index 73ded7c72482b8355372201e2a55f35c5cc123b1..64215f2380f12fc5d3903c381ad02ce309c0129e 100644 --- a/arch/csky/abiv2/inc/abi/ckmmu.h +++ b/arch/csky/abiv2/inc/abi/ckmmu.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_CKMMUV2_H #define __ASM_CSKY_CKMMUV2_H @@ -78,8 +77,13 @@ static inline void tlb_read(void) static inline void tlb_invalid_all(void) { #ifdef CONFIG_CPU_HAS_TLBI - asm volatile("tlbi.alls\n":::"memory"); sync_is(); + asm volatile( + "tlbi.alls \n" + "sync.i \n" + : + : + : "memory"); #else mtcr("cr<8, 15>", 0x04000000); #endif @@ -88,8 +92,13 @@ static inline void tlb_invalid_all(void) static inline void local_tlb_invalid_all(void) { #ifdef CONFIG_CPU_HAS_TLBI - asm volatile("tlbi.all\n":::"memory"); sync_is(); + asm volatile( + "tlbi.all \n" + "sync.i \n" + : + : + : "memory"); #else tlb_invalid_all(); #endif @@ -100,16 +109,31 @@ static inline void tlb_invalid_indexed(void) mtcr("cr<8, 15>", 0x02000000); } -static inline void setup_pgd(unsigned long pgd, bool kernel) +#define NOP32 ".long 0x4820c400\n" + +static inline void setup_pgd(pgd_t *pgd, int asid) { - if (kernel) - mtcr("cr<28, 15>", pgd | BIT(0)); - else - mtcr("cr<29, 15>", pgd | BIT(0)); +#ifdef CONFIG_CPU_HAS_TLBI + sync_is(); +#else + mb(); +#endif + asm volatile( +#ifdef CONFIG_CPU_HAS_TLBI + "mtcr %1, cr<28, 15> \n" +#endif + "mtcr %1, cr<29, 15> \n" + "mtcr %0, cr< 4, 15> \n" + ".rept 64 \n" + NOP32 + ".endr \n" + : + :"r"(asid), "r"(__pa(pgd) | BIT(0)) + :"memory"); } -static inline unsigned long get_pgd(void) +static inline pgd_t *get_pgd(void) { - return mfcr("cr<29, 15>") & ~BIT(0); + return __va(mfcr("cr<29, 15>") & ~BIT(0)); } #endif /* __ASM_CSKY_CKMMUV2_H */ diff --git a/arch/csky/abiv2/inc/abi/entry.h b/arch/csky/abiv2/inc/abi/entry.h index bedcc6f06bba23bbd99992b83bab860b38bdc7af..cca63e699b5806c81992165b18fbac1fefdda0fa 100644 --- a/arch/csky/abiv2/inc/abi/entry.h +++ b/arch/csky/abiv2/inc/abi/entry.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_ENTRY_H #define __ASM_CSKY_ENTRY_H @@ -26,6 +25,9 @@ stw tls, (sp, 0) stw lr, (sp, 4) + RD_MEH lr + WR_MEH lr + mfcr lr, epc movi tls, \epc_inc add lr, tls @@ -231,6 +233,16 @@ mtcr \rx, cr<8, 15> .endm +#ifdef CONFIG_PAGE_OFFSET_80000000 +#define MSA_SET cr<30, 15> +#define MSA_CLR cr<31, 15> +#endif + +#ifdef CONFIG_PAGE_OFFSET_A0000000 +#define MSA_SET cr<31, 15> +#define MSA_CLR cr<30, 15> +#endif + .macro SETUP_MMU /* Init psr and enable ee */ lrw r6, DEFAULT_PSR_VALUE @@ -281,15 +293,15 @@ * 31 - 29 | 28 - 9 | 8 | 7 | 6 | 5 | 4 | 3 | 2 | 1 | 0 * BA Reserved SH WA B SO SEC C D V */ - mfcr r6, cr<30, 15> /* Get MSA0 */ + mfcr r6, MSA_SET /* Get MSA */ 2: lsri r6, 29 lsli r6, 29 addi r6, 0x1ce - mtcr r6, cr<30, 15> /* Set MSA0 */ + mtcr r6, MSA_SET /* Set MSA */ movi r6, 0 - mtcr r6, cr<31, 15> /* Clr MSA1 */ + mtcr r6, MSA_CLR /* Clr MSA */ /* enable MMU */ mfcr r6, cr18 diff --git a/arch/csky/abiv2/inc/abi/fpu.h b/arch/csky/abiv2/inc/abi/fpu.h index 09e2700a36936e35479eaccd4691772216ee13ce..aabb793550132dca373361a97031b912e44afa4f 100644 --- a/arch/csky/abiv2/inc/abi/fpu.h +++ b/arch/csky/abiv2/inc/abi/fpu.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_FPU_H #define __ASM_CSKY_FPU_H diff --git a/arch/csky/abiv2/inc/abi/page.h b/arch/csky/abiv2/inc/abi/page.h index 0a70cb553dca9817d456dcfc9c59c1d3b48d1f40..cf005f13cd15d34ac237f9e71aee7aa25ef9532a 100644 --- a/arch/csky/abiv2/inc/abi/page.h +++ b/arch/csky/abiv2/inc/abi/page.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. static inline void clear_user_page(void *addr, unsigned long vaddr, struct page *page) diff --git a/arch/csky/abiv2/inc/abi/pgtable-bits.h b/arch/csky/abiv2/inc/abi/pgtable-bits.h index 137f7932c83b3be191e9971da02bce7270b9ad62..7e7f389f546fbdc6c336e5f84e62b7ca352af6fc 100644 --- a/arch/csky/abiv2/inc/abi/pgtable-bits.h +++ b/arch/csky/abiv2/inc/abi/pgtable-bits.h @@ -1,37 +1,48 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_PGTABLE_BITS_H #define __ASM_CSKY_PGTABLE_BITS_H /* implemented in software */ #define _PAGE_ACCESSED (1<<7) -#define PAGE_ACCESSED_BIT (7) - #define _PAGE_READ (1<<8) #define _PAGE_WRITE (1<<9) #define _PAGE_PRESENT (1<<10) - #define _PAGE_MODIFIED (1<<11) -#define PAGE_MODIFIED_BIT (11) /* implemented in hardware */ #define _PAGE_GLOBAL (1<<0) - #define _PAGE_VALID (1<<1) -#define PAGE_VALID_BIT (1) - #define _PAGE_DIRTY (1<<2) -#define PAGE_DIRTY_BIT (2) #define _PAGE_SO (1<<5) #define _PAGE_BUF (1<<6) - #define _PAGE_CACHE (1<<3) - #define _CACHE_MASK _PAGE_CACHE -#define _CACHE_CACHED (_PAGE_VALID | _PAGE_CACHE | _PAGE_BUF) -#define _CACHE_UNCACHED (_PAGE_VALID) +#define _CACHE_CACHED (_PAGE_CACHE | _PAGE_BUF) +#define _CACHE_UNCACHED (0) + +#define _PAGE_PROT_NONE _PAGE_WRITE + +/* + * Encode and decode a swap entry + * + * Format of swap PTE: + * bit 0: _PAGE_GLOBAL (zero) + * bit 1: _PAGE_VALID (zero) + * bit 2 - 6: swap type + * bit 7 - 8: swap offset[0 - 1] + * bit 9: _PAGE_WRITE (zero) + * bit 10: _PAGE_PRESENT (zero) + * bit 11 - 31: swap offset[2 - 22] + */ +#define __swp_type(x) (((x).val >> 2) & 0x1f) +#define __swp_offset(x) ((((x).val >> 7) & 0x3) | \ + (((x).val >> 9) & 0x7ffffc)) +#define __swp_entry(type, offset) ((swp_entry_t) { \ + ((type & 0x1f) << 2) | \ + ((offset & 0x3) << 7) | \ + ((offset & 0x7ffffc) << 9)}) #endif /* __ASM_CSKY_PGTABLE_BITS_H */ diff --git a/arch/csky/abiv2/inc/abi/reg_ops.h b/arch/csky/abiv2/inc/abi/reg_ops.h index ae82c3f26a6b8986f36b7088bcc0902fca2f1766..49ba18a647511123f4c5a8c9698abda0583856ec 100644 --- a/arch/csky/abiv2/inc/abi/reg_ops.h +++ b/arch/csky/abiv2/inc/abi/reg_ops.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ABI_REG_OPS_H #define __ABI_REG_OPS_H diff --git a/arch/csky/abiv2/inc/abi/regdef.h b/arch/csky/abiv2/inc/abi/regdef.h index d7328bbc1ce7ad7c44a6d4934a24f619d744a422..0933addbc27b75e7b3d9d19f43325c93e70ccca0 100644 --- a/arch/csky/abiv2/inc/abi/regdef.h +++ b/arch/csky/abiv2/inc/abi/regdef.h @@ -1,10 +1,14 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_REGDEF_H #define __ASM_CSKY_REGDEF_H +#ifdef __ASSEMBLY__ #define syscallid r7 +#else +#define syscallid "r7" +#endif + #define regs_syscallid(regs) regs->regs[3] #define regs_fp(regs) regs->regs[4] diff --git a/arch/csky/abiv2/inc/abi/switch_context.h b/arch/csky/abiv2/inc/abi/switch_context.h index 73a81245a3b390a4aa485c98b8b4f97f1b2560bb..5dd5c3f4ee7ebaf373db61616b6372b2769e7ec5 100644 --- a/arch/csky/abiv2/inc/abi/switch_context.h +++ b/arch/csky/abiv2/inc/abi/switch_context.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ABI_CSKY_PTRACE_H #define __ABI_CSKY_PTRACE_H diff --git a/arch/csky/abiv2/inc/abi/vdso.h b/arch/csky/abiv2/inc/abi/vdso.h index b60d4a07032636e862840ee0d0432e94a3b97b67..40fd10d893ff2239ca254090d54427ef06137b84 100644 --- a/arch/csky/abiv2/inc/abi/vdso.h +++ b/arch/csky/abiv2/inc/abi/vdso.h @@ -3,21 +3,7 @@ #ifndef __ABI_CSKY_VDSO_H #define __ABI_CSKY_VDSO_H -#include +/* movi r7, 173 */ +#define SET_SYSCALL_ID .long 0x008bea07 -static inline int setup_vdso_page(unsigned short *ptr) -{ - int err = 0; - - /* movi r7, 173 */ - err |= __put_user(0xea07, ptr); - err |= __put_user(0x008b, ptr+1); - - /* trap 0 */ - err |= __put_user(0xc000, ptr+2); - err |= __put_user(0x2020, ptr+3); - - return err; -} - -#endif /* __ABI_CSKY_STRING_H */ +#endif /* __ABI_CSKY_VDSO_H */ diff --git a/arch/csky/abiv2/sysdep.h b/arch/csky/abiv2/sysdep.h index bbbedfd3477740cf0a9c1c810ee1735da1cb31bf..61abe9201c50461f7e4ed88931aec796cbb64e86 100644 --- a/arch/csky/abiv2/sysdep.h +++ b/arch/csky/abiv2/sysdep.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __SYSDEP_H #define __SYSDEP_H diff --git a/arch/csky/include/asm/addrspace.h b/arch/csky/include/asm/addrspace.h index d1c2ede692edb787475e540a993d52e1afb3afc5..6fc05d44536c81098bc24e8314c6987b270e55b0 100644 --- a/arch/csky/include/asm/addrspace.h +++ b/arch/csky/include/asm/addrspace.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_ADDRSPACE_H #define __ASM_CSKY_ADDRSPACE_H diff --git a/arch/csky/include/asm/atomic.h b/arch/csky/include/asm/atomic.h deleted file mode 100644 index e369d73b13e3254570201bf70ae773aa5d6535ce..0000000000000000000000000000000000000000 --- a/arch/csky/include/asm/atomic.h +++ /dev/null @@ -1,212 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ - -#ifndef __ASM_CSKY_ATOMIC_H -#define __ASM_CSKY_ATOMIC_H - -#include -#include -#include - -#ifdef CONFIG_CPU_HAS_LDSTEX - -#define __atomic_add_unless __atomic_add_unless -static inline int __atomic_add_unless(atomic_t *v, int a, int u) -{ - unsigned long tmp, ret; - - smp_mb(); - - asm volatile ( - "1: ldex.w %0, (%3) \n" - " mov %1, %0 \n" - " cmpne %0, %4 \n" - " bf 2f \n" - " add %0, %2 \n" - " stex.w %0, (%3) \n" - " bez %0, 1b \n" - "2: \n" - : "=&r" (tmp), "=&r" (ret) - : "r" (a), "r"(&v->counter), "r"(u) - : "memory"); - - if (ret != u) - smp_mb(); - - return ret; -} - -#define ATOMIC_OP(op, c_op) \ -static inline void atomic_##op(int i, atomic_t *v) \ -{ \ - unsigned long tmp; \ - \ - asm volatile ( \ - "1: ldex.w %0, (%2) \n" \ - " " #op " %0, %1 \n" \ - " stex.w %0, (%2) \n" \ - " bez %0, 1b \n" \ - : "=&r" (tmp) \ - : "r" (i), "r"(&v->counter) \ - : "memory"); \ -} - -#define ATOMIC_OP_RETURN(op, c_op) \ -static inline int atomic_##op##_return(int i, atomic_t *v) \ -{ \ - unsigned long tmp, ret; \ - \ - smp_mb(); \ - asm volatile ( \ - "1: ldex.w %0, (%3) \n" \ - " " #op " %0, %2 \n" \ - " mov %1, %0 \n" \ - " stex.w %0, (%3) \n" \ - " bez %0, 1b \n" \ - : "=&r" (tmp), "=&r" (ret) \ - : "r" (i), "r"(&v->counter) \ - : "memory"); \ - smp_mb(); \ - \ - return ret; \ -} - -#define ATOMIC_FETCH_OP(op, c_op) \ -static inline int atomic_fetch_##op(int i, atomic_t *v) \ -{ \ - unsigned long tmp, ret; \ - \ - smp_mb(); \ - asm volatile ( \ - "1: ldex.w %0, (%3) \n" \ - " mov %1, %0 \n" \ - " " #op " %0, %2 \n" \ - " stex.w %0, (%3) \n" \ - " bez %0, 1b \n" \ - : "=&r" (tmp), "=&r" (ret) \ - : "r" (i), "r"(&v->counter) \ - : "memory"); \ - smp_mb(); \ - \ - return ret; \ -} - -#else /* CONFIG_CPU_HAS_LDSTEX */ - -#include - -#define __atomic_add_unless __atomic_add_unless -static inline int __atomic_add_unless(atomic_t *v, int a, int u) -{ - unsigned long tmp, ret, flags; - - raw_local_irq_save(flags); - - asm volatile ( - " ldw %0, (%3) \n" - " mov %1, %0 \n" - " cmpne %0, %4 \n" - " bf 2f \n" - " add %0, %2 \n" - " stw %0, (%3) \n" - "2: \n" - : "=&r" (tmp), "=&r" (ret) - : "r" (a), "r"(&v->counter), "r"(u) - : "memory"); - - raw_local_irq_restore(flags); - - return ret; -} - -#define ATOMIC_OP(op, c_op) \ -static inline void atomic_##op(int i, atomic_t *v) \ -{ \ - unsigned long tmp, flags; \ - \ - raw_local_irq_save(flags); \ - \ - asm volatile ( \ - " ldw %0, (%2) \n" \ - " " #op " %0, %1 \n" \ - " stw %0, (%2) \n" \ - : "=&r" (tmp) \ - : "r" (i), "r"(&v->counter) \ - : "memory"); \ - \ - raw_local_irq_restore(flags); \ -} - -#define ATOMIC_OP_RETURN(op, c_op) \ -static inline int atomic_##op##_return(int i, atomic_t *v) \ -{ \ - unsigned long tmp, ret, flags; \ - \ - raw_local_irq_save(flags); \ - \ - asm volatile ( \ - " ldw %0, (%3) \n" \ - " " #op " %0, %2 \n" \ - " stw %0, (%3) \n" \ - " mov %1, %0 \n" \ - : "=&r" (tmp), "=&r" (ret) \ - : "r" (i), "r"(&v->counter) \ - : "memory"); \ - \ - raw_local_irq_restore(flags); \ - \ - return ret; \ -} - -#define ATOMIC_FETCH_OP(op, c_op) \ -static inline int atomic_fetch_##op(int i, atomic_t *v) \ -{ \ - unsigned long tmp, ret, flags; \ - \ - raw_local_irq_save(flags); \ - \ - asm volatile ( \ - " ldw %0, (%3) \n" \ - " mov %1, %0 \n" \ - " " #op " %0, %2 \n" \ - " stw %0, (%3) \n" \ - : "=&r" (tmp), "=&r" (ret) \ - : "r" (i), "r"(&v->counter) \ - : "memory"); \ - \ - raw_local_irq_restore(flags); \ - \ - return ret; \ -} - -#endif /* CONFIG_CPU_HAS_LDSTEX */ - -#define atomic_add_return atomic_add_return -ATOMIC_OP_RETURN(add, +) -#define atomic_sub_return atomic_sub_return -ATOMIC_OP_RETURN(sub, -) - -#define atomic_fetch_add atomic_fetch_add -ATOMIC_FETCH_OP(add, +) -#define atomic_fetch_sub atomic_fetch_sub -ATOMIC_FETCH_OP(sub, -) -#define atomic_fetch_and atomic_fetch_and -ATOMIC_FETCH_OP(and, &) -#define atomic_fetch_or atomic_fetch_or -ATOMIC_FETCH_OP(or, |) -#define atomic_fetch_xor atomic_fetch_xor -ATOMIC_FETCH_OP(xor, ^) - -#define atomic_and atomic_and -ATOMIC_OP(and, &) -#define atomic_or atomic_or -ATOMIC_OP(or, |) -#define atomic_xor atomic_xor -ATOMIC_OP(xor, ^) - -#undef ATOMIC_FETCH_OP -#undef ATOMIC_OP_RETURN -#undef ATOMIC_OP - -#include - -#endif /* __ASM_CSKY_ATOMIC_H */ diff --git a/arch/csky/include/asm/barrier.h b/arch/csky/include/asm/barrier.h index a430e7fddf350cff5e5f24b5719b66b54b3da426..84fc600c8b453416f03fc5509838541d58d4e3b1 100644 --- a/arch/csky/include/asm/barrier.h +++ b/arch/csky/include/asm/barrier.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_BARRIER_H #define __ASM_CSKY_BARRIER_H @@ -8,6 +7,61 @@ #define nop() asm volatile ("nop\n":::"memory") +#ifdef CONFIG_SMP + +/* + * bar.brwarws: ordering barrier for all load/store instructions + * before/after + * + * |31|30 26|25 21|20 16|15 10|9 5|4 0| + * 1 10000 00000 00000 100001 00001 0 bw br aw ar + * + * b: before + * a: after + * r: read + * w: write + * + * Here are all combinations: + * + * bar.brw + * bar.br + * bar.bw + * bar.arw + * bar.ar + * bar.aw + * bar.brwarw + * bar.brarw + * bar.bwarw + * bar.brwar + * bar.brwaw + * bar.brar + * bar.bwaw + */ +#define __bar_brw() asm volatile (".long 0x842cc000\n":::"memory") +#define __bar_br() asm volatile (".long 0x8424c000\n":::"memory") +#define __bar_bw() asm volatile (".long 0x8428c000\n":::"memory") +#define __bar_arw() asm volatile (".long 0x8423c000\n":::"memory") +#define __bar_ar() asm volatile (".long 0x8421c000\n":::"memory") +#define __bar_aw() asm volatile (".long 0x8422c000\n":::"memory") +#define __bar_brwarw() asm volatile (".long 0x842fc000\n":::"memory") +#define __bar_brarw() asm volatile (".long 0x8427c000\n":::"memory") +#define __bar_bwarw() asm volatile (".long 0x842bc000\n":::"memory") +#define __bar_brwar() asm volatile (".long 0x842dc000\n":::"memory") +#define __bar_brwaw() asm volatile (".long 0x842ec000\n":::"memory") +#define __bar_brar() asm volatile (".long 0x8425c000\n":::"memory") +#define __bar_brar() asm volatile (".long 0x8425c000\n":::"memory") +#define __bar_bwaw() asm volatile (".long 0x842ac000\n":::"memory") + +#define __smp_mb() __bar_brwarw() +#define __smp_rmb() __bar_brar() +#define __smp_wmb() __bar_bwaw() + +#define ACQUIRE_FENCE ".long 0x8427c000\n" +#define __smp_acquire_fence() __bar_brarw() +#define __smp_release_fence() __bar_brwaw() + +#endif /* CONFIG_SMP */ + /* * sync: completion barrier, all sync.xx instructions * guarantee the last response recieved by bus transaction @@ -15,31 +69,14 @@ * sync.s: inherit from sync, but also shareable to other cores * sync.i: inherit from sync, but also flush cpu pipeline * sync.is: the same with sync.i + sync.s - * - * bar.brwarw: ordering barrier for all load/store instructions before it - * bar.brwarws: ordering barrier for all load/store instructions before it - * and shareable to other cores - * bar.brar: ordering barrier for all load instructions before it - * bar.brars: ordering barrier for all load instructions before it - * and shareable to other cores - * bar.bwaw: ordering barrier for all store instructions before it - * bar.bwaws: ordering barrier for all store instructions before it - * and shareable to other cores */ +#define mb() asm volatile ("sync\n":::"memory") #ifdef CONFIG_CPU_HAS_CACHEV2 -#define mb() asm volatile ("sync.s\n":::"memory") - -#ifdef CONFIG_SMP -#define __smp_mb() asm volatile ("bar.brwarws\n":::"memory") -#define __smp_rmb() asm volatile ("bar.brars\n":::"memory") -#define __smp_wmb() asm volatile ("bar.bwaws\n":::"memory") -#endif /* CONFIG_SMP */ - -#define sync_is() asm volatile ("sync.is\n":::"memory") - -#else /* !CONFIG_CPU_HAS_CACHEV2 */ -#define mb() asm volatile ("sync\n":::"memory") +/* + * Using three sync.is to prevent speculative PTW + */ +#define sync_is() asm volatile ("sync.is\nsync.is\nsync.is\n":::"memory") #endif #include diff --git a/arch/csky/include/asm/bitops.h b/arch/csky/include/asm/bitops.h index 43b9838bff63d153a1942c1d292f8bc81af70599..91818787d860925defd3fc71cb3cefe87fe02d17 100644 --- a/arch/csky/include/asm/bitops.h +++ b/arch/csky/include/asm/bitops.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_BITOPS_H #define __ASM_CSKY_BITOPS_H diff --git a/arch/csky/include/asm/bug.h b/arch/csky/include/asm/bug.h index 33ebd16b9c7874863d0b59e9361e59116040c5a4..03f1a5f9184a68cc634be46a072c8338e497cf3b 100644 --- a/arch/csky/include/asm/bug.h +++ b/arch/csky/include/asm/bug.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_BUG_H #define __ASM_CSKY_BUG_H @@ -21,6 +20,8 @@ do { \ struct pt_regs; void die(struct pt_regs *regs, const char *str); +void do_trap(struct pt_regs *regs, int signo, int code, unsigned long addr); + void show_regs(struct pt_regs *regs); void show_code(struct pt_regs *regs); diff --git a/arch/csky/include/asm/cacheflush.h b/arch/csky/include/asm/cacheflush.h index f0b8f25429a27f723c191888f624b53c0d778529..d0f9eafe8988ea1f5187f4b61e15f6c1d0bd981b 100644 --- a/arch/csky/include/asm/cacheflush.h +++ b/arch/csky/include/asm/cacheflush.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_CACHEFLUSH_H #define __ASM_CSKY_CACHEFLUSH_H diff --git a/arch/csky/include/asm/checksum.h b/arch/csky/include/asm/checksum.h index 7685824291b1dcebce99045b2c1e57f19e7f61d5..aa12ef4b90806c26e017196e7aeea973382dd400 100644 --- a/arch/csky/include/asm/checksum.h +++ b/arch/csky/include/asm/checksum.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_CHECKSUM_H #define __ASM_CSKY_CHECKSUM_H diff --git a/arch/csky/include/asm/clocksource.h b/arch/csky/include/asm/clocksource.h new file mode 100644 index 0000000000000000000000000000000000000000..54da0e49efa146f6ba2d755b7418850fc514f01f --- /dev/null +++ b/arch/csky/include/asm/clocksource.h @@ -0,0 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef __ASM_VDSO_CSKY_CLOCKSOURCE_H +#define __ASM_VDSO_CSKY_CLOCKSOURCE_H + +#include + +#endif diff --git a/arch/csky/include/asm/cmpxchg.h b/arch/csky/include/asm/cmpxchg.h index 89224530a0eed162e970bbc877576d7d899c1535..dabc8e46ce7b400c484de3118a4c7b3c4d6320d2 100644 --- a/arch/csky/include/asm/cmpxchg.h +++ b/arch/csky/include/asm/cmpxchg.h @@ -3,12 +3,12 @@ #ifndef __ASM_CSKY_CMPXCHG_H #define __ASM_CSKY_CMPXCHG_H -#ifdef CONFIG_CPU_HAS_LDSTEX +#ifdef CONFIG_SMP #include extern void __bad_xchg(void); -#define __xchg(new, ptr, size) \ +#define __xchg_relaxed(new, ptr, size) \ ({ \ __typeof__(ptr) __ptr = (ptr); \ __typeof__(new) __new = (new); \ @@ -16,7 +16,6 @@ extern void __bad_xchg(void); unsigned long tmp; \ switch (size) { \ case 4: \ - smp_mb(); \ asm volatile ( \ "1: ldex.w %0, (%3) \n" \ " mov %1, %2 \n" \ @@ -25,7 +24,6 @@ extern void __bad_xchg(void); : "=&r" (__ret), "=&r" (tmp) \ : "r" (__new), "r"(__ptr) \ :); \ - smp_mb(); \ break; \ default: \ __bad_xchg(); \ @@ -33,9 +31,10 @@ extern void __bad_xchg(void); __ret; \ }) -#define xchg(ptr, x) (__xchg((x), (ptr), sizeof(*(ptr)))) +#define xchg_relaxed(ptr, x) \ + (__xchg_relaxed((x), (ptr), sizeof(*(ptr)))) -#define __cmpxchg(ptr, old, new, size) \ +#define __cmpxchg_relaxed(ptr, old, new, size) \ ({ \ __typeof__(ptr) __ptr = (ptr); \ __typeof__(new) __new = (new); \ @@ -44,7 +43,6 @@ extern void __bad_xchg(void); __typeof__(*(ptr)) __ret; \ switch (size) { \ case 4: \ - smp_mb(); \ asm volatile ( \ "1: ldex.w %0, (%3) \n" \ " cmpne %0, %4 \n" \ @@ -56,7 +54,6 @@ extern void __bad_xchg(void); : "=&r" (__ret), "=&r" (__tmp) \ : "r" (__new), "r"(__ptr), "r"(__old) \ :); \ - smp_mb(); \ break; \ default: \ __bad_xchg(); \ @@ -64,8 +61,18 @@ extern void __bad_xchg(void); __ret; \ }) -#define cmpxchg(ptr, o, n) \ - (__cmpxchg((ptr), (o), (n), sizeof(*(ptr)))) +#define cmpxchg_relaxed(ptr, o, n) \ + (__cmpxchg_relaxed((ptr), (o), (n), sizeof(*(ptr)))) + +#define cmpxchg(ptr, o, n) \ +({ \ + __typeof__(*(ptr)) __ret; \ + __smp_release_fence(); \ + __ret = cmpxchg_relaxed(ptr, o, n); \ + __smp_acquire_fence(); \ + __ret; \ +}) + #else #include #endif diff --git a/arch/csky/include/asm/elf.h b/arch/csky/include/asm/elf.h index eb2cc5a673b5acc0bbed47d2e0265c9098b55685..48b83e283ed47287c9fedc251061e534d1092a30 100644 --- a/arch/csky/include/asm/elf.h +++ b/arch/csky/include/asm/elf.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_ELF_H #define __ASM_CSKY_ELF_H diff --git a/arch/csky/include/asm/fixmap.h b/arch/csky/include/asm/fixmap.h index 4b589cc209000e09d7f424f2498352eb1507864b..49a77cbbe2a9cffbcbc4a4006404fca2349f903c 100644 --- a/arch/csky/include/asm/fixmap.h +++ b/arch/csky/include/asm/fixmap.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_FIXMAP_H #define __ASM_CSKY_FIXMAP_H diff --git a/arch/csky/include/asm/ftrace.h b/arch/csky/include/asm/ftrace.h index fae72b0b1374dfc279dea8ca23f2eee1611105a3..9b86341731b69f7776e9bf977f6bfc2fdda92820 100644 --- a/arch/csky/include/asm/ftrace.h +++ b/arch/csky/include/asm/ftrace.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_FTRACE_H #define __ASM_CSKY_FTRACE_H diff --git a/arch/csky/include/asm/futex.h b/arch/csky/include/asm/futex.h new file mode 100644 index 0000000000000000000000000000000000000000..6cfd312723fac012dc03d3a8c7e5808b9157e15d --- /dev/null +++ b/arch/csky/include/asm/futex.h @@ -0,0 +1,121 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef __ASM_CSKY_FUTEX_H +#define __ASM_CSKY_FUTEX_H + +#ifndef CONFIG_SMP +#include +#else +#include +#include +#include +#include + +#define __futex_atomic_op(insn, ret, oldval, uaddr, oparg) \ +{ \ + u32 tmp; \ + \ + __atomic_pre_full_fence(); \ + \ + __asm__ __volatile__ ( \ + "1: ldex.w %[ov], %[u] \n" \ + " "insn" \n" \ + "2: stex.w %[t], %[u] \n" \ + " bez %[t], 1b \n" \ + " br 4f \n" \ + "3: mov %[r], %[e] \n" \ + "4: \n" \ + " .section __ex_table,\"a\" \n" \ + " .balign 4 \n" \ + " .long 1b, 3b \n" \ + " .long 2b, 3b \n" \ + " .previous \n" \ + : [r] "+r" (ret), [ov] "=&r" (oldval), \ + [u] "+m" (*uaddr), [t] "=&r" (tmp) \ + : [op] "Jr" (oparg), [e] "jr" (-EFAULT) \ + : "memory"); \ + \ + __atomic_post_full_fence(); \ +} + +static inline int +arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *uaddr) +{ + int oldval = 0, ret = 0; + + if (!access_ok(uaddr, sizeof(u32))) + return -EFAULT; + + switch (op) { + case FUTEX_OP_SET: + __futex_atomic_op("mov %[t], %[ov]", + ret, oldval, uaddr, oparg); + break; + case FUTEX_OP_ADD: + __futex_atomic_op("add %[t], %[ov], %[op]", + ret, oldval, uaddr, oparg); + break; + case FUTEX_OP_OR: + __futex_atomic_op("or %[t], %[ov], %[op]", + ret, oldval, uaddr, oparg); + break; + case FUTEX_OP_ANDN: + __futex_atomic_op("and %[t], %[ov], %[op]", + ret, oldval, uaddr, ~oparg); + break; + case FUTEX_OP_XOR: + __futex_atomic_op("xor %[t], %[ov], %[op]", + ret, oldval, uaddr, oparg); + break; + default: + ret = -ENOSYS; + } + + if (!ret) + *oval = oldval; + + return ret; +} + + + +static inline int +futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, + u32 oldval, u32 newval) +{ + int ret = 0; + u32 val, tmp; + + if (!access_ok(uaddr, sizeof(u32))) + return -EFAULT; + + __atomic_pre_full_fence(); + + __asm__ __volatile__ ( + "1: ldex.w %[v], %[u] \n" + " cmpne %[v], %[ov] \n" + " bt 4f \n" + " mov %[t], %[nv] \n" + "2: stex.w %[t], %[u] \n" + " bez %[t], 1b \n" + " br 4f \n" + "3: mov %[r], %[e] \n" + "4: \n" + " .section __ex_table,\"a\" \n" + " .balign 4 \n" + " .long 1b, 3b \n" + " .long 2b, 3b \n" + " .previous \n" + : [r] "+r" (ret), [v] "=&r" (val), [u] "+m" (*uaddr), + [t] "=&r" (tmp) + : [ov] "Jr" (oldval), [nv] "Jr" (newval), [e] "Jr" (-EFAULT) + : "memory"); + + __atomic_post_full_fence(); + + *uval = val; + return ret; +} + +#endif /* CONFIG_SMP */ +#endif /* __ASM_CSKY_FUTEX_H */ diff --git a/arch/csky/include/asm/highmem.h b/arch/csky/include/asm/highmem.h index 1f4ed3f4c0d9bcaa41adcbd6a3c520dd7a9f7317..1ed810effb3d1ee90d73d50e299fa157ab9c86f1 100644 --- a/arch/csky/include/asm/highmem.h +++ b/arch/csky/include/asm/highmem.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_HIGHMEM_H #define __ASM_CSKY_HIGHMEM_H diff --git a/arch/csky/include/asm/io.h b/arch/csky/include/asm/io.h index e909587f24c56e7ce095c3013653edaca5e886a2..f82654053dc056da95f43c3402741cee9263c0b2 100644 --- a/arch/csky/include/asm/io.h +++ b/arch/csky/include/asm/io.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_IO_H #define __ASM_CSKY_IO_H diff --git a/arch/csky/include/asm/memory.h b/arch/csky/include/asm/memory.h index a65c6759f53753307774b38d7e22d02939f6bde5..d12179801ae3cff612749a3624c2bc2ed3b95404 100644 --- a/arch/csky/include/asm/memory.h +++ b/arch/csky/include/asm/memory.h @@ -10,7 +10,7 @@ #define FIXADDR_TOP _AC(0xffffc000, UL) #define PKMAP_BASE _AC(0xff800000, UL) -#define VMALLOC_START _AC(0xc0008000, UL) +#define VMALLOC_START (PAGE_OFFSET + LOWMEM_LIMIT + (PAGE_SIZE * 8)) #define VMALLOC_END (PKMAP_BASE - (PAGE_SIZE * 2)) #ifdef CONFIG_HAVE_TCM diff --git a/arch/csky/include/asm/mmu.h b/arch/csky/include/asm/mmu.h index 26fbb1d15df08ea711f73efe26b39bc356312cc0..d78321901d06306ff92059a127806c1a38971563 100644 --- a/arch/csky/include/asm/mmu.h +++ b/arch/csky/include/asm/mmu.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_MMU_H #define __ASM_CSKY_MMU_H diff --git a/arch/csky/include/asm/mmu_context.h b/arch/csky/include/asm/mmu_context.h index b227d29393a8fc5b92c6ea753d7444f4ea6701f2..95d99b30792c8067c0229514f8c149dd86185607 100644 --- a/arch/csky/include/asm/mmu_context.h +++ b/arch/csky/include/asm/mmu_context.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_MMU_CONTEXT_H #define __ASM_CSKY_MMU_CONTEXT_H @@ -14,12 +13,6 @@ #include #include -#define TLBMISS_HANDLER_SETUP_PGD(pgd) \ - setup_pgd(__pa(pgd), false) - -#define TLBMISS_HANDLER_SETUP_PGD_KERNEL(pgd) \ - setup_pgd(__pa(pgd), true) - #define ASID_MASK ((1 << CONFIG_CPU_ASID_BITS) - 1) #define cpu_asid(mm) (atomic64_read(&mm->context.asid) & ASID_MASK) @@ -36,8 +29,7 @@ switch_mm(struct mm_struct *prev, struct mm_struct *next, if (prev != next) check_and_switch_context(next, cpu); - TLBMISS_HANDLER_SETUP_PGD(next->pgd); - write_mmu_entryhi(next->context.asid.counter); + setup_pgd(next->pgd, next->context.asid.counter); flush_icache_deferred(next); } diff --git a/arch/csky/include/asm/page.h b/arch/csky/include/asm/page.h index 9b98bf31d57cec0aae47a219dbe6b3fd2db8c263..3b91fc3cf36f9e5b12d1dfdade4a59684f9729b1 100644 --- a/arch/csky/include/asm/page.h +++ b/arch/csky/include/asm/page.h @@ -24,7 +24,7 @@ * address region. We use them mapping kernel 1GB direct-map address area and * for more than 1GB of memory we use highmem. */ -#define PAGE_OFFSET 0x80000000 +#define PAGE_OFFSET CONFIG_PAGE_OFFSET #define SSEG_SIZE 0x20000000 #define LOWMEM_LIMIT (SSEG_SIZE * 2) diff --git a/arch/csky/include/asm/perf_event.h b/arch/csky/include/asm/perf_event.h index 572093e110017156b7052cd2a5e6c80615f2b79b..249905d8a4e893bd1126a7ab1d11969a28c5e860 100644 --- a/arch/csky/include/asm/perf_event.h +++ b/arch/csky/include/asm/perf_event.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_PERF_EVENT_H #define __ASM_CSKY_PERF_EVENT_H diff --git a/arch/csky/include/asm/pgalloc.h b/arch/csky/include/asm/pgalloc.h index d58d8146b729c4f8259a2bc1ce2cd805e5b39f41..cd211aabbefd1462bc00537b6c4f8afec8d97f9d 100644 --- a/arch/csky/include/asm/pgalloc.h +++ b/arch/csky/include/asm/pgalloc.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_PGALLOC_H #define __ASM_CSKY_PGALLOC_H @@ -71,7 +70,7 @@ do { \ } while (0) extern void pagetable_init(void); -extern void pre_mmu_init(void); +extern void mmu_init(unsigned long min_pfn, unsigned long max_pfn); extern void pre_trap_init(void); #endif /* __ASM_CSKY_PGALLOC_H */ diff --git a/arch/csky/include/asm/pgtable.h b/arch/csky/include/asm/pgtable.h index 2002cb7f105361e1fd57fa8037afbb44da239325..0d60367b6bfa7db06c10fe639e6cdd8f75fff48b 100644 --- a/arch/csky/include/asm/pgtable.h +++ b/arch/csky/include/asm/pgtable.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_PGTABLE_H #define __ASM_CSKY_PGTABLE_H @@ -14,7 +13,7 @@ #define PGDIR_SIZE (1UL << PGDIR_SHIFT) #define PGDIR_MASK (~(PGDIR_SIZE-1)) -#define USER_PTRS_PER_PGD (0x80000000UL/PGDIR_SIZE) +#define USER_PTRS_PER_PGD (PAGE_OFFSET/PGDIR_SIZE) #define FIRST_USER_ADDRESS 0UL /* @@ -34,23 +33,13 @@ #define pmd_page(pmd) (pfn_to_page(pmd_phys(pmd) >> PAGE_SHIFT)) #define pte_clear(mm, addr, ptep) set_pte((ptep), \ - (((unsigned int) addr & PAGE_OFFSET) ? __pte(_PAGE_GLOBAL) : __pte(0))) + (((unsigned int) addr >= PAGE_OFFSET) ? __pte(_PAGE_GLOBAL) : __pte(0))) #define pte_none(pte) (!(pte_val(pte) & ~_PAGE_GLOBAL)) #define pte_present(pte) (pte_val(pte) & _PAGE_PRESENT) #define pte_pfn(x) ((unsigned long)((x).pte_low >> PAGE_SHIFT)) #define pfn_pte(pfn, prot) __pte(((unsigned long long)(pfn) << PAGE_SHIFT) \ | pgprot_val(prot)) -#define __READABLE (_PAGE_READ | _PAGE_VALID | _PAGE_ACCESSED) -#define __WRITEABLE (_PAGE_WRITE | _PAGE_DIRTY | _PAGE_MODIFIED) - -#define _PAGE_CHG_MASK (PAGE_MASK | _PAGE_ACCESSED | _PAGE_MODIFIED | \ - _CACHE_MASK) - -#define __swp_type(x) (((x).val >> 4) & 0xff) -#define __swp_offset(x) ((x).val >> 12) -#define __swp_entry(type, offset) ((swp_entry_t) {((type) << 4) | \ - ((offset) << 12) }) #define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val(pte) }) #define __swp_entry_to_pte(x) ((pte_t) { (x).val }) @@ -59,41 +48,52 @@ pgprot_val(pgprot)) /* - * CSKY can't do page protection for execute, and considers that the same like - * read. Also, write permissions imply read permissions. This is the closest - * we can get by reasonable means.. + * C-SKY only has VALID and DIRTY bit in hardware. So we need to use the + * two bits emulate PRESENT, READ, WRITE, EXEC, MODIFIED, ACCESSED. */ -#define PAGE_NONE __pgprot(_PAGE_PRESENT | _CACHE_CACHED) -#define PAGE_SHARED __pgprot(_PAGE_PRESENT | _PAGE_READ | _PAGE_WRITE | \ +#define _PAGE_BASE (_PAGE_PRESENT | _PAGE_ACCESSED) + +#define PAGE_NONE __pgprot(_PAGE_PROT_NONE) +#define PAGE_READ __pgprot(_PAGE_BASE | _PAGE_READ | \ _CACHE_CACHED) -#define PAGE_COPY __pgprot(_PAGE_PRESENT | _PAGE_READ | _CACHE_CACHED) -#define PAGE_READONLY __pgprot(_PAGE_PRESENT | _PAGE_READ | _CACHE_CACHED) -#define PAGE_KERNEL __pgprot(_PAGE_PRESENT | __READABLE | __WRITEABLE | \ - _PAGE_GLOBAL | _CACHE_CACHED) -#define PAGE_USERIO __pgprot(_PAGE_PRESENT | _PAGE_READ | _PAGE_WRITE | \ +#define PAGE_WRITE __pgprot(_PAGE_BASE | _PAGE_READ | _PAGE_WRITE | \ _CACHE_CACHED) +#define PAGE_SHARED PAGE_WRITE + +#define PAGE_KERNEL __pgprot(_PAGE_BASE | _PAGE_READ | _PAGE_VALID | \ + _PAGE_WRITE | _PAGE_DIRTY | _PAGE_MODIFIED | \ + _PAGE_GLOBAL | \ + _CACHE_CACHED) + +#define _PAGE_IOREMAP (_PAGE_BASE | _PAGE_READ | _PAGE_VALID | \ + _PAGE_WRITE | _PAGE_DIRTY | _PAGE_MODIFIED | \ + _PAGE_GLOBAL | \ + _CACHE_UNCACHED | _PAGE_SO) + +#define _PAGE_CHG_MASK (~(unsigned long) \ + (_PAGE_PRESENT | _PAGE_READ | _PAGE_WRITE | \ + _CACHE_MASK | _PAGE_GLOBAL)) -#define _PAGE_IOREMAP \ - (_PAGE_PRESENT | __READABLE | __WRITEABLE | _PAGE_GLOBAL | \ - _CACHE_UNCACHED | _PAGE_SO) +#define MAX_SWAPFILES_CHECK() \ + BUILD_BUG_ON(MAX_SWAPFILES_SHIFT != 5) #define __P000 PAGE_NONE -#define __P001 PAGE_READONLY -#define __P010 PAGE_COPY -#define __P011 PAGE_COPY -#define __P100 PAGE_READONLY -#define __P101 PAGE_READONLY -#define __P110 PAGE_COPY -#define __P111 PAGE_COPY +#define __P001 PAGE_READ +#define __P010 PAGE_READ +#define __P011 PAGE_READ +#define __P100 PAGE_READ +#define __P101 PAGE_READ +#define __P110 PAGE_READ +#define __P111 PAGE_READ #define __S000 PAGE_NONE -#define __S001 PAGE_READONLY -#define __S010 PAGE_SHARED -#define __S011 PAGE_SHARED -#define __S100 PAGE_READONLY -#define __S101 PAGE_READONLY -#define __S110 PAGE_SHARED -#define __S111 PAGE_SHARED +#define __S001 PAGE_READ +#define __S010 PAGE_WRITE +#define __S011 PAGE_WRITE +#define __S100 PAGE_READ +#define __S101 PAGE_READ +#define __S110 PAGE_WRITE +#define __S111 PAGE_WRITE extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)]; #define ZERO_PAGE(vaddr) (virt_to_page(empty_zero_page)) diff --git a/arch/csky/include/asm/processor.h b/arch/csky/include/asm/processor.h index 4800f6563abbfb74ec346741d358cb36daea8eae..9e933021fe8e08335ebfb26dd4a902e2588c2e78 100644 --- a/arch/csky/include/asm/processor.h +++ b/arch/csky/include/asm/processor.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_PROCESSOR_H #define __ASM_CSKY_PROCESSOR_H @@ -28,7 +27,7 @@ extern struct cpuinfo_csky cpu_data[]; * for a 64 bit kernel expandable to 8192EB, of which the current CSKY * implementations will "only" be able to use 1TB ... */ -#define TASK_SIZE 0x7fff8000UL +#define TASK_SIZE (PAGE_OFFSET - (PAGE_SIZE * 8)) #ifdef __KERNEL__ #define STACK_TOP TASK_SIZE diff --git a/arch/csky/include/asm/ptrace.h b/arch/csky/include/asm/ptrace.h index 91ceb1b454c99df3807993520ff87e14ff137ede..4202aab6df42692d771e9edd980ebdb22af6ddad 100644 --- a/arch/csky/include/asm/ptrace.h +++ b/arch/csky/include/asm/ptrace.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_PTRACE_H #define __ASM_CSKY_PTRACE_H diff --git a/arch/csky/include/asm/segment.h b/arch/csky/include/asm/segment.h index 79ede9b1a6467f7d56fedc63803db1248d157714..589e8321dc14212e25a4347424a9e432298030f6 100644 --- a/arch/csky/include/asm/segment.h +++ b/arch/csky/include/asm/segment.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_SEGMENT_H #define __ASM_CSKY_SEGMENT_H @@ -10,7 +9,7 @@ typedef struct { #define KERNEL_DS ((mm_segment_t) { 0xFFFFFFFF }) -#define USER_DS ((mm_segment_t) { 0x80000000UL }) +#define USER_DS ((mm_segment_t) { PAGE_OFFSET }) #define get_fs() (current_thread_info()->addr_limit) #define set_fs(x) (current_thread_info()->addr_limit = (x)) #define uaccess_kernel() (get_fs().seg == KERNEL_DS.seg) diff --git a/arch/csky/include/asm/shmparam.h b/arch/csky/include/asm/shmparam.h index efafe4c79fedef24817ada8833c760df885508cd..2fe6cea0dae9e049e4591dded6fcf61a1b0090d7 100644 --- a/arch/csky/include/asm/shmparam.h +++ b/arch/csky/include/asm/shmparam.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_SHMPARAM_H #define __ASM_CSKY_SHMPARAM_H diff --git a/arch/csky/include/asm/spinlock.h b/arch/csky/include/asm/spinlock.h index 7cf3f2b34ceaff1c00f8120d84680697e9a76fc5..69f5aa249c5f9004610b189130ce93c8a58a558b 100644 --- a/arch/csky/include/asm/spinlock.h +++ b/arch/csky/include/asm/spinlock.h @@ -6,8 +6,6 @@ #include #include -#ifdef CONFIG_QUEUED_RWLOCKS - /* * Ticket-based spin-locking. */ @@ -88,169 +86,4 @@ static inline int arch_spin_is_contended(arch_spinlock_t *lock) #include -/* See include/linux/spinlock.h */ -#define smp_mb__after_spinlock() smp_mb() - -#else /* CONFIG_QUEUED_RWLOCKS */ - -/* - * Test-and-set spin-locking. - */ -static inline void arch_spin_lock(arch_spinlock_t *lock) -{ - u32 *p = &lock->lock; - u32 tmp; - - asm volatile ( - "1: ldex.w %0, (%1) \n" - " bnez %0, 1b \n" - " movi %0, 1 \n" - " stex.w %0, (%1) \n" - " bez %0, 1b \n" - : "=&r" (tmp) - : "r"(p) - : "cc"); - smp_mb(); -} - -static inline void arch_spin_unlock(arch_spinlock_t *lock) -{ - smp_mb(); - WRITE_ONCE(lock->lock, 0); -} - -static inline int arch_spin_trylock(arch_spinlock_t *lock) -{ - u32 *p = &lock->lock; - u32 tmp; - - asm volatile ( - "1: ldex.w %0, (%1) \n" - " bnez %0, 2f \n" - " movi %0, 1 \n" - " stex.w %0, (%1) \n" - " bez %0, 1b \n" - " movi %0, 0 \n" - "2: \n" - : "=&r" (tmp) - : "r"(p) - : "cc"); - - if (!tmp) - smp_mb(); - - return !tmp; -} - -#define arch_spin_is_locked(x) (READ_ONCE((x)->lock) != 0) - -/* - * read lock/unlock/trylock - */ -static inline void arch_read_lock(arch_rwlock_t *lock) -{ - u32 *p = &lock->lock; - u32 tmp; - - asm volatile ( - "1: ldex.w %0, (%1) \n" - " blz %0, 1b \n" - " addi %0, 1 \n" - " stex.w %0, (%1) \n" - " bez %0, 1b \n" - : "=&r" (tmp) - : "r"(p) - : "cc"); - smp_mb(); -} - -static inline void arch_read_unlock(arch_rwlock_t *lock) -{ - u32 *p = &lock->lock; - u32 tmp; - - smp_mb(); - asm volatile ( - "1: ldex.w %0, (%1) \n" - " subi %0, 1 \n" - " stex.w %0, (%1) \n" - " bez %0, 1b \n" - : "=&r" (tmp) - : "r"(p) - : "cc"); -} - -static inline int arch_read_trylock(arch_rwlock_t *lock) -{ - u32 *p = &lock->lock; - u32 tmp; - - asm volatile ( - "1: ldex.w %0, (%1) \n" - " blz %0, 2f \n" - " addi %0, 1 \n" - " stex.w %0, (%1) \n" - " bez %0, 1b \n" - " movi %0, 0 \n" - "2: \n" - : "=&r" (tmp) - : "r"(p) - : "cc"); - - if (!tmp) - smp_mb(); - - return !tmp; -} - -/* - * write lock/unlock/trylock - */ -static inline void arch_write_lock(arch_rwlock_t *lock) -{ - u32 *p = &lock->lock; - u32 tmp; - - asm volatile ( - "1: ldex.w %0, (%1) \n" - " bnez %0, 1b \n" - " subi %0, 1 \n" - " stex.w %0, (%1) \n" - " bez %0, 1b \n" - : "=&r" (tmp) - : "r"(p) - : "cc"); - smp_mb(); -} - -static inline void arch_write_unlock(arch_rwlock_t *lock) -{ - smp_mb(); - WRITE_ONCE(lock->lock, 0); -} - -static inline int arch_write_trylock(arch_rwlock_t *lock) -{ - u32 *p = &lock->lock; - u32 tmp; - - asm volatile ( - "1: ldex.w %0, (%1) \n" - " bnez %0, 2f \n" - " subi %0, 1 \n" - " stex.w %0, (%1) \n" - " bez %0, 1b \n" - " movi %0, 0 \n" - "2: \n" - : "=&r" (tmp) - : "r"(p) - : "cc"); - - if (!tmp) - smp_mb(); - - return !tmp; -} - -#endif /* CONFIG_QUEUED_RWLOCKS */ #endif /* __ASM_CSKY_SPINLOCK_H */ diff --git a/arch/csky/include/asm/spinlock_types.h b/arch/csky/include/asm/spinlock_types.h index 88b82438b1820c0134e035a8b03367cb88ae94a5..8ff0f6ff3a006eb398a542052cc26a9949622815 100644 --- a/arch/csky/include/asm/spinlock_types.h +++ b/arch/csky/include/asm/spinlock_types.h @@ -22,16 +22,6 @@ typedef struct { #define __ARCH_SPIN_LOCK_UNLOCKED { { 0 } } -#ifdef CONFIG_QUEUED_RWLOCKS #include -#else /* CONFIG_NR_CPUS > 2 */ - -typedef struct { - u32 lock; -} arch_rwlock_t; - -#define __ARCH_RW_LOCK_UNLOCKED { 0 } - -#endif /* CONFIG_QUEUED_RWLOCKS */ #endif /* __ASM_CSKY_SPINLOCK_TYPES_H */ diff --git a/arch/csky/include/asm/string.h b/arch/csky/include/asm/string.h index 73142de18355b9e1ca70aa32d64b9c67c58d01f1..a0d81e9d6b8f64ab5ab1636b0311801493b93790 100644 --- a/arch/csky/include/asm/string.h +++ b/arch/csky/include/asm/string.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef _CSKY_STRING_MM_H_ #define _CSKY_STRING_MM_H_ diff --git a/arch/csky/include/asm/switch_to.h b/arch/csky/include/asm/switch_to.h index 35a39e88933da6836967cfdfcffd5184237644e3..731e466415e252de544fe91789ef3149fe538bf1 100644 --- a/arch/csky/include/asm/switch_to.h +++ b/arch/csky/include/asm/switch_to.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_SWITCH_TO_H #define __ASM_CSKY_SWITCH_TO_H diff --git a/arch/csky/include/asm/syscalls.h b/arch/csky/include/asm/syscalls.h index 5d48e5e0082e29a02677603ca96af351cc0043da..ea9ce6138b9b5877fe7ffe8ff2d44baffda3aae5 100644 --- a/arch/csky/include/asm/syscalls.h +++ b/arch/csky/include/asm/syscalls.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_SYSCALLS_H #define __ASM_CSKY_SYSCALLS_H diff --git a/arch/csky/include/asm/thread_info.h b/arch/csky/include/asm/thread_info.h index 21456a3737c2afa7288b303e6d597d8c16f5ae62..8c349a8f904d90b7352ef512e7b004b17c7c4406 100644 --- a/arch/csky/include/asm/thread_info.h +++ b/arch/csky/include/asm/thread_info.h @@ -1,12 +1,10 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef _ASM_CSKY_THREAD_INFO_H #define _ASM_CSKY_THREAD_INFO_H #ifndef __ASSEMBLY__ -#include #include #include #include diff --git a/arch/csky/include/asm/tlb.h b/arch/csky/include/asm/tlb.h index fdff9b8d70c811256ce73534c4d32fe5bddda719..3498e65f59f8eb15e45b24d8dfa9844d4b4cf940 100644 --- a/arch/csky/include/asm/tlb.h +++ b/arch/csky/include/asm/tlb.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_TLB_H #define __ASM_CSKY_TLB_H diff --git a/arch/csky/include/asm/tlbflush.h b/arch/csky/include/asm/tlbflush.h index 6845b06677035dab74c1c87ef0c73fe474705bfe..407160b4fde7da0a91b78f618cb94a1f5ec878b7 100644 --- a/arch/csky/include/asm/tlbflush.h +++ b/arch/csky/include/asm/tlbflush.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_TLBFLUSH_H #define __ASM_TLBFLUSH_H diff --git a/arch/csky/include/asm/traps.h b/arch/csky/include/asm/traps.h index 1c081805b96226d2e6327e54612e7cdd5e1c4213..421a4195e2feb3bbee92f341bd00f0ecf96e7245 100644 --- a/arch/csky/include/asm/traps.h +++ b/arch/csky/include/asm/traps.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_TRAPS_H #define __ASM_CSKY_TRAPS_H diff --git a/arch/csky/include/asm/uaccess.h b/arch/csky/include/asm/uaccess.h index 1633ffe5ae15a7ef2d3c44c1211791cbfafea788..3dec272e1fa31366a8364dd0ed7fad27e986f120 100644 --- a/arch/csky/include/asm/uaccess.h +++ b/arch/csky/include/asm/uaccess.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_UACCESS_H #define __ASM_CSKY_UACCESS_H diff --git a/arch/csky/include/asm/unistd.h b/arch/csky/include/asm/unistd.h index da7a182956157860626d6426a8f4c8497e33b1be..9cf97de9a26df0acad24137cdc3f531a81efeb07 100644 --- a/arch/csky/include/asm/unistd.h +++ b/arch/csky/include/asm/unistd.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #include diff --git a/arch/csky/include/asm/vdso.h b/arch/csky/include/asm/vdso.h index d963d691f3a1f75e18348639aa0181d173df8540..eb5142f9c564c86cc92335eeb71c373493336a76 100644 --- a/arch/csky/include/asm/vdso.h +++ b/arch/csky/include/asm/vdso.h @@ -3,10 +3,25 @@ #ifndef __ASM_CSKY_VDSO_H #define __ASM_CSKY_VDSO_H -#include +#include -struct csky_vdso { - unsigned short rt_signal_retcode[4]; +#ifndef GENERIC_TIME_VSYSCALL +struct vdso_data { }; +#endif + +/* + * The VDSO symbols are mapped into Linux so we can just use regular symbol + * addressing to get their offsets in userspace. The symbols are mapped at an + * offset of 0, but since the linker must support setting weak undefined + * symbols to the absolute address 0 it also happens to support other low + * addresses even when the code model suggests those low addresses would not + * otherwise be availiable. + */ +#define VDSO_SYMBOL(base, name) \ +({ \ + extern const char __vdso_##name[]; \ + (void __user *)((unsigned long)(base) + __vdso_##name); \ +}) #endif /* __ASM_CSKY_VDSO_H */ diff --git a/arch/csky/include/asm/vdso/clocksource.h b/arch/csky/include/asm/vdso/clocksource.h new file mode 100644 index 0000000000000000000000000000000000000000..dfca7b4724b798ea0e0f05ea7559a4570f8fa5f7 --- /dev/null +++ b/arch/csky/include/asm/vdso/clocksource.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef __ASM_VDSO_CSKY_CLOCKSOURCE_H +#define __ASM_VDSO_CSKY_CLOCKSOURCE_H + +#define VDSO_ARCH_CLOCKMODES \ + VDSO_CLOCKMODE_ARCHTIMER + +#endif /* __ASM_VDSO_CSKY_CLOCKSOURCE_H */ diff --git a/arch/csky/include/asm/vdso/gettimeofday.h b/arch/csky/include/asm/vdso/gettimeofday.h new file mode 100644 index 0000000000000000000000000000000000000000..6c4f1446944f94dc3f02a193260998231edea566 --- /dev/null +++ b/arch/csky/include/asm/vdso/gettimeofday.h @@ -0,0 +1,114 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef __ASM_VDSO_CSKY_GETTIMEOFDAY_H +#define __ASM_VDSO_CSKY_GETTIMEOFDAY_H + +#ifndef __ASSEMBLY__ + +#include +#include +#include +#include + +#define VDSO_HAS_CLOCK_GETRES 1 + +static __always_inline +int gettimeofday_fallback(struct __kernel_old_timeval *_tv, + struct timezone *_tz) +{ + register struct __kernel_old_timeval *tv asm("a0") = _tv; + register struct timezone *tz asm("a1") = _tz; + register long ret asm("a0"); + register long nr asm(syscallid) = __NR_gettimeofday; + + asm volatile ("trap 0\n" + : "=r" (ret) + : "r"(tv), "r"(tz), "r"(nr) + : "memory"); + + return ret; +} + +static __always_inline +long clock_gettime_fallback(clockid_t _clkid, struct __kernel_timespec *_ts) +{ + register clockid_t clkid asm("a0") = _clkid; + register struct __kernel_timespec *ts asm("a1") = _ts; + register long ret asm("a0"); + register long nr asm(syscallid) = __NR_clock_gettime64; + + asm volatile ("trap 0\n" + : "=r" (ret) + : "r"(clkid), "r"(ts), "r"(nr) + : "memory"); + + return ret; +} + +static __always_inline +long clock_gettime32_fallback(clockid_t _clkid, struct old_timespec32 *_ts) +{ + register clockid_t clkid asm("a0") = _clkid; + register struct old_timespec32 *ts asm("a1") = _ts; + register long ret asm("a0"); + register long nr asm(syscallid) = __NR_clock_gettime; + + asm volatile ("trap 0\n" + : "=r" (ret) + : "r"(clkid), "r"(ts), "r"(nr) + : "memory"); + + return ret; +} + +static __always_inline +int clock_getres_fallback(clockid_t _clkid, struct __kernel_timespec *_ts) +{ + register clockid_t clkid asm("a0") = _clkid; + register struct __kernel_timespec *ts asm("a1") = _ts; + register long ret asm("a0"); + register long nr asm(syscallid) = __NR_clock_getres_time64; + + asm volatile ("trap 0\n" + : "=r" (ret) + : "r"(clkid), "r"(ts), "r"(nr) + : "memory"); + + return ret; +} + +static __always_inline +int clock_getres32_fallback(clockid_t _clkid, struct old_timespec32 *_ts) +{ + register clockid_t clkid asm("a0") = _clkid; + register struct old_timespec32 *ts asm("a1") = _ts; + register long ret asm("a0"); + register long nr asm(syscallid) = __NR_clock_getres; + + asm volatile ("trap 0\n" + : "=r" (ret) + : "r"(clkid), "r"(ts), "r"(nr) + : "memory"); + + return ret; +} + +uint64_t csky_pmu_read_cc(void); +static __always_inline u64 __arch_get_hw_counter(s32 clock_mode, + const struct vdso_data *vd) +{ +#ifdef CONFIG_CSKY_PMU_V1 + return csky_pmu_read_cc(); +#else + return 0; +#endif +} + +static __always_inline const struct vdso_data *__arch_get_vdso_data(void) +{ + return _vdso_data; +} + +#endif /* !__ASSEMBLY__ */ + +#endif /* __ASM_VDSO_CSKY_GETTIMEOFDAY_H */ diff --git a/arch/csky/include/asm/vdso/processor.h b/arch/csky/include/asm/vdso/processor.h new file mode 100644 index 0000000000000000000000000000000000000000..39a6b561d0cc846cfb39a962c17e46104ae04807 --- /dev/null +++ b/arch/csky/include/asm/vdso/processor.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#ifndef __ASM_VDSO_CSKY_PROCESSOR_H +#define __ASM_VDSO_CSKY_PROCESSOR_H + +#ifndef __ASSEMBLY__ + +#define cpu_relax() barrier() + +#endif /* __ASSEMBLY__ */ + +#endif /* __ASM_VDSO_CSKY_PROCESSOR_H */ diff --git a/arch/csky/include/asm/vdso/vsyscall.h b/arch/csky/include/asm/vdso/vsyscall.h new file mode 100644 index 0000000000000000000000000000000000000000..c276211a7c4d56c48c0330e309c6e5a5c5c09dc9 --- /dev/null +++ b/arch/csky/include/asm/vdso/vsyscall.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef __ASM_VDSO_CSKY_VSYSCALL_H +#define __ASM_VDSO_CSKY_VSYSCALL_H + +#ifndef __ASSEMBLY__ + +#include + +extern struct vdso_data *vdso_data; + +static __always_inline struct vdso_data *__csky_get_k_vdso_data(void) +{ + return vdso_data; +} +#define __arch_get_k_vdso_data __csky_get_k_vdso_data + +#include + +#endif /* !__ASSEMBLY__ */ + +#endif /* __ASM_VDSO_CSKY_VSYSCALL_H */ diff --git a/arch/csky/include/uapi/asm/byteorder.h b/arch/csky/include/uapi/asm/byteorder.h index d150cd66487333a1c9b3f3b62ca73fd04c6823f9..1aedd513b65ac37cb6666242356d0892a394d063 100644 --- a/arch/csky/include/uapi/asm/byteorder.h +++ b/arch/csky/include/uapi/asm/byteorder.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_BYTEORDER_H #define __ASM_CSKY_BYTEORDER_H diff --git a/arch/csky/include/uapi/asm/perf_regs.h b/arch/csky/include/uapi/asm/perf_regs.h index 49d4e147a5596ff8f5683b3373b21dcaf59ba4d2..d0a8ac6a1b77d4ae8828ad98443763659e1e19ff 100644 --- a/arch/csky/include/uapi/asm/perf_regs.h +++ b/arch/csky/include/uapi/asm/perf_regs.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -// Copyright (C) 2019 Hangzhou C-SKY Microsystems co.,ltd. #ifndef _ASM_CSKY_PERF_REGS_H #define _ASM_CSKY_PERF_REGS_H diff --git a/arch/csky/include/uapi/asm/ptrace.h b/arch/csky/include/uapi/asm/ptrace.h index 66b2268e324eb56ca618b9df15ebfe179ade5994..3be9c14334a6e342ac536b25d4444a0617216dc4 100644 --- a/arch/csky/include/uapi/asm/ptrace.h +++ b/arch/csky/include/uapi/asm/ptrace.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef _CSKY_PTRACE_H #define _CSKY_PTRACE_H diff --git a/arch/csky/include/uapi/asm/sigcontext.h b/arch/csky/include/uapi/asm/sigcontext.h index 670c020f2cb89da19e45758c99b8220c27f47314..859afb60247711b34f9a3422bb5bfbadf99123ef 100644 --- a/arch/csky/include/uapi/asm/sigcontext.h +++ b/arch/csky/include/uapi/asm/sigcontext.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #ifndef __ASM_CSKY_SIGCONTEXT_H #define __ASM_CSKY_SIGCONTEXT_H diff --git a/arch/csky/include/uapi/asm/unistd.h b/arch/csky/include/uapi/asm/unistd.h index ba40189297338d1de36b9cff54d76f0746b344c6..7ff6a2466af10d9655433503376aa90de0a3911d 100644 --- a/arch/csky/include/uapi/asm/unistd.h +++ b/arch/csky/include/uapi/asm/unistd.h @@ -1,5 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. #define __ARCH_WANT_STAT64 #define __ARCH_WANT_NEW_STAT diff --git a/arch/csky/kernel/Makefile b/arch/csky/kernel/Makefile index 37f37c0e934a8f36c95f7e2e0654ecf63b08e503..6c0f36010ed0509a8b1a89dbc0a518a36146da2b 100644 --- a/arch/csky/kernel/Makefile +++ b/arch/csky/kernel/Makefile @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0-only extra-y := head.o vmlinux.lds -obj-y += entry.o atomic.o signal.o traps.o irq.o time.o vdso.o +obj-y += entry.o atomic.o signal.o traps.o irq.o time.o vdso.o vdso/ obj-y += power.o syscall.o syscall_table.o setup.o obj-y += process.o cpu-probe.o ptrace.o stacktrace.o obj-y += probes/ diff --git a/arch/csky/kernel/atomic.S b/arch/csky/kernel/atomic.S index 3821ef9b75672d8a5af90839ffa7f95cfdb4da50..e73e548f78559c7a99f99a998b53bd9b43d26971 100644 --- a/arch/csky/kernel/atomic.S +++ b/arch/csky/kernel/atomic.S @@ -14,6 +14,10 @@ */ ENTRY(csky_cmpxchg) USPTOKSP + + RD_MEH a3 + WR_MEH a3 + mfcr a3, epc addi a3, TRAP0_SIZE @@ -36,11 +40,11 @@ ENTRY(csky_cmpxchg) 2: sync.is #else -1: +GLOBAL(csky_cmpxchg_ldw) ldw a3, (a2) cmpne a0, a3 bt16 3f -2: +GLOBAL(csky_cmpxchg_stw) stw a1, (a2) 3: #endif @@ -55,19 +59,3 @@ ENTRY(csky_cmpxchg) KSPTOUSP rte END(csky_cmpxchg) - -#ifndef CONFIG_CPU_HAS_LDSTEX -/* - * Called from tlbmodified exception - */ -ENTRY(csky_cmpxchg_fixup) - mfcr a0, epc - lrw a1, 2b - cmpne a1, a0 - bt 1f - subi a1, (2b - 1b) - stw a1, (sp, LSAVE_PC) -1: - rts -END(csky_cmpxchg_fixup) -#endif diff --git a/arch/csky/kernel/entry.S b/arch/csky/kernel/entry.S index 5a5cabd076e1664bafa95848e6dc197a8f50d16b..c1bd7a6b4ab6c1f6455d9f8d26beda6fa09953e6 100644 --- a/arch/csky/kernel/entry.S +++ b/arch/csky/kernel/entry.S @@ -13,10 +13,6 @@ #include #include -#define PTE_INDX_MSK 0xffc -#define PTE_INDX_SHIFT 10 -#define _PGDIR_SHIFT 22 - .macro zero_fp #ifdef CONFIG_STACKTRACE movi r8, 0 @@ -41,108 +37,15 @@ #endif .endm -.macro tlbop_begin name, val0, val1, val2 -ENTRY(csky_\name) - mtcr a3, ss2 - mtcr r6, ss3 - mtcr a2, ss4 - - RD_PGDR r6 - RD_MEH a3 -#ifdef CONFIG_CPU_HAS_TLBI - tlbi.vaas a3 - sync.is - - btsti a3, 31 - bf 1f - RD_PGDR_K r6 -1: -#else - bgeni a2, 31 - WR_MCIR a2 - bgeni a2, 25 - WR_MCIR a2 -#endif - bclri r6, 0 - lrw a2, va_pa_offset - ld.w a2, (a2, 0) - subu r6, a2 - bseti r6, 31 - - mov a2, a3 - lsri a2, _PGDIR_SHIFT - lsli a2, 2 - addu r6, a2 - ldw r6, (r6) - - lrw a2, va_pa_offset - ld.w a2, (a2, 0) - subu r6, a2 - bseti r6, 31 - - lsri a3, PTE_INDX_SHIFT - lrw a2, PTE_INDX_MSK - and a3, a2 - addu r6, a3 - ldw a3, (r6) - - movi a2, (_PAGE_PRESENT | \val0) - and a3, a2 - cmpne a3, a2 - bt \name - - /* First read/write the page, just update the flags */ - ldw a3, (r6) - bgeni a2, PAGE_VALID_BIT - bseti a2, PAGE_ACCESSED_BIT - bseti a2, \val1 - bseti a2, \val2 - or a3, a2 - stw a3, (r6) - - /* Some cpu tlb-hardrefill bypass the cache */ -#ifdef CONFIG_CPU_NEED_TLBSYNC - movi a2, 0x22 - bseti a2, 6 - mtcr r6, cr22 - mtcr a2, cr17 - sync -#endif - - mfcr a3, ss2 - mfcr r6, ss3 - mfcr a2, ss4 - rte -\name: - mfcr a3, ss2 - mfcr r6, ss3 - mfcr a2, ss4 +.text +ENTRY(csky_pagefault) SAVE_ALL 0 -.endm -.macro tlbop_end is_write zero_fp context_tracking - RD_MEH a2 - psrset ee, ie + psrset ee mov a0, sp - movi a1, \is_write jbsr do_page_fault jmpi ret_from_exception -.endm - -.text - -tlbop_begin tlbinvalidl, _PAGE_READ, PAGE_VALID_BIT, PAGE_ACCESSED_BIT -tlbop_end 0 - -tlbop_begin tlbinvalids, _PAGE_WRITE, PAGE_DIRTY_BIT, PAGE_MODIFIED_BIT -tlbop_end 1 - -tlbop_begin tlbmodified, _PAGE_WRITE, PAGE_DIRTY_BIT, PAGE_MODIFIED_BIT -#ifndef CONFIG_CPU_HAS_LDSTEX -jbsr csky_cmpxchg_fixup -#endif -tlbop_end 1 ENTRY(csky_systemcall) SAVE_ALL TRAP0_SIZE @@ -314,6 +217,9 @@ ENTRY(csky_trap) ENTRY(csky_get_tls) USPTOKSP + RD_MEH a0 + WR_MEH a0 + /* increase epc for continue */ mfcr a0, epc addi a0, TRAP0_SIZE diff --git a/arch/csky/kernel/head.S b/arch/csky/kernel/head.S index 17ed9d2504807dfa385f5da0ef380b635a388c99..7e3e4f15b0523b8f6124776a8291f07354f1dee6 100644 --- a/arch/csky/kernel/head.S +++ b/arch/csky/kernel/head.S @@ -21,10 +21,16 @@ END(_start) ENTRY(_start_smp_secondary) SETUP_MMU - /* copy msa1 from CPU0 */ - lrw r6, secondary_msa1 +#ifdef CONFIG_PAGE_OFFSET_80000000 + lrw r6, secondary_msa1 ld.w r6, (r6, 0) mtcr r6, cr<31, 15> +#endif + + lrw r6, secondary_pgd + ld.w r6, (r6, 0) + mtcr r6, cr<28, 15> + mtcr r6, cr<29, 15> /* set stack point */ lrw r6, secondary_stack diff --git a/arch/csky/kernel/perf_event.c b/arch/csky/kernel/perf_event.c index 1a29f1157449a96e1190b9e65c5017c35c0b44af..e5f18420ce641970da50f5419e6a370521d5438a 100644 --- a/arch/csky/kernel/perf_event.c +++ b/arch/csky/kernel/perf_event.c @@ -87,7 +87,7 @@ static int csky_pmu_irq; }) /* cycle counter */ -static uint64_t csky_pmu_read_cc(void) +uint64_t csky_pmu_read_cc(void) { uint32_t lo, hi, tmp; uint64_t result; @@ -1319,7 +1319,7 @@ int csky_pmu_device_probe(struct platform_device *pdev, pr_notice("[perf] PMU request irq fail!\n"); } - ret = cpuhp_setup_state(CPUHP_AP_PERF_ONLINE, "AP_PERF_ONLINE", + ret = cpuhp_setup_state(CPUHP_AP_PERF_CSKY_ONLINE, "AP_PERF_ONLINE", csky_pmu_starting_cpu, csky_pmu_dying_cpu); if (ret) { diff --git a/arch/csky/kernel/probes/simulate-insn.c b/arch/csky/kernel/probes/simulate-insn.c index 4e464fed52ec63eb3e1071d602b851234f8dcf89..d6e8d092c9b7f4f7cd9d30a9a9ff05d8b27d3ba2 100644 --- a/arch/csky/kernel/probes/simulate-insn.c +++ b/arch/csky/kernel/probes/simulate-insn.c @@ -274,9 +274,9 @@ void __kprobes simulate_bnezad32(u32 opcode, long addr, struct pt_regs *regs) { unsigned long tmp = opcode & 0x1f; - unsigned long val; + long val; - csky_insn_reg_get_val(regs, tmp, &val); + csky_insn_reg_get_val(regs, tmp, (unsigned long *)&val); val -= 1; @@ -286,7 +286,7 @@ simulate_bnezad32(u32 opcode, long addr, struct pt_regs *regs) } else instruction_pointer_set(regs, addr + 4); - csky_insn_reg_set_val(regs, tmp, val); + csky_insn_reg_set_val(regs, tmp, (unsigned long)val); } void __kprobes @@ -297,13 +297,11 @@ simulate_bhsz32(u32 opcode, long addr, struct pt_regs *regs) csky_insn_reg_get_val(regs, tmp, &val); - if (val >= 0) { + if ((long) val >= 0) { instruction_pointer_set(regs, addr + sign_extend32((opcode & 0xffff0000) >> 15, 15)); } else instruction_pointer_set(regs, addr + 4); - - csky_insn_reg_set_val(regs, tmp, val); } void __kprobes @@ -314,13 +312,11 @@ simulate_bhz32(u32 opcode, long addr, struct pt_regs *regs) csky_insn_reg_get_val(regs, tmp, &val); - if (val > 0) { + if ((long) val > 0) { instruction_pointer_set(regs, addr + sign_extend32((opcode & 0xffff0000) >> 15, 15)); } else instruction_pointer_set(regs, addr + 4); - - csky_insn_reg_set_val(regs, tmp, val); } void __kprobes @@ -331,13 +327,11 @@ simulate_blsz32(u32 opcode, long addr, struct pt_regs *regs) csky_insn_reg_get_val(regs, tmp, &val); - if (val <= 0) { + if ((long) val <= 0) { instruction_pointer_set(regs, addr + sign_extend32((opcode & 0xffff0000) >> 15, 15)); } else instruction_pointer_set(regs, addr + 4); - - csky_insn_reg_set_val(regs, tmp, val); } void __kprobes @@ -348,13 +342,11 @@ simulate_blz32(u32 opcode, long addr, struct pt_regs *regs) csky_insn_reg_get_val(regs, tmp, &val); - if (val < 0) { + if ((long) val < 0) { instruction_pointer_set(regs, addr + sign_extend32((opcode & 0xffff0000) >> 15, 15)); } else instruction_pointer_set(regs, addr + 4); - - csky_insn_reg_set_val(regs, tmp, val); } void __kprobes diff --git a/arch/csky/kernel/process.c b/arch/csky/kernel/process.c index 69af6bc87e6473d2c0c868cf364893e3593f8adf..3d0ca22cd0e2e6075c4bcdcb5d69d6e55f5131f1 100644 --- a/arch/csky/kernel/process.c +++ b/arch/csky/kernel/process.c @@ -49,7 +49,7 @@ int copy_thread(unsigned long clone_flags, /* setup thread.sp for switch_to !!! */ p->thread.sp = (unsigned long)childstack; - if (unlikely(p->flags & PF_KTHREAD)) { + if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { memset(childregs, 0, sizeof(struct pt_regs)); childstack->r15 = (unsigned long) ret_from_kernel_thread; childstack->r10 = kthread_arg; diff --git a/arch/csky/kernel/ptrace.c b/arch/csky/kernel/ptrace.c index d822144906ac1362d361a959c7c2f1cfec0aa6a8..0105ac81b4328adc090342543cd4601c048aed7d 100644 --- a/arch/csky/kernel/ptrace.c +++ b/arch/csky/kernel/ptrace.c @@ -22,6 +22,7 @@ #include #include +#include #define CREATE_TRACE_POINTS #include @@ -83,7 +84,7 @@ static int gpr_get(struct task_struct *target, /* Abiv1 regs->tls is fake and we need sync here. */ regs->tls = task_thread_info(target)->tp_value; - return membuf_write(&to, regs, sizeof(regs)); + return membuf_write(&to, regs, sizeof(*regs)); } static int gpr_set(struct task_struct *target, @@ -343,6 +344,124 @@ asmlinkage void syscall_trace_exit(struct pt_regs *regs) trace_sys_exit(regs, syscall_get_return_value(current, regs)); } +#ifdef CONFIG_CPU_CK860 +static void show_iutlb(void) +{ + int entry, i; + unsigned long flags; + unsigned long oldpid; + unsigned long entryhi[16], entrylo0[16], entrylo1[16]; + + oldpid = read_mmu_entryhi(); + + entry = 0x8000; + + local_irq_save(flags); + + for (i = 0; i < 16; i++) { + write_mmu_index(entry); + tlb_read(); + entryhi[i] = read_mmu_entryhi(); + entrylo0[i] = read_mmu_entrylo0(); + entrylo1[i] = read_mmu_entrylo1(); + + entry++; + } + + local_irq_restore(flags); + + write_mmu_entryhi(oldpid); + + printk("\n\n\n"); + for (i = 0; i < 16; i++) + printk("iutlb[%d]: entryhi - 0x%lx; entrylo0 - 0x%lx;" + " entrylo1 - 0x%lx\n", + i, entryhi[i], entrylo0[i], entrylo1[i]); + printk("\n\n\n"); +} + +static void show_dutlb(void) +{ + int entry, i; + unsigned long flags; + unsigned long oldpid; + unsigned long entryhi[16], entrylo0[16], entrylo1[16]; + + oldpid = read_mmu_entryhi(); + + entry = 0x4000; + + local_irq_save(flags); + + for (i = 0; i < 16; i++) { + write_mmu_index(entry); + tlb_read(); + entryhi[i] = read_mmu_entryhi(); + entrylo0[i] = read_mmu_entrylo0(); + entrylo1[i] = read_mmu_entrylo1(); + + entry++; + } + + local_irq_restore(flags); + + write_mmu_entryhi(oldpid); + + printk("\n\n\n"); + for (i = 0; i < 16; i++) + printk("dutlb[%d]: entryhi - 0x%lx; entrylo0 - 0x%lx;" + " entrylo1 - 0x%lx\n", + i, entryhi[i], entrylo0[i], entrylo1[i]); + printk("\n\n\n"); +} + +static unsigned long entryhi[1024], entrylo0[1024], entrylo1[1024]; +static void show_jtlb(void) +{ + int entry; + unsigned long flags; + unsigned long oldpid; + + oldpid = read_mmu_entryhi(); + + entry = 0; + + local_irq_save(flags); + while (entry < 1024) { + write_mmu_index(entry); + tlb_read(); + entryhi[entry] = read_mmu_entryhi(); + entrylo0[entry] = read_mmu_entrylo0(); + entrylo1[entry] = read_mmu_entrylo1(); + + entry++; + } + local_irq_restore(flags); + + write_mmu_entryhi(oldpid); + + printk("\n\n\n"); + + for (entry = 0; entry < 1024; entry++) + printk("jtlb[%x]: entryhi - 0x%lx; entrylo0 - 0x%lx;" + " entrylo1 - 0x%lx\n", + entry, entryhi[entry], entrylo0[entry], entrylo1[entry]); + printk("\n\n\n"); +} + +static void show_tlb(void) +{ + show_iutlb(); + show_dutlb(); + show_jtlb(); +} +#else +static void show_tlb(void) +{ + return; +} +#endif + void show_regs(struct pt_regs *fp) { pr_info("\nCURRENT PROCESS:\n\n"); @@ -363,9 +482,10 @@ void show_regs(struct pt_regs *fp) pr_info("PC: 0x%08lx (%pS)\n", (long)fp->pc, (void *)fp->pc); pr_info("LR: 0x%08lx (%pS)\n", (long)fp->lr, (void *)fp->lr); - pr_info("SP: 0x%08lx\n", (long)fp); - pr_info("orig_a0: 0x%08lx\n", fp->orig_a0); + pr_info("SP: 0x%08lx\n", (long)fp->usp); pr_info("PSR: 0x%08lx\n", (long)fp->sr); + pr_info("orig_a0: 0x%08lx\n", fp->orig_a0); + pr_info("PT_REGS: 0x%08lx\n", (long)fp); pr_info(" a0: 0x%08lx a1: 0x%08lx a2: 0x%08lx a3: 0x%08lx\n", fp->a0, fp->a1, fp->a2, fp->a3); @@ -395,5 +515,7 @@ void show_regs(struct pt_regs *fp) fp->regs[8], fp->regs[9]); #endif + show_tlb(); + return; } diff --git a/arch/csky/kernel/setup.c b/arch/csky/kernel/setup.c index e4cab16056d6067ac2e41490ddad7f6f2f7db15d..e93bc6f74432ba259b48fab50499e9a6b51e2c1f 100644 --- a/arch/csky/kernel/setup.c +++ b/arch/csky/kernel/setup.c @@ -45,13 +45,17 @@ static void __init csky_memblock_init(void) if (size >= lowmem_size) { max_low_pfn = min_low_pfn + lowmem_size; +#ifdef CONFIG_PAGE_OFFSET_80000000 write_mmu_msa1(read_mmu_msa0() + SSEG_SIZE); +#endif } else if (size > sseg_size) { max_low_pfn = min_low_pfn + sseg_size; } max_zone_pfn[ZONE_NORMAL] = max_low_pfn; + mmu_init(min_low_pfn, max_low_pfn); + #ifdef CONFIG_HIGHMEM max_zone_pfn[ZONE_HIGHMEM] = max_pfn; @@ -101,16 +105,26 @@ void __init setup_arch(char **cmdline_p) unsigned long va_pa_offset; EXPORT_SYMBOL(va_pa_offset); +static inline unsigned long read_mmu_msa(void) +{ +#ifdef CONFIG_PAGE_OFFSET_80000000 + return read_mmu_msa0(); +#endif + +#ifdef CONFIG_PAGE_OFFSET_A0000000 + return read_mmu_msa1(); +#endif +} + asmlinkage __visible void __init csky_start(unsigned int unused, void *dtb_start) { /* Clean up bss section */ memset(__bss_start, 0, __bss_stop - __bss_start); - va_pa_offset = read_mmu_msa0() & ~(SSEG_SIZE - 1); + va_pa_offset = read_mmu_msa() & ~(SSEG_SIZE - 1); pre_trap_init(); - pre_mmu_init(); if (dtb_start == NULL) early_init_dt_scan(__dtb_start); diff --git a/arch/csky/kernel/signal.c b/arch/csky/kernel/signal.c index 37ea64ed3c12bc96ab86a0caa953a0f32c309bc8..312f046d452d87e19f6da7230d4ab50db1140ed3 100644 --- a/arch/csky/kernel/signal.c +++ b/arch/csky/kernel/signal.c @@ -134,7 +134,6 @@ setup_rt_frame(struct ksignal *ksig, sigset_t *set, struct pt_regs *regs) { struct rt_sigframe *frame; int err = 0; - struct csky_vdso *vdso = current->mm->context.vdso; frame = get_sigframe(ksig, regs, sizeof(*frame)); if (!access_ok(frame, sizeof(*frame))) @@ -152,7 +151,8 @@ setup_rt_frame(struct ksignal *ksig, sigset_t *set, struct pt_regs *regs) return -EFAULT; /* Set up to return from userspace. */ - regs->lr = (unsigned long)(vdso->rt_signal_retcode); + regs->lr = (unsigned long)VDSO_SYMBOL( + current->mm->context.vdso, rt_sigreturn); /* * Set up registers for signal handler. diff --git a/arch/csky/kernel/smp.c b/arch/csky/kernel/smp.c index 041d0de6a1b6771de342bf1ed74292f93f87d872..0f9f5eef9338695b0b43dab09d198c70b2962395 100644 --- a/arch/csky/kernel/smp.c +++ b/arch/csky/kernel/smp.c @@ -203,8 +203,8 @@ volatile unsigned int secondary_hint; volatile unsigned int secondary_hint2; volatile unsigned int secondary_ccr; volatile unsigned int secondary_stack; - -unsigned long secondary_msa1; +volatile unsigned int secondary_msa1; +volatile unsigned int secondary_pgd; int __cpu_up(unsigned int cpu, struct task_struct *tidle) { @@ -216,6 +216,7 @@ int __cpu_up(unsigned int cpu, struct task_struct *tidle) secondary_hint2 = mfcr("cr<21, 1>"); secondary_ccr = mfcr("cr18"); secondary_msa1 = read_mmu_msa1(); + secondary_pgd = mfcr("cr<29, 15>"); /* * Because other CPUs are in reset status, we must flush data @@ -262,8 +263,6 @@ void csky_start_secondary(void) flush_tlb_all(); write_mmu_pagemask(0); - TLBMISS_HANDLER_SETUP_PGD(swapper_pg_dir); - TLBMISS_HANDLER_SETUP_PGD_KERNEL(swapper_pg_dir); #ifdef CONFIG_CPU_HAS_FPU init_fpu(); diff --git a/arch/csky/kernel/traps.c b/arch/csky/kernel/traps.c index 959a917c989d85c0587f67a39e404899c929612a..e5fbf8653a2158825d660b4e8332f0baab836cda 100644 --- a/arch/csky/kernel/traps.c +++ b/arch/csky/kernel/traps.c @@ -39,9 +39,7 @@ asmlinkage void csky_cmpxchg(void); asmlinkage void csky_get_tls(void); asmlinkage void csky_irq(void); -asmlinkage void csky_tlbinvalidl(void); -asmlinkage void csky_tlbinvalids(void); -asmlinkage void csky_tlbmodified(void); +asmlinkage void csky_pagefault(void); /* Defined in head.S */ asmlinkage void _start_smp_secondary(void); @@ -66,9 +64,9 @@ void __init trap_init(void) VEC_INIT(VEC_TRAP3, csky_get_tls); /* setup MMU TLB exception */ - VEC_INIT(VEC_TLBINVALIDL, csky_tlbinvalidl); - VEC_INIT(VEC_TLBINVALIDS, csky_tlbinvalids); - VEC_INIT(VEC_TLBMODIFIED, csky_tlbmodified); + VEC_INIT(VEC_TLBINVALIDL, csky_pagefault); + VEC_INIT(VEC_TLBINVALIDS, csky_pagefault); + VEC_INIT(VEC_TLBMODIFIED, csky_pagefault); #ifdef CONFIG_CPU_HAS_FPU init_fpu(); diff --git a/arch/csky/kernel/vdso.c b/arch/csky/kernel/vdso.c index abc3dbc658d4979aa9260561a73b60225796561a..16c20d64d16532b1d1498a007193c7916a6bb15a 100644 --- a/arch/csky/kernel/vdso.c +++ b/arch/csky/kernel/vdso.c @@ -1,86 +1,107 @@ // SPDX-License-Identifier: GPL-2.0 // Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. -#include -#include -#include -#include -#include #include #include -#include -#include -#include +#include +#include +#include +#include +#ifdef GENERIC_TIME_VSYSCALL +#include +#else #include -#include +#endif -static struct page *vdso_page; +extern char vdso_start[], vdso_end[]; -static int __init init_vdso(void) -{ - struct csky_vdso *vdso; - int err = 0; - - vdso_page = alloc_page(GFP_KERNEL); - if (!vdso_page) - panic("Cannot allocate vdso"); +static unsigned int vdso_pages; +static struct page **vdso_pagelist; - vdso = vmap(&vdso_page, 1, 0, PAGE_KERNEL); - if (!vdso) - panic("Cannot map vdso"); +/* + * The vDSO data page. + */ +static union { + struct vdso_data data; + u8 page[PAGE_SIZE]; +} vdso_data_store __page_aligned_data; +struct vdso_data *vdso_data = &vdso_data_store.data; - clear_page(vdso); - - err = setup_vdso_page(vdso->rt_signal_retcode); - if (err) - panic("Cannot set signal return code, err: %x.", err); +static int __init vdso_init(void) +{ + unsigned int i; + + vdso_pages = (vdso_end - vdso_start) >> PAGE_SHIFT; + vdso_pagelist = + kcalloc(vdso_pages + 1, sizeof(struct page *), GFP_KERNEL); + if (unlikely(vdso_pagelist == NULL)) { + pr_err("vdso: pagelist allocation failed\n"); + return -ENOMEM; + } - dcache_wb_range((unsigned long)vdso, (unsigned long)vdso + 16); + for (i = 0; i < vdso_pages; i++) { + struct page *pg; - vunmap(vdso); + pg = virt_to_page(vdso_start + (i << PAGE_SHIFT)); + vdso_pagelist[i] = pg; + } + vdso_pagelist[i] = virt_to_page(vdso_data); return 0; } -subsys_initcall(init_vdso); +arch_initcall(vdso_init); -int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp) +int arch_setup_additional_pages(struct linux_binprm *bprm, + int uses_interp) { - int ret; - unsigned long addr; struct mm_struct *mm = current->mm; + unsigned long vdso_base, vdso_len; + int ret; - mmap_write_lock(mm); + vdso_len = (vdso_pages + 1) << PAGE_SHIFT; - addr = get_unmapped_area(NULL, STACK_TOP, PAGE_SIZE, 0, 0); - if (IS_ERR_VALUE(addr)) { - ret = addr; - goto up_fail; + mmap_write_lock(mm); + vdso_base = get_unmapped_area(NULL, 0, vdso_len, 0, 0); + if (IS_ERR_VALUE(vdso_base)) { + ret = vdso_base; + goto end; } - ret = install_special_mapping( - mm, - addr, - PAGE_SIZE, - VM_READ|VM_EXEC|VM_MAYREAD|VM_MAYWRITE|VM_MAYEXEC, - &vdso_page); - if (ret) - goto up_fail; + /* + * Put vDSO base into mm struct. We need to do this before calling + * install_special_mapping or the perf counter mmap tracking code + * will fail to recognise it as a vDSO (since arch_vma_name fails). + */ + mm->context.vdso = (void *)vdso_base; + + ret = + install_special_mapping(mm, vdso_base, vdso_pages << PAGE_SHIFT, + (VM_READ | VM_EXEC | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC), + vdso_pagelist); + + if (unlikely(ret)) { + mm->context.vdso = NULL; + goto end; + } - mm->context.vdso = (void *)addr; + vdso_base += (vdso_pages << PAGE_SHIFT); + ret = install_special_mapping(mm, vdso_base, PAGE_SIZE, + (VM_READ | VM_MAYREAD), &vdso_pagelist[vdso_pages]); -up_fail: + if (unlikely(ret)) + mm->context.vdso = NULL; +end: mmap_write_unlock(mm); return ret; } const char *arch_vma_name(struct vm_area_struct *vma) { - if (vma->vm_mm == NULL) - return NULL; - - if (vma->vm_start == (long)vma->vm_mm->context.vdso) + if (vma->vm_mm && (vma->vm_start == (long)vma->vm_mm->context.vdso)) return "[vdso]"; - else - return NULL; + if (vma->vm_mm && (vma->vm_start == + (long)vma->vm_mm->context.vdso + PAGE_SIZE)) + return "[vdso_data]"; + return NULL; } diff --git a/arch/x86/platform/sfi/Makefile b/arch/csky/kernel/vdso/.gitignore similarity index 58% rename from arch/x86/platform/sfi/Makefile rename to arch/csky/kernel/vdso/.gitignore index 4eba24c2af6725d814cfe4494250e5b5bf965ec0..3a19def868eccd8c5544a6629592be0932204646 100644 --- a/arch/x86/platform/sfi/Makefile +++ b/arch/csky/kernel/vdso/.gitignore @@ -1,2 +1,4 @@ # SPDX-License-Identifier: GPL-2.0-only -obj-$(CONFIG_SFI) += sfi.o +vdso.lds +*.tmp +vdso-syms.S diff --git a/arch/csky/kernel/vdso/Makefile b/arch/csky/kernel/vdso/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..0b6909f10667936b86834883035584fe9d13e8dc --- /dev/null +++ b/arch/csky/kernel/vdso/Makefile @@ -0,0 +1,72 @@ +# SPDX-License-Identifier: GPL-2.0-only + +# Absolute relocation type $(ARCH_REL_TYPE_ABS) needs to be defined before +# the inclusion of generic Makefile. +ARCH_REL_TYPE_ABS := R_CKCORE_ADDR32|R_CKCORE_JUMP_SLOT +include $(srctree)/lib/vdso/Makefile + +# Symbols present in the vdso +vdso-syms += rt_sigreturn +vdso-syms += vgettimeofday + +# Files to link into the vdso +obj-vdso = $(patsubst %, %.o, $(vdso-syms)) note.o + +ifneq ($(c-gettimeofday-y),) + CFLAGS_vgettimeofday.o += -include $(c-gettimeofday-y) +endif + +ccflags-y := -fno-stack-protector -DBUILD_VDSO32 + +# Build rules +targets := $(obj-vdso) vdso.so vdso.so.dbg vdso.lds vdso-dummy.o +obj-vdso := $(addprefix $(obj)/, $(obj-vdso)) + +obj-y += vdso.o vdso-syms.o +CPPFLAGS_vdso.lds += -P -C -U$(ARCH) + +# Disable gcov profiling for VDSO code +GCOV_PROFILE := n +KCOV_INSTRUMENT := n + +# Force dependency +$(obj)/vdso.o: $(obj)/vdso.so + +SYSCFLAGS_vdso.so.dbg = $(c_flags) +$(obj)/vdso.so.dbg: $(src)/vdso.lds $(obj-vdso) FORCE + $(call if_changed,vdsold) +SYSCFLAGS_vdso.so.dbg = -shared -s -Wl,-soname=linux-vdso.so.1 \ + -Wl,--build-id=sha1 -Wl,--hash-style=both + +$(obj)/vdso-syms.S: $(obj)/vdso.so FORCE + $(call if_changed,so2s) + +# strip rule for the .so file +$(obj)/%.so: OBJCOPYFLAGS := -S +$(obj)/%.so: $(obj)/%.so.dbg FORCE + $(call if_changed,objcopy) + +# actual build commands +# The DSO images are built using a special linker script +# Make sure only to export the intended __vdso_xxx symbol offsets. +quiet_cmd_vdsold = VDSOLD $@ + cmd_vdsold = $(CC) $(KBUILD_CFLAGS) $(call cc-option, -no-pie) -nostdlib -nostartfiles $(SYSCFLAGS_$(@F)) \ + -Wl,-T,$(filter-out FORCE,$^) -o $@.tmp && \ + $(CROSS_COMPILE)objcopy \ + $(patsubst %, -G __vdso_%, $(vdso-syms)) $@.tmp $@ && \ + rm $@.tmp + +# Extracts symbol offsets from the VDSO, converting them into an assembly file +# that contains the same symbols at the same offsets. +quiet_cmd_so2s = SO2S $@ + cmd_so2s = $(NM) -D $< | $(srctree)/$(src)/so2s.sh > $@ + +# install commands for the unstripped file +quiet_cmd_vdso_install = INSTALL $@ + cmd_vdso_install = cp $(obj)/$@.dbg $(MODLIB)/vdso/$@ + +vdso.so: $(obj)/vdso.so.dbg + @mkdir -p $(MODLIB)/vdso + $(call cmd,vdso_install) + +vdso_install: vdso.so diff --git a/arch/csky/kernel/vdso/note.S b/arch/csky/kernel/vdso/note.S new file mode 100644 index 0000000000000000000000000000000000000000..2a956c942211140ea137a7203142d54d07144078 --- /dev/null +++ b/arch/csky/kernel/vdso/note.S @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * This supplies .note.* sections to go into the PT_NOTE inside the vDSO text. + * Here we can supply some information useful to userland. + */ + +#include +#include + +ELFNOTE_START(Linux, 0, "a") + .long LINUX_VERSION_CODE +ELFNOTE_END diff --git a/arch/csky/kernel/vdso/rt_sigreturn.S b/arch/csky/kernel/vdso/rt_sigreturn.S new file mode 100644 index 0000000000000000000000000000000000000000..0a6bd1216118c184bf6153255886e0cee6fe6fc6 --- /dev/null +++ b/arch/csky/kernel/vdso/rt_sigreturn.S @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#include +#include +#include + + .text +ENTRY(__vdso_rt_sigreturn) + .cfi_startproc + .cfi_signal_frame + SET_SYSCALL_ID + trap 0 + .cfi_endproc +ENDPROC(__vdso_rt_sigreturn) diff --git a/arch/csky/kernel/vdso/so2s.sh b/arch/csky/kernel/vdso/so2s.sh new file mode 100755 index 0000000000000000000000000000000000000000..69da3d529c6da36b80981c508842a1165aa83a5f --- /dev/null +++ b/arch/csky/kernel/vdso/so2s.sh @@ -0,0 +1,5 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0+ + +sed 's!\([0-9a-f]*\) T \([a-z0-9_]*\)\(@@LINUX_5.10\)*!.global \2\n.set \2,0x\1!' \ +| grep '^\.' diff --git a/arch/csky/kernel/vdso/vdso.S b/arch/csky/kernel/vdso/vdso.S new file mode 100644 index 0000000000000000000000000000000000000000..5162ca0694949a38e19b05d964eb58946871c898 --- /dev/null +++ b/arch/csky/kernel/vdso/vdso.S @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#include +#include +#include + + __PAGE_ALIGNED_DATA + + .globl vdso_start, vdso_end + .balign PAGE_SIZE +vdso_start: + .incbin "arch/csky/kernel/vdso/vdso.so" + .balign PAGE_SIZE +vdso_end: + + .previous diff --git a/arch/csky/kernel/vdso/vdso.lds.S b/arch/csky/kernel/vdso/vdso.lds.S new file mode 100644 index 0000000000000000000000000000000000000000..590a6c79fff7d78dc22be83cc9fec113a2ff3bd5 --- /dev/null +++ b/arch/csky/kernel/vdso/vdso.lds.S @@ -0,0 +1,58 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#include + +OUTPUT_ARCH(csky) + +SECTIONS +{ + PROVIDE(_vdso_data = . + PAGE_SIZE); + . = SIZEOF_HEADERS; + + .hash : { *(.hash) } :text + .gnu.hash : { *(.gnu.hash) } + .dynsym : { *(.dynsym) } + .dynstr : { *(.dynstr) } + .gnu.version : { *(.gnu.version) } + .gnu.version_d : { *(.gnu.version_d) } + .gnu.version_r : { *(.gnu.version_r) } + + .note : { *(.note.*) } :text :note + .dynamic : { *(.dynamic) } :text :dynamic + + .eh_frame_hdr : { *(.eh_frame_hdr) } :text :eh_frame_hdr + .eh_frame : { KEEP (*(.eh_frame)) } :text + + .rodata : { *(.rodata .rodata.* .gnu.linkonce.r.*) } + + . = 0x800; + .text : { *(.text .text.*) } :text + + .data : { + *(.got.plt) *(.got) + *(.data .data.* .gnu.linkonce.d.*) + *(.dynbss) + *(.bss .bss.* .gnu.linkonce.b.*) + } +} + +PHDRS +{ + text PT_LOAD FLAGS(5) FILEHDR PHDRS; /* PF_R|PF_X */ + dynamic PT_DYNAMIC FLAGS(4); /* PF_R */ + note PT_NOTE FLAGS(4); /* PF_R */ + eh_frame_hdr PT_GNU_EH_FRAME; +} + +VERSION +{ + LINUX_5.10 { + global: + __vdso_rt_sigreturn; + __vdso_clock_gettime; + __vdso_clock_gettime64; + __vdso_gettimeofday; + __vdso_clock_getres; + local: *; + }; +} diff --git a/arch/csky/kernel/vdso/vgettimeofday.c b/arch/csky/kernel/vdso/vgettimeofday.c new file mode 100644 index 0000000000000000000000000000000000000000..da491832c098a0c77d64917de15e8759d0b5d690 --- /dev/null +++ b/arch/csky/kernel/vdso/vgettimeofday.c @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include +#include + +int __vdso_clock_gettime(clockid_t clock, + struct old_timespec32 *ts) +{ + return __cvdso_clock_gettime32(clock, ts); +} + +int __vdso_clock_gettime64(clockid_t clock, + struct __kernel_timespec *ts) +{ + return __cvdso_clock_gettime(clock, ts); +} + +int __vdso_gettimeofday(struct __kernel_old_timeval *tv, + struct timezone *tz) +{ + return __cvdso_gettimeofday(tv, tz); +} + +int __vdso_clock_getres(clockid_t clock_id, + struct old_timespec32 *res) +{ + return __cvdso_clock_getres_time32(clock_id, res); +} diff --git a/arch/csky/kernel/vmlinux.lds.S b/arch/csky/kernel/vmlinux.lds.S index f03033e17c29e76462bc0ea166957ee4a3119779..e8b1a4a497980a2bef01497fa07426267d85b75b 100644 --- a/arch/csky/kernel/vmlinux.lds.S +++ b/arch/csky/kernel/vmlinux.lds.S @@ -33,6 +33,7 @@ SECTIONS .text : AT(ADDR(.text) - LOAD_OFFSET) { _text = .; + VBR_BASE IRQENTRY_TEXT SOFTIRQENTRY_TEXT TEXT_TEXT @@ -104,7 +105,6 @@ SECTIONS EXCEPTION_TABLE(L1_CACHE_BYTES) BSS_SECTION(L1_CACHE_BYTES, PAGE_SIZE, L1_CACHE_BYTES) - VBR_BASE _end = . ; STABS_DEBUG diff --git a/arch/csky/mm/fault.c b/arch/csky/mm/fault.c index 081b178b41b14c5c2cac1ea2fd1f78067d60b363..1482de56f4f7ecaa7116bd7ada9cb4e6b2a455da 100644 --- a/arch/csky/mm/fault.c +++ b/arch/csky/mm/fault.c @@ -1,29 +1,10 @@ // SPDX-License-Identifier: GPL-2.0 // Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd. -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include #include -#include -#include #include - -#include -#include -#include -#include +#include +#include int fixup_exception(struct pt_regs *regs) { @@ -39,180 +20,287 @@ int fixup_exception(struct pt_regs *regs) return 0; } -/* - * This routine handles page faults. It determines the address, - * and the problem, and then passes it off to one of the appropriate - * routines. - */ -asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long write, - unsigned long mmu_meh) +static inline bool is_write(struct pt_regs *regs) { - struct vm_area_struct *vma = NULL; - struct task_struct *tsk = current; - struct mm_struct *mm = tsk->mm; - int si_code; - int fault; - unsigned long address = mmu_meh & PAGE_MASK; + switch (trap_no(regs)) { + case VEC_TLBINVALIDS: + return true; + case VEC_TLBMODIFIED: + return true; + } - if (kprobe_page_fault(regs, tsk->thread.trap_no)) + return false; +} + +#ifdef CONFIG_CPU_HAS_LDSTEX +static inline void csky_cmpxchg_fixup(struct pt_regs *regs) +{ + return; +} +#else +extern unsigned long csky_cmpxchg_ldw; +extern unsigned long csky_cmpxchg_stw; +static inline void csky_cmpxchg_fixup(struct pt_regs *regs) +{ + if (trap_no(regs) != VEC_TLBMODIFIED) return; - si_code = SEGV_MAPERR; + if (instruction_pointer(regs) == csky_cmpxchg_stw) + instruction_pointer_set(regs, csky_cmpxchg_ldw); + return; +} +#endif + +static inline void no_context(struct pt_regs *regs, unsigned long addr) +{ + current->thread.trap_no = trap_no(regs); + + /* Are we prepared to handle this kernel fault? */ + if (fixup_exception(regs)) + return; -#ifndef CONFIG_CPU_HAS_TLBI /* - * We fault-in kernel-space virtual memory on-demand. The - * 'reference' page table is init_mm.pgd. - * - * NOTE! We MUST NOT take any locks for this case. We may - * be in an interrupt or a critical region, and should - * only copy the information from the master page table, - * nothing more. + * Oops. The kernel tried to access some bad page. We'll have to + * terminate things with extreme prejudice. */ - if (unlikely(address >= VMALLOC_START) && - unlikely(address <= VMALLOC_END)) { - /* - * Synchronize this task's top level page-table - * with the 'reference' page table. - * - * Do _not_ use "tsk" here. We might be inside - * an interrupt in the middle of a task switch.. - */ - int offset = pgd_index(address); - pgd_t *pgd, *pgd_k; - pud_t *pud, *pud_k; - pmd_t *pmd, *pmd_k; - pte_t *pte_k; + bust_spinlocks(1); + pr_alert("Unable to handle kernel paging request at virtual " + "addr 0x%08lx, pc: 0x%08lx\n", addr, regs->pc); + die(regs, "Oops"); + do_exit(SIGKILL); +} - unsigned long pgd_base; +static inline void mm_fault_error(struct pt_regs *regs, unsigned long addr, vm_fault_t fault) +{ + current->thread.trap_no = trap_no(regs); - pgd_base = (unsigned long)__va(get_pgd()); - pgd = (pgd_t *)pgd_base + offset; - pgd_k = init_mm.pgd + offset; + if (fault & VM_FAULT_OOM) { + /* + * We ran out of memory, call the OOM killer, and return the userspace + * (which will retry the fault, or kill us if we got oom-killed). + */ + if (!user_mode(regs)) { + no_context(regs, addr); + return; + } + pagefault_out_of_memory(); + return; + } else if (fault & VM_FAULT_SIGBUS) { + /* Kernel mode? Handle exceptions or die */ + if (!user_mode(regs)) { + no_context(regs, addr); + return; + } + do_trap(regs, SIGBUS, BUS_ADRERR, addr); + return; + } + BUG(); +} - if (!pgd_present(*pgd_k)) - goto no_context; - set_pgd(pgd, *pgd_k); +static inline void bad_area(struct pt_regs *regs, struct mm_struct *mm, int code, unsigned long addr) +{ + /* + * Something tried to access memory that isn't in our memory map. + * Fix it, but check if it's kernel or user first. + */ + mmap_read_unlock(mm); + /* User mode accesses just cause a SIGSEGV */ + if (user_mode(regs)) { + do_trap(regs, SIGSEGV, code, addr); + return; + } - pud = (pud_t *)pgd; - pud_k = (pud_t *)pgd_k; - if (!pud_present(*pud_k)) - goto no_context; + no_context(regs, addr); +} - pmd = pmd_offset(pud, address); - pmd_k = pmd_offset(pud_k, address); - if (!pmd_present(*pmd_k)) - goto no_context; - set_pmd(pmd, *pmd_k); +static inline void vmalloc_fault(struct pt_regs *regs, int code, unsigned long addr) +{ + pgd_t *pgd, *pgd_k; + pud_t *pud, *pud_k; + pmd_t *pmd, *pmd_k; + pte_t *pte_k; + int offset; - pte_k = pte_offset_kernel(pmd_k, address); - if (!pte_present(*pte_k)) - goto no_context; + /* User mode accesses just cause a SIGSEGV */ + if (user_mode(regs)) { + do_trap(regs, SIGSEGV, code, addr); return; } -#endif - perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); /* - * If we're in an interrupt or have no user - * context, we must not take the fault.. + * Synchronize this task's top level page-table + * with the 'reference' page table. + * + * Do _not_ use "tsk" here. We might be inside + * an interrupt in the middle of a task switch.. */ - if (in_atomic() || !mm) - goto bad_area_nosemaphore; + offset = pgd_index(addr); - mmap_read_lock(mm); - vma = find_vma(mm, address); - if (!vma) - goto bad_area; - if (vma->vm_start <= address) - goto good_area; - if (!(vma->vm_flags & VM_GROWSDOWN)) - goto bad_area; - if (expand_stack(vma, address)) - goto bad_area; - /* - * Ok, we have a good vm_area for this memory access, so - * we can handle it.. - */ -good_area: - si_code = SEGV_ACCERR; + pgd = get_pgd() + offset; + pgd_k = init_mm.pgd + offset; - if (write) { + if (!pgd_present(*pgd_k)) { + no_context(regs, addr); + return; + } + set_pgd(pgd, *pgd_k); + + pud = (pud_t *)pgd; + pud_k = (pud_t *)pgd_k; + if (!pud_present(*pud_k)) { + no_context(regs, addr); + return; + } + + pmd = pmd_offset(pud, addr); + pmd_k = pmd_offset(pud_k, addr); + if (!pmd_present(*pmd_k)) { + no_context(regs, addr); + return; + } + set_pmd(pmd, *pmd_k); + + pte_k = pte_offset_kernel(pmd_k, addr); + if (!pte_present(*pte_k)) { + no_context(regs, addr); + return; + } + + flush_tlb_one(addr); +} + +static inline bool access_error(struct pt_regs *regs, struct vm_area_struct *vma) +{ + if (is_write(regs)) { if (!(vma->vm_flags & VM_WRITE)) - goto bad_area; + return true; } else { if (unlikely(!vma_is_accessible(vma))) - goto bad_area; + return true; } + return false; +} + +/* + * This routine handles page faults. It determines the address and the + * problem, and then passes it off to one of the appropriate routines. + */ +asmlinkage void do_page_fault(struct pt_regs *regs) +{ + struct task_struct *tsk; + struct vm_area_struct *vma; + struct mm_struct *mm; + unsigned long addr = read_mmu_entryhi() & PAGE_MASK; + unsigned int flags = FAULT_FLAG_DEFAULT; + int code = SEGV_MAPERR; + vm_fault_t fault; + + tsk = current; + mm = tsk->mm; + + csky_cmpxchg_fixup(regs); + + if (kprobe_page_fault(regs, tsk->thread.trap_no)) + return; /* - * If for any reason at all we couldn't handle the fault, - * make sure we exit gracefully rather than endlessly redo - * the fault. + * Fault-in kernel-space virtual memory on-demand. + * The 'reference' page table is init_mm.pgd. + * + * NOTE! We MUST NOT take any locks for this case. We may + * be in an interrupt or a critical region, and should + * only copy the information from the master page table, + * nothing more. */ - fault = handle_mm_fault(vma, address, write ? FAULT_FLAG_WRITE : 0, - regs); - if (unlikely(fault & VM_FAULT_ERROR)) { - if (fault & VM_FAULT_OOM) - goto out_of_memory; - else if (fault & VM_FAULT_SIGBUS) - goto do_sigbus; - else if (fault & VM_FAULT_SIGSEGV) - goto bad_area; - BUG(); + if (unlikely((addr >= VMALLOC_START) && (addr <= VMALLOC_END))) { + vmalloc_fault(regs, code, addr); + return; } - mmap_read_unlock(mm); - return; + + /* Enable interrupts if they were enabled in the parent context. */ + if (likely(regs->sr & BIT(6))) + local_irq_enable(); /* - * Something tried to access memory that isn't in our memory map.. - * Fix it, but check if it's kernel or user first.. + * If we're in an interrupt, have no user context, or are running + * in an atomic region, then we must not take the fault. */ -bad_area: - mmap_read_unlock(mm); - -bad_area_nosemaphore: - /* User mode accesses just cause a SIGSEGV */ - if (user_mode(regs)) { - tsk->thread.trap_no = trap_no(regs); - force_sig_fault(SIGSEGV, si_code, (void __user *)address); + if (unlikely(faulthandler_disabled() || !mm)) { + no_context(regs, addr); return; } -no_context: - tsk->thread.trap_no = trap_no(regs); + if (user_mode(regs)) + flags |= FAULT_FLAG_USER; - /* Are we prepared to handle this kernel fault? */ - if (fixup_exception(regs)) + perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, addr); + + if (is_write(regs)) + flags |= FAULT_FLAG_WRITE; +retry: + mmap_read_lock(mm); + vma = find_vma(mm, addr); + if (unlikely(!vma)) { + bad_area(regs, mm, code, addr); + return; + } + if (likely(vma->vm_start <= addr)) + goto good_area; + if (unlikely(!(vma->vm_flags & VM_GROWSDOWN))) { + bad_area(regs, mm, code, addr); return; + } + if (unlikely(expand_stack(vma, addr))) { + bad_area(regs, mm, code, addr); + return; + } /* - * Oops. The kernel tried to access some bad page. We'll have to - * terminate things with extreme prejudice. + * Ok, we have a good vm_area for this memory access, so + * we can handle it. */ - bust_spinlocks(1); - pr_alert("Unable to handle kernel paging request at virtual " - "address 0x%08lx, pc: 0x%08lx\n", address, regs->pc); - die(regs, "Oops"); +good_area: + code = SEGV_ACCERR; + + if (unlikely(access_error(regs, vma))) { + bad_area(regs, mm, code, addr); + return; + } -out_of_memory: - tsk->thread.trap_no = trap_no(regs); + /* + * If for any reason at all we could not handle the fault, + * make sure we exit gracefully rather than endlessly redo + * the fault. + */ + fault = handle_mm_fault(vma, addr, flags, regs); /* - * We ran out of memory, call the OOM killer, and return the userspace - * (which will retry the fault, or kill us if we got oom-killed). + * If we need to retry but a fatal signal is pending, handle the + * signal first. We do not need to release the mmap_lock because it + * would already be released in __lock_page_or_retry in mm/filemap.c. */ - pagefault_out_of_memory(); - return; + if (fault_signal_pending(fault, regs)) { + if (!user_mode(regs)) + no_context(regs, addr); + return; + } -do_sigbus: - tsk->thread.trap_no = trap_no(regs); + if (unlikely((fault & VM_FAULT_RETRY) && (flags & FAULT_FLAG_ALLOW_RETRY))) { + flags |= FAULT_FLAG_TRIED; - mmap_read_unlock(mm); + /* + * No need to mmap_read_unlock(mm) as we would + * have already released it in __lock_page_or_retry + * in mm/filemap.c. + */ + goto retry; + } - /* Kernel mode? Handle exceptions or die */ - if (!user_mode(regs)) - goto no_context; + mmap_read_unlock(mm); - force_sig_fault(SIGBUS, BUS_ADRERR, (void __user *)address); + if (unlikely(fault & VM_FAULT_ERROR)) { + mm_fault_error(regs, addr, fault); + return; + } + return; } diff --git a/arch/csky/mm/init.c b/arch/csky/mm/init.c index af627128314f81867a10af034e1923936a59f195..894050a8ce093b91bd6ba504f1d3065cf1308c84 100644 --- a/arch/csky/mm/init.c +++ b/arch/csky/mm/init.c @@ -28,9 +28,15 @@ #include #include #include +#include + +#define PTRS_KERN_TABLE \ + ((PTRS_PER_PGD - USER_PTRS_PER_PGD) * PTRS_PER_PTE) pgd_t swapper_pg_dir[PTRS_PER_PGD] __page_aligned_bss; pte_t invalid_pte_table[PTRS_PER_PTE] __page_aligned_bss; +pte_t kernel_pte_tables[PTRS_KERN_TABLE] __page_aligned_bss; + EXPORT_SYMBOL(invalid_pte_table); unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)] __page_aligned_bss; @@ -80,9 +86,9 @@ void __init mem_init(void) #ifdef CONFIG_HIGHMEM unsigned long tmp; - max_mapnr = highend_pfn; + set_max_mapnr(highend_pfn - ARCH_PFN_OFFSET); #else - max_mapnr = max_low_pfn; + set_max_mapnr(max_low_pfn - ARCH_PFN_OFFSET); #endif high_memory = (void *) __va(max_low_pfn << PAGE_SHIFT); @@ -104,24 +110,9 @@ void __init mem_init(void) mem_init_print_info(NULL); } -extern char __init_begin[], __init_end[]; - void free_initmem(void) { - unsigned long addr; - - addr = (unsigned long) &__init_begin; - - while (addr < (unsigned long) &__init_end) { - ClearPageReserved(virt_to_page(addr)); - init_page_count(virt_to_page(addr)); - free_page(addr); - totalram_pages_inc(); - addr += PAGE_SIZE; - } - - pr_info("Freeing unused kernel memory: %dk freed\n", - ((unsigned int)&__init_end - (unsigned int)&__init_begin) >> 10); + free_initmem_default(-1); } void pgd_init(unsigned long *p) @@ -130,20 +121,35 @@ void pgd_init(unsigned long *p) for (i = 0; i < PTRS_PER_PGD; i++) p[i] = __pa(invalid_pte_table); + + flush_tlb_all(); + local_icache_inv_all(NULL); } -void __init pre_mmu_init(void) +void __init mmu_init(unsigned long min_pfn, unsigned long max_pfn) { - /* - * Setup page-table and enable TLB-hardrefill - */ + int i; + + for (i = 0; i < USER_PTRS_PER_PGD; i++) + swapper_pg_dir[i].pgd = __pa(invalid_pte_table); + + for (i = USER_PTRS_PER_PGD; i < PTRS_PER_PGD; i++) + swapper_pg_dir[i].pgd = + __pa(kernel_pte_tables + (PTRS_PER_PTE * (i - USER_PTRS_PER_PGD))); + + for (i = 0; i < PTRS_KERN_TABLE; i++) + set_pte(&kernel_pte_tables[i], __pte(_PAGE_GLOBAL)); + + for (i = min_pfn; i < max_pfn; i++) + set_pte(&kernel_pte_tables[i - PFN_DOWN(va_pa_offset)], pfn_pte(i, PAGE_KERNEL)); + flush_tlb_all(); - pgd_init((unsigned long *)swapper_pg_dir); - TLBMISS_HANDLER_SETUP_PGD(swapper_pg_dir); - TLBMISS_HANDLER_SETUP_PGD_KERNEL(swapper_pg_dir); + local_icache_inv_all(NULL); /* Setup page mask to 4k */ write_mmu_pagemask(0); + + setup_pgd(swapper_pg_dir, 0); } void __init fixrange_init(unsigned long start, unsigned long end, diff --git a/arch/csky/mm/tlb.c b/arch/csky/mm/tlb.c index ed151238111228ca6d1c851a895c9413e5be1dd1..9234c5e5ceafd5e573dde6c6047913aa9fa2b14e 100644 --- a/arch/csky/mm/tlb.c +++ b/arch/csky/mm/tlb.c @@ -24,7 +24,13 @@ void flush_tlb_all(void) void flush_tlb_mm(struct mm_struct *mm) { #ifdef CONFIG_CPU_HAS_TLBI - asm volatile("tlbi.asids %0"::"r"(cpu_asid(mm))); + sync_is(); + asm volatile( + "tlbi.asids %0 \n" + "sync.i \n" + : + : "r" (cpu_asid(mm)) + : "memory"); #else tlb_invalid_all(); #endif @@ -53,11 +59,17 @@ void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, end &= TLB_ENTRY_SIZE_MASK; #ifdef CONFIG_CPU_HAS_TLBI + sync_is(); while (start < end) { - asm volatile("tlbi.vas %0"::"r"(start | newpid)); + asm volatile( + "tlbi.vas %0 \n" + : + : "r" (start | newpid) + : "memory"); + start += 2*PAGE_SIZE; } - sync_is(); + asm volatile("sync.i\n"); #else { unsigned long flags, oldpid; @@ -87,11 +99,17 @@ void flush_tlb_kernel_range(unsigned long start, unsigned long end) end &= TLB_ENTRY_SIZE_MASK; #ifdef CONFIG_CPU_HAS_TLBI + sync_is(); while (start < end) { - asm volatile("tlbi.vaas %0"::"r"(start)); + asm volatile( + "tlbi.vaas %0 \n" + : + : "r" (start) + : "memory"); + start += 2*PAGE_SIZE; } - sync_is(); + asm volatile("sync.i\n"); #else { unsigned long flags, oldpid; @@ -121,8 +139,13 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long addr) addr &= TLB_ENTRY_SIZE_MASK; #ifdef CONFIG_CPU_HAS_TLBI - asm volatile("tlbi.vas %0"::"r"(addr | newpid)); sync_is(); + asm volatile( + "tlbi.vas %0 \n" + "sync.i \n" + : + : "r" (addr | newpid) + : "memory"); #else { int oldpid, idx; @@ -147,8 +170,13 @@ void flush_tlb_one(unsigned long addr) addr &= TLB_ENTRY_SIZE_MASK; #ifdef CONFIG_CPU_HAS_TLBI - asm volatile("tlbi.vaas %0"::"r"(addr)); sync_is(); + asm volatile( + "tlbi.vaas %0 \n" + "sync.i \n" + : + : "r" (addr) + : "memory"); #else { int oldpid, idx; diff --git a/arch/h8300/kernel/process.c b/arch/h8300/kernel/process.c index bc1364db58feb01e82c1fd407dea4ee0a8611944..46b1342ce515b3a33b63a9559786fbe55f2770ce 100644 --- a/arch/h8300/kernel/process.c +++ b/arch/h8300/kernel/process.c @@ -112,7 +112,7 @@ int copy_thread(unsigned long clone_flags, unsigned long usp, childregs = (struct pt_regs *) (THREAD_SIZE + task_stack_page(p)) - 1; - if (unlikely(p->flags & PF_KTHREAD)) { + if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { memset(childregs, 0, sizeof(struct pt_regs)); childregs->retpc = (unsigned long) ret_from_kernel_thread; childregs->er4 = topstk; /* arg */ diff --git a/arch/hexagon/configs/comet_defconfig b/arch/hexagon/configs/comet_defconfig index e324f65f41e76e0c67a92a2919d6d21ad7c215e8..f19ae2ab0aaa0f6e4613ffa27d38c58d7bfbd845 100644 --- a/arch/hexagon/configs/comet_defconfig +++ b/arch/hexagon/configs/comet_defconfig @@ -1,7 +1,6 @@ CONFIG_SMP=y CONFIG_DEFAULT_MMAP_MIN_ADDR=0 CONFIG_HZ_100=y -CONFIG_EXPERIMENTAL=y CONFIG_CROSS_COMPILE="hexagon-" CONFIG_LOCALVERSION="-smp" # CONFIG_LOCALVERSION_AUTO is not set diff --git a/arch/hexagon/kernel/process.c b/arch/hexagon/kernel/process.c index 6a980cba7b294ce709c5b7e971a065c0d451b7b2..c61165c99ae0ba9d45c79e6f3ab9323ef72cebc4 100644 --- a/arch/hexagon/kernel/process.c +++ b/arch/hexagon/kernel/process.c @@ -73,7 +73,7 @@ int copy_thread(unsigned long clone_flags, unsigned long usp, unsigned long arg, sizeof(*ss)); ss->lr = (unsigned long)ret_from_fork; p->thread.switch_sp = ss; - if (unlikely(p->flags & PF_KTHREAD)) { + if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { memset(childregs, 0, sizeof(struct pt_regs)); /* r24 <- fn, r25 <- arg */ ss->r24 = usp; diff --git a/arch/ia64/Makefile b/arch/ia64/Makefile index f3328a29e881ba6808bfe1c8c2cd4dbbce88cc7a..467b7e7f967c607eda09e8ba24e6c587b518a184 100644 --- a/arch/ia64/Makefile +++ b/arch/ia64/Makefile @@ -14,7 +14,6 @@ KBUILD_DEFCONFIG := generic_defconfig NM := $(CROSS_COMPILE)nm -B -READELF := $(CROSS_COMPILE)readelf CHECKFLAGS += -D__ia64=1 -D__ia64__=1 -D_LP64 -D__LP64__ @@ -85,9 +84,3 @@ define archhelp echo ' install - Install compressed kernel image' echo '* unwcheck - Check vmlinux for invalid unwind info' endef - -archprepare: make_nr_irqs_h -PHONY += make_nr_irqs_h - -make_nr_irqs_h: - $(Q)$(MAKE) $(build)=arch/ia64/kernel include/generated/nr-irqs.h diff --git a/arch/ia64/include/asm/irq.h b/arch/ia64/include/asm/irq.h index 5acf52e908722c0a3cbeaaba96ed7e310a042707..0eccf33dfe8be99e7a22c1061fd2d2e88e66cc0a 100644 --- a/arch/ia64/include/asm/irq.h +++ b/arch/ia64/include/asm/irq.h @@ -14,7 +14,9 @@ #include #include -#include +#include + +#define NR_IRQS IA64_NATIVE_NR_IRQS static __inline__ int irq_canonicalize (int irq) diff --git a/arch/ia64/include/asm/mca.h b/arch/ia64/include/asm/mca.h index 726df17f1b5166c6157f9b532dd8efd525cb326b..05805249296cd5234ac8caceef7e29302a1c23fc 100644 --- a/arch/ia64/include/asm/mca.h +++ b/arch/ia64/include/asm/mca.h @@ -14,13 +14,10 @@ #if !defined(__ASSEMBLY__) -#include +#include +#include #include - -#include -#include -#include -#include +#include #define IA64_MCA_RENDEZ_TIMEOUT (20 * 1000) /* value in milliseconds - 20 seconds */ @@ -83,7 +80,7 @@ struct ia64_sal_os_state { /* common */ unsigned long sal_ra; /* Return address in SAL, physical */ unsigned long sal_gp; /* GP of the SAL - physical */ - pal_min_state_area_t *pal_min_state; /* from R17. physical in asm, virtual in C */ + struct pal_min_state_area *pal_min_state; /* from R17. physical in asm, virtual in C */ /* Previous values of IA64_KR(CURRENT) and IA64_KR(CURRENT_STACK). * Note: if the MCA/INIT recovery code wants to resume to a new context * then it must change these values to reflect the new kernel stack. diff --git a/arch/ia64/include/asm/pal.h b/arch/ia64/include/asm/pal.h index f9d2b3b2dfad4f7a81601642bbd65ceb26b45c93..b1d87955e8cc9cecdaadddb31e200a2ee98ed8e5 100644 --- a/arch/ia64/include/asm/pal.h +++ b/arch/ia64/include/asm/pal.h @@ -750,7 +750,7 @@ typedef union pal_mc_error_info_u { * for PAL. */ -typedef struct pal_min_state_area_s { +struct pal_min_state_area { u64 pmsa_nat_bits; /* nat bits for saved GRs */ u64 pmsa_gr[15]; /* GR1 - GR15 */ u64 pmsa_bank0_gr[16]; /* GR16 - GR31 */ @@ -766,7 +766,7 @@ typedef struct pal_min_state_area_s { u64 pmsa_xfs; /* previous ifs */ u64 pmsa_br1; /* branch register 1 */ u64 pmsa_reserved[70]; /* pal_min_state_area should total to 1KB */ -} pal_min_state_area_t; +}; struct ia64_pal_retval { diff --git a/arch/ia64/include/asm/pgtable.h b/arch/ia64/include/asm/pgtable.h index 779b6972aa84b38186cacc00402ae7c0e22bfbc1..9b4efe89e62d8793a832c354ffe3ede83e172ee9 100644 --- a/arch/ia64/include/asm/pgtable.h +++ b/arch/ia64/include/asm/pgtable.h @@ -517,12 +517,6 @@ extern struct page *zero_page_memmap_ptr; __changed; \ }) #endif - -# ifdef CONFIG_VIRTUAL_MEM_MAP - /* arch mem_map init routine is needed due to holes in a virtual mem_map */ - extern void memmap_init (unsigned long size, int nid, unsigned long zone, - unsigned long start_pfn); -# endif /* CONFIG_VIRTUAL_MEM_MAP */ # endif /* !__ASSEMBLY__ */ /* diff --git a/arch/ia64/include/asm/sal.h b/arch/ia64/include/asm/sal.h index 08f5b6aaed7324096e0d07cc39503c7d65b1dc9d..78f4f7b40435e5d3d6bfa975608005255b8506b4 100644 --- a/arch/ia64/include/asm/sal.h +++ b/arch/ia64/include/asm/sal.h @@ -385,7 +385,7 @@ typedef struct sal_processor_static_info { fr : 1, reserved : 58; } valid; - pal_min_state_area_t min_state_area; + struct pal_min_state_area min_state_area; u64 br[8]; u64 cr[128]; u64 ar[128]; diff --git a/arch/ia64/kernel/Makefile b/arch/ia64/kernel/Makefile index c89bd5f8cbf8311975c7bcbb181a635010cf8387..78717819131cc2bc2a8101d0247ab7bf04c11dbf 100644 --- a/arch/ia64/kernel/Makefile +++ b/arch/ia64/kernel/Makefile @@ -47,8 +47,3 @@ CFLAGS_traps.o += -mfixed-range=f2-f5,f16-f31 # The gate DSO image is built using a special linker script. include $(src)/Makefile.gate - -include/generated/nr-irqs.h: arch/$(SRCARCH)/kernel/nr-irqs.s FORCE - $(call filechk,offsets,__ASM_NR_IRQS_H__) - -targets += nr-irqs.s diff --git a/arch/ia64/kernel/asm-offsets.c b/arch/ia64/kernel/asm-offsets.c index fb0deb8a42218ad2514db46904dd28daf86ece61..be3b90fef2e9218bc1351bf7c6dc9a06544490a6 100644 --- a/arch/ia64/kernel/asm-offsets.c +++ b/arch/ia64/kernel/asm-offsets.c @@ -245,23 +245,23 @@ void foo(void) BLANK(); DEFINE(IA64_PMSA_GR_OFFSET, - offsetof (struct pal_min_state_area_s, pmsa_gr)); + offsetof(struct pal_min_state_area, pmsa_gr)); DEFINE(IA64_PMSA_BANK1_GR_OFFSET, - offsetof (struct pal_min_state_area_s, pmsa_bank1_gr)); + offsetof(struct pal_min_state_area, pmsa_bank1_gr)); DEFINE(IA64_PMSA_PR_OFFSET, - offsetof (struct pal_min_state_area_s, pmsa_pr)); + offsetof(struct pal_min_state_area, pmsa_pr)); DEFINE(IA64_PMSA_BR0_OFFSET, - offsetof (struct pal_min_state_area_s, pmsa_br0)); + offsetof(struct pal_min_state_area, pmsa_br0)); DEFINE(IA64_PMSA_RSC_OFFSET, - offsetof (struct pal_min_state_area_s, pmsa_rsc)); + offsetof(struct pal_min_state_area, pmsa_rsc)); DEFINE(IA64_PMSA_IIP_OFFSET, - offsetof (struct pal_min_state_area_s, pmsa_iip)); + offsetof(struct pal_min_state_area, pmsa_iip)); DEFINE(IA64_PMSA_IPSR_OFFSET, - offsetof (struct pal_min_state_area_s, pmsa_ipsr)); + offsetof(struct pal_min_state_area, pmsa_ipsr)); DEFINE(IA64_PMSA_IFS_OFFSET, - offsetof (struct pal_min_state_area_s, pmsa_ifs)); + offsetof(struct pal_min_state_area, pmsa_ifs)); DEFINE(IA64_PMSA_XIP_OFFSET, - offsetof (struct pal_min_state_area_s, pmsa_xip)); + offsetof(struct pal_min_state_area, pmsa_xip)); BLANK(); /* used by fsys_gettimeofday in arch/ia64/kernel/fsys.S */ diff --git a/arch/ia64/kernel/crash.c b/arch/ia64/kernel/crash.c index 4f47741005d26e160671556529f3985ee8509307..76730f34685c451141c4a3890e6b126b732e4712 100644 --- a/arch/ia64/kernel/crash.c +++ b/arch/ia64/kernel/crash.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include diff --git a/arch/ia64/kernel/efi.c b/arch/ia64/kernel/efi.c index dd7fd750bb934925ca94b85910548b8eacb2fb72..c5fe21de46a81dbccd8d57a4869a93ff412da05a 100644 --- a/arch/ia64/kernel/efi.c +++ b/arch/ia64/kernel/efi.c @@ -40,6 +40,7 @@ #include #include #include +#include #include #include diff --git a/arch/ia64/kernel/mca.c b/arch/ia64/kernel/mca.c index 0fea266b4d39447d34eb5fcf40828fa2538c3841..d4cae2fc69ca333ae03fe342aa3f04f0ce389653 100644 --- a/arch/ia64/kernel/mca.c +++ b/arch/ia64/kernel/mca.c @@ -97,6 +97,7 @@ #include #include #include +#include #include #include @@ -895,7 +896,7 @@ static void finish_pt_regs(struct pt_regs *regs, struct ia64_sal_os_state *sos, unsigned long *nat) { - const pal_min_state_area_t *ms = sos->pal_min_state; + const struct pal_min_state_area *ms = sos->pal_min_state; const u64 *bank; /* If ipsr.ic then use pmsa_{iip,ipsr,ifs}, else use @@ -971,7 +972,7 @@ ia64_mca_modify_original_stack(struct pt_regs *regs, char *p; ia64_va va; extern char ia64_leave_kernel[]; /* Need asm address, not function descriptor */ - const pal_min_state_area_t *ms = sos->pal_min_state; + const struct pal_min_state_area *ms = sos->pal_min_state; struct task_struct *previous_current; struct pt_regs *old_regs; struct switch_stack *old_sw; diff --git a/arch/ia64/kernel/mca_drv.c b/arch/ia64/kernel/mca_drv.c index 4d0ab323dee8c7d65cb0db3a1adb1b186970cda6..36a69b4e61690a615d8f5f90e7beb6bcb511d7c4 100644 --- a/arch/ia64/kernel/mca_drv.c +++ b/arch/ia64/kernel/mca_drv.c @@ -496,7 +496,7 @@ recover_from_read_error(slidx_table_t *slidx, struct ia64_sal_os_state *sos) { u64 target_identifier; - pal_min_state_area_t *pmsa; + struct pal_min_state_area *pmsa; struct ia64_psr *psr1, *psr2; ia64_fptr_t *mca_hdlr_bh = (ia64_fptr_t*)mca_handler_bhhook; diff --git a/arch/ia64/kernel/nr-irqs.c b/arch/ia64/kernel/nr-irqs.c deleted file mode 100644 index f2633b22d3be2ba1256c742e3865802b5e5df113..0000000000000000000000000000000000000000 --- a/arch/ia64/kernel/nr-irqs.c +++ /dev/null @@ -1,22 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * calculate - * NR_IRQS = max(IA64_NATIVE_NR_IRQS, XEN_NR_IRQS, FOO_NR_IRQS...) - * depending on config. - * This must be calculated before processing asm-offset.c. - */ - -#define ASM_OFFSETS_C 1 - -#include -#include -#include - -void foo(void) -{ - union paravirt_nr_irqs_max { - char ia64_native_nr_irqs[IA64_NATIVE_NR_IRQS]; - }; - - DEFINE(NR_IRQS, sizeof (union paravirt_nr_irqs_max)); -} diff --git a/arch/ia64/kernel/process.c b/arch/ia64/kernel/process.c index 4ebbfa076a26a0c2645d40eae55df3f67cd3c16e..7e1a1525e2026f243fd386f875d44bf4ef509de6 100644 --- a/arch/ia64/kernel/process.c +++ b/arch/ia64/kernel/process.c @@ -338,7 +338,7 @@ copy_thread(unsigned long clone_flags, unsigned long user_stack_base, ia64_drop_fpu(p); /* don't pick up stale state from a CPU's fph */ - if (unlikely(p->flags & PF_KTHREAD)) { + if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { if (unlikely(!user_stack_base)) { /* fork_idle() called us */ return 0; diff --git a/arch/ia64/kernel/syscalls/Makefile b/arch/ia64/kernel/syscalls/Makefile index 813a58cba39ca129088cb0af87ac523605a4b9eb..bf4bda0f63eb6f98aa91989e080ece4dbd6988c1 100644 --- a/arch/ia64/kernel/syscalls/Makefile +++ b/arch/ia64/kernel/syscalls/Makefile @@ -5,7 +5,7 @@ uapi := arch/$(SRCARCH)/include/generated/uapi/asm _dummy := $(shell [ -d '$(uapi)' ] || mkdir -p '$(uapi)') \ $(shell [ -d '$(kapi)' ] || mkdir -p '$(kapi)') -syscall := $(srctree)/$(src)/syscall.tbl +syscall := $(src)/syscall.tbl syshdr := $(srctree)/$(src)/syscallhdr.sh systbl := $(srctree)/$(src)/syscalltbl.sh @@ -22,19 +22,20 @@ quiet_cmd_systbl = SYSTBL $@ '$(systbl_offset_$(basetarget))' syshdr_offset_unistd_64 := __NR_Linux -$(uapi)/unistd_64.h: $(syscall) $(syshdr) +$(uapi)/unistd_64.h: $(syscall) $(syshdr) FORCE $(call if_changed,syshdr) systbl_offset_syscall_table := 1024 -$(kapi)/syscall_table.h: $(syscall) $(systbl) +$(kapi)/syscall_table.h: $(syscall) $(systbl) FORCE $(call if_changed,systbl) uapisyshdr-y += unistd_64.h kapisyshdr-y += syscall_table.h -targets += $(uapisyshdr-y) $(kapisyshdr-y) +uapisyshdr-y := $(addprefix $(uapi)/, $(uapisyshdr-y)) +kapisyshdr-y := $(addprefix $(kapi)/, $(kapisyshdr-y)) +targets += $(addprefix ../../../../, $(uapisyshdr-y) $(kapisyshdr-y)) PHONY += all -all: $(addprefix $(uapi)/,$(uapisyshdr-y)) -all: $(addprefix $(kapi)/,$(kapisyshdr-y)) +all: $(uapisyshdr-y) $(kapisyshdr-y) @: diff --git a/arch/ia64/kernel/syscalls/syscall.tbl b/arch/ia64/kernel/syscalls/syscall.tbl index bfc00f2bd437236823e16abf22f68ca1bc6bebdb..d89231166e19efbdae049f28da2194e34969da58 100644 --- a/arch/ia64/kernel/syscalls/syscall.tbl +++ b/arch/ia64/kernel/syscalls/syscall.tbl @@ -362,3 +362,4 @@ 439 common faccessat2 sys_faccessat2 440 common process_madvise sys_process_madvise 441 common epoll_pwait2 sys_epoll_pwait2 +442 common mount_setattr sys_mount_setattr diff --git a/arch/ia64/mm/init.c b/arch/ia64/mm/init.c index b19f47a5a305124245d561538d6b77f3826871d2..16d0d7d2265784b5818ee739632adb7057540054 100644 --- a/arch/ia64/mm/init.c +++ b/arch/ia64/mm/init.c @@ -536,18 +536,20 @@ virtual_memmap_init(u64 start, u64 end, void *arg) / sizeof(struct page)); if (map_start < map_end) - memmap_init_zone((unsigned long)(map_end - map_start), + memmap_init_range((unsigned long)(map_end - map_start), args->nid, args->zone, page_to_pfn(map_start), page_to_pfn(map_end), MEMINIT_EARLY, NULL, MIGRATE_MOVABLE); return 0; } -void __meminit -memmap_init (unsigned long size, int nid, unsigned long zone, - unsigned long start_pfn) +void __meminit memmap_init_zone(struct zone *zone) { + int nid = zone_to_nid(zone), zone_id = zone_idx(zone); + unsigned long start_pfn = zone->zone_start_pfn; + unsigned long size = zone->spanned_pages; + if (!vmem_map) { - memmap_init_zone(size, nid, zone, start_pfn, start_pfn + size, + memmap_init_range(size, nid, zone_id, start_pfn, start_pfn + size, MEMINIT_EARLY, NULL, MIGRATE_MOVABLE); } else { struct page *start; @@ -557,7 +559,7 @@ memmap_init (unsigned long size, int nid, unsigned long zone, args.start = start; args.end = start + size; args.nid = nid; - args.zone = zone; + args.zone = zone_id; efi_memmap_walk(virtual_memmap_init, &args); } diff --git a/arch/m68k/coldfire/clk.c b/arch/m68k/coldfire/clk.c index 7bc666e482ebe82f400d8d184ec8c8b8c388c04d..076a9caa9557b1bbb54070071ab31892d6ff6f42 100644 --- a/arch/m68k/coldfire/clk.c +++ b/arch/m68k/coldfire/clk.c @@ -90,6 +90,10 @@ EXPORT_SYMBOL(clk_get); int clk_enable(struct clk *clk) { unsigned long flags; + + if (!clk) + return -EINVAL; + spin_lock_irqsave(&clk_lock, flags); if ((clk->enabled++ == 0) && clk->clk_ops) clk->clk_ops->enable(clk); diff --git a/arch/m68k/kernel/process.c b/arch/m68k/kernel/process.c index 08359a6e058f632294989db6800dcddd5626a0f2..da83cc83e79123b92654fb99f1dbbeb972593d09 100644 --- a/arch/m68k/kernel/process.c +++ b/arch/m68k/kernel/process.c @@ -157,7 +157,7 @@ int copy_thread(unsigned long clone_flags, unsigned long usp, unsigned long arg, */ p->thread.fs = get_fs().seg; - if (unlikely(p->flags & PF_KTHREAD)) { + if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { /* kernel thread */ memset(frame, 0, sizeof(struct fork_frame)); frame->regs.sr = PS_S; diff --git a/arch/m68k/kernel/syscalls/Makefile b/arch/m68k/kernel/syscalls/Makefile index 659faefdcb1da5f0ffbafc40fd598dd98d9a5fde..285aaba832d93936c3f2b7edb2938dbfe13fc21f 100644 --- a/arch/m68k/kernel/syscalls/Makefile +++ b/arch/m68k/kernel/syscalls/Makefile @@ -5,7 +5,7 @@ uapi := arch/$(SRCARCH)/include/generated/uapi/asm _dummy := $(shell [ -d '$(uapi)' ] || mkdir -p '$(uapi)') \ $(shell [ -d '$(kapi)' ] || mkdir -p '$(kapi)') -syscall := $(srctree)/$(src)/syscall.tbl +syscall := $(src)/syscall.tbl syshdr := $(srctree)/$(src)/syscallhdr.sh systbl := $(srctree)/$(src)/syscalltbl.sh @@ -21,18 +21,19 @@ quiet_cmd_systbl = SYSTBL $@ '$(systbl_abi_$(basetarget))' \ '$(systbl_offset_$(basetarget))' -$(uapi)/unistd_32.h: $(syscall) $(syshdr) +$(uapi)/unistd_32.h: $(syscall) $(syshdr) FORCE $(call if_changed,syshdr) -$(kapi)/syscall_table.h: $(syscall) $(systbl) +$(kapi)/syscall_table.h: $(syscall) $(systbl) FORCE $(call if_changed,systbl) uapisyshdr-y += unistd_32.h kapisyshdr-y += syscall_table.h -targets += $(uapisyshdr-y) $(kapisyshdr-y) +uapisyshdr-y := $(addprefix $(uapi)/, $(uapisyshdr-y)) +kapisyshdr-y := $(addprefix $(kapi)/, $(kapisyshdr-y)) +targets += $(addprefix ../../../../, $(uapisyshdr-y) $(kapisyshdr-y)) PHONY += all -all: $(addprefix $(uapi)/,$(uapisyshdr-y)) -all: $(addprefix $(kapi)/,$(kapisyshdr-y)) +all: $(uapisyshdr-y) $(kapisyshdr-y) @: diff --git a/arch/m68k/kernel/syscalls/syscall.tbl b/arch/m68k/kernel/syscalls/syscall.tbl index 7fe4e45c864c4f36ca7cdd6f410590541f5e6429..72bde6707dd33c311b5f239542279cc554bccb2b 100644 --- a/arch/m68k/kernel/syscalls/syscall.tbl +++ b/arch/m68k/kernel/syscalls/syscall.tbl @@ -441,3 +441,4 @@ 439 common faccessat2 sys_faccessat2 440 common process_madvise sys_process_madvise 441 common epoll_pwait2 sys_epoll_pwait2 +442 common mount_setattr sys_mount_setattr diff --git a/arch/microblaze/Kconfig b/arch/microblaze/Kconfig index 25a5a3fb14aa1296f56eb3bc8097d689c3986557..0660f47012bcbe9e38fd60d91246446c9ab4d112 100644 --- a/arch/microblaze/Kconfig +++ b/arch/microblaze/Kconfig @@ -38,7 +38,6 @@ config MICROBLAZE select OF_EARLY_FLATTREE select PCI_DOMAINS_GENERIC if PCI select PCI_SYSCALL if PCI - select TRACING_SUPPORT select VIRT_TO_BUS select CPU_NO_EFFICIENT_FFS select MMU_GATHER_NO_RANGE diff --git a/arch/microblaze/kernel/module.c b/arch/microblaze/kernel/module.c index 9f12e3c2bb42a3197b59b2e58e1002ca9bf0123f..e5db3a57b9e30d9ece1e2fe169bf9a10b7d462b8 100644 --- a/arch/microblaze/kernel/module.c +++ b/arch/microblaze/kernel/module.c @@ -24,9 +24,6 @@ int apply_relocate_add(Elf32_Shdr *sechdrs, const char *strtab, Elf32_Sym *sym; unsigned long int *location; unsigned long int value; -#if __GNUC__ < 4 - unsigned long int old_value; -#endif pr_debug("Applying add relocation section %u to %u\n", relsec, sechdrs[relsec].sh_info); @@ -49,40 +46,17 @@ int apply_relocate_add(Elf32_Shdr *sechdrs, const char *strtab, */ case R_MICROBLAZE_32: -#if __GNUC__ < 4 - old_value = *location; - *location = value + old_value; - - pr_debug("R_MICROBLAZE_32 (%08lx->%08lx)\n", - old_value, value); -#else *location = value; -#endif break; case R_MICROBLAZE_64: -#if __GNUC__ < 4 - /* Split relocs only required/used pre gcc4.1.1 */ - old_value = ((location[0] & 0x0000FFFF) << 16) | - (location[1] & 0x0000FFFF); - value += old_value; -#endif location[0] = (location[0] & 0xFFFF0000) | (value >> 16); location[1] = (location[1] & 0xFFFF0000) | (value & 0xFFFF); -#if __GNUC__ < 4 - pr_debug("R_MICROBLAZE_64 (%08lx->%08lx)\n", - old_value, value); -#endif break; case R_MICROBLAZE_64_PCREL: -#if __GNUC__ < 4 - old_value = (location[0] & 0xFFFF) << 16 | - (location[1] & 0xFFFF); - value -= old_value; -#endif value -= (unsigned long int)(location) + 4; location[0] = (location[0] & 0xFFFF0000) | (value >> 16); diff --git a/arch/microblaze/kernel/process.c b/arch/microblaze/kernel/process.c index 657c2beb665ea27440c9a2b0992f19e350f49708..62aa237180b6724a121e2399bbd12b738d532839 100644 --- a/arch/microblaze/kernel/process.c +++ b/arch/microblaze/kernel/process.c @@ -59,7 +59,7 @@ int copy_thread(unsigned long clone_flags, unsigned long usp, unsigned long arg, struct pt_regs *childregs = task_pt_regs(p); struct thread_info *ti = task_thread_info(p); - if (unlikely(p->flags & PF_KTHREAD)) { + if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { /* if we're creating a new kernel thread then just zeroing all * the registers. That's OK for a brand new thread.*/ memset(childregs, 0, sizeof(struct pt_regs)); diff --git a/arch/microblaze/kernel/syscalls/Makefile b/arch/microblaze/kernel/syscalls/Makefile index 659faefdcb1da5f0ffbafc40fd598dd98d9a5fde..285aaba832d93936c3f2b7edb2938dbfe13fc21f 100644 --- a/arch/microblaze/kernel/syscalls/Makefile +++ b/arch/microblaze/kernel/syscalls/Makefile @@ -5,7 +5,7 @@ uapi := arch/$(SRCARCH)/include/generated/uapi/asm _dummy := $(shell [ -d '$(uapi)' ] || mkdir -p '$(uapi)') \ $(shell [ -d '$(kapi)' ] || mkdir -p '$(kapi)') -syscall := $(srctree)/$(src)/syscall.tbl +syscall := $(src)/syscall.tbl syshdr := $(srctree)/$(src)/syscallhdr.sh systbl := $(srctree)/$(src)/syscalltbl.sh @@ -21,18 +21,19 @@ quiet_cmd_systbl = SYSTBL $@ '$(systbl_abi_$(basetarget))' \ '$(systbl_offset_$(basetarget))' -$(uapi)/unistd_32.h: $(syscall) $(syshdr) +$(uapi)/unistd_32.h: $(syscall) $(syshdr) FORCE $(call if_changed,syshdr) -$(kapi)/syscall_table.h: $(syscall) $(systbl) +$(kapi)/syscall_table.h: $(syscall) $(systbl) FORCE $(call if_changed,systbl) uapisyshdr-y += unistd_32.h kapisyshdr-y += syscall_table.h -targets += $(uapisyshdr-y) $(kapisyshdr-y) +uapisyshdr-y := $(addprefix $(uapi)/, $(uapisyshdr-y)) +kapisyshdr-y := $(addprefix $(kapi)/, $(kapisyshdr-y)) +targets += $(addprefix ../../../../, $(uapisyshdr-y) $(kapisyshdr-y)) PHONY += all -all: $(addprefix $(uapi)/,$(uapisyshdr-y)) -all: $(addprefix $(kapi)/,$(kapisyshdr-y)) +all: $(uapisyshdr-y) $(kapisyshdr-y) @: diff --git a/arch/microblaze/kernel/syscalls/syscall.tbl b/arch/microblaze/kernel/syscalls/syscall.tbl index a522adf194abe3a3cd982a92888b58365d3858f7..d603a5ec933865b1876267e6e1bf5e57e0044137 100644 --- a/arch/microblaze/kernel/syscalls/syscall.tbl +++ b/arch/microblaze/kernel/syscalls/syscall.tbl @@ -447,3 +447,4 @@ 439 common faccessat2 sys_faccessat2 440 common process_madvise sys_process_madvise 441 common epoll_pwait2 sys_epoll_pwait2 +442 common mount_setattr sys_mount_setattr diff --git a/arch/microblaze/kernel/vmlinux.lds.S b/arch/microblaze/kernel/vmlinux.lds.S index df07b3d06cd6b4c4fc256a74d050229dc0c56e5c..fb31747ec092b898383f496209c0a945cab71f4c 100644 --- a/arch/microblaze/kernel/vmlinux.lds.S +++ b/arch/microblaze/kernel/vmlinux.lds.S @@ -45,7 +45,7 @@ SECTIONS { _etext = . ; } - . = ALIGN (4) ; + . = ALIGN (8) ; __fdt_blob : AT(ADDR(__fdt_blob) - LOAD_OFFSET) { _fdt_start = . ; /* place for fdt blob */ *(__fdt_blob) ; /* Any link-placed DTB */ diff --git a/arch/mips/bmips/setup.c b/arch/mips/bmips/setup.c index 95f8f10d8697c7d7670a30185ab160397c95eba3..31bcfa4e08b998b25b196f98cbb82bdf6f6a42e5 100644 --- a/arch/mips/bmips/setup.c +++ b/arch/mips/bmips/setup.c @@ -196,4 +196,4 @@ static int __init plat_dev_init(void) return 0; } -device_initcall(plat_dev_init); +arch_initcall(plat_dev_init); diff --git a/arch/mips/configs/nlm_xlp_defconfig b/arch/mips/configs/nlm_xlp_defconfig index 72a211d2d556fd64b6689f3b822304307814b7bd..32c290611723257a2c3dac84a4ac6f8ced2726d6 100644 --- a/arch/mips/configs/nlm_xlp_defconfig +++ b/arch/mips/configs/nlm_xlp_defconfig @@ -549,7 +549,6 @@ CONFIG_PRINTK_TIME=y CONFIG_DEBUG_INFO=y # CONFIG_ENABLE_MUST_CHECK is not set CONFIG_FRAME_WARN=1024 -CONFIG_UNUSED_SYMBOLS=y CONFIG_DEBUG_MEMORY_INIT=y CONFIG_DETECT_HUNG_TASK=y CONFIG_SCHEDSTATS=y diff --git a/arch/mips/configs/nlm_xlr_defconfig b/arch/mips/configs/nlm_xlr_defconfig index 4ecb157e56d4272c17dd7abff83c941d54046296..bf9b9244929ecdbdea5915400a29abf7ebca7181 100644 --- a/arch/mips/configs/nlm_xlr_defconfig +++ b/arch/mips/configs/nlm_xlr_defconfig @@ -500,7 +500,6 @@ CONFIG_CRC7=m CONFIG_PRINTK_TIME=y CONFIG_DEBUG_INFO=y # CONFIG_ENABLE_MUST_CHECK is not set -CONFIG_UNUSED_SYMBOLS=y CONFIG_DEBUG_MEMORY_INIT=y CONFIG_DETECT_HUNG_TASK=y CONFIG_SCHEDSTATS=y diff --git a/arch/mips/kernel/process.c b/arch/mips/kernel/process.c index af4c862ec5ff668b18f633206cd66775a2c247db..7efa0d1a4c2bccecf7918ca762a656c08e3508d2 100644 --- a/arch/mips/kernel/process.c +++ b/arch/mips/kernel/process.c @@ -120,7 +120,7 @@ int copy_thread(unsigned long clone_flags, unsigned long usp, /* Put the stack after the struct pt_regs. */ childksp = (unsigned long) childregs; p->thread.cp0_status = (read_c0_status() & ~(ST0_CU2|ST0_CU1)) | ST0_KERNEL_CUMASK; - if (unlikely(p->flags & PF_KTHREAD)) { + if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { /* kernel thread */ unsigned long status = p->thread.cp0_status; memset(childregs, 0, sizeof(struct pt_regs)); diff --git a/arch/mips/kernel/r4k-bugs64.c b/arch/mips/kernel/r4k-bugs64.c index 1ff19f1ea5caffcd16383e8691b997896827be1e..35729c9e6cfa116cd599c0f2cb34469ca8431c23 100644 --- a/arch/mips/kernel/r4k-bugs64.c +++ b/arch/mips/kernel/r4k-bugs64.c @@ -18,7 +18,7 @@ static char bug64hit[] __initdata = "reliable operation impossible!\n%s"; static char nowar[] __initdata = - "Please report to ."; + "Please report to ."; static char r4kwar[] __initdata = "Enable CPU_R4000_WORKAROUNDS to rectify."; static char daddiwar[] __initdata = diff --git a/arch/mips/kernel/syscalls/Makefile b/arch/mips/kernel/syscalls/Makefile index a1ce8b7dbcfaf8c8b4ad3e8cec9d41a56d8b4a1f..51f8b805f2ed480777248cca4212d100529b2cdb 100644 --- a/arch/mips/kernel/syscalls/Makefile +++ b/arch/mips/kernel/syscalls/Makefile @@ -5,9 +5,9 @@ uapi := arch/$(SRCARCH)/include/generated/uapi/asm _dummy := $(shell [ -d '$(uapi)' ] || mkdir -p '$(uapi)') \ $(shell [ -d '$(kapi)' ] || mkdir -p '$(kapi)') -syscalln32 := $(srctree)/$(src)/syscall_n32.tbl -syscalln64 := $(srctree)/$(src)/syscall_n64.tbl -syscallo32 := $(srctree)/$(src)/syscall_o32.tbl +syscalln32 := $(src)/syscall_n32.tbl +syscalln64 := $(src)/syscall_n64.tbl +syscallo32 := $(src)/syscall_o32.tbl syshdr := $(srctree)/$(src)/syscallhdr.sh sysnr := $(srctree)/$(src)/syscallnr.sh systbl := $(srctree)/$(src)/syscalltbl.sh @@ -31,50 +31,50 @@ quiet_cmd_systbl = SYSTBL $@ '$(systbl_offset_$(basetarget))' syshdr_offset_unistd_n32 := __NR_Linux -$(uapi)/unistd_n32.h: $(syscalln32) $(syshdr) +$(uapi)/unistd_n32.h: $(syscalln32) $(syshdr) FORCE $(call if_changed,syshdr) syshdr_offset_unistd_n64 := __NR_Linux -$(uapi)/unistd_n64.h: $(syscalln64) $(syshdr) +$(uapi)/unistd_n64.h: $(syscalln64) $(syshdr) FORCE $(call if_changed,syshdr) syshdr_offset_unistd_o32 := __NR_Linux -$(uapi)/unistd_o32.h: $(syscallo32) $(syshdr) +$(uapi)/unistd_o32.h: $(syscallo32) $(syshdr) FORCE $(call if_changed,syshdr) sysnr_pfx_unistd_nr_n32 := N32 sysnr_offset_unistd_nr_n32 := 6000 -$(kapi)/unistd_nr_n32.h: $(syscalln32) $(sysnr) +$(kapi)/unistd_nr_n32.h: $(syscalln32) $(sysnr) FORCE $(call if_changed,sysnr) sysnr_pfx_unistd_nr_n64 := 64 sysnr_offset_unistd_nr_n64 := 5000 -$(kapi)/unistd_nr_n64.h: $(syscalln64) $(sysnr) +$(kapi)/unistd_nr_n64.h: $(syscalln64) $(sysnr) FORCE $(call if_changed,sysnr) sysnr_pfx_unistd_nr_o32 := O32 sysnr_offset_unistd_nr_o32 := 4000 -$(kapi)/unistd_nr_o32.h: $(syscallo32) $(sysnr) +$(kapi)/unistd_nr_o32.h: $(syscallo32) $(sysnr) FORCE $(call if_changed,sysnr) systbl_abi_syscall_table_32_o32 := 32_o32 systbl_offset_syscall_table_32_o32 := 4000 -$(kapi)/syscall_table_32_o32.h: $(syscallo32) $(systbl) +$(kapi)/syscall_table_32_o32.h: $(syscallo32) $(systbl) FORCE $(call if_changed,systbl) systbl_abi_syscall_table_64_n32 := 64_n32 systbl_offset_syscall_table_64_n32 := 6000 -$(kapi)/syscall_table_64_n32.h: $(syscalln32) $(systbl) +$(kapi)/syscall_table_64_n32.h: $(syscalln32) $(systbl) FORCE $(call if_changed,systbl) systbl_abi_syscall_table_64_n64 := 64_n64 systbl_offset_syscall_table_64_n64 := 5000 -$(kapi)/syscall_table_64_n64.h: $(syscalln64) $(systbl) +$(kapi)/syscall_table_64_n64.h: $(syscalln64) $(systbl) FORCE $(call if_changed,systbl) systbl_abi_syscall_table_64_o32 := 64_o32 systbl_offset_syscall_table_64_o32 := 4000 -$(kapi)/syscall_table_64_o32.h: $(syscallo32) $(systbl) +$(kapi)/syscall_table_64_o32.h: $(syscallo32) $(systbl) FORCE $(call if_changed,systbl) uapisyshdr-y += unistd_n32.h \ @@ -88,9 +88,10 @@ kapisyshdr-y += syscall_table_32_o32.h \ unistd_nr_n64.h \ unistd_nr_o32.h -targets += $(uapisyshdr-y) $(kapisyshdr-y) +uapisyshdr-y := $(addprefix $(uapi)/, $(uapisyshdr-y)) +kapisyshdr-y := $(addprefix $(kapi)/, $(kapisyshdr-y)) +targets += $(addprefix ../../../../, $(uapisyshdr-y) $(kapisyshdr-y)) PHONY += all -all: $(addprefix $(uapi)/,$(uapisyshdr-y)) -all: $(addprefix $(kapi)/,$(kapisyshdr-y)) +all: $(uapisyshdr-y) $(kapisyshdr-y) @: diff --git a/arch/mips/kernel/syscalls/syscall_n32.tbl b/arch/mips/kernel/syscalls/syscall_n32.tbl index 0f03ad223f332cfe755c019fada0fbbaf958149c..8fd8c1790941c66048d8f90763dc29f9bdd74571 100644 --- a/arch/mips/kernel/syscalls/syscall_n32.tbl +++ b/arch/mips/kernel/syscalls/syscall_n32.tbl @@ -380,3 +380,4 @@ 439 n32 faccessat2 sys_faccessat2 440 n32 process_madvise sys_process_madvise 441 n32 epoll_pwait2 compat_sys_epoll_pwait2 +442 n32 mount_setattr sys_mount_setattr diff --git a/arch/mips/kernel/syscalls/syscall_n64.tbl b/arch/mips/kernel/syscalls/syscall_n64.tbl index 91649690b52f1b25ac229ad4e224366861ab66b9..169f21438065b58d0722445fb1f4afe16ff16a27 100644 --- a/arch/mips/kernel/syscalls/syscall_n64.tbl +++ b/arch/mips/kernel/syscalls/syscall_n64.tbl @@ -356,3 +356,4 @@ 439 n64 faccessat2 sys_faccessat2 440 n64 process_madvise sys_process_madvise 441 n64 epoll_pwait2 sys_epoll_pwait2 +442 n64 mount_setattr sys_mount_setattr diff --git a/arch/mips/kernel/syscalls/syscall_o32.tbl b/arch/mips/kernel/syscalls/syscall_o32.tbl index 4bad0c40aed6bcfa46294cd0be3d3566137085e0..090d29ca80ff8fefb7ef4f962f69371189c3ae61 100644 --- a/arch/mips/kernel/syscalls/syscall_o32.tbl +++ b/arch/mips/kernel/syscalls/syscall_o32.tbl @@ -429,3 +429,4 @@ 439 o32 faccessat2 sys_faccessat2 440 o32 process_madvise sys_process_madvise 441 o32 epoll_pwait2 sys_epoll_pwait2 compat_sys_epoll_pwait2 +442 o32 mount_setattr sys_mount_setattr diff --git a/arch/mips/lib/iomap-pci.c b/arch/mips/lib/iomap-pci.c index 210f5a95ecb100131e7a247311211093a5612efc..a9cb28813f0bc3ebee5be26d6f48f87e824a1fd9 100644 --- a/arch/mips/lib/iomap-pci.c +++ b/arch/mips/lib/iomap-pci.c @@ -32,7 +32,7 @@ void __iomem *__pci_ioport_map(struct pci_dev *dev, sprintf(name, "%04x:%02x", pci_domain_nr(bus), bus->number); printk(KERN_WARNING "io_map_base of root PCI bus %s unset. " "Trying to continue but you better\nfix this issue or " - "report it to linux-mips@linux-mips.org or your " + "report it to linux-mips@vger.kernel.org or your " "vendor.\n", name); #ifdef CONFIG_PCI_DOMAINS panic("To avoid data corruption io_map_base MUST be set with " diff --git a/arch/mips/loongson64/Platform b/arch/mips/loongson64/Platform index e2354e128d9a05a2514b3ea28b790005bbda9348..3e660d6d3c2bd5b3f5dd511c08b86ea811b11777 100644 --- a/arch/mips/loongson64/Platform +++ b/arch/mips/loongson64/Platform @@ -13,7 +13,7 @@ cflags-$(CONFIG_CPU_LOONGSON64) += -Wa,--trap # can't easily be used safely within the kbuild framework. # ifeq ($(call cc-ifversion, -ge, 0409, y), y) - ifeq ($(call ld-ifversion, -ge, 225000000, y), y) + ifeq ($(call ld-ifversion, -ge, 22500, y), y) cflags-$(CONFIG_CPU_LOONGSON64) += \ $(call cc-option,-march=loongson3a -U_MIPS_ISA -D_MIPS_ISA=_MIPS_ISA_MIPS64) else diff --git a/arch/mips/mm/cache.c b/arch/mips/mm/cache.c index 1754498b0717440bda0571725d6327f1b9885b8a..7719d632df8dfa62af229eb80d2d5ae76ba32a51 100644 --- a/arch/mips/mm/cache.c +++ b/arch/mips/mm/cache.c @@ -157,29 +157,31 @@ unsigned long _page_cachable_default; EXPORT_SYMBOL(_page_cachable_default); #define PM(p) __pgprot(_page_cachable_default | (p)) +#define PVA(p) PM(_PAGE_VALID | _PAGE_ACCESSED | (p)) static inline void setup_protection_map(void) { protection_map[0] = PM(_PAGE_PRESENT | _PAGE_NO_EXEC | _PAGE_NO_READ); - protection_map[1] = PM(_PAGE_PRESENT | _PAGE_NO_EXEC); - protection_map[2] = PM(_PAGE_PRESENT | _PAGE_NO_EXEC | _PAGE_NO_READ); - protection_map[3] = PM(_PAGE_PRESENT | _PAGE_NO_EXEC); - protection_map[4] = PM(_PAGE_PRESENT); - protection_map[5] = PM(_PAGE_PRESENT); - protection_map[6] = PM(_PAGE_PRESENT); - protection_map[7] = PM(_PAGE_PRESENT); + protection_map[1] = PVA(_PAGE_PRESENT | _PAGE_NO_EXEC); + protection_map[2] = PVA(_PAGE_PRESENT | _PAGE_NO_EXEC | _PAGE_NO_READ); + protection_map[3] = PVA(_PAGE_PRESENT | _PAGE_NO_EXEC); + protection_map[4] = PVA(_PAGE_PRESENT); + protection_map[5] = PVA(_PAGE_PRESENT); + protection_map[6] = PVA(_PAGE_PRESENT); + protection_map[7] = PVA(_PAGE_PRESENT); protection_map[8] = PM(_PAGE_PRESENT | _PAGE_NO_EXEC | _PAGE_NO_READ); - protection_map[9] = PM(_PAGE_PRESENT | _PAGE_NO_EXEC); - protection_map[10] = PM(_PAGE_PRESENT | _PAGE_NO_EXEC | _PAGE_WRITE | + protection_map[9] = PVA(_PAGE_PRESENT | _PAGE_NO_EXEC); + protection_map[10] = PVA(_PAGE_PRESENT | _PAGE_NO_EXEC | _PAGE_WRITE | _PAGE_NO_READ); - protection_map[11] = PM(_PAGE_PRESENT | _PAGE_NO_EXEC | _PAGE_WRITE); - protection_map[12] = PM(_PAGE_PRESENT); - protection_map[13] = PM(_PAGE_PRESENT); - protection_map[14] = PM(_PAGE_PRESENT | _PAGE_WRITE); - protection_map[15] = PM(_PAGE_PRESENT | _PAGE_WRITE); + protection_map[11] = PVA(_PAGE_PRESENT | _PAGE_NO_EXEC | _PAGE_WRITE); + protection_map[12] = PVA(_PAGE_PRESENT); + protection_map[13] = PVA(_PAGE_PRESENT); + protection_map[14] = PVA(_PAGE_PRESENT); + protection_map[15] = PVA(_PAGE_PRESENT); } +#undef _PVA #undef PM void cpu_cache_init(void) diff --git a/arch/mips/mm/pgtable-32.c b/arch/mips/mm/pgtable-32.c index bd4b0656add3016c0b69200db063d8ec5bdadad5..61891af250197bd721a377045bf80ec7296799ee 100644 --- a/arch/mips/mm/pgtable-32.c +++ b/arch/mips/mm/pgtable-32.c @@ -45,7 +45,6 @@ void set_pmd_at(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp, pmd_t pmd) { *pmdp = pmd; - flush_tlb_all(); } #endif /* defined(CONFIG_TRANSPARENT_HUGEPAGE) */ diff --git a/arch/mips/mm/pgtable-64.c b/arch/mips/mm/pgtable-64.c index 183ff9f9c026b6c65e7c588bedb852b38ee99bf7..7536f7804c440cc6c8b1779d0b49dd2e715e66cc 100644 --- a/arch/mips/mm/pgtable-64.c +++ b/arch/mips/mm/pgtable-64.c @@ -100,7 +100,6 @@ void set_pmd_at(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp, pmd_t pmd) { *pmdp = pmd; - flush_tlb_all(); } void __init pagetable_init(void) diff --git a/arch/mips/sgi-ip32/ip32-irq.c b/arch/mips/sgi-ip32/ip32-irq.c index 1bbd5bfb545835773b806c4923d5a578ace01f1e..e21ea1de05e31953ce51f04122512cd27b2d9c46 100644 --- a/arch/mips/sgi-ip32/ip32-irq.c +++ b/arch/mips/sgi-ip32/ip32-irq.c @@ -343,7 +343,7 @@ static void ip32_unknown_interrupt(void) printk("Register dump:\n"); show_regs(get_irq_regs()); - printk("Please mail this report to linux-mips@linux-mips.org\n"); + printk("Please mail this report to linux-mips@vger.kernel.org\n"); printk("Spinning..."); while(1) ; } diff --git a/arch/mips/vdso/Kconfig b/arch/mips/vdso/Kconfig index 7aec721398d59e4b5d271d98d7de0a9a43a4a039..a665f6108cb5a3c477fb0f2ce06e081c92055489 100644 --- a/arch/mips/vdso/Kconfig +++ b/arch/mips/vdso/Kconfig @@ -12,7 +12,7 @@ # the lack of relocations. As such, we disable the VDSO for microMIPS builds. config MIPS_LD_CAN_LINK_VDSO - def_bool LD_VERSION >= 225000000 || LD_IS_LLD + def_bool LD_VERSION >= 22500 || LD_IS_LLD config MIPS_DISABLE_VDSO def_bool CPU_MICROMIPS || (!CPU_MIPSR6 && !MIPS_LD_CAN_LINK_VDSO) diff --git a/arch/nds32/configs/defconfig b/arch/nds32/configs/defconfig index 40313a63507570b6088ebce45719b04134186fd7..f9a89cf00aa696134ef64322fef1505bcf5f8302 100644 --- a/arch/nds32/configs/defconfig +++ b/arch/nds32/configs/defconfig @@ -1,4 +1,3 @@ -CONFIG_CROSS_COMPILE="nds32le-linux-" CONFIG_SYSVIPC=y CONFIG_POSIX_MQUEUE=y CONFIG_HIGH_RES_TIMERS=y diff --git a/arch/nds32/kernel/process.c b/arch/nds32/kernel/process.c index e01ad5d1722450f2f4b54e16e615b4c9cf4dea40..c1327e552ec6c68a66ea708c127e7fc7e00d1df3 100644 --- a/arch/nds32/kernel/process.c +++ b/arch/nds32/kernel/process.c @@ -156,7 +156,7 @@ int copy_thread(unsigned long clone_flags, unsigned long stack_start, memset(&p->thread.cpu_context, 0, sizeof(struct cpu_context)); - if (unlikely(p->flags & PF_KTHREAD)) { + if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { memset(childregs, 0, sizeof(struct pt_regs)); /* kernel thread fn */ p->thread.cpu_context.r6 = stack_start; diff --git a/arch/nds32/kernel/setup.c b/arch/nds32/kernel/setup.c index c356e484dcab3300504078cd17b3759f1db36c6b..af82e996f412ecf898de9097c7a084079a5108ff 100644 --- a/arch/nds32/kernel/setup.c +++ b/arch/nds32/kernel/setup.c @@ -52,7 +52,7 @@ EXPORT_SYMBOL(elf_hwcap); /* * The following string table, must sync with HWCAP_xx bitmask, - * which is defined in + * which is defined above */ static const char *hwcap_str[] = { "mfusr_pc", diff --git a/arch/nds32/kernel/time.c b/arch/nds32/kernel/time.c index ac9d78ce3a818926dd61bd23cddfa10e495b7a1e..574a3d0a853980a9458ed236ad4fcb9bd242890d 100644 --- a/arch/nds32/kernel/time.c +++ b/arch/nds32/kernel/time.c @@ -2,7 +2,7 @@ // Copyright (C) 2005-2017 Andes Technology Corporation #include -#include +#include void __init time_init(void) { diff --git a/arch/nds32/kernel/traps.c b/arch/nds32/kernel/traps.c index 6a9772ba73927696c05d5a866ef08696bd7c2bba..ee0d9ae192a5041a8a6494b93f9373f5876737fe 100644 --- a/arch/nds32/kernel/traps.c +++ b/arch/nds32/kernel/traps.c @@ -25,17 +25,8 @@ extern void show_pte(struct mm_struct *mm, unsigned long addr); void dump_mem(const char *lvl, unsigned long bottom, unsigned long top) { unsigned long first; - mm_segment_t fs; int i; - /* - * We need to switch to kernel mode so that we can use __get_user - * to safely read from kernel space. Note that we now dump the - * code first, just in case the backtrace kills us. - */ - fs = get_fs(); - set_fs(KERNEL_DS); - pr_emerg("%s(0x%08lx to 0x%08lx)\n", lvl, bottom, top); for (first = bottom & ~31; first < top; first += 32) { @@ -48,7 +39,9 @@ void dump_mem(const char *lvl, unsigned long bottom, unsigned long top) for (p = first, i = 0; i < 8 && p < top; i++, p += 4) { if (p >= bottom && p < top) { unsigned long val; - if (__get_user(val, (unsigned long *)p) == 0) + + if (get_kernel_nofault(val, + (unsigned long *)p) == 0) sprintf(str + i * 9, " %08lx", val); else sprintf(str + i * 9, " ????????"); @@ -56,46 +49,10 @@ void dump_mem(const char *lvl, unsigned long bottom, unsigned long top) } pr_emerg("%s%04lx:%s\n", lvl, first & 0xffff, str); } - - set_fs(fs); } EXPORT_SYMBOL(dump_mem); -static void dump_instr(struct pt_regs *regs) -{ - unsigned long addr = instruction_pointer(regs); - mm_segment_t fs; - char str[sizeof("00000000 ") * 5 + 2 + 1], *p = str; - int i; - - return; - /* - * We need to switch to kernel mode so that we can use __get_user - * to safely read from kernel space. Note that we now dump the - * code first, just in case the backtrace kills us. - */ - fs = get_fs(); - set_fs(KERNEL_DS); - - pr_emerg("Code: "); - for (i = -4; i < 1; i++) { - unsigned int val, bad; - - bad = __get_user(val, &((u32 *) addr)[i]); - - if (!bad) { - p += sprintf(p, i == 0 ? "(%08x) " : "%08x ", val); - } else { - p += sprintf(p, "bad PC value"); - break; - } - } - pr_emerg("Code: %s\n", str); - - set_fs(fs); -} - #define LOOP_TIMES (100) static void __dump(struct task_struct *tsk, unsigned long *base_reg, const char *loglvl) @@ -179,7 +136,6 @@ void die(const char *str, struct pt_regs *regs, int err) if (!user_mode(regs) || in_interrupt()) { dump_mem("Stack: ", regs->sp, (regs->sp + PAGE_SIZE) & PAGE_MASK); - dump_instr(regs); dump_stack(); } diff --git a/arch/nios2/kernel/entry.S b/arch/nios2/kernel/entry.S index da8442450e4603c847c27fa901ba924a740a1df0..0794cd7803dfe0fe68e7ee4105bfa4ba805a329e 100644 --- a/arch/nios2/kernel/entry.S +++ b/arch/nios2/kernel/entry.S @@ -389,7 +389,10 @@ ENTRY(ret_from_interrupt) */ ENTRY(sys_clone) SAVE_SWITCH_STACK + subi sp, sp, 4 /* make space for tls pointer */ + stw r8, 0(sp) /* pass tls pointer (r8) via stack (5th argument) */ call nios2_clone + addi sp, sp, 4 RESTORE_SWITCH_STACK ret diff --git a/arch/nios2/kernel/process.c b/arch/nios2/kernel/process.c index 50b4eb19a6cc9962198ddbe12e1ba02bf2324d4b..c5f916ca6845f02876a9826a0b0fa9dcfa265715 100644 --- a/arch/nios2/kernel/process.c +++ b/arch/nios2/kernel/process.c @@ -109,7 +109,7 @@ int copy_thread(unsigned long clone_flags, unsigned long usp, unsigned long arg, struct switch_stack *childstack = ((struct switch_stack *)childregs) - 1; - if (unlikely(p->flags & PF_KTHREAD)) { + if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { memset(childstack, 0, sizeof(struct switch_stack) + sizeof(struct pt_regs)); diff --git a/arch/nios2/kernel/setup.c b/arch/nios2/kernel/setup.c index 3c6e3c813a0bc05650c8a404f4f373d0d24b31de..d2f21957e99c7f035dffddba39b5f0e9cd63a32f 100644 --- a/arch/nios2/kernel/setup.c +++ b/arch/nios2/kernel/setup.c @@ -32,8 +32,6 @@ EXPORT_SYMBOL(memory_start); unsigned long memory_end; EXPORT_SYMBOL(memory_end); -unsigned long memory_size; - static struct pt_regs fake_regs = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; @@ -141,16 +139,22 @@ asmlinkage void __init nios2_boot_init(unsigned r4, unsigned r5, unsigned r6, parse_early_param(); } +static void __init find_limits(unsigned long *min, unsigned long *max_low, + unsigned long *max_high) +{ + *max_low = PFN_DOWN(memblock_get_current_limit()); + *min = PFN_UP(memblock_start_of_DRAM()); + *max_high = PFN_DOWN(memblock_end_of_DRAM()); +} + void __init setup_arch(char **cmdline_p) { int dram_start; console_verbose(); - dram_start = memblock_start_of_DRAM(); - memory_size = memblock_phys_mem_size(); - memory_start = PAGE_ALIGN((unsigned long)__pa(_end)); - memory_end = (unsigned long) CONFIG_NIOS2_MEM_BASE + memory_size; + memory_start = memblock_start_of_DRAM(); + memory_end = memblock_end_of_DRAM(); init_mm.start_code = (unsigned long) _stext; init_mm.end_code = (unsigned long) _etext; @@ -161,11 +165,10 @@ void __init setup_arch(char **cmdline_p) /* Keep a copy of command line */ *cmdline_p = boot_command_line; - min_low_pfn = PFN_UP(memory_start); - max_low_pfn = PFN_DOWN(memory_end); + find_limits(&min_low_pfn, &max_low_pfn, &max_pfn); max_mapnr = max_low_pfn; - memblock_reserve(dram_start, memory_start - dram_start); + memblock_reserve(__pa_symbol(_stext), _end - _stext); #ifdef CONFIG_BLK_DEV_INITRD if (initrd_start) { memblock_reserve(virt_to_phys((void *)initrd_start), diff --git a/arch/nios2/kernel/sys_nios2.c b/arch/nios2/kernel/sys_nios2.c index cd390ec4f88bf5689557148cac034d63c9e68313..b1ca85699952135678c814053b94acc4ff388138 100644 --- a/arch/nios2/kernel/sys_nios2.c +++ b/arch/nios2/kernel/sys_nios2.c @@ -22,6 +22,7 @@ asmlinkage int sys_cacheflush(unsigned long addr, unsigned long len, unsigned int op) { struct vm_area_struct *vma; + struct mm_struct *mm = current->mm; if (len == 0) return 0; @@ -34,16 +35,22 @@ asmlinkage int sys_cacheflush(unsigned long addr, unsigned long len, if (addr + len < addr) return -EFAULT; + if (mmap_read_lock_killable(mm)) + return -EINTR; + /* * Verify that the specified address region actually belongs * to this process. */ - vma = find_vma(current->mm, addr); - if (vma == NULL || addr < vma->vm_start || addr + len > vma->vm_end) + vma = find_vma(mm, addr); + if (vma == NULL || addr < vma->vm_start || addr + len > vma->vm_end) { + mmap_read_unlock(mm); return -EFAULT; + } flush_cache_range(vma, addr, addr + len); + mmap_read_unlock(mm); return 0; } diff --git a/arch/openrisc/Kbuild b/arch/openrisc/Kbuild new file mode 100644 index 0000000000000000000000000000000000000000..4234b4c03e725af424ddacdd3bf742ea584cd452 --- /dev/null +++ b/arch/openrisc/Kbuild @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-y += lib/ kernel/ mm/ +obj-y += boot/dts/ diff --git a/arch/openrisc/Makefile b/arch/openrisc/Makefile index bf10141c7426844fe505a0bb8f5da0b73472aaca..410e7abfac698262379fda147854606d2d027289 100644 --- a/arch/openrisc/Makefile +++ b/arch/openrisc/Makefile @@ -24,6 +24,10 @@ LIBGCC := $(shell $(CC) $(KBUILD_CFLAGS) -print-libgcc-file-name) KBUILD_CFLAGS += -pipe -ffixed-r10 -D__linux__ +all: vmlinux.bin + +boot := arch/$(ARCH)/boot + ifeq ($(CONFIG_OPENRISC_HAVE_INST_MUL),y) KBUILD_CFLAGS += $(call cc-option,-mhard-mul) else @@ -38,14 +42,13 @@ endif head-y := arch/openrisc/kernel/head.o -core-y += arch/openrisc/lib/ \ - arch/openrisc/kernel/ \ - arch/openrisc/mm/ +core-y += arch/openrisc/ libs-y += $(LIBGCC) -ifneq '$(CONFIG_OPENRISC_BUILTIN_DTB)' '""' -BUILTIN_DTB := y -else -BUILTIN_DTB := n -endif -core-$(BUILTIN_DTB) += arch/openrisc/boot/dts/ +PHONY += vmlinux.bin + +vmlinux.bin: vmlinux + $(Q)$(MAKE) $(build)=$(boot) $(boot)/$@ + +archclean: + $(Q)$(MAKE) $(clean)=$(boot) diff --git a/arch/openrisc/boot/.gitignore b/arch/openrisc/boot/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..007d6fea3145df3eccef45dce063b2439a70e874 --- /dev/null +++ b/arch/openrisc/boot/.gitignore @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0 +vmlinux.bin diff --git a/arch/openrisc/boot/Makefile b/arch/openrisc/boot/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..5b28538f4dd1b49daa8ed8ce7c30dc0d995a13aa --- /dev/null +++ b/arch/openrisc/boot/Makefile @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Makefile for bootable kernel images +# + +targets += vmlinux.bin + +OBJCOPYFLAGS_vmlinux.bin := -O binary +$(obj)/vmlinux.bin: vmlinux FORCE + $(call if_changed,objcopy) diff --git a/arch/openrisc/kernel/process.c b/arch/openrisc/kernel/process.c index 3c98728cce2491c80e586945760b4603e1714c71..eb62429681fc806e5b48658d05488b3aa1e826c4 100644 --- a/arch/openrisc/kernel/process.c +++ b/arch/openrisc/kernel/process.c @@ -34,6 +34,7 @@ #include #include #include +#include #include #include @@ -49,10 +50,16 @@ */ struct thread_info *current_thread_info_set[NR_CPUS] = { &init_thread_info, }; -void machine_restart(void) +void machine_restart(char *cmd) { - printk(KERN_INFO "*** MACHINE RESTART ***\n"); - __asm__("l.nop 1"); + do_kernel_restart(cmd); + + /* Give a grace period for failure to restart of 1s */ + mdelay(1000); + + /* Whoops - the platform was unable to reboot. Tell the user! */ + pr_emerg("Reboot failed -- System halted\n"); + while (1); } /* @@ -167,7 +174,7 @@ copy_thread(unsigned long clone_flags, unsigned long usp, unsigned long arg, sp -= sizeof(struct pt_regs); kregs = (struct pt_regs *)sp; - if (unlikely(p->flags & PF_KTHREAD)) { + if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { memset(kregs, 0, sizeof(struct pt_regs)); kregs->gpr[20] = usp; /* fn, kernel thread */ kregs->gpr[22] = arg; diff --git a/arch/openrisc/kernel/smp.c b/arch/openrisc/kernel/smp.c index 29c82ef2e207ce098918a6a7a682311ab2d75eba..48e1092a64de35c7502a5b3ce91108358d297c05 100644 --- a/arch/openrisc/kernel/smp.c +++ b/arch/openrisc/kernel/smp.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -60,22 +61,32 @@ void __init smp_prepare_boot_cpu(void) void __init smp_init_cpus(void) { - int i; + struct device_node *cpu; + u32 cpu_id; - for (i = 0; i < NR_CPUS; i++) - set_cpu_possible(i, true); + for_each_of_cpu_node(cpu) { + if (of_property_read_u32(cpu, "reg", &cpu_id)) { + pr_warn("%s missing reg property", cpu->full_name); + continue; + } + + if (cpu_id < NR_CPUS) + set_cpu_possible(cpu_id, true); + } } void __init smp_prepare_cpus(unsigned int max_cpus) { - int i; + unsigned int cpu; /* * Initialise the present map, which describes the set of CPUs * actually populated at the present time. */ - for (i = 0; i < max_cpus; i++) - set_cpu_present(i, true); + for_each_possible_cpu(cpu) { + if (cpu < max_cpus) + set_cpu_present(cpu, true); + } } void __init smp_cpus_done(unsigned int max_cpus) diff --git a/arch/parisc/Kconfig b/arch/parisc/Kconfig index ecef9aff9d7296789e9d694ad68c13317e0536e2..4e53ac46e857e5d9485ce8fcd2d011eea5cd0dae 100644 --- a/arch/parisc/Kconfig +++ b/arch/parisc/Kconfig @@ -61,8 +61,10 @@ config PARISC select HAVE_KRETPROBES select HAVE_DYNAMIC_FTRACE if $(cc-option,-fpatchable-function-entry=1,1) select HAVE_FTRACE_MCOUNT_RECORD if HAVE_DYNAMIC_FTRACE + select FTRACE_MCOUNT_USE_PATCHABLE_FUNCTION_ENTRY if DYNAMIC_FTRACE select HAVE_KPROBES_ON_FTRACE select HAVE_DYNAMIC_FTRACE_WITH_REGS + select HAVE_SOFTIRQ_ON_OWN_STACK if IRQSTACKS select SET_FS help diff --git a/arch/parisc/configs/generic-32bit_defconfig b/arch/parisc/configs/generic-32bit_defconfig index 3cbcfad5f7249d53cac0977e06323add28be0b31..7611d48c599e01269d0dd3222e752dff0c3a86db 100644 --- a/arch/parisc/configs/generic-32bit_defconfig +++ b/arch/parisc/configs/generic-32bit_defconfig @@ -22,7 +22,6 @@ CONFIG_PCI_LBA=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y CONFIG_MODULE_FORCE_UNLOAD=y -CONFIG_UNUSED_SYMBOLS=y # CONFIG_BLK_DEV_BSG is not set # CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set CONFIG_BINFMT_MISC=m diff --git a/arch/parisc/configs/generic-64bit_defconfig b/arch/parisc/configs/generic-64bit_defconfig index 8f81fcbf04c413ed6b9ec798974d413a1b5b7b7f..53054b81461a10e10f1e1e7bf13c7c44384a0566 100644 --- a/arch/parisc/configs/generic-64bit_defconfig +++ b/arch/parisc/configs/generic-64bit_defconfig @@ -31,7 +31,6 @@ CONFIG_MODULE_FORCE_LOAD=y CONFIG_MODULE_UNLOAD=y CONFIG_MODULE_FORCE_UNLOAD=y CONFIG_MODVERSIONS=y -CONFIG_UNUSED_SYMBOLS=y CONFIG_BLK_DEV_INTEGRITY=y CONFIG_BINFMT_MISC=m # CONFIG_COMPACTION is not set diff --git a/arch/parisc/include/asm/hardirq.h b/arch/parisc/include/asm/hardirq.h index fad29aa6f45fa7f1fb179a751fa0c573ab916d31..1e4fbd0fd944808c3be46ac53fe462053ed4b826 100644 --- a/arch/parisc/include/asm/hardirq.h +++ b/arch/parisc/include/asm/hardirq.h @@ -12,10 +12,6 @@ #include #include -#ifdef CONFIG_IRQSTACKS -#define __ARCH_HAS_DO_SOFTIRQ -#endif - typedef struct { unsigned int __softirq_pending; unsigned int kernel_stack_usage; diff --git a/arch/parisc/kernel/irq.c b/arch/parisc/kernel/irq.c index 1dfb439b0692838b8ac6aebd1521070816a16957..0d46b19dc4d3d77fd643bb82bd960115382fb105 100644 --- a/arch/parisc/kernel/irq.c +++ b/arch/parisc/kernel/irq.c @@ -17,6 +17,7 @@ #include #include +#include #include #include diff --git a/arch/parisc/kernel/process.c b/arch/parisc/kernel/process.c index fda1c1a6a444dd00bb10711950148a2d16a7299a..b144fbe29bc16517a81fa200de7be2304ed29796 100644 --- a/arch/parisc/kernel/process.c +++ b/arch/parisc/kernel/process.c @@ -200,7 +200,7 @@ copy_thread(unsigned long clone_flags, unsigned long usp, extern void * const ret_from_kernel_thread; extern void * const child_return; - if (unlikely(p->flags & PF_KTHREAD)) { + if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { /* kernel thread */ memset(cregs, 0, sizeof(struct pt_regs)); if (!usp) /* idle thread */ diff --git a/arch/parisc/kernel/syscalls/Makefile b/arch/parisc/kernel/syscalls/Makefile index c22a21c39f308429ccaaa7d3b1d29dc23010a1fb..283f64407b07a40944e842c44a15ed2629b85db9 100644 --- a/arch/parisc/kernel/syscalls/Makefile +++ b/arch/parisc/kernel/syscalls/Makefile @@ -5,7 +5,7 @@ uapi := arch/$(SRCARCH)/include/generated/uapi/asm _dummy := $(shell [ -d '$(uapi)' ] || mkdir -p '$(uapi)') \ $(shell [ -d '$(kapi)' ] || mkdir -p '$(kapi)') -syscall := $(srctree)/$(src)/syscall.tbl +syscall := $(src)/syscall.tbl syshdr := $(srctree)/$(src)/syscallhdr.sh systbl := $(srctree)/$(src)/syscalltbl.sh @@ -22,24 +22,24 @@ quiet_cmd_systbl = SYSTBL $@ '$(systbl_offset_$(basetarget))' syshdr_abis_unistd_32 := common,32 -$(uapi)/unistd_32.h: $(syscall) $(syshdr) +$(uapi)/unistd_32.h: $(syscall) $(syshdr) FORCE $(call if_changed,syshdr) syshdr_abis_unistd_64 := common,64 -$(uapi)/unistd_64.h: $(syscall) $(syshdr) +$(uapi)/unistd_64.h: $(syscall) $(syshdr) FORCE $(call if_changed,syshdr) systbl_abis_syscall_table_32 := common,32 -$(kapi)/syscall_table_32.h: $(syscall) $(systbl) +$(kapi)/syscall_table_32.h: $(syscall) $(systbl) FORCE $(call if_changed,systbl) systbl_abis_syscall_table_64 := common,64 -$(kapi)/syscall_table_64.h: $(syscall) $(systbl) +$(kapi)/syscall_table_64.h: $(syscall) $(systbl) FORCE $(call if_changed,systbl) systbl_abis_syscall_table_c32 := common,32 systbl_abi_syscall_table_c32 := c32 -$(kapi)/syscall_table_c32.h: $(syscall) $(systbl) +$(kapi)/syscall_table_c32.h: $(syscall) $(systbl) FORCE $(call if_changed,systbl) uapisyshdr-y += unistd_32.h unistd_64.h @@ -47,9 +47,10 @@ kapisyshdr-y += syscall_table_32.h \ syscall_table_64.h \ syscall_table_c32.h -targets += $(uapisyshdr-y) $(kapisyshdr-y) +uapisyshdr-y := $(addprefix $(uapi)/, $(uapisyshdr-y)) +kapisyshdr-y := $(addprefix $(kapi)/, $(kapisyshdr-y)) +targets += $(addprefix ../../../../, $(uapisyshdr-y) $(kapisyshdr-y)) PHONY += all -all: $(addprefix $(uapi)/,$(uapisyshdr-y)) -all: $(addprefix $(kapi)/,$(kapisyshdr-y)) +all: $(uapisyshdr-y) $(kapisyshdr-y) @: diff --git a/arch/parisc/kernel/syscalls/syscall.tbl b/arch/parisc/kernel/syscalls/syscall.tbl index 6bcc31966b4460b28a863384f97ea5c97d7b027c..271a9251968345de67e53a2f2f9ad1ba5aa855dd 100644 --- a/arch/parisc/kernel/syscalls/syscall.tbl +++ b/arch/parisc/kernel/syscalls/syscall.tbl @@ -439,3 +439,4 @@ 439 common faccessat2 sys_faccessat2 440 common process_madvise sys_process_madvise 441 common epoll_pwait2 sys_epoll_pwait2 compat_sys_epoll_pwait2 +442 common mount_setattr sys_mount_setattr diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig index 9141f03060ceacd640534938dab529ea831eae5a..386ae12d8523b4af1aa20dd7237d6e7c57d202ab 100644 --- a/arch/powerpc/Kconfig +++ b/arch/powerpc/Kconfig @@ -196,7 +196,6 @@ config PPC select HAVE_STACKPROTECTOR if PPC64 && $(cc-option,-mstack-protector-guard=tls -mstack-protector-guard-reg=r13) select HAVE_STACKPROTECTOR if PPC32 && $(cc-option,-mstack-protector-guard=tls -mstack-protector-guard-reg=r2) select HAVE_CONTEXT_TRACKING if PPC64 - select HAVE_TIF_NOHZ if PPC64 select HAVE_DEBUG_KMEMLEAK select HAVE_DEBUG_STACKOVERFLOW select HAVE_DYNAMIC_FTRACE @@ -236,6 +235,7 @@ config PPC select MMU_GATHER_PAGE_SIZE select HAVE_REGS_AND_STACK_ACCESS_API select HAVE_RELIABLE_STACKTRACE if PPC_BOOK3S_64 && CPU_LITTLE_ENDIAN + select HAVE_SOFTIRQ_ON_OWN_STACK select HAVE_SYSCALL_TRACEPOINTS select HAVE_VIRT_CPU_ACCOUNTING select HAVE_IRQ_TIME_ACCOUNTING @@ -503,18 +503,14 @@ config HOTPLUG_CPU Say N if you are unsure. config PPC_QUEUED_SPINLOCKS - bool "Queued spinlocks" + bool "Queued spinlocks" if EXPERT depends on SMP + default PPC_BOOK3S_64 help Say Y here to use queued spinlocks which give better scalability and fairness on large SMP and NUMA systems without harming single threaded performance. - This option is currently experimental, the code is more complex and - less tested so it defaults to "N" for the moment. - - If unsure, say "N". - config ARCH_CPU_PROBE_RELEASE def_bool y depends on HOTPLUG_CPU @@ -718,18 +714,6 @@ config ARCH_MEMORY_PROBE def_bool y depends on MEMORY_HOTPLUG -config STDBINUTILS - bool "Using standard binutils settings" - depends on 44x - default y - help - Turning this option off allows you to select 256KB PAGE_SIZE on 44x. - Note, that kernel will be able to run only those applications, - which had been compiled using binutils later than 2.17.50.0.3 with - '-zmax-page-size' set to 256K (the default is 64K). Or, if using - the older binutils, you can patch them with a trivial patch, which - changes the ELF_MAXPAGESIZE definition from 0x10000 to 0x40000. - choice prompt "Page size" default PPC_4K_PAGES @@ -769,17 +753,15 @@ config PPC_64K_PAGES select HAVE_ARCH_SOFT_DIRTY if PPC_BOOK3S_64 config PPC_256K_PAGES - bool "256k page size" - depends on 44x && !STDBINUTILS + bool "256k page size (Requires non-standard binutils settings)" + depends on 44x && !PPC_47x help Make the page size 256k. - As the ELF standard only requires alignment to support page - sizes up to 64k, you will need to compile all of your user - space applications with a non-standard binutils settings - (see the STDBINUTILS description for details). - - Say N unless you know what you are doing. + The kernel will only be able to run applications that have been + compiled with '-zmax-page-size' set to 256K (the default is 64K) using + binutils later than 2.17.50.0.3, or by patching the ELF_MAXPAGESIZE + definition from 0x10000 to 0x40000 in older versions. endchoice diff --git a/arch/powerpc/Kconfig.debug b/arch/powerpc/Kconfig.debug index b88900f4832fdf7a798b024d59f283032fd85d35..ae084357994e87b5f07294ab9543b8a80d7d94a0 100644 --- a/arch/powerpc/Kconfig.debug +++ b/arch/powerpc/Kconfig.debug @@ -88,6 +88,7 @@ config PPC_IRQ_SOFT_MASK_DEBUG config XMON bool "Include xmon kernel debugger" depends on DEBUG_KERNEL + select CONSOLE_POLL if SERIAL_CPM_CONSOLE help Include in-kernel hooks for the xmon kernel monitor/debugger. Unless you are intending to debug the kernel, say N here. diff --git a/arch/powerpc/Makefile b/arch/powerpc/Makefile index b959fdaec713968ee270c670cc3a889faf4b6b20..5f8544cf724a4fff9b2aed3f0f487e047fd8ee44 100644 --- a/arch/powerpc/Makefile +++ b/arch/powerpc/Makefile @@ -65,7 +65,7 @@ UTS_MACHINE := $(subst $(space),,$(machine-y)) ifdef CONFIG_PPC32 KBUILD_LDFLAGS_MODULE += arch/powerpc/lib/crtsavres.o else -ifeq ($(call ld-ifversion, -ge, 225000000, y),y) +ifeq ($(call ld-ifversion, -ge, 22500, y),y) # Have the linker provide sfpr if possible. # There is a corresponding test in arch/powerpc/lib/Makefile KBUILD_LDFLAGS_MODULE += --save-restore-funcs diff --git a/arch/powerpc/configs/44x/akebono_defconfig b/arch/powerpc/configs/44x/akebono_defconfig index 72b8f93a9bdd68c898e96bbfd2470043e7f0bf03..4bc549c6edc5a4e9bacfcf5e2b7d074c569bff9e 100644 --- a/arch/powerpc/configs/44x/akebono_defconfig +++ b/arch/powerpc/configs/44x/akebono_defconfig @@ -20,6 +20,7 @@ CONFIG_IRQ_ALL_CPUS=y # CONFIG_COMPACTION is not set # CONFIG_SUSPEND is not set CONFIG_NET=y +CONFIG_NETDEVICES=y CONFIG_PACKET=y CONFIG_UNIX=y CONFIG_INET=y @@ -40,7 +41,9 @@ CONFIG_BLK_DEV_RAM_SIZE=35000 # CONFIG_SCSI_PROC_FS is not set CONFIG_BLK_DEV_SD=y # CONFIG_SCSI_LOWLEVEL is not set +CONFIG_ATA=y # CONFIG_SATA_PMP is not set +CONFIG_SATA_AHCI_PLATFORM=y # CONFIG_ATA_SFF is not set # CONFIG_NET_VENDOR_3COM is not set # CONFIG_NET_VENDOR_ADAPTEC is not set @@ -97,6 +100,8 @@ CONFIG_USB_OHCI_HCD=y # CONFIG_USB_OHCI_HCD_PCI is not set CONFIG_USB_STORAGE=y CONFIG_MMC=y +CONFIG_MMC_SDHCI=y +CONFIG_MMC_SDHCI_PLTFM=y CONFIG_RTC_CLASS=y CONFIG_RTC_DRV_M41T80=y CONFIG_EXT2_FS=y diff --git a/arch/powerpc/configs/ppc6xx_defconfig b/arch/powerpc/configs/ppc6xx_defconfig index 10c055eaebf06b9c3af4d48ec4b090f9cc371ae1..6677ac0da45a5ec036c19a5f0418e0448a0fcd4a 100644 --- a/arch/powerpc/configs/ppc6xx_defconfig +++ b/arch/powerpc/configs/ppc6xx_defconfig @@ -1071,7 +1071,6 @@ CONFIG_NLS_ISO8859_15=m CONFIG_NLS_KOI8_R=m CONFIG_NLS_KOI8_U=m CONFIG_DEBUG_INFO=y -CONFIG_UNUSED_SYMBOLS=y CONFIG_HEADERS_INSTALL=y CONFIG_MAGIC_SYSRQ=y CONFIG_DEBUG_KERNEL=y diff --git a/arch/powerpc/crypto/sha256-spe-glue.c b/arch/powerpc/crypto/sha256-spe-glue.c index a6e650a97d8f4b0de168dd7f546177797d140175..ffedea7e4befb73d6dd4fbf6f33a02277a21eb10 100644 --- a/arch/powerpc/crypto/sha256-spe-glue.c +++ b/arch/powerpc/crypto/sha256-spe-glue.c @@ -129,7 +129,7 @@ static int ppc_spe_sha256_update(struct shash_desc *desc, const u8 *data, src += bytes; len -= bytes; - }; + } memcpy((char *)sctx->buf, src, len); return 0; diff --git a/arch/powerpc/include/asm/asm-prototypes.h b/arch/powerpc/include/asm/asm-prototypes.h index d0b832cbbec830fd010cc591bf714868ae46fe23..939f3c94c8f392adf3e58be533c8f46569d66ba5 100644 --- a/arch/powerpc/include/asm/asm-prototypes.h +++ b/arch/powerpc/include/asm/asm-prototypes.h @@ -56,35 +56,6 @@ int exit_vmx_usercopy(void); int enter_vmx_ops(void); void *exit_vmx_ops(void *dest); -/* Traps */ -long machine_check_early(struct pt_regs *regs); -long hmi_exception_realmode(struct pt_regs *regs); -void SMIException(struct pt_regs *regs); -void handle_hmi_exception(struct pt_regs *regs); -void instruction_breakpoint_exception(struct pt_regs *regs); -void RunModeException(struct pt_regs *regs); -void single_step_exception(struct pt_regs *regs); -void program_check_exception(struct pt_regs *regs); -void alignment_exception(struct pt_regs *regs); -void StackOverflow(struct pt_regs *regs); -void stack_overflow_exception(struct pt_regs *regs); -void kernel_fp_unavailable_exception(struct pt_regs *regs); -void altivec_unavailable_exception(struct pt_regs *regs); -void vsx_unavailable_exception(struct pt_regs *regs); -void fp_unavailable_tm(struct pt_regs *regs); -void altivec_unavailable_tm(struct pt_regs *regs); -void vsx_unavailable_tm(struct pt_regs *regs); -void facility_unavailable_exception(struct pt_regs *regs); -void TAUException(struct pt_regs *regs); -void altivec_assist_exception(struct pt_regs *regs); -void unrecoverable_exception(struct pt_regs *regs); -void kernel_bad_stack(struct pt_regs *regs); -void system_reset_exception(struct pt_regs *regs); -void machine_check_exception(struct pt_regs *regs); -void emulation_assist_interrupt(struct pt_regs *regs); -long do_slb_fault(struct pt_regs *regs, unsigned long ea); -void do_bad_slb_fault(struct pt_regs *regs, unsigned long ea, long err); - /* signals, syscalls and interrupts */ long sys_swapcontext(struct ucontext __user *old_ctx, struct ucontext __user *new_ctx, diff --git a/arch/powerpc/include/asm/book3s/32/kup.h b/arch/powerpc/include/asm/book3s/32/kup.h index a0117a9d5b06cd45ac4ed317d0a2410cc890e0ad..73bc5d2c431dfaa42ba1f9904087f99b652fff57 100644 --- a/arch/powerpc/include/asm/book3s/32/kup.h +++ b/arch/powerpc/include/asm/book3s/32/kup.h @@ -95,12 +95,12 @@ static inline void kuap_update_sr(u32 sr, u32 addr, u32 end) addr &= 0xf0000000; /* align addr to start of segment */ barrier(); /* make sure thread.kuap is updated before playing with SRs */ while (addr < end) { - mtsrin(sr, addr); + mtsr(sr, addr); sr += 0x111; /* next VSID */ sr &= 0xf0ffffff; /* clear VSID overflow */ addr += 0x10000000; /* address of next segment */ } - isync(); /* Context sync required after mtsrin() */ + isync(); /* Context sync required after mtsr() */ } static __always_inline void allow_user_access(void __user *to, const void __user *from, @@ -122,7 +122,7 @@ static __always_inline void allow_user_access(void __user *to, const void __user end = min(addr + size, TASK_SIZE); current->thread.kuap = (addr & 0xf0000000) | ((((end - 1) >> 28) + 1) & 0xf); - kuap_update_sr(mfsrin(addr) & ~SR_KS, addr, end); /* Clear Ks */ + kuap_update_sr(mfsr(addr) & ~SR_KS, addr, end); /* Clear Ks */ } static __always_inline void prevent_user_access(void __user *to, const void __user *from, @@ -151,7 +151,7 @@ static __always_inline void prevent_user_access(void __user *to, const void __us } current->thread.kuap = 0; - kuap_update_sr(mfsrin(addr) | SR_KS, addr, end); /* set Ks */ + kuap_update_sr(mfsr(addr) | SR_KS, addr, end); /* set Ks */ } static inline unsigned long prevent_user_access_return(void) diff --git a/arch/powerpc/include/asm/book3s/32/mmu-hash.h b/arch/powerpc/include/asm/book3s/32/mmu-hash.h index 685c589e723f525a2ce37f9eb69d7509f9d189f3..b85f8e114a9c0f52f15f78251ed43db90d1afa19 100644 --- a/arch/powerpc/include/asm/book3s/32/mmu-hash.h +++ b/arch/powerpc/include/asm/book3s/32/mmu-hash.h @@ -94,7 +94,7 @@ typedef struct { } mm_context_t; void update_bats(void); -static inline void cleanup_cpu_mmu_context(void) { }; +static inline void cleanup_cpu_mmu_context(void) { } /* patch sites */ extern s32 patch__hash_page_A0, patch__hash_page_A1, patch__hash_page_A2; diff --git a/arch/powerpc/include/asm/book3s/64/kup.h b/arch/powerpc/include/asm/book3s/64/kup.h index 7d1ef7b9754e7c450af4eba56c7531efbaff6f50..8bd90505089624294d010f39a27f89b185176ebf 100644 --- a/arch/powerpc/include/asm/book3s/64/kup.h +++ b/arch/powerpc/include/asm/book3s/64/kup.h @@ -339,7 +339,7 @@ static inline unsigned long get_kuap(void) * This has no effect in terms of actually blocking things on hash, * so it doesn't break anything. */ - if (!early_mmu_has_feature(MMU_FTR_BOOK3S_KUAP)) + if (!mmu_has_feature(MMU_FTR_BOOK3S_KUAP)) return AMR_KUAP_BLOCKED; return mfspr(SPRN_AMR); @@ -347,7 +347,7 @@ static inline unsigned long get_kuap(void) static inline void set_kuap(unsigned long value) { - if (!early_mmu_has_feature(MMU_FTR_BOOK3S_KUAP)) + if (!mmu_has_feature(MMU_FTR_BOOK3S_KUAP)) return; /* diff --git a/arch/powerpc/include/asm/book3s/64/mmu-hash.h b/arch/powerpc/include/asm/book3s/64/mmu-hash.h index 066b1d34c7bca4562ffbb614ac84a47b957206b0..f911bdb68d8bb52b1df8011654ca1b6d55a65c5b 100644 --- a/arch/powerpc/include/asm/book3s/64/mmu-hash.h +++ b/arch/powerpc/include/asm/book3s/64/mmu-hash.h @@ -454,6 +454,8 @@ static inline unsigned long hpt_hash(unsigned long vpn, #define HPTE_NOHPTE_UPDATE 0x2 #define HPTE_USE_KERNEL_KEY 0x4 +long hpte_insert_repeating(unsigned long hash, unsigned long vpn, unsigned long pa, + unsigned long rlags, unsigned long vflags, int psize, int ssize); extern int __hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid, pte_t *ptep, unsigned long trap, unsigned long flags, int ssize, int subpage_prot); @@ -467,6 +469,8 @@ extern int hash_page_mm(struct mm_struct *mm, unsigned long ea, unsigned long flags); extern int hash_page(unsigned long ea, unsigned long access, unsigned long trap, unsigned long dsisr); +void low_hash_fault(struct pt_regs *regs, unsigned long address, int rc); +int __hash_page(unsigned long trap, unsigned long ea, unsigned long dsisr, unsigned long msr); int __hash_page_huge(unsigned long ea, unsigned long access, unsigned long vsid, pte_t *ptep, unsigned long trap, unsigned long flags, int ssize, unsigned int shift, unsigned int mmu_psize); @@ -521,6 +525,7 @@ void slb_dump_contents(struct slb_entry *slb_ptr); extern void slb_vmalloc_update(void); extern void slb_set_size(u16 size); +void preload_new_slb_context(unsigned long start, unsigned long sp); #endif /* __ASSEMBLY__ */ /* diff --git a/arch/powerpc/include/asm/book3s/64/mmu.h b/arch/powerpc/include/asm/book3s/64/mmu.h index 995bbcdd0ef82c51e8a822de38c1e48f3aa56aa5..eace8c3f7b0a1bcc3c1a3e8a558899d5abda3c27 100644 --- a/arch/powerpc/include/asm/book3s/64/mmu.h +++ b/arch/powerpc/include/asm/book3s/64/mmu.h @@ -239,7 +239,7 @@ static inline void setup_initial_memory_limit(phys_addr_t first_memblock_base, #ifdef CONFIG_PPC_PSERIES extern void radix_init_pseries(void); #else -static inline void radix_init_pseries(void) { }; +static inline void radix_init_pseries(void) { } #endif #ifdef CONFIG_HOTPLUG_CPU diff --git a/arch/powerpc/include/asm/book3s/64/pgtable.h b/arch/powerpc/include/asm/book3s/64/pgtable.h index a398866816297bdbae0fa808b656a8215a21f68f..058601efbc8a3bf9df2110d250857f74ac8898cf 100644 --- a/arch/powerpc/include/asm/book3s/64/pgtable.h +++ b/arch/powerpc/include/asm/book3s/64/pgtable.h @@ -388,11 +388,28 @@ static inline int __ptep_test_and_clear_young(struct mm_struct *mm, #define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG #define ptep_test_and_clear_young(__vma, __addr, __ptep) \ ({ \ - int __r; \ - __r = __ptep_test_and_clear_young((__vma)->vm_mm, __addr, __ptep); \ - __r; \ + __ptep_test_and_clear_young((__vma)->vm_mm, __addr, __ptep); \ }) +/* + * On Book3S CPUs, clearing the accessed bit without a TLB flush + * doesn't cause data corruption. [ It could cause incorrect + * page aging and the (mistaken) reclaim of hot pages, but the + * chance of that should be relatively low. ] + * + * So as a performance optimization don't flush the TLB when + * clearing the accessed bit, it will eventually be flushed by + * a context switch or a VM operation anyway. [ In the rare + * event of it not getting flushed for a long time the delay + * shouldn't really matter because there's no real memory + * pressure for swapout to react to. ] + */ +#define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH +#define ptep_clear_flush_young ptep_test_and_clear_young + +#define __HAVE_ARCH_PMDP_CLEAR_YOUNG_FLUSH +#define pmdp_clear_flush_young pmdp_test_and_clear_young + static inline int __pte_write(pte_t pte) { return !!(pte_raw(pte) & cpu_to_be64(_PAGE_WRITE)); diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h index 94439e0cefc9c222c31c32538ab9ac0b9f26f39e..8b33601cdb9d74cbaa8027190a20d7eaa5174afd 100644 --- a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h +++ b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h @@ -35,7 +35,7 @@ extern void radix__flush_pwc_lpid(unsigned int lpid); extern void radix__flush_all_lpid(unsigned int lpid); extern void radix__flush_all_lpid_guest(unsigned int lpid); #else -static inline void radix__tlbiel_all(unsigned int action) { WARN_ON(1); }; +static inline void radix__tlbiel_all(unsigned int action) { WARN_ON(1); } static inline void radix__flush_tlb_lpid_page(unsigned int lpid, unsigned long addr, unsigned long page_size) diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush.h b/arch/powerpc/include/asm/book3s/64/tlbflush.h index dcb5c3839d2f7b74da16497cbbd5fd1d2d6a64ed..215973b4cb26986ddcf1b0ad82eea4a3dd82bf8d 100644 --- a/arch/powerpc/include/asm/book3s/64/tlbflush.h +++ b/arch/powerpc/include/asm/book3s/64/tlbflush.h @@ -31,7 +31,7 @@ static inline void tlbiel_all(void) hash__tlbiel_all(TLB_INVAL_SCOPE_GLOBAL); } #else -static inline void tlbiel_all(void) { BUG(); }; +static inline void tlbiel_all(void) { BUG(); } #endif static inline void tlbiel_all_lpid(bool radix) diff --git a/arch/powerpc/include/asm/bug.h b/arch/powerpc/include/asm/bug.h index 464f8ca8a5c970ec06f7f034391baf7fda27acc5..d1635ffbb179e7e0f21bd49398780fd8f38e576e 100644 --- a/arch/powerpc/include/asm/bug.h +++ b/arch/powerpc/include/asm/bug.h @@ -111,12 +111,15 @@ #ifndef __ASSEMBLY__ struct pt_regs; -extern int do_page_fault(struct pt_regs *, unsigned long, unsigned long); -extern void bad_page_fault(struct pt_regs *, unsigned long, int); -void __bad_page_fault(struct pt_regs *regs, unsigned long address, int sig); +long do_page_fault(struct pt_regs *); +long hash__do_page_fault(struct pt_regs *); +void bad_page_fault(struct pt_regs *, int); +void __bad_page_fault(struct pt_regs *regs, int sig); +void do_bad_page_fault_segv(struct pt_regs *regs); extern void _exception(int, struct pt_regs *, int, unsigned long); extern void _exception_pkey(struct pt_regs *, unsigned long, int); extern void die(const char *, struct pt_regs *, long); +void die_mce(const char *str, struct pt_regs *regs, long err); extern bool die_will_crash(void); extern void panic_flush_kmsg_start(void); extern void panic_flush_kmsg_end(void); diff --git a/arch/powerpc/include/asm/cacheflush.h b/arch/powerpc/include/asm/cacheflush.h index 138e46d8c04e587ab36ec3ac8f2049fcf1d763e3..f63495109f634216039b0333c93070064ee153b0 100644 --- a/arch/powerpc/include/asm/cacheflush.h +++ b/arch/powerpc/include/asm/cacheflush.h @@ -8,6 +8,12 @@ #include #include +/* + * This flag is used to indicate that the page pointed to by a pte is clean + * and does not require cleaning before returning it to the user. + */ +#define PG_dcache_clean PG_arch_1 + #ifdef CONFIG_PPC_BOOK3S_64 /* * Book3s has no ptesync after setting a pte, so without this ptesync it's diff --git a/arch/powerpc/include/asm/cputime.h b/arch/powerpc/include/asm/cputime.h index ed75d1c318e3124af7ec195902c2051db3fc14fa..504f7fe6711a32783b148e9e59734a78bf4f3b97 100644 --- a/arch/powerpc/include/asm/cputime.h +++ b/arch/powerpc/include/asm/cputime.h @@ -87,6 +87,17 @@ static notrace inline void account_cpu_user_exit(void) acct->starttime_user = tb; } +static notrace inline void account_stolen_time(void) +{ +#ifdef CONFIG_PPC_SPLPAR + if (firmware_has_feature(FW_FEATURE_SPLPAR)) { + struct lppaca *lp = local_paca->lppaca_ptr; + + if (unlikely(local_paca->dtl_ridx != be64_to_cpu(lp->dtl_idx))) + accumulate_stolen_time(); + } +#endif +} #endif /* __KERNEL__ */ #else /* CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */ @@ -96,5 +107,8 @@ static inline void account_cpu_user_entry(void) static inline void account_cpu_user_exit(void) { } +static notrace inline void account_stolen_time(void) +{ +} #endif /* CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */ #endif /* __POWERPC_CPUTIME_H */ diff --git a/arch/powerpc/include/asm/debug.h b/arch/powerpc/include/asm/debug.h index ec57daf87f409639376ab3fa16d8f8a381162514..86a14736c76c36fa7e04de7e5a2525b73f8958b1 100644 --- a/arch/powerpc/include/asm/debug.h +++ b/arch/powerpc/include/asm/debug.h @@ -50,10 +50,6 @@ bool ppc_breakpoint_available(void); #ifdef CONFIG_PPC_ADV_DEBUG_REGS extern void do_send_trap(struct pt_regs *regs, unsigned long address, unsigned long error_code, int brkpt); -#else - -extern void do_break(struct pt_regs *regs, unsigned long address, - unsigned long error_code); #endif #endif /* _ASM_POWERPC_DEBUG_H */ diff --git a/arch/powerpc/include/asm/firmware.h b/arch/powerpc/include/asm/firmware.h index aa6a5ef5d4830c8ff97983c392f9ae604139ef87..7604673787d6792018d1b31c5d0c5e72ae754b83 100644 --- a/arch/powerpc/include/asm/firmware.h +++ b/arch/powerpc/include/asm/firmware.h @@ -137,7 +137,7 @@ extern unsigned int __start___fw_ftr_fixup, __stop___fw_ftr_fixup; #ifdef CONFIG_PPC_PSERIES void pseries_probe_fw_features(void); #else -static inline void pseries_probe_fw_features(void) { }; +static inline void pseries_probe_fw_features(void) { } #endif #endif /* __ASSEMBLY__ */ diff --git a/arch/powerpc/include/asm/hugetlb.h b/arch/powerpc/include/asm/hugetlb.h index 013165e6261861591d778833979f45807a8af514..f18c543bc01de36adee0ddc670d19cc0aac75b0f 100644 --- a/arch/powerpc/include/asm/hugetlb.h +++ b/arch/powerpc/include/asm/hugetlb.h @@ -17,8 +17,6 @@ extern bool hugetlb_disabled; void hugetlbpage_init_default(void); -void flush_dcache_icache_hugepage(struct page *page); - int slice_is_hugepage_only_range(struct mm_struct *mm, unsigned long addr, unsigned long len); diff --git a/arch/powerpc/include/asm/hw_irq.h b/arch/powerpc/include/asm/hw_irq.h index 0363734ff56e0f9fc0e2c6ddb013cea24d7cbbd6..56a98936a6a93734c4993133cdf6060b7bc3ee79 100644 --- a/arch/powerpc/include/asm/hw_irq.h +++ b/arch/powerpc/include/asm/hw_irq.h @@ -38,6 +38,8 @@ #define PACA_IRQ_MUST_HARD_MASK (PACA_IRQ_EE) #endif +#endif /* CONFIG_PPC64 */ + /* * flags for paca->irq_soft_mask */ @@ -46,18 +48,56 @@ #define IRQS_PMI_DISABLED 2 #define IRQS_ALL_DISABLED (IRQS_DISABLED | IRQS_PMI_DISABLED) -#endif /* CONFIG_PPC64 */ - #ifndef __ASSEMBLY__ -extern void replay_system_reset(void); -extern void replay_soft_interrupts(void); +static inline void __hard_irq_enable(void) +{ + if (IS_ENABLED(CONFIG_BOOKE) || IS_ENABLED(CONFIG_40x)) + wrtee(MSR_EE); + else if (IS_ENABLED(CONFIG_PPC_8xx)) + wrtspr(SPRN_EIE); + else if (IS_ENABLED(CONFIG_PPC_BOOK3S_64)) + __mtmsrd(MSR_EE | MSR_RI, 1); + else + mtmsr(mfmsr() | MSR_EE); +} -extern void timer_interrupt(struct pt_regs *); -extern void timer_broadcast_interrupt(void); -extern void performance_monitor_exception(struct pt_regs *regs); -extern void WatchdogException(struct pt_regs *regs); -extern void unknown_exception(struct pt_regs *regs); +static inline void __hard_irq_disable(void) +{ + if (IS_ENABLED(CONFIG_BOOKE) || IS_ENABLED(CONFIG_40x)) + wrtee(0); + else if (IS_ENABLED(CONFIG_PPC_8xx)) + wrtspr(SPRN_EID); + else if (IS_ENABLED(CONFIG_PPC_BOOK3S_64)) + __mtmsrd(MSR_RI, 1); + else + mtmsr(mfmsr() & ~MSR_EE); +} + +static inline void __hard_EE_RI_disable(void) +{ + if (IS_ENABLED(CONFIG_BOOKE) || IS_ENABLED(CONFIG_40x)) + wrtee(0); + else if (IS_ENABLED(CONFIG_PPC_8xx)) + wrtspr(SPRN_NRI); + else if (IS_ENABLED(CONFIG_PPC_BOOK3S_64)) + __mtmsrd(0, 1); + else + mtmsr(mfmsr() & ~(MSR_EE | MSR_RI)); +} + +static inline void __hard_RI_enable(void) +{ + if (IS_ENABLED(CONFIG_BOOKE) || IS_ENABLED(CONFIG_40x)) + return; + + if (IS_ENABLED(CONFIG_PPC_8xx)) + wrtspr(SPRN_EID); + else if (IS_ENABLED(CONFIG_PPC_BOOK3S_64)) + __mtmsrd(MSR_RI, 1); + else + mtmsr(mfmsr() | MSR_RI); +} #ifdef CONFIG_PPC64 #include @@ -221,18 +261,6 @@ static inline bool arch_irqs_disabled(void) #endif /* CONFIG_PPC_BOOK3S */ -#ifdef CONFIG_PPC_BOOK3E -#define __hard_irq_enable() wrtee(MSR_EE) -#define __hard_irq_disable() wrtee(0) -#define __hard_EE_RI_disable() wrtee(0) -#define __hard_RI_enable() do { } while (0) -#else -#define __hard_irq_enable() __mtmsrd(MSR_EE|MSR_RI, 1) -#define __hard_irq_disable() __mtmsrd(MSR_RI, 1) -#define __hard_EE_RI_disable() __mtmsrd(0, 1) -#define __hard_RI_enable() __mtmsrd(MSR_RI, 1) -#endif - #define hard_irq_disable() do { \ unsigned long flags; \ __hard_irq_disable(); \ @@ -296,8 +324,17 @@ extern void irq_set_pending_from_srr1(unsigned long srr1); extern void force_external_irq_replay(void); +static inline void irq_soft_mask_regs_set_state(struct pt_regs *regs, unsigned long val) +{ + regs->softe = val; +} #else /* CONFIG_PPC64 */ +static inline notrace unsigned long irq_soft_mask_return(void) +{ + return 0; +} + static inline unsigned long arch_local_save_flags(void) { return mfmsr(); @@ -327,22 +364,12 @@ static inline unsigned long arch_local_irq_save(void) static inline void arch_local_irq_disable(void) { - if (IS_ENABLED(CONFIG_BOOKE)) - wrtee(0); - else if (IS_ENABLED(CONFIG_PPC_8xx)) - wrtspr(SPRN_EID); - else - mtmsr(mfmsr() & ~MSR_EE); + __hard_irq_disable(); } static inline void arch_local_irq_enable(void) { - if (IS_ENABLED(CONFIG_BOOKE)) - wrtee(MSR_EE); - else if (IS_ENABLED(CONFIG_PPC_8xx)) - wrtspr(SPRN_EIE); - else - mtmsr(mfmsr() | MSR_EE); + __hard_irq_enable(); } static inline bool arch_irqs_disabled_flags(unsigned long flags) @@ -364,6 +391,9 @@ static inline bool arch_irq_disabled_regs(struct pt_regs *regs) static inline void may_hard_irq_enable(void) { } +static inline void irq_soft_mask_regs_set_state(struct pt_regs *regs, unsigned long val) +{ +} #endif /* CONFIG_PPC64 */ #define ARCH_IRQ_INIT_FLAGS IRQ_NOREQUEST diff --git a/arch/powerpc/include/asm/interrupt.h b/arch/powerpc/include/asm/interrupt.h new file mode 100644 index 0000000000000000000000000000000000000000..aedfba29e43a5e0a39b52fa406eabf4dcda82e2e --- /dev/null +++ b/arch/powerpc/include/asm/interrupt.h @@ -0,0 +1,449 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#ifndef _ASM_POWERPC_INTERRUPT_H +#define _ASM_POWERPC_INTERRUPT_H + +#include +#include +#include +#include +#include +#include + +struct interrupt_state { +#ifdef CONFIG_PPC_BOOK3E_64 + enum ctx_state ctx_state; +#endif +}; + +static inline void booke_restore_dbcr0(void) +{ +#ifdef CONFIG_PPC_ADV_DEBUG_REGS + unsigned long dbcr0 = current->thread.debug.dbcr0; + + if (IS_ENABLED(CONFIG_PPC32) && unlikely(dbcr0 & DBCR0_IDM)) { + mtspr(SPRN_DBSR, -1); + mtspr(SPRN_DBCR0, global_dbcr0[smp_processor_id()]); + } +#endif +} + +static inline void interrupt_enter_prepare(struct pt_regs *regs, struct interrupt_state *state) +{ + /* + * Book3E reconciles irq soft mask in asm + */ +#ifdef CONFIG_PPC_BOOK3S_64 + if (irq_soft_mask_set_return(IRQS_ALL_DISABLED) == IRQS_ENABLED) + trace_hardirqs_off(); + local_paca->irq_happened |= PACA_IRQ_HARD_DIS; + + if (user_mode(regs)) { + CT_WARN_ON(ct_state() != CONTEXT_USER); + user_exit_irqoff(); + + account_cpu_user_entry(); + account_stolen_time(); + } else { + /* + * CT_WARN_ON comes here via program_check_exception, + * so avoid recursion. + */ + if (TRAP(regs) != 0x700) + CT_WARN_ON(ct_state() != CONTEXT_KERNEL); + } +#endif + +#ifdef CONFIG_PPC_BOOK3E_64 + state->ctx_state = exception_enter(); + if (user_mode(regs)) + account_cpu_user_entry(); +#endif +} + +/* + * Care should be taken to note that interrupt_exit_prepare and + * interrupt_async_exit_prepare do not necessarily return immediately to + * regs context (e.g., if regs is usermode, we don't necessarily return to + * user mode). Other interrupts might be taken between here and return, + * context switch / preemption may occur in the exit path after this, or a + * signal may be delivered, etc. + * + * The real interrupt exit code is platform specific, e.g., + * interrupt_exit_user_prepare / interrupt_exit_kernel_prepare for 64s. + * + * However interrupt_nmi_exit_prepare does return directly to regs, because + * NMIs do not do "exit work" or replay soft-masked interrupts. + */ +static inline void interrupt_exit_prepare(struct pt_regs *regs, struct interrupt_state *state) +{ +#ifdef CONFIG_PPC_BOOK3E_64 + exception_exit(state->ctx_state); +#endif + + /* + * Book3S exits to user via interrupt_exit_user_prepare(), which does + * context tracking, which is a cleaner way to handle PREEMPT=y + * and avoid context entry/exit in e.g., preempt_schedule_irq()), + * which is likely to be where the core code wants to end up. + * + * The above comment explains why we can't do the + * + * if (user_mode(regs)) + * user_exit_irqoff(); + * + * sequence here. + */ +} + +static inline void interrupt_async_enter_prepare(struct pt_regs *regs, struct interrupt_state *state) +{ +#ifdef CONFIG_PPC_BOOK3S_64 + if (cpu_has_feature(CPU_FTR_CTRL) && + !test_thread_local_flags(_TLF_RUNLATCH)) + __ppc64_runlatch_on(); +#endif + + interrupt_enter_prepare(regs, state); + irq_enter(); +} + +static inline void interrupt_async_exit_prepare(struct pt_regs *regs, struct interrupt_state *state) +{ + irq_exit(); + interrupt_exit_prepare(regs, state); +} + +struct interrupt_nmi_state { +#ifdef CONFIG_PPC64 +#ifdef CONFIG_PPC_BOOK3S_64 + u8 irq_soft_mask; + u8 irq_happened; +#endif + u8 ftrace_enabled; +#endif +}; + +static inline void interrupt_nmi_enter_prepare(struct pt_regs *regs, struct interrupt_nmi_state *state) +{ +#ifdef CONFIG_PPC64 +#ifdef CONFIG_PPC_BOOK3S_64 + state->irq_soft_mask = local_paca->irq_soft_mask; + state->irq_happened = local_paca->irq_happened; + + /* + * Set IRQS_ALL_DISABLED unconditionally so irqs_disabled() does + * the right thing, and set IRQ_HARD_DIS. We do not want to reconcile + * because that goes through irq tracing which we don't want in NMI. + */ + local_paca->irq_soft_mask = IRQS_ALL_DISABLED; + local_paca->irq_happened |= PACA_IRQ_HARD_DIS; + + /* Don't do any per-CPU operations until interrupt state is fixed */ +#endif + /* Allow DEC and PMI to be traced when they are soft-NMI */ + if (TRAP(regs) != 0x900 && TRAP(regs) != 0xf00 && TRAP(regs) != 0x260) { + state->ftrace_enabled = this_cpu_get_ftrace_enabled(); + this_cpu_set_ftrace_enabled(0); + } +#endif + + /* + * Do not use nmi_enter() for pseries hash guest taking a real-mode + * NMI because not everything it touches is within the RMA limit. + */ + if (!IS_ENABLED(CONFIG_PPC_BOOK3S_64) || + !firmware_has_feature(FW_FEATURE_LPAR) || + radix_enabled() || (mfmsr() & MSR_DR)) + nmi_enter(); +} + +static inline void interrupt_nmi_exit_prepare(struct pt_regs *regs, struct interrupt_nmi_state *state) +{ + if (!IS_ENABLED(CONFIG_PPC_BOOK3S_64) || + !firmware_has_feature(FW_FEATURE_LPAR) || + radix_enabled() || (mfmsr() & MSR_DR)) + nmi_exit(); + +#ifdef CONFIG_PPC64 + if (TRAP(regs) != 0x900 && TRAP(regs) != 0xf00 && TRAP(regs) != 0x260) + this_cpu_set_ftrace_enabled(state->ftrace_enabled); + +#ifdef CONFIG_PPC_BOOK3S_64 + /* Check we didn't change the pending interrupt mask. */ + WARN_ON_ONCE((state->irq_happened | PACA_IRQ_HARD_DIS) != local_paca->irq_happened); + local_paca->irq_happened = state->irq_happened; + local_paca->irq_soft_mask = state->irq_soft_mask; +#endif +#endif +} + +/* + * Don't use noinstr here like x86, but rather add NOKPROBE_SYMBOL to each + * function definition. The reason for this is the noinstr section is placed + * after the main text section, i.e., very far away from the interrupt entry + * asm. That creates problems with fitting linker stubs when building large + * kernels. + */ +#define interrupt_handler __visible noinline notrace __no_kcsan __no_sanitize_address + +/** + * DECLARE_INTERRUPT_HANDLER_RAW - Declare raw interrupt handler function + * @func: Function name of the entry point + * @returns: Returns a value back to asm caller + */ +#define DECLARE_INTERRUPT_HANDLER_RAW(func) \ + __visible long func(struct pt_regs *regs) + +/** + * DEFINE_INTERRUPT_HANDLER_RAW - Define raw interrupt handler function + * @func: Function name of the entry point + * @returns: Returns a value back to asm caller + * + * @func is called from ASM entry code. + * + * This is a plain function which does no tracing, reconciling, etc. + * The macro is written so it acts as function definition. Append the + * body with a pair of curly brackets. + * + * raw interrupt handlers must not enable or disable interrupts, or + * schedule, tracing and instrumentation (ftrace, lockdep, etc) would + * not be advisable either, although may be possible in a pinch, the + * trace will look odd at least. + * + * A raw handler may call one of the other interrupt handler functions + * to be converted into that interrupt context without these restrictions. + * + * On PPC64, _RAW handlers may return with fast_interrupt_return. + * + * Specific handlers may have additional restrictions. + */ +#define DEFINE_INTERRUPT_HANDLER_RAW(func) \ +static __always_inline long ____##func(struct pt_regs *regs); \ + \ +interrupt_handler long func(struct pt_regs *regs) \ +{ \ + long ret; \ + \ + ret = ____##func (regs); \ + \ + return ret; \ +} \ +NOKPROBE_SYMBOL(func); \ + \ +static __always_inline long ____##func(struct pt_regs *regs) + +/** + * DECLARE_INTERRUPT_HANDLER - Declare synchronous interrupt handler function + * @func: Function name of the entry point + */ +#define DECLARE_INTERRUPT_HANDLER(func) \ + __visible void func(struct pt_regs *regs) + +/** + * DEFINE_INTERRUPT_HANDLER - Define synchronous interrupt handler function + * @func: Function name of the entry point + * + * @func is called from ASM entry code. + * + * The macro is written so it acts as function definition. Append the + * body with a pair of curly brackets. + */ +#define DEFINE_INTERRUPT_HANDLER(func) \ +static __always_inline void ____##func(struct pt_regs *regs); \ + \ +interrupt_handler void func(struct pt_regs *regs) \ +{ \ + struct interrupt_state state; \ + \ + interrupt_enter_prepare(regs, &state); \ + \ + ____##func (regs); \ + \ + interrupt_exit_prepare(regs, &state); \ +} \ +NOKPROBE_SYMBOL(func); \ + \ +static __always_inline void ____##func(struct pt_regs *regs) + +/** + * DECLARE_INTERRUPT_HANDLER_RET - Declare synchronous interrupt handler function + * @func: Function name of the entry point + * @returns: Returns a value back to asm caller + */ +#define DECLARE_INTERRUPT_HANDLER_RET(func) \ + __visible long func(struct pt_regs *regs) + +/** + * DEFINE_INTERRUPT_HANDLER_RET - Define synchronous interrupt handler function + * @func: Function name of the entry point + * @returns: Returns a value back to asm caller + * + * @func is called from ASM entry code. + * + * The macro is written so it acts as function definition. Append the + * body with a pair of curly brackets. + */ +#define DEFINE_INTERRUPT_HANDLER_RET(func) \ +static __always_inline long ____##func(struct pt_regs *regs); \ + \ +interrupt_handler long func(struct pt_regs *regs) \ +{ \ + struct interrupt_state state; \ + long ret; \ + \ + interrupt_enter_prepare(regs, &state); \ + \ + ret = ____##func (regs); \ + \ + interrupt_exit_prepare(regs, &state); \ + \ + return ret; \ +} \ +NOKPROBE_SYMBOL(func); \ + \ +static __always_inline long ____##func(struct pt_regs *regs) + +/** + * DECLARE_INTERRUPT_HANDLER_ASYNC - Declare asynchronous interrupt handler function + * @func: Function name of the entry point + */ +#define DECLARE_INTERRUPT_HANDLER_ASYNC(func) \ + __visible void func(struct pt_regs *regs) + +/** + * DEFINE_INTERRUPT_HANDLER_ASYNC - Define asynchronous interrupt handler function + * @func: Function name of the entry point + * + * @func is called from ASM entry code. + * + * The macro is written so it acts as function definition. Append the + * body with a pair of curly brackets. + */ +#define DEFINE_INTERRUPT_HANDLER_ASYNC(func) \ +static __always_inline void ____##func(struct pt_regs *regs); \ + \ +interrupt_handler void func(struct pt_regs *regs) \ +{ \ + struct interrupt_state state; \ + \ + interrupt_async_enter_prepare(regs, &state); \ + \ + ____##func (regs); \ + \ + interrupt_async_exit_prepare(regs, &state); \ +} \ +NOKPROBE_SYMBOL(func); \ + \ +static __always_inline void ____##func(struct pt_regs *regs) + +/** + * DECLARE_INTERRUPT_HANDLER_NMI - Declare NMI interrupt handler function + * @func: Function name of the entry point + * @returns: Returns a value back to asm caller + */ +#define DECLARE_INTERRUPT_HANDLER_NMI(func) \ + __visible long func(struct pt_regs *regs) + +/** + * DEFINE_INTERRUPT_HANDLER_NMI - Define NMI interrupt handler function + * @func: Function name of the entry point + * @returns: Returns a value back to asm caller + * + * @func is called from ASM entry code. + * + * The macro is written so it acts as function definition. Append the + * body with a pair of curly brackets. + */ +#define DEFINE_INTERRUPT_HANDLER_NMI(func) \ +static __always_inline long ____##func(struct pt_regs *regs); \ + \ +interrupt_handler long func(struct pt_regs *regs) \ +{ \ + struct interrupt_nmi_state state; \ + long ret; \ + \ + interrupt_nmi_enter_prepare(regs, &state); \ + \ + ret = ____##func (regs); \ + \ + interrupt_nmi_exit_prepare(regs, &state); \ + \ + return ret; \ +} \ +NOKPROBE_SYMBOL(func); \ + \ +static __always_inline long ____##func(struct pt_regs *regs) + + +/* Interrupt handlers */ +/* kernel/traps.c */ +DECLARE_INTERRUPT_HANDLER_NMI(system_reset_exception); +#ifdef CONFIG_PPC_BOOK3S_64 +DECLARE_INTERRUPT_HANDLER_ASYNC(machine_check_exception); +#else +DECLARE_INTERRUPT_HANDLER_NMI(machine_check_exception); +#endif +DECLARE_INTERRUPT_HANDLER(SMIException); +DECLARE_INTERRUPT_HANDLER(handle_hmi_exception); +DECLARE_INTERRUPT_HANDLER(unknown_exception); +DECLARE_INTERRUPT_HANDLER_ASYNC(unknown_async_exception); +DECLARE_INTERRUPT_HANDLER(instruction_breakpoint_exception); +DECLARE_INTERRUPT_HANDLER(RunModeException); +DECLARE_INTERRUPT_HANDLER(single_step_exception); +DECLARE_INTERRUPT_HANDLER(program_check_exception); +DECLARE_INTERRUPT_HANDLER(emulation_assist_interrupt); +DECLARE_INTERRUPT_HANDLER(alignment_exception); +DECLARE_INTERRUPT_HANDLER(StackOverflow); +DECLARE_INTERRUPT_HANDLER(stack_overflow_exception); +DECLARE_INTERRUPT_HANDLER(kernel_fp_unavailable_exception); +DECLARE_INTERRUPT_HANDLER(altivec_unavailable_exception); +DECLARE_INTERRUPT_HANDLER(vsx_unavailable_exception); +DECLARE_INTERRUPT_HANDLER(facility_unavailable_exception); +DECLARE_INTERRUPT_HANDLER(fp_unavailable_tm); +DECLARE_INTERRUPT_HANDLER(altivec_unavailable_tm); +DECLARE_INTERRUPT_HANDLER(vsx_unavailable_tm); +DECLARE_INTERRUPT_HANDLER_NMI(performance_monitor_exception_nmi); +DECLARE_INTERRUPT_HANDLER_ASYNC(performance_monitor_exception_async); +DECLARE_INTERRUPT_HANDLER_RAW(performance_monitor_exception); +DECLARE_INTERRUPT_HANDLER(DebugException); +DECLARE_INTERRUPT_HANDLER(altivec_assist_exception); +DECLARE_INTERRUPT_HANDLER(CacheLockingException); +DECLARE_INTERRUPT_HANDLER(SPEFloatingPointException); +DECLARE_INTERRUPT_HANDLER(SPEFloatingPointRoundException); +DECLARE_INTERRUPT_HANDLER(unrecoverable_exception); +DECLARE_INTERRUPT_HANDLER(WatchdogException); +DECLARE_INTERRUPT_HANDLER(kernel_bad_stack); + +/* slb.c */ +DECLARE_INTERRUPT_HANDLER_RAW(do_slb_fault); +DECLARE_INTERRUPT_HANDLER(do_bad_slb_fault); + +/* hash_utils.c */ +DECLARE_INTERRUPT_HANDLER_RAW(do_hash_fault); + +/* fault.c */ +DECLARE_INTERRUPT_HANDLER_RET(do_page_fault); +DECLARE_INTERRUPT_HANDLER(do_bad_page_fault_segv); + +/* process.c */ +DECLARE_INTERRUPT_HANDLER(do_break); + +/* time.c */ +DECLARE_INTERRUPT_HANDLER_ASYNC(timer_interrupt); + +/* mce.c */ +DECLARE_INTERRUPT_HANDLER_NMI(machine_check_early); +DECLARE_INTERRUPT_HANDLER_NMI(hmi_exception_realmode); + +DECLARE_INTERRUPT_HANDLER_ASYNC(TAUException); + +void replay_system_reset(void); +void replay_soft_interrupts(void); + +static inline void interrupt_cond_local_irq_enable(struct pt_regs *regs) +{ + if (!arch_irq_disabled_regs(regs)) + local_irq_enable(); +} + +#endif /* _ASM_POWERPC_INTERRUPT_H */ diff --git a/arch/powerpc/include/asm/irq.h b/arch/powerpc/include/asm/irq.h index 4f983ca4030a40e9355f403818009b7e9c707a86..f3f264e441a79e9569ede340c8c5dd3a45cbd697 100644 --- a/arch/powerpc/include/asm/irq.h +++ b/arch/powerpc/include/asm/irq.h @@ -37,8 +37,6 @@ extern int distribute_irqs; struct pt_regs; -#define __ARCH_HAS_DO_SOFTIRQ - #if defined(CONFIG_BOOKE) || defined(CONFIG_40x) /* * Per-cpu stacks for handling critical, debug and machine check diff --git a/arch/powerpc/include/asm/kexec.h b/arch/powerpc/include/asm/kexec.h index 55d6ede30c19afb7716a27ee423a3d3c58009506..9ab344d29a545957c8577b47ff34f3047559583f 100644 --- a/arch/powerpc/include/asm/kexec.h +++ b/arch/powerpc/include/asm/kexec.h @@ -136,6 +136,7 @@ int load_crashdump_segments_ppc64(struct kimage *image, int setup_purgatory_ppc64(struct kimage *image, const void *slave_code, const void *fdt, unsigned long kernel_load_addr, unsigned long fdt_load_addr); +unsigned int kexec_fdt_totalsize_ppc64(struct kimage *image); int setup_new_fdt_ppc64(const struct kimage *image, void *fdt, unsigned long initrd_load_addr, unsigned long initrd_len, const char *cmdline); diff --git a/arch/powerpc/include/asm/kup.h b/arch/powerpc/include/asm/kup.h index bf221a2a523e79e62a5e19ba7d8b86334731f2c5..7ec21af49a45c5b3e0588bd291f1bc7192bcb123 100644 --- a/arch/powerpc/include/asm/kup.h +++ b/arch/powerpc/include/asm/kup.h @@ -91,6 +91,7 @@ static __always_inline void setup_kup(void) static inline void allow_read_from_user(const void __user *from, unsigned long size) { + barrier_nospec(); allow_user_access(NULL, from, size, KUAP_READ); } @@ -102,6 +103,7 @@ static inline void allow_write_to_user(void __user *to, unsigned long size) static inline void allow_read_write_user(void __user *to, const void __user *from, unsigned long size) { + barrier_nospec(); allow_user_access(to, from, size, KUAP_READ_WRITE); } diff --git a/arch/powerpc/include/asm/kvm_book3s.h b/arch/powerpc/include/asm/kvm_book3s.h index d32ec9ae73bd4150b64c90ed2691f1426de73e7d..2f5f919f6cd302646827509179d802461efa2a2b 100644 --- a/arch/powerpc/include/asm/kvm_book3s.h +++ b/arch/powerpc/include/asm/kvm_book3s.h @@ -277,6 +277,13 @@ extern int kvmppc_hcall_impl_hv_realmode(unsigned long cmd); extern void kvmppc_copy_to_svcpu(struct kvm_vcpu *vcpu); extern void kvmppc_copy_from_svcpu(struct kvm_vcpu *vcpu); +long kvmppc_read_intr(void); +void kvmppc_bad_interrupt(struct pt_regs *regs); +void kvmhv_p9_set_lpcr(struct kvm_split_mode *sip); +void kvmhv_p9_restore_lpcr(struct kvm_split_mode *sip); +void kvmppc_set_msr_hv(struct kvm_vcpu *vcpu, u64 msr); +void kvmppc_inject_interrupt_hv(struct kvm_vcpu *vcpu, int vec, u64 srr1_flags); + #ifdef CONFIG_PPC_TRANSACTIONAL_MEM void kvmppc_save_tm_pr(struct kvm_vcpu *vcpu); void kvmppc_restore_tm_pr(struct kvm_vcpu *vcpu); diff --git a/arch/powerpc/include/asm/kvm_ppc.h b/arch/powerpc/include/asm/kvm_ppc.h index df4bda867bab75b30529f834cb1f19f9b9858c7b..8aacd76bb702b4b8a192f98e5a23abb909490d74 100644 --- a/arch/powerpc/include/asm/kvm_ppc.h +++ b/arch/powerpc/include/asm/kvm_ppc.h @@ -629,9 +629,9 @@ extern int h_ipi_redirect; static inline struct kvmppc_passthru_irqmap *kvmppc_get_passthru_irqmap( struct kvm *kvm) { return NULL; } -static inline void kvmppc_alloc_host_rm_ops(void) {}; -static inline void kvmppc_free_host_rm_ops(void) {}; -static inline void kvmppc_free_pimap(struct kvm *kvm) {}; +static inline void kvmppc_alloc_host_rm_ops(void) {} +static inline void kvmppc_free_host_rm_ops(void) {} +static inline void kvmppc_free_pimap(struct kvm *kvm) {} static inline int kvmppc_xics_rm_complete(struct kvm_vcpu *vcpu, u32 hcall) { return 0; } static inline int kvmppc_xics_enabled(struct kvm_vcpu *vcpu) @@ -883,9 +883,9 @@ static inline void kvmppc_mmu_flush_icache(kvm_pfn_t pfn) /* Clear i-cache for new pages */ page = pfn_to_page(pfn); - if (!test_bit(PG_arch_1, &page->flags)) { + if (!test_bit(PG_dcache_clean, &page->flags)) { flush_dcache_icache_page(page); - set_bit(PG_arch_1, &page->flags); + set_bit(PG_dcache_clean, &page->flags); } } diff --git a/arch/powerpc/include/asm/machdep.h b/arch/powerpc/include/asm/machdep.h index cf6ebbc16cb47c8d28804b33d1a280250d3ec3ee..764f2732a82182b7738bf2c1cb2829dab7870796 100644 --- a/arch/powerpc/include/asm/machdep.h +++ b/arch/powerpc/include/asm/machdep.h @@ -59,6 +59,9 @@ struct machdep_calls { int (*pcibios_root_bridge_prepare)(struct pci_host_bridge *bridge); + /* finds all the pci_controllers present at boot */ + void (*discover_phbs)(void); + /* To setup PHBs when using automatic OF platform driver for PCI */ int (*pci_setup_phb)(struct pci_controller *host); diff --git a/arch/powerpc/include/asm/mce.h b/arch/powerpc/include/asm/mce.h index e6c27ae843dc455cfc17cfbe68308078db9d2380..331d944280b8190007474c045b774b69edf09d12 100644 --- a/arch/powerpc/include/asm/mce.h +++ b/arch/powerpc/include/asm/mce.h @@ -204,7 +204,18 @@ struct mce_error_info { bool ignore_event; }; -#define MAX_MC_EVT 100 +#define MAX_MC_EVT 10 + +struct mce_info { + int mce_nest_count; + struct machine_check_event mce_event[MAX_MC_EVT]; + /* Queue for delayed MCE events. */ + int mce_queue_count; + struct machine_check_event mce_event_queue[MAX_MC_EVT]; + /* Queue for delayed MCE UE events. */ + int mce_ue_count; + struct machine_check_event mce_ue_event_queue[MAX_MC_EVT]; +}; /* Release flags for get_mce_event() */ #define MCE_EVENT_RELEASE true @@ -234,4 +245,11 @@ long __machine_check_early_realmode_p8(struct pt_regs *regs); long __machine_check_early_realmode_p9(struct pt_regs *regs); long __machine_check_early_realmode_p10(struct pt_regs *regs); #endif /* CONFIG_PPC_BOOK3S_64 */ + +#ifdef CONFIG_PPC_BOOK3S_64 +void mce_init(void); +#else +static inline void mce_init(void) { }; +#endif /* CONFIG_PPC_BOOK3S_64 */ + #endif /* __ASM_PPC64_MCE_H__ */ diff --git a/arch/powerpc/include/asm/mmu_context.h b/arch/powerpc/include/asm/mmu_context.h index d5821834dba9645da64d14ee6d4b09c68e1ba678..652ce85f9410a8840d842389d80e7b6b501f3cb7 100644 --- a/arch/powerpc/include/asm/mmu_context.h +++ b/arch/powerpc/include/asm/mmu_context.h @@ -282,9 +282,6 @@ static inline bool arch_vma_access_permitted(struct vm_area_struct *vma, } #define pkey_mm_init(mm) -#define thread_pkey_regs_save(thread) -#define thread_pkey_regs_restore(new_thread, old_thread) -#define thread_pkey_regs_init(thread) #define arch_dup_pkeys(oldmm, mm) static inline u64 pte_to_hpte_pkey_bits(u64 pteflags, unsigned long flags) diff --git a/arch/powerpc/include/asm/nmi.h b/arch/powerpc/include/asm/nmi.h index 84b4cfe73edddec59abd875d6e807dce7c251a2e..160abcb8e9fab22c22c4bf0b6269b1098afe9901 100644 --- a/arch/powerpc/include/asm/nmi.h +++ b/arch/powerpc/include/asm/nmi.h @@ -4,6 +4,7 @@ #ifdef CONFIG_PPC_WATCHDOG extern void arch_touch_nmi_watchdog(void); +long soft_nmi_interrupt(struct pt_regs *regs); #else static inline void arch_touch_nmi_watchdog(void) {} #endif diff --git a/arch/powerpc/include/asm/paca.h b/arch/powerpc/include/asm/paca.h index 9454d29ff4b47d27b2cf6bdf9f882da76377c584..ec18ac818e3a6f7bbec3e858b7c4c94d971ce36a 100644 --- a/arch/powerpc/include/asm/paca.h +++ b/arch/powerpc/include/asm/paca.h @@ -29,6 +29,7 @@ #include #include #include +#include #include @@ -108,8 +109,7 @@ struct paca_struct { */ /* used for most interrupts/exceptions */ u64 exgen[EX_SIZE] __attribute__((aligned(0x80))); - u64 exslb[EX_SIZE]; /* used for SLB/segment table misses - * on the linear mapping */ + /* SLB related definitions */ u16 vmalloc_sllp; u8 slb_cache_ptr; @@ -273,6 +273,9 @@ struct paca_struct { #ifdef CONFIG_MMIOWB struct mmiowb_state mmiowb_state; #endif +#ifdef CONFIG_PPC_BOOK3S_64 + struct mce_info *mce_info; +#endif /* CONFIG_PPC_BOOK3S_64 */ } ____cacheline_aligned; extern void copy_mm_to_paca(struct mm_struct *mm); @@ -285,9 +288,9 @@ extern void free_unused_pacas(void); #else /* CONFIG_PPC64 */ -static inline void allocate_paca_ptrs(void) { }; -static inline void allocate_paca(int cpu) { }; -static inline void free_unused_pacas(void) { }; +static inline void allocate_paca_ptrs(void) { } +static inline void allocate_paca(int cpu) { } +static inline void free_unused_pacas(void) { } #endif /* CONFIG_PPC64 */ diff --git a/arch/powerpc/include/asm/paravirt.h b/arch/powerpc/include/asm/paravirt.h index edc08f04aef7745045f9d76fc4491adb851fa759..5d1726bb28e79ee95988a0ecd921cd39eecefc9d 100644 --- a/arch/powerpc/include/asm/paravirt.h +++ b/arch/powerpc/include/asm/paravirt.h @@ -10,6 +10,7 @@ #endif #ifdef CONFIG_PPC_SPLPAR +#include #include #include diff --git a/arch/powerpc/include/asm/perf_event.h b/arch/powerpc/include/asm/perf_event.h index daec64d41b44781576f7369601fb551622912386..164e910bf654f3d6e4e13ad27a46c2450f0ae10b 100644 --- a/arch/powerpc/include/asm/perf_event.h +++ b/arch/powerpc/include/asm/perf_event.h @@ -14,6 +14,7 @@ #include #else static inline bool is_sier_available(void) { return false; } +static inline unsigned long get_pmcs_ext_regs(int idx) { return 0; } #endif #ifdef CONFIG_FSL_EMB_PERF_EVENT @@ -40,6 +41,7 @@ static inline bool is_sier_available(void) { return false; } /* To support perf_regs sier update */ extern bool is_sier_available(void); +extern unsigned long get_pmcs_ext_regs(int idx); /* To define perf extended regs mask value */ extern u64 PERF_REG_EXTENDED_MASK; #define PERF_REG_EXTENDED_MASK PERF_REG_EXTENDED_MASK diff --git a/arch/powerpc/include/asm/perf_event_server.h b/arch/powerpc/include/asm/perf_event_server.h index 3b7baba01c92df94578c41af503f9c4adc71a191..00e7e671bb4ba9048652763db40aeefb966f222a 100644 --- a/arch/powerpc/include/asm/perf_event_server.h +++ b/arch/powerpc/include/asm/perf_event_server.h @@ -36,9 +36,9 @@ struct power_pmu { unsigned long test_adder; int (*compute_mmcr)(u64 events[], int n_ev, unsigned int hwc[], struct mmcr_regs *mmcr, - struct perf_event *pevents[]); + struct perf_event *pevents[], u32 flags); int (*get_constraint)(u64 event_id, unsigned long *mskp, - unsigned long *valp); + unsigned long *valp, u64 event_config1); int (*get_alternatives)(u64 event_id, unsigned int flags, u64 alt[]); void (*get_mem_data_src)(union perf_mem_data_src *dsrc, @@ -83,6 +83,7 @@ struct power_pmu { #define PPMU_NO_SIAR 0x00000100 /* Do not use SIAR */ #define PPMU_ARCH_31 0x00000200 /* Has MMCR3, SIER2 and SIER3 */ #define PPMU_P10_DD1 0x00000400 /* Is power10 DD1 processor version */ +#define PPMU_HAS_ATTR_CONFIG1 0x00000800 /* Using config1 attribute */ /* * Values for flags to get_alternatives() diff --git a/arch/powerpc/include/asm/pgtable.h b/arch/powerpc/include/asm/pgtable.h index f7613f43c9cf23911e98eaf571a637c4a641aaf4..4eed82172e3348efa832c5ad7ffbcf04909e195a 100644 --- a/arch/powerpc/include/asm/pgtable.h +++ b/arch/powerpc/include/asm/pgtable.h @@ -162,6 +162,9 @@ static inline bool is_ioremap_addr(const void *x) return addr >= IOREMAP_BASE && addr < IOREMAP_END; } + +struct seq_file; +void arch_report_meminfo(struct seq_file *m); #endif /* CONFIG_PPC64 */ #endif /* __ASSEMBLY__ */ diff --git a/arch/powerpc/include/asm/pkeys.h b/arch/powerpc/include/asm/pkeys.h index a7951049e1299b07cc3e9e304c4ea05f053f03e8..59a2c7dbc78fbf3ebf58f79507fb7710a2337b2e 100644 --- a/arch/powerpc/include/asm/pkeys.h +++ b/arch/powerpc/include/asm/pkeys.h @@ -169,10 +169,4 @@ static inline bool arch_pkeys_enabled(void) } extern void pkey_mm_init(struct mm_struct *mm); -extern bool arch_supports_pkeys(int cap); -extern unsigned int arch_usable_pkeys(void); -extern void thread_pkey_regs_save(struct thread_struct *thread); -extern void thread_pkey_regs_restore(struct thread_struct *new_thread, - struct thread_struct *old_thread); -extern void thread_pkey_regs_init(struct thread_struct *thread); #endif /*_ASM_POWERPC_KEYS_H */ diff --git a/arch/powerpc/include/asm/ppc-pci.h b/arch/powerpc/include/asm/ppc-pci.h index 7f4be5a05eb3f4c341b81c73e60ec8a2f891a375..2b9edbf6e929b86cd6859ec9c1211f2bdaccd8ac 100644 --- a/arch/powerpc/include/asm/ppc-pci.h +++ b/arch/powerpc/include/asm/ppc-pci.h @@ -13,10 +13,6 @@ extern unsigned long isa_io_base; -extern void pci_setup_phb_io(struct pci_controller *hose, int primary); -extern void pci_setup_phb_io_dynamic(struct pci_controller *hose, int primary); - - extern struct list_head hose_list; extern struct pci_dev *isa_bridge_pcidev; /* may be NULL if no ISA bus */ @@ -32,9 +28,6 @@ struct pci_dn; void *pci_traverse_device_nodes(struct device_node *start, void *(*fn)(struct device_node *, void *), void *data); -void *traverse_pci_dn(struct pci_dn *root, - void *(*fn)(struct pci_dn *, void *), - void *data); extern void pci_devs_phb_init_dynamic(struct pci_controller *phb); /* From rtas_pci.h */ diff --git a/arch/powerpc/include/asm/ppc_asm.h b/arch/powerpc/include/asm/ppc_asm.h index cc1bca57133272b0c1eea88e5b014ba4146c3522..3dceb64fc9af24855d76b6aa9df7a99cb78217c9 100644 --- a/arch/powerpc/include/asm/ppc_asm.h +++ b/arch/powerpc/include/asm/ppc_asm.h @@ -25,7 +25,6 @@ #ifndef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE #define ACCOUNT_CPU_USER_ENTRY(ptr, ra, rb) #define ACCOUNT_CPU_USER_EXIT(ptr, ra, rb) -#define ACCOUNT_STOLEN_TIME #else #define ACCOUNT_CPU_USER_ENTRY(ptr, ra, rb) \ MFTB(ra); /* get timebase */ \ @@ -44,29 +43,6 @@ PPC_LL ra, ACCOUNT_SYSTEM_TIME(ptr); \ add ra,ra,rb; /* add on to system time */ \ PPC_STL ra, ACCOUNT_SYSTEM_TIME(ptr) - -#ifdef CONFIG_PPC_SPLPAR -#define ACCOUNT_STOLEN_TIME \ -BEGIN_FW_FTR_SECTION; \ - beq 33f; \ - /* from user - see if there are any DTL entries to process */ \ - ld r10,PACALPPACAPTR(r13); /* get ptr to VPA */ \ - ld r11,PACA_DTL_RIDX(r13); /* get log read index */ \ - addi r10,r10,LPPACA_DTLIDX; \ - LDX_BE r10,0,r10; /* get log write index */ \ - cmpd cr1,r11,r10; \ - beq+ cr1,33f; \ - bl accumulate_stolen_time; \ - ld r12,_MSR(r1); \ - andi. r10,r12,MSR_PR; /* Restore cr0 (coming from user) */ \ -33: \ -END_FW_FTR_SECTION_IFSET(FW_FEATURE_SPLPAR) - -#else /* CONFIG_PPC_SPLPAR */ -#define ACCOUNT_STOLEN_TIME - -#endif /* CONFIG_PPC_SPLPAR */ - #endif /* CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */ /* diff --git a/arch/powerpc/include/asm/ptrace.h b/arch/powerpc/include/asm/ptrace.h index 58f9dc060a7b40de875c893e11a3adce828044d5..975ba260006a46755670884e9f15363ac4683c30 100644 --- a/arch/powerpc/include/asm/ptrace.h +++ b/arch/powerpc/include/asm/ptrace.h @@ -70,6 +70,9 @@ struct pt_regs }; #endif + +#define STACK_FRAME_WITH_PT_REGS (STACK_FRAME_OVERHEAD + sizeof(struct pt_regs)) + #ifdef __powerpc64__ /* @@ -229,6 +232,11 @@ static inline bool trap_is_scv(struct pt_regs *regs) return (IS_ENABLED(CONFIG_PPC_BOOK3S_64) && TRAP(regs) == 0x3000); } +static inline bool trap_is_unsupported_scv(struct pt_regs *regs) +{ + return IS_ENABLED(CONFIG_PPC_BOOK3S_64) && TRAP(regs) == 0x7ff0; +} + static inline bool trap_is_syscall(struct pt_regs *regs) { return (trap_is_scv(regs) || TRAP(regs) == 0xc00); diff --git a/arch/powerpc/include/asm/reg.h b/arch/powerpc/include/asm/reg.h index e40a921d78f96e82a6897712c2e822f07fbf06a6..da103e92c1126aa4e6e2410cfa3a510bd8c5624b 100644 --- a/arch/powerpc/include/asm/reg.h +++ b/arch/powerpc/include/asm/reg.h @@ -1375,6 +1375,7 @@ #define mtmsr(v) asm volatile("mtmsr %0" : \ : "r" ((unsigned long)(v)) \ : "memory") +#define __mtmsrd(v, l) BUILD_BUG() #define __MTMSR "mtmsr" #endif @@ -1413,13 +1414,24 @@ static inline void msr_check_and_clear(unsigned long bits) } #ifdef CONFIG_PPC32 -#define mfsrin(v) ({unsigned int rval; \ - asm volatile("mfsrin %0,%1" : "=r" (rval) : "r" (v)); \ - rval;}) +static inline u32 mfsr(u32 idx) +{ + u32 val; + + if (__builtin_constant_p(idx)) + asm volatile("mfsr %0, %1" : "=r" (val): "i" (idx >> 28)); + else + asm volatile("mfsrin %0, %1" : "=r" (val): "r" (idx)); -static inline void mtsrin(u32 val, u32 idx) + return val; +} + +static inline void mtsr(u32 val, u32 idx) { - asm volatile("mtsrin %0, %1" : : "r" (val), "r" (idx)); + if (__builtin_constant_p(idx)) + asm volatile("mtsr %1, %0" : : "r" (val), "i" (idx >> 28)); + else + asm volatile("mtsrin %0, %1" : : "r" (val), "r" (idx)); } #endif diff --git a/arch/powerpc/include/asm/reg_booke.h b/arch/powerpc/include/asm/reg_booke.h index 262782f08fd48e23029313aecec175ef0f2e866b..17b8dcd9a40dedf137c1842bbbc6fe504e3e83d0 100644 --- a/arch/powerpc/include/asm/reg_booke.h +++ b/arch/powerpc/include/asm/reg_booke.h @@ -691,6 +691,9 @@ #define mttmr(rn, v) asm volatile(MTTMR(rn, %0) : \ : "r" ((unsigned long)(v)) \ : "memory") + +extern unsigned long global_dbcr0[]; + #endif /* !__ASSEMBLY__ */ #endif /* __ASM_POWERPC_REG_BOOKE_H__ */ diff --git a/arch/powerpc/include/asm/rtas.h b/arch/powerpc/include/asm/rtas.h index 332e1000ca0ff503a2f079b52c4ebe703b74f819..658448ca5b8ae97e65914f9e73fd07e938c81fb4 100644 --- a/arch/powerpc/include/asm/rtas.h +++ b/arch/powerpc/include/asm/rtas.h @@ -369,7 +369,7 @@ void rtas_initialize(void); #else static inline int page_is_rtas_user_buf(unsigned long pfn) { return 0;} static inline void pSeries_coalesce_init(void) { } -static inline void rtas_initialize(void) { }; +static inline void rtas_initialize(void) { } #endif extern int call_rtas(const char *, int, int, unsigned long *, ...); diff --git a/arch/powerpc/include/asm/setup.h b/arch/powerpc/include/asm/setup.h index a466749703f1fbde304ea51df8abd21d60c82d28..e89bfebd4e00e06e4d9833b23b9366442fa65e30 100644 --- a/arch/powerpc/include/asm/setup.h +++ b/arch/powerpc/include/asm/setup.h @@ -58,7 +58,7 @@ void do_rfi_flush_fixups(enum l1d_flush_type types); #ifdef CONFIG_PPC_BARRIER_NOSPEC void setup_barrier_nospec(void); #else -static inline void setup_barrier_nospec(void) { }; +static inline void setup_barrier_nospec(void) { } #endif void do_uaccess_flush_fixups(enum l1d_flush_type types); void do_entry_flush_fixups(enum l1d_flush_type types); @@ -68,13 +68,13 @@ extern bool barrier_nospec_enabled; #ifdef CONFIG_PPC_BARRIER_NOSPEC void do_barrier_nospec_fixups_range(bool enable, void *start, void *end); #else -static inline void do_barrier_nospec_fixups_range(bool enable, void *start, void *end) { }; +static inline void do_barrier_nospec_fixups_range(bool enable, void *start, void *end) { } #endif #ifdef CONFIG_PPC_FSL_BOOK3E void setup_spectre_v2(void); #else -static inline void setup_spectre_v2(void) {}; +static inline void setup_spectre_v2(void) {} #endif void do_btb_flush_fixups(void); diff --git a/arch/powerpc/include/asm/simple_spinlock.h b/arch/powerpc/include/asm/simple_spinlock.h index 9c3c305343331f7455fae7767c5a77aef4530dfb..5b862de29dff6c4316a17823e3b66a758814f654 100644 --- a/arch/powerpc/include/asm/simple_spinlock.h +++ b/arch/powerpc/include/asm/simple_spinlock.h @@ -90,8 +90,8 @@ static inline int arch_spin_trylock(arch_spinlock_t *lock) void splpar_spin_yield(arch_spinlock_t *lock); void splpar_rw_yield(arch_rwlock_t *lock); #else /* SPLPAR */ -static inline void splpar_spin_yield(arch_spinlock_t *lock) {}; -static inline void splpar_rw_yield(arch_rwlock_t *lock) {}; +static inline void splpar_spin_yield(arch_spinlock_t *lock) {} +static inline void splpar_rw_yield(arch_rwlock_t *lock) {} #endif static inline void spin_yield(arch_spinlock_t *lock) diff --git a/arch/powerpc/include/asm/smp.h b/arch/powerpc/include/asm/smp.h index c4e2d53acd2be4760d10632fe9dfb42c8fcfbbec..7a13bc20f0a0ce3ef95a6eb9921b40a3a2ba2e69 100644 --- a/arch/powerpc/include/asm/smp.h +++ b/arch/powerpc/include/asm/smp.h @@ -236,7 +236,7 @@ static inline void set_hard_smp_processor_id(int cpu, int phys) #if defined(CONFIG_PPC64) && (defined(CONFIG_SMP) || defined(CONFIG_KEXEC_CORE)) extern void smp_release_cpus(void); #else -static inline void smp_release_cpus(void) { }; +static inline void smp_release_cpus(void) { } #endif extern int smt_enabled_at_boot; diff --git a/arch/powerpc/include/asm/thread_info.h b/arch/powerpc/include/asm/thread_info.h index 3d8a47af7a25885cab19caa77b683fc95ed9eba4..386d576673a1df56b6e18c73477d73e09ecf0eae 100644 --- a/arch/powerpc/include/asm/thread_info.h +++ b/arch/powerpc/include/asm/thread_info.h @@ -94,7 +94,6 @@ void arch_setup_new_exec(void); #define TIF_PATCH_PENDING 6 /* pending live patching update */ #define TIF_SYSCALL_AUDIT 7 /* syscall auditing active */ #define TIF_SINGLESTEP 8 /* singlestepping active */ -#define TIF_NOHZ 9 /* in adaptive nohz mode */ #define TIF_SECCOMP 10 /* secure computing */ #define TIF_RESTOREALL 11 /* Restore all regs (implies NOERROR) */ #define TIF_NOERROR 12 /* Force successful syscall return */ @@ -128,11 +127,10 @@ void arch_setup_new_exec(void); #define _TIF_UPROBE (1< #include +#include #include #include #include #ifdef CONFIG_SMP -void doorbell_exception(struct pt_regs *regs) +DEFINE_INTERRUPT_HANDLER_ASYNC(doorbell_exception) { struct pt_regs *old_regs = set_irq_regs(regs); - irq_enter(); trace_doorbell_entry(regs); ppc_msgsync(); @@ -35,13 +35,12 @@ void doorbell_exception(struct pt_regs *regs) smp_ipi_demux_relaxed(); /* already performed the barrier */ trace_doorbell_exit(regs); - irq_exit(); + set_irq_regs(old_regs); } #else /* CONFIG_SMP */ -void doorbell_exception(struct pt_regs *regs) +DEFINE_INTERRUPT_HANDLER_ASYNC(doorbell_exception) { printk(KERN_WARNING "Received doorbell on non-smp system\n"); } #endif /* CONFIG_SMP */ - diff --git a/arch/powerpc/kernel/eeh.c b/arch/powerpc/kernel/eeh.c index 813713c9120c0f85f0c9599f55647f66dbb668db..cd60bc1c8701138f8e2e3682e6f3aaafc88ff62f 100644 --- a/arch/powerpc/kernel/eeh.c +++ b/arch/powerpc/kernel/eeh.c @@ -1596,6 +1596,35 @@ static int proc_eeh_show(struct seq_file *m, void *v) } #ifdef CONFIG_DEBUG_FS + + +static struct pci_dev *eeh_debug_lookup_pdev(struct file *filp, + const char __user *user_buf, + size_t count, loff_t *ppos) +{ + uint32_t domain, bus, dev, fn; + struct pci_dev *pdev; + char buf[20]; + int ret; + + memset(buf, 0, sizeof(buf)); + ret = simple_write_to_buffer(buf, sizeof(buf)-1, ppos, user_buf, count); + if (!ret) + return ERR_PTR(-EFAULT); + + ret = sscanf(buf, "%x:%x:%x.%x", &domain, &bus, &dev, &fn); + if (ret != 4) { + pr_err("%s: expected 4 args, got %d\n", __func__, ret); + return ERR_PTR(-EINVAL); + } + + pdev = pci_get_domain_bus_and_slot(domain, bus, (dev << 3) | fn); + if (!pdev) + return ERR_PTR(-ENODEV); + + return pdev; +} + static int eeh_enable_dbgfs_set(void *data, u64 val) { if (val) @@ -1688,26 +1717,13 @@ static ssize_t eeh_dev_check_write(struct file *filp, const char __user *user_buf, size_t count, loff_t *ppos) { - uint32_t domain, bus, dev, fn; struct pci_dev *pdev; struct eeh_dev *edev; - char buf[20]; int ret; - memset(buf, 0, sizeof(buf)); - ret = simple_write_to_buffer(buf, sizeof(buf)-1, ppos, user_buf, count); - if (!ret) - return -EFAULT; - - ret = sscanf(buf, "%x:%x:%x.%x", &domain, &bus, &dev, &fn); - if (ret != 4) { - pr_err("%s: expected 4 args, got %d\n", __func__, ret); - return -EINVAL; - } - - pdev = pci_get_domain_bus_and_slot(domain, bus, (dev << 3) | fn); - if (!pdev) - return -ENODEV; + pdev = eeh_debug_lookup_pdev(filp, user_buf, count, ppos); + if (IS_ERR(pdev)) + return PTR_ERR(pdev); edev = pci_dev_to_eeh_dev(pdev); if (!edev) { @@ -1717,8 +1733,8 @@ static ssize_t eeh_dev_check_write(struct file *filp, } ret = eeh_dev_check_failure(edev); - pci_info(pdev, "eeh_dev_check_failure(%04x:%02x:%02x.%01x) = %d\n", - domain, bus, dev, fn, ret); + pci_info(pdev, "eeh_dev_check_failure(%s) = %d\n", + pci_name(pdev), ret); pci_dev_put(pdev); @@ -1829,25 +1845,12 @@ static ssize_t eeh_dev_break_write(struct file *filp, const char __user *user_buf, size_t count, loff_t *ppos) { - uint32_t domain, bus, dev, fn; struct pci_dev *pdev; - char buf[20]; int ret; - memset(buf, 0, sizeof(buf)); - ret = simple_write_to_buffer(buf, sizeof(buf)-1, ppos, user_buf, count); - if (!ret) - return -EFAULT; - - ret = sscanf(buf, "%x:%x:%x.%x", &domain, &bus, &dev, &fn); - if (ret != 4) { - pr_err("%s: expected 4 args, got %d\n", __func__, ret); - return -EINVAL; - } - - pdev = pci_get_domain_bus_and_slot(domain, bus, (dev << 3) | fn); - if (!pdev) - return -ENODEV; + pdev = eeh_debug_lookup_pdev(filp, user_buf, count, ppos); + if (IS_ERR(pdev)) + return PTR_ERR(pdev); ret = eeh_debugfs_break_device(pdev); pci_dev_put(pdev); @@ -1865,6 +1868,53 @@ static const struct file_operations eeh_dev_break_fops = { .read = eeh_debugfs_dev_usage, }; +static ssize_t eeh_dev_can_recover(struct file *filp, + const char __user *user_buf, + size_t count, loff_t *ppos) +{ + struct pci_driver *drv; + struct pci_dev *pdev; + size_t ret; + + pdev = eeh_debug_lookup_pdev(filp, user_buf, count, ppos); + if (IS_ERR(pdev)) + return PTR_ERR(pdev); + + /* + * In order for error recovery to work the driver needs to implement + * .error_detected(), so it can quiesce IO to the device, and + * .slot_reset() so it can re-initialise the device after a reset. + * + * Ideally they'd implement .resume() too, but some drivers which + * we need to support (notably IPR) don't so I guess we can tolerate + * that. + * + * .mmio_enabled() is mostly there as a work-around for devices which + * take forever to re-init after a hot reset. Implementing that is + * strictly optional. + */ + drv = pci_dev_driver(pdev); + if (drv && + drv->err_handler && + drv->err_handler->error_detected && + drv->err_handler->slot_reset) { + ret = count; + } else { + ret = -EOPNOTSUPP; + } + + pci_dev_put(pdev); + + return ret; +} + +static const struct file_operations eeh_dev_can_recover_fops = { + .open = simple_open, + .llseek = no_llseek, + .write = eeh_dev_can_recover, + .read = eeh_debugfs_dev_usage, +}; + #endif static int __init eeh_init_proc(void) @@ -1889,6 +1939,9 @@ static int __init eeh_init_proc(void) debugfs_create_file_unsafe("eeh_force_recover", 0600, powerpc_debugfs_root, NULL, &eeh_force_recover_fops); + debugfs_create_file_unsafe("eeh_dev_can_recover", 0600, + powerpc_debugfs_root, NULL, + &eeh_dev_can_recover_fops); eeh_cache_debugfs_init(); #endif } diff --git a/arch/powerpc/kernel/entry_32.S b/arch/powerpc/kernel/entry_32.S index 1c9b0ccc2172e8480da768e953550f57b83dfd24..78c430b7f9d943316c8c538ee8ab164c3f32d995 100644 --- a/arch/powerpc/kernel/entry_32.S +++ b/arch/powerpc/kernel/entry_32.S @@ -175,14 +175,11 @@ transfer_to_handler: addi r11,r11,global_dbcr0@l #ifdef CONFIG_SMP lwz r9,TASK_CPU(r2) - slwi r9,r9,3 + slwi r9,r9,2 add r11,r11,r9 #endif lwz r12,0(r11) mtspr SPRN_DBCR0,r12 - lwz r12,4(r11) - addi r12,r12,-1 - stw r12,4(r11) #endif b 3f @@ -276,8 +273,7 @@ reenable_mmu: * We save a bunch of GPRs, * r3 can be different from GPR3(r1) at this point, r9 and r11 * contains the old MSR and handler address respectively, - * r4 & r5 can contain page fault arguments that need to be passed - * along as well. r0, r6-r8, r12, CCR, CTR, XER etc... are left + * r0, r4-r8, r12, CCR, CTR, XER etc... are left * clobbered as they aren't useful past this point. */ @@ -285,15 +281,11 @@ reenable_mmu: stw r9,8(r1) stw r11,12(r1) stw r3,16(r1) - stw r4,20(r1) - stw r5,24(r1) /* If we are disabling interrupts (normal case), simply log it with * lockdep */ 1: bl trace_hardirqs_off - lwz r5,24(r1) - lwz r4,20(r1) lwz r3,16(r1) lwz r11,12(r1) lwz r9,8(r1) @@ -334,132 +326,29 @@ stack_ovf: _ASM_NOKPROBE_SYMBOL(stack_ovf) #endif -#ifdef CONFIG_TRACE_IRQFLAGS -trace_syscall_entry_irq_off: - /* - * Syscall shouldn't happen while interrupts are disabled, - * so let's do a warning here. - */ -0: trap - EMIT_BUG_ENTRY 0b,__FILE__,__LINE__, BUGFLAG_WARNING - bl trace_hardirqs_on - - /* Now enable for real */ - LOAD_REG_IMMEDIATE(r10, MSR_KERNEL | MSR_EE) - mtmsr r10 - - REST_GPR(0, r1) - REST_4GPRS(3, r1) - REST_2GPRS(7, r1) - b DoSyscall -#endif /* CONFIG_TRACE_IRQFLAGS */ - .globl transfer_to_syscall transfer_to_syscall: -#ifdef CONFIG_TRACE_IRQFLAGS - andi. r12,r9,MSR_EE - beq- trace_syscall_entry_irq_off -#endif /* CONFIG_TRACE_IRQFLAGS */ + SAVE_NVGPRS(r1) +#ifdef CONFIG_PPC_BOOK3S_32 + kuep_lock r11, r12 +#endif -/* - * Handle a system call. - */ - .stabs "arch/powerpc/kernel/",N_SO,0,0,0f - .stabs "entry_32.S",N_SO,0,0,0f -0: - -_GLOBAL(DoSyscall) - stw r3,ORIG_GPR3(r1) - li r12,0 - stw r12,RESULT(r1) -#ifdef CONFIG_TRACE_IRQFLAGS - /* Make sure interrupts are enabled */ - mfmsr r11 - andi. r12,r11,MSR_EE - /* We came in with interrupts disabled, we WARN and mark them enabled - * for lockdep now */ -0: tweqi r12, 0 - EMIT_BUG_ENTRY 0b,__FILE__,__LINE__, BUGFLAG_WARNING -#endif /* CONFIG_TRACE_IRQFLAGS */ - lwz r11,TI_FLAGS(r2) - andi. r11,r11,_TIF_SYSCALL_DOTRACE - bne- syscall_dotrace -syscall_dotrace_cont: - cmplwi 0,r0,NR_syscalls - lis r10,sys_call_table@h - ori r10,r10,sys_call_table@l - slwi r0,r0,2 - bge- 66f - - barrier_nospec_asm - /* - * Prevent the load of the handler below (based on the user-passed - * system call number) being speculatively executed until the test - * against NR_syscalls and branch to .66f above has - * committed. - */ + /* Calling convention has r9 = orig r0, r10 = regs */ + addi r10,r1,STACK_FRAME_OVERHEAD + mr r9,r0 + stw r10,THREAD+PT_REGS(r2) + bl system_call_exception - lwzx r10,r10,r0 /* Fetch system call handler [ptr] */ - mtlr r10 - addi r9,r1,STACK_FRAME_OVERHEAD - PPC440EP_ERR42 - blrl /* Call handler */ - .globl ret_from_syscall ret_from_syscall: -#ifdef CONFIG_DEBUG_RSEQ - /* Check whether the syscall is issued inside a restartable sequence */ - stw r3,GPR3(r1) - addi r3,r1,STACK_FRAME_OVERHEAD - bl rseq_syscall - lwz r3,GPR3(r1) -#endif - mr r6,r3 - /* disable interrupts so current_thread_info()->flags can't change */ - LOAD_REG_IMMEDIATE(r10,MSR_KERNEL) /* doesn't include MSR_EE */ - /* Note: We don't bother telling lockdep about it */ - mtmsr r10 - lwz r9,TI_FLAGS(r2) - li r8,-MAX_ERRNO - andi. r0,r9,(_TIF_SYSCALL_DOTRACE|_TIF_SINGLESTEP|_TIF_USER_WORK_MASK|_TIF_PERSYSCALL_MASK) - bne- syscall_exit_work - cmplw 0,r3,r8 - blt+ syscall_exit_cont - lwz r11,_CCR(r1) /* Load CR */ - neg r3,r3 - oris r11,r11,0x1000 /* Set SO bit in CR */ - stw r11,_CCR(r1) -syscall_exit_cont: - lwz r8,_MSR(r1) -#ifdef CONFIG_TRACE_IRQFLAGS - /* If we are going to return from the syscall with interrupts - * off, we trace that here. It shouldn't normally happen. - */ - andi. r10,r8,MSR_EE - bne+ 1f - stw r3,GPR3(r1) - bl trace_hardirqs_off - lwz r3,GPR3(r1) -1: -#endif /* CONFIG_TRACE_IRQFLAGS */ -#if defined(CONFIG_4xx) || defined(CONFIG_BOOKE) - /* If the process has its own DBCR0 value, load it up. The internal - debug mode bit tells us that dbcr0 should be loaded. */ - lwz r0,THREAD+THREAD_DBCR0(r2) - andis. r10,r0,DBCR0_IDM@h - bnel- load_dbcr0 -#endif + addi r4,r1,STACK_FRAME_OVERHEAD + li r5,0 + bl syscall_exit_prepare #ifdef CONFIG_PPC_47x lis r4,icache_44x_need_flush@ha lwz r5,icache_44x_need_flush@l(r4) cmplwi cr0,r5,0 bne- 2f #endif /* CONFIG_PPC_47x */ -1: -BEGIN_FTR_SECTION - lwarx r7,0,r1 -END_FTR_SECTION_IFSET(CPU_FTR_NEED_PAIRED_STWCX) - stwcx. r0,0,r1 /* to clear the reservation */ - ACCOUNT_CPU_USER_EXIT(r2, r5, r7) #ifdef CONFIG_PPC_BOOK3S_32 kuep_unlock r5, r7 #endif @@ -467,21 +356,36 @@ END_FTR_SECTION_IFSET(CPU_FTR_NEED_PAIRED_STWCX) lwz r4,_LINK(r1) lwz r5,_CCR(r1) mtlr r4 - mtcr r5 lwz r7,_NIP(r1) - lwz r2,GPR2(r1) - lwz r1,GPR1(r1) + lwz r8,_MSR(r1) + cmpwi r3,0 + lwz r3,GPR3(r1) syscall_exit_finish: -#if defined(CONFIG_PPC_8xx) && defined(CONFIG_PERF_EVENTS) - mtspr SPRN_NRI, r0 -#endif mtspr SPRN_SRR0,r7 mtspr SPRN_SRR1,r8 + + bne 3f + mtcr r5 + +1: lwz r2,GPR2(r1) + lwz r1,GPR1(r1) rfi #ifdef CONFIG_40x b . /* Prevent prefetch past rfi */ #endif -_ASM_NOKPROBE_SYMBOL(syscall_exit_finish) + +3: mtcr r5 + lwz r4,_CTR(r1) + lwz r5,_XER(r1) + REST_NVGPRS(r1) + mtctr r4 + mtxer r5 + lwz r0,GPR0(r1) + lwz r3,GPR3(r1) + REST_8GPRS(4,r1) + lwz r12,GPR12(r1) + b 1b + #ifdef CONFIG_44x 2: li r7,0 iccci r0,r0 @@ -489,9 +393,6 @@ _ASM_NOKPROBE_SYMBOL(syscall_exit_finish) b 1b #endif /* CONFIG_44x */ -66: li r3,-ENOSYS - b ret_from_syscall - .globl ret_from_fork ret_from_fork: REST_NVGPRS(r1) @@ -510,157 +411,6 @@ ret_from_kernel_thread: li r3,0 b ret_from_syscall -/* Traced system call support */ -syscall_dotrace: - SAVE_NVGPRS(r1) - li r0,0xc00 - stw r0,_TRAP(r1) - addi r3,r1,STACK_FRAME_OVERHEAD - bl do_syscall_trace_enter - /* - * Restore argument registers possibly just changed. - * We use the return value of do_syscall_trace_enter - * for call number to look up in the table (r0). - */ - mr r0,r3 - lwz r3,GPR3(r1) - lwz r4,GPR4(r1) - lwz r5,GPR5(r1) - lwz r6,GPR6(r1) - lwz r7,GPR7(r1) - lwz r8,GPR8(r1) - REST_NVGPRS(r1) - - cmplwi r0,NR_syscalls - /* Return code is already in r3 thanks to do_syscall_trace_enter() */ - bge- ret_from_syscall - b syscall_dotrace_cont - -syscall_exit_work: - andi. r0,r9,_TIF_RESTOREALL - beq+ 0f - REST_NVGPRS(r1) - b 2f -0: cmplw 0,r3,r8 - blt+ 1f - andi. r0,r9,_TIF_NOERROR - bne- 1f - lwz r11,_CCR(r1) /* Load CR */ - neg r3,r3 - oris r11,r11,0x1000 /* Set SO bit in CR */ - stw r11,_CCR(r1) - -1: stw r6,RESULT(r1) /* Save result */ - stw r3,GPR3(r1) /* Update return value */ -2: andi. r0,r9,(_TIF_PERSYSCALL_MASK) - beq 4f - - /* Clear per-syscall TIF flags if any are set. */ - - li r11,_TIF_PERSYSCALL_MASK - addi r12,r2,TI_FLAGS -3: lwarx r8,0,r12 - andc r8,r8,r11 - stwcx. r8,0,r12 - bne- 3b - -4: /* Anything which requires enabling interrupts? */ - andi. r0,r9,(_TIF_SYSCALL_DOTRACE|_TIF_SINGLESTEP) - beq ret_from_except - - /* Re-enable interrupts. There is no need to trace that with - * lockdep as we are supposed to have IRQs on at this point - */ - ori r10,r10,MSR_EE - mtmsr r10 - - /* Save NVGPRS if they're not saved already */ - lwz r4,_TRAP(r1) - andi. r4,r4,1 - beq 5f - SAVE_NVGPRS(r1) - li r4,0xc00 - stw r4,_TRAP(r1) -5: - addi r3,r1,STACK_FRAME_OVERHEAD - bl do_syscall_trace_leave - b ret_from_except_full - - /* - * System call was called from kernel. We get here with SRR1 in r9. - * Mark the exception as recoverable once we have retrieved SRR0, - * trap a warning and return ENOSYS with CR[SO] set. - */ - .globl ret_from_kernel_syscall -ret_from_kernel_syscall: - mfspr r9, SPRN_SRR0 - mfspr r10, SPRN_SRR1 -#if !defined(CONFIG_4xx) && !defined(CONFIG_BOOKE) - LOAD_REG_IMMEDIATE(r11, MSR_KERNEL & ~(MSR_IR|MSR_DR)) - mtmsr r11 -#endif - -0: trap - EMIT_BUG_ENTRY 0b,__FILE__,__LINE__, BUGFLAG_WARNING - - li r3, ENOSYS - crset so -#if defined(CONFIG_PPC_8xx) && defined(CONFIG_PERF_EVENTS) - mtspr SPRN_NRI, r0 -#endif - mtspr SPRN_SRR0, r9 - mtspr SPRN_SRR1, r10 - rfi -#ifdef CONFIG_40x - b . /* Prevent prefetch past rfi */ -#endif -_ASM_NOKPROBE_SYMBOL(ret_from_kernel_syscall) - -/* - * The fork/clone functions need to copy the full register set into - * the child process. Therefore we need to save all the nonvolatile - * registers (r13 - r31) before calling the C code. - */ - .globl ppc_fork -ppc_fork: - SAVE_NVGPRS(r1) - lwz r0,_TRAP(r1) - rlwinm r0,r0,0,0,30 /* clear LSB to indicate full */ - stw r0,_TRAP(r1) /* register set saved */ - b sys_fork - - .globl ppc_vfork -ppc_vfork: - SAVE_NVGPRS(r1) - lwz r0,_TRAP(r1) - rlwinm r0,r0,0,0,30 /* clear LSB to indicate full */ - stw r0,_TRAP(r1) /* register set saved */ - b sys_vfork - - .globl ppc_clone -ppc_clone: - SAVE_NVGPRS(r1) - lwz r0,_TRAP(r1) - rlwinm r0,r0,0,0,30 /* clear LSB to indicate full */ - stw r0,_TRAP(r1) /* register set saved */ - b sys_clone - - .globl ppc_clone3 -ppc_clone3: - SAVE_NVGPRS(r1) - lwz r0,_TRAP(r1) - rlwinm r0,r0,0,0,30 /* clear LSB to indicate full */ - stw r0,_TRAP(r1) /* register set saved */ - b sys_clone3 - - .globl ppc_swapcontext -ppc_swapcontext: - SAVE_NVGPRS(r1) - lwz r0,_TRAP(r1) - rlwinm r0,r0,0,0,30 /* clear LSB to indicate full */ - stw r0,_TRAP(r1) /* register set saved */ - b sys_swapcontext - /* * Top-level page fault handling. * This is in assembler because if do_page_fault tells us that @@ -670,10 +420,6 @@ ppc_swapcontext: .globl handle_page_fault handle_page_fault: addi r3,r1,STACK_FRAME_OVERHEAD -#ifdef CONFIG_PPC_BOOK3S_32 - andis. r0,r5,DSISR_DABRMATCH@h - bne- handle_dabr_fault -#endif bl do_page_fault cmpwi r3,0 beq+ ret_from_except @@ -681,23 +427,11 @@ handle_page_fault: lwz r0,_TRAP(r1) clrrwi r0,r0,1 stw r0,_TRAP(r1) - mr r5,r3 + mr r4,r3 /* err arg for bad_page_fault */ addi r3,r1,STACK_FRAME_OVERHEAD - lwz r4,_DAR(r1) bl __bad_page_fault b ret_from_except_full -#ifdef CONFIG_PPC_BOOK3S_32 - /* We have a data breakpoint exception - handle it */ -handle_dabr_fault: - SAVE_NVGPRS(r1) - lwz r0,_TRAP(r1) - clrrwi r0,r0,1 - stw r0,_TRAP(r1) - bl do_break - b ret_from_except_full -#endif - /* * This routine switches between two different tasks. The process * state of one is saved on its kernel stack. Then the state @@ -1237,14 +971,11 @@ load_dbcr0: addi r11,r11,global_dbcr0@l #ifdef CONFIG_SMP lwz r9,TASK_CPU(r2) - slwi r9,r9,3 + slwi r9,r9,2 add r11,r11,r9 #endif stw r10,0(r11) mtspr SPRN_DBCR0,r0 - lwz r10,4(r11) - addi r10,r10,1 - stw r10,4(r11) li r11,-1 mtspr SPRN_DBSR,r11 /* clear all pending debug events */ blr @@ -1253,7 +984,7 @@ load_dbcr0: .align 4 .global global_dbcr0 global_dbcr0: - .space 8*NR_CPUS + .space 4*NR_CPUS .previous #endif /* !(CONFIG_4xx || CONFIG_BOOKE) */ diff --git a/arch/powerpc/kernel/entry_64.S b/arch/powerpc/kernel/entry_64.S index 33ddfeef4fe9ed4a8eb13f38964cc195e896d7c4..6c4d9e276c4d5b8835a150e3b8d775eb384ecfc3 100644 --- a/arch/powerpc/kernel/entry_64.S +++ b/arch/powerpc/kernel/entry_64.S @@ -108,7 +108,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_TM) li r11,\trapnr std r11,_TRAP(r1) std r12,_CCR(r1) - std r3,ORIG_GPR3(r1) addi r10,r1,STACK_FRAME_OVERHEAD ld r11,exception_marker@toc(r2) std r11,-16(r10) /* "regshere" marker */ @@ -225,6 +224,12 @@ _ASM_NOKPROBE_SYMBOL(system_call_vectored_emulate) b system_call_vectored_common #endif + .balign IFETCH_ALIGN_BYTES + .globl system_call_common_real +system_call_common_real: + ld r10,PACAKMSR(r13) /* get MSR value for kernel */ + mtmsrd r10 + .balign IFETCH_ALIGN_BYTES .globl system_call_common system_call_common: @@ -278,7 +283,6 @@ END_BTB_FLUSH_SECTION std r10,_LINK(r1) std r11,_TRAP(r1) std r12,_CCR(r1) - std r3,ORIG_GPR3(r1) addi r10,r1,STACK_FRAME_OVERHEAD ld r11,exception_marker@toc(r2) std r11,-16(r10) /* "regshere" marker */ diff --git a/arch/powerpc/kernel/exceptions-64e.S b/arch/powerpc/kernel/exceptions-64e.S index 74d07dc0bb48d166175923bf1418d6325b09df45..e8eb9992a27038d078256a745c0342287c89f163 100644 --- a/arch/powerpc/kernel/exceptions-64e.S +++ b/arch/powerpc/kernel/exceptions-64e.S @@ -398,7 +398,6 @@ exc_##n##_common: \ std r10,_NIP(r1); /* save SRR0 to stackframe */ \ std r11,_MSR(r1); /* save SRR1 to stackframe */ \ beq 2f; /* if from kernel mode */ \ - ACCOUNT_CPU_USER_ENTRY(r13,r10,r11);/* accounting (uses cr0+eq) */ \ 2: ld r3,excf+EX_R10(r13); /* get back r10 */ \ ld r4,excf+EX_R11(r13); /* get back r11 */ \ mfspr r5,scratch; /* get back r13 */ \ @@ -791,7 +790,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) EXCEPTION_COMMON_CRIT(0xd00) std r14,_DSISR(r1) addi r3,r1,STACK_FRAME_OVERHEAD - mr r4,r14 ld r14,PACA_EXCRIT+EX_R14(r13) ld r15,PACA_EXCRIT+EX_R15(r13) bl save_nvgprs @@ -864,7 +862,6 @@ kernel_dbg_exc: INTS_DISABLE std r14,_DSISR(r1) addi r3,r1,STACK_FRAME_OVERHEAD - mr r4,r14 ld r14,PACA_EXDBG+EX_R14(r13) ld r15,PACA_EXDBG+EX_R15(r13) bl save_nvgprs @@ -1011,8 +1008,6 @@ storage_fault_common: std r14,_DAR(r1) std r15,_DSISR(r1) addi r3,r1,STACK_FRAME_OVERHEAD - mr r4,r14 - mr r5,r15 ld r14,PACA_EXGEN+EX_R14(r13) ld r15,PACA_EXGEN+EX_R15(r13) bl do_page_fault @@ -1020,9 +1015,8 @@ storage_fault_common: bne- 1f b ret_from_except_lite 1: bl save_nvgprs - mr r5,r3 + mr r4,r3 addi r3,r1,STACK_FRAME_OVERHEAD - ld r4,_DAR(r1) bl __bad_page_fault b ret_from_except diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S index 6e53f76387374bc7f0485246bbc018f7ed21947f..60d3051a8bc8ad3638d9d0696441311f08fd9e89 100644 --- a/arch/powerpc/kernel/exceptions-64s.S +++ b/arch/powerpc/kernel/exceptions-64s.S @@ -139,7 +139,6 @@ name: #define IKVM_VIRT .L_IKVM_VIRT_\name\() /* Virt entry tests KVM */ #define ISTACK .L_ISTACK_\name\() /* Set regular kernel stack */ #define __ISTACK(name) .L_ISTACK_ ## name -#define IRECONCILE .L_IRECONCILE_\name\() /* Do RECONCILE_IRQ_STATE */ #define IKUAP .L_IKUAP_\name\() /* Do KUAP lock */ #define INT_DEFINE_BEGIN(n) \ @@ -203,9 +202,6 @@ do_define_int n .ifndef ISTACK ISTACK=1 .endif - .ifndef IRECONCILE - IRECONCILE=1 - .endif .ifndef IKUAP IKUAP=1 .endif @@ -581,7 +577,6 @@ DEFINE_FIXED_SYMBOL(\name\()_common_real) kuap_save_amr_and_lock r9, r10, cr1, cr0 .endif beq 101f /* if from kernel mode */ - ACCOUNT_CPU_USER_ENTRY(r13, r9, r10) BEGIN_FTR_SECTION ld r9,IAREA+EX_PPR(r13) /* Read PPR from paca */ std r9,_PPR(r1) @@ -649,14 +644,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_CFAR) ld r11,exception_marker@toc(r2) std r10,RESULT(r1) /* clear regs->result */ std r11,STACK_FRAME_OVERHEAD-16(r1) /* mark the frame */ - - .if ISTACK - ACCOUNT_STOLEN_TIME - .endif - - .if IRECONCILE - RECONCILE_IRQ_STATE(r10, r11) - .endif .endm /* @@ -705,14 +692,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_CFAR) ld r1,GPR1(r1) .endm -#define RUNLATCH_ON \ -BEGIN_FTR_SECTION \ - ld r3, PACA_THREAD_INFO(r13); \ - ld r4,TI_LOCAL_FLAGS(r3); \ - andi. r0,r4,_TLF_RUNLATCH; \ - beql ppc64_runlatch_on_trampoline; \ -END_FTR_SECTION_IFSET(CPU_FTR_CTRL) - /* * When the idle code in power4_idle puts the CPU into NAP mode, * it has to do so in a loop, and relies on the external interrupt @@ -935,7 +914,6 @@ INT_DEFINE_BEGIN(system_reset) */ ISET_RI=0 ISTACK=0 - IRECONCILE=0 IKVM_REAL=1 INT_DEFINE_END(system_reset) @@ -1022,20 +1000,6 @@ EXC_COMMON_BEGIN(system_reset_common) ld r1,PACA_NMI_EMERG_SP(r13) subi r1,r1,INT_FRAME_SIZE __GEN_COMMON_BODY system_reset - /* - * Set IRQS_ALL_DISABLED unconditionally so irqs_disabled() does - * the right thing. We do not want to reconcile because that goes - * through irq tracing which we don't want in NMI. - * - * Save PACAIRQHAPPENED to RESULT (otherwise unused), and set HARD_DIS - * as we are running with MSR[EE]=0. - */ - li r10,IRQS_ALL_DISABLED - stb r10,PACAIRQSOFTMASK(r13) - lbz r10,PACAIRQHAPPENED(r13) - std r10,RESULT(r1) - ori r10,r10,PACA_IRQ_HARD_DIS - stb r10,PACAIRQHAPPENED(r13) addi r3,r1,STACK_FRAME_OVERHEAD bl system_reset_exception @@ -1051,14 +1015,6 @@ EXC_COMMON_BEGIN(system_reset_common) subi r10,r10,1 sth r10,PACA_IN_NMI(r13) - /* - * Restore soft mask settings. - */ - ld r10,RESULT(r1) - stb r10,PACAIRQHAPPENED(r13) - ld r10,SOFTE(r1) - stb r10,PACAIRQSOFTMASK(r13) - kuap_kernel_restore r9, r10 EXCEPTION_RESTORE_REGS RFI_TO_USER_OR_KERNEL @@ -1123,7 +1079,6 @@ INT_DEFINE_BEGIN(machine_check_early) ISTACK=0 IDAR=1 IDSISR=1 - IRECONCILE=0 IKUAP=0 /* We don't touch AMR here, we never go to virtual mode */ INT_DEFINE_END(machine_check_early) @@ -1205,30 +1160,11 @@ END_FTR_SECTION_IFSET(CPU_FTR_HVMODE) li r10,MSR_RI mtmsrd r10,1 - /* - * Set IRQS_ALL_DISABLED and save PACAIRQHAPPENED (see - * system_reset_common) - */ - li r10,IRQS_ALL_DISABLED - stb r10,PACAIRQSOFTMASK(r13) - lbz r10,PACAIRQHAPPENED(r13) - std r10,RESULT(r1) - ori r10,r10,PACA_IRQ_HARD_DIS - stb r10,PACAIRQHAPPENED(r13) - addi r3,r1,STACK_FRAME_OVERHEAD bl machine_check_early std r3,RESULT(r1) /* Save result */ ld r12,_MSR(r1) - /* - * Restore soft mask settings. - */ - ld r10,RESULT(r1) - stb r10,PACAIRQHAPPENED(r13) - ld r10,SOFTE(r1) - stb r10,PACAIRQSOFTMASK(r13) - #ifdef CONFIG_PPC_P7_NAP /* * Check if thread was in power saving mode. We come here when any @@ -1401,14 +1337,15 @@ END_FTR_SECTION_IFSET(CPU_FTR_HVMODE) * * Handling: * - Hash MMU - * Go to do_hash_page first to see if the HPT can be filled from an entry in - * the Linux page table. Hash faults can hit in kernel mode in a fairly + * Go to do_hash_fault, which attempts to fill the HPT from an entry in the + * Linux page table. Hash faults can hit in kernel mode in a fairly * arbitrary state (e.g., interrupts disabled, locks held) when accessing * "non-bolted" regions, e.g., vmalloc space. However these should always be - * backed by Linux page tables. + * backed by Linux page table entries. * - * If none is found, do a Linux page fault. Linux page faults can happen in - * kernel mode due to user copy operations of course. + * If no entry is found the Linux page fault handler is invoked (by + * do_hash_fault). Linux page faults can happen in kernel mode due to user + * copy operations of course. * * KVM: The KVM HDSI handler may perform a load with MSR[DR]=1 in guest * MMU context, which may cause a DSI in the host, which must go to the @@ -1437,15 +1374,24 @@ EXC_VIRT_BEGIN(data_access, 0x4300, 0x80) EXC_VIRT_END(data_access, 0x4300, 0x80) EXC_COMMON_BEGIN(data_access_common) GEN_COMMON data_access - ld r4,_DAR(r1) - ld r5,_DSISR(r1) + ld r4,_DSISR(r1) + addi r3,r1,STACK_FRAME_OVERHEAD + andis. r0,r4,DSISR_DABRMATCH@h + bne- 1f BEGIN_MMU_FTR_SECTION - ld r6,_MSR(r1) - li r3,0x300 - b do_hash_page /* Try to handle as hpte fault */ + bl do_hash_fault MMU_FTR_SECTION_ELSE - b handle_page_fault + bl do_page_fault ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) + b interrupt_return + +1: bl do_break + /* + * do_break() may have changed the NV GPRS while handling a breakpoint. + * If so, we need to restore them with their updated values. + */ + REST_NVGPRS(r1) + b interrupt_return GEN_KVM data_access @@ -1466,14 +1412,9 @@ ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) * on user-handler data structures. * * KVM: Same as 0x300, DSLB must test for KVM guest. - * - * A dedicated save area EXSLB is used (XXX: but it actually need not be - * these days, we could use EXGEN). */ INT_DEFINE_BEGIN(data_access_slb) IVEC=0x380 - IAREA=PACA_EXSLB - IRECONCILE=0 IDAR=1 IKVM_SKIP=1 IKVM_REAL=1 @@ -1487,10 +1428,9 @@ EXC_VIRT_BEGIN(data_access_slb, 0x4380, 0x80) EXC_VIRT_END(data_access_slb, 0x4380, 0x80) EXC_COMMON_BEGIN(data_access_slb_common) GEN_COMMON data_access_slb - ld r4,_DAR(r1) - addi r3,r1,STACK_FRAME_OVERHEAD BEGIN_MMU_FTR_SECTION /* HPT case, do SLB fault */ + addi r3,r1,STACK_FRAME_OVERHEAD bl do_slb_fault cmpdi r3,0 bne- 1f @@ -1501,9 +1441,6 @@ MMU_FTR_SECTION_ELSE li r3,-EFAULT ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) std r3,RESULT(r1) - RECONCILE_IRQ_STATE(r10, r11) - ld r4,_DAR(r1) - ld r5,RESULT(r1) addi r3,r1,STACK_FRAME_OVERHEAD bl do_bad_slb_fault b interrupt_return @@ -1538,15 +1475,13 @@ EXC_VIRT_BEGIN(instruction_access, 0x4400, 0x80) EXC_VIRT_END(instruction_access, 0x4400, 0x80) EXC_COMMON_BEGIN(instruction_access_common) GEN_COMMON instruction_access - ld r4,_DAR(r1) - ld r5,_DSISR(r1) + addi r3,r1,STACK_FRAME_OVERHEAD BEGIN_MMU_FTR_SECTION - ld r6,_MSR(r1) - li r3,0x400 - b do_hash_page /* Try to handle as hpte fault */ + bl do_hash_fault MMU_FTR_SECTION_ELSE - b handle_page_fault + bl do_page_fault ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) + b interrupt_return GEN_KVM instruction_access @@ -1562,8 +1497,6 @@ ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) */ INT_DEFINE_BEGIN(instruction_access_slb) IVEC=0x480 - IAREA=PACA_EXSLB - IRECONCILE=0 IISIDE=1 IDAR=1 #ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE @@ -1579,10 +1512,9 @@ EXC_VIRT_BEGIN(instruction_access_slb, 0x4480, 0x80) EXC_VIRT_END(instruction_access_slb, 0x4480, 0x80) EXC_COMMON_BEGIN(instruction_access_slb_common) GEN_COMMON instruction_access_slb - ld r4,_DAR(r1) - addi r3,r1,STACK_FRAME_OVERHEAD BEGIN_MMU_FTR_SECTION /* HPT case, do SLB fault */ + addi r3,r1,STACK_FRAME_OVERHEAD bl do_slb_fault cmpdi r3,0 bne- 1f @@ -1593,9 +1525,6 @@ MMU_FTR_SECTION_ELSE li r3,-EFAULT ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) std r3,RESULT(r1) - RECONCILE_IRQ_STATE(r10, r11) - ld r4,_DAR(r1) - ld r5,RESULT(r1) addi r3,r1,STACK_FRAME_OVERHEAD bl do_bad_slb_fault b interrupt_return @@ -1643,7 +1572,6 @@ EXC_VIRT_END(hardware_interrupt, 0x4500, 0x100) EXC_COMMON_BEGIN(hardware_interrupt_common) GEN_COMMON hardware_interrupt FINISH_NAP - RUNLATCH_ON addi r3,r1,STACK_FRAME_OVERHEAD bl do_IRQ b interrupt_return @@ -1697,6 +1625,51 @@ INT_DEFINE_BEGIN(program_check) INT_DEFINE_END(program_check) EXC_REAL_BEGIN(program_check, 0x700, 0x100) + +#ifdef CONFIG_CPU_LITTLE_ENDIAN + /* + * There's a short window during boot where although the kernel is + * running little endian, any exceptions will cause the CPU to switch + * back to big endian. For example a WARN() boils down to a trap + * instruction, which will cause a program check, and we end up here but + * with the CPU in big endian mode. The first instruction of the program + * check handler (in GEN_INT_ENTRY below) is an mtsprg, which when + * executed in the wrong endian is an lhzu with a ~3GB displacement from + * r3. The content of r3 is random, so that is a load from some random + * location, and depending on the system can easily lead to a checkstop, + * or an infinitely recursive page fault. + * + * So to handle that case we have a trampoline here that can detect we + * are in the wrong endian and flip us back to the correct endian. We + * can't flip MSR[LE] using mtmsr, so we have to use rfid. That requires + * backing up SRR0/1 as well as a GPR. To do that we use SPRG0/2/3, as + * SPRG1 is already used for the paca. SPRG3 is user readable, but this + * trampoline is only active very early in boot, and SPRG3 will be + * reinitialised in vdso_getcpu_init() before userspace starts. + */ +BEGIN_FTR_SECTION + tdi 0,0,0x48 // Trap never, or in reverse endian: b . + 8 + b 1f // Skip trampoline if endian is correct + .long 0xa643707d // mtsprg 0, r11 Backup r11 + .long 0xa6027a7d // mfsrr0 r11 + .long 0xa643727d // mtsprg 2, r11 Backup SRR0 in SPRG2 + .long 0xa6027b7d // mfsrr1 r11 + .long 0xa643737d // mtsprg 3, r11 Backup SRR1 in SPRG3 + .long 0xa600607d // mfmsr r11 + .long 0x01006b69 // xori r11, r11, 1 Invert MSR[LE] + .long 0xa6037b7d // mtsrr1 r11 + .long 0x34076039 // li r11, 0x734 + .long 0xa6037a7d // mtsrr0 r11 + .long 0x2400004c // rfid + mfsprg r11, 3 + mtsrr1 r11 // Restore SRR1 + mfsprg r11, 2 + mtsrr0 r11 // Restore SRR0 + mfsprg r11, 0 // Restore r11 +1: +END_FTR_SECTION(0, 1) // nop out after boot +#endif /* CONFIG_CPU_LITTLE_ENDIAN */ + GEN_INT_ENTRY program_check, virt=0 EXC_REAL_END(program_check, 0x700, 0x100) EXC_VIRT_BEGIN(program_check, 0x4700, 0x100) @@ -1755,7 +1728,6 @@ EXC_COMMON_BEGIN(program_check_common) */ INT_DEFINE_BEGIN(fp_unavailable) IVEC=0x800 - IRECONCILE=0 #ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE IKVM_REAL=1 #endif @@ -1770,7 +1742,6 @@ EXC_VIRT_END(fp_unavailable, 0x4800, 0x100) EXC_COMMON_BEGIN(fp_unavailable_common) GEN_COMMON fp_unavailable bne 1f /* if from user, just load it up */ - RECONCILE_IRQ_STATE(r10, r11) addi r3,r1,STACK_FRAME_OVERHEAD bl kernel_fp_unavailable_exception 0: trap @@ -1789,7 +1760,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_TM) b fast_interrupt_return #ifdef CONFIG_PPC_TRANSACTIONAL_MEM 2: /* User process was in a transaction */ - RECONCILE_IRQ_STATE(r10, r11) addi r3,r1,STACK_FRAME_OVERHEAD bl fp_unavailable_tm b interrupt_return @@ -1832,7 +1802,6 @@ EXC_VIRT_END(decrementer, 0x4900, 0x80) EXC_COMMON_BEGIN(decrementer_common) GEN_COMMON decrementer FINISH_NAP - RUNLATCH_ON addi r3,r1,STACK_FRAME_OVERHEAD bl timer_interrupt b interrupt_return @@ -1854,7 +1823,6 @@ INT_DEFINE_BEGIN(hdecrementer) IVEC=0x980 IHSRR=1 ISTACK=0 - IRECONCILE=0 IKVM_REAL=1 IKVM_VIRT=1 INT_DEFINE_END(hdecrementer) @@ -1919,12 +1887,11 @@ EXC_VIRT_END(doorbell_super, 0x4a00, 0x100) EXC_COMMON_BEGIN(doorbell_super_common) GEN_COMMON doorbell_super FINISH_NAP - RUNLATCH_ON addi r3,r1,STACK_FRAME_OVERHEAD #ifdef CONFIG_PPC_DOORBELL bl doorbell_exception #else - bl unknown_exception + bl unknown_async_exception #endif b interrupt_return @@ -2001,12 +1968,9 @@ END_FTR_SECTION_IFSET(CPU_FTR_REAL_LE) HMT_MEDIUM .if ! \virt - __LOAD_HANDLER(r10, system_call_common) - mtspr SPRN_SRR0,r10 - ld r10,PACAKMSR(r13) - mtspr SPRN_SRR1,r10 - RFI_TO_KERNEL - b . /* prevent speculative execution */ + __LOAD_HANDLER(r10, system_call_common_real) + mtctr r10 + bctr .else li r10,MSR_RI mtmsrd r10,1 /* Set RI (EE=0) */ @@ -2137,9 +2101,7 @@ EXC_COMMON_BEGIN(h_data_storage_common) GEN_COMMON h_data_storage addi r3,r1,STACK_FRAME_OVERHEAD BEGIN_MMU_FTR_SECTION - ld r4,_DAR(r1) - li r5,SIGSEGV - bl bad_page_fault + bl do_bad_page_fault_segv MMU_FTR_SECTION_ELSE bl unknown_exception ALT_MMU_FTR_SECTION_END_IFSET(MMU_FTR_TYPE_RADIX) @@ -2230,7 +2192,6 @@ INT_DEFINE_BEGIN(hmi_exception_early) IHSRR=1 IREALMODE_COMMON=1 ISTACK=0 - IRECONCILE=0 IKUAP=0 /* We don't touch AMR here, we never go to virtual mode */ IKVM_REAL=1 INT_DEFINE_END(hmi_exception_early) @@ -2277,7 +2238,6 @@ EXC_COMMON_BEGIN(hmi_exception_early_common) EXC_COMMON_BEGIN(hmi_exception_common) GEN_COMMON hmi_exception FINISH_NAP - RUNLATCH_ON addi r3,r1,STACK_FRAME_OVERHEAD bl handle_hmi_exception b interrupt_return @@ -2307,12 +2267,11 @@ EXC_VIRT_END(h_doorbell, 0x4e80, 0x20) EXC_COMMON_BEGIN(h_doorbell_common) GEN_COMMON h_doorbell FINISH_NAP - RUNLATCH_ON addi r3,r1,STACK_FRAME_OVERHEAD #ifdef CONFIG_PPC_DOORBELL bl doorbell_exception #else - bl unknown_exception + bl unknown_async_exception #endif b interrupt_return @@ -2341,7 +2300,6 @@ EXC_VIRT_END(h_virt_irq, 0x4ea0, 0x20) EXC_COMMON_BEGIN(h_virt_irq_common) GEN_COMMON h_virt_irq FINISH_NAP - RUNLATCH_ON addi r3,r1,STACK_FRAME_OVERHEAD bl do_IRQ b interrupt_return @@ -2388,7 +2346,6 @@ EXC_VIRT_END(performance_monitor, 0x4f00, 0x20) EXC_COMMON_BEGIN(performance_monitor_common) GEN_COMMON performance_monitor FINISH_NAP - RUNLATCH_ON addi r3,r1,STACK_FRAME_OVERHEAD bl performance_monitor_exception b interrupt_return @@ -2404,7 +2361,6 @@ EXC_COMMON_BEGIN(performance_monitor_common) */ INT_DEFINE_BEGIN(altivec_unavailable) IVEC=0xf20 - IRECONCILE=0 #ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE IKVM_REAL=1 #endif @@ -2434,7 +2390,6 @@ BEGIN_FTR_SECTION b fast_interrupt_return #ifdef CONFIG_PPC_TRANSACTIONAL_MEM 2: /* User process was in a transaction */ - RECONCILE_IRQ_STATE(r10, r11) addi r3,r1,STACK_FRAME_OVERHEAD bl altivec_unavailable_tm b interrupt_return @@ -2442,7 +2397,6 @@ BEGIN_FTR_SECTION 1: END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) #endif - RECONCILE_IRQ_STATE(r10, r11) addi r3,r1,STACK_FRAME_OVERHEAD bl altivec_unavailable_exception b interrupt_return @@ -2458,7 +2412,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) */ INT_DEFINE_BEGIN(vsx_unavailable) IVEC=0xf40 - IRECONCILE=0 #ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE IKVM_REAL=1 #endif @@ -2487,7 +2440,6 @@ BEGIN_FTR_SECTION b load_up_vsx #ifdef CONFIG_PPC_TRANSACTIONAL_MEM 2: /* User process was in a transaction */ - RECONCILE_IRQ_STATE(r10, r11) addi r3,r1,STACK_FRAME_OVERHEAD bl vsx_unavailable_tm b interrupt_return @@ -2495,7 +2447,6 @@ BEGIN_FTR_SECTION 1: END_FTR_SECTION_IFSET(CPU_FTR_VSX) #endif - RECONCILE_IRQ_STATE(r10, r11) addi r3,r1,STACK_FRAME_OVERHEAD bl vsx_unavailable_exception b interrupt_return @@ -2830,7 +2781,6 @@ EXC_VIRT_NONE(0x5800, 0x100) INT_DEFINE_BEGIN(soft_nmi) IVEC=0x900 ISTACK=0 - IRECONCILE=0 /* Soft-NMI may fire under local_irq_disable */ INT_DEFINE_END(soft_nmi) /* @@ -2849,17 +2799,6 @@ EXC_COMMON_BEGIN(soft_nmi_common) subi r1,r1,INT_FRAME_SIZE __GEN_COMMON_BODY soft_nmi - /* - * Set IRQS_ALL_DISABLED and save PACAIRQHAPPENED (see - * system_reset_common) - */ - li r10,IRQS_ALL_DISABLED - stb r10,PACAIRQSOFTMASK(r13) - lbz r10,PACAIRQHAPPENED(r13) - std r10,RESULT(r1) - ori r10,r10,PACA_IRQ_HARD_DIS - stb r10,PACAIRQHAPPENED(r13) - addi r3,r1,STACK_FRAME_OVERHEAD bl soft_nmi_interrupt @@ -2867,14 +2806,6 @@ EXC_COMMON_BEGIN(soft_nmi_common) li r9,0 mtmsrd r9,1 - /* - * Restore soft mask settings. - */ - ld r10,RESULT(r1) - stb r10,PACAIRQHAPPENED(r13) - ld r10,SOFTE(r1) - stb r10,PACAIRQSOFTMASK(r13) - kuap_kernel_restore r9, r10 EXCEPTION_RESTORE_REGS hsrr=0 RFI_TO_KERNEL @@ -3148,9 +3079,6 @@ kvmppc_skip_Hinterrupt: * come here. */ -EXC_COMMON_BEGIN(ppc64_runlatch_on_trampoline) - b __ppc64_runlatch_on - USE_FIXED_SECTION(virt_trampolines) /* * All code below __end_interrupts is treated as soft-masked. If @@ -3221,99 +3149,3 @@ disable_machine_check: RFI_TO_KERNEL 1: mtlr r0 blr - -/* - * Hash table stuff - */ - .balign IFETCH_ALIGN_BYTES -do_hash_page: -#ifdef CONFIG_PPC_BOOK3S_64 - lis r0,(DSISR_BAD_FAULT_64S | DSISR_DABRMATCH | DSISR_KEYFAULT)@h - ori r0,r0,DSISR_BAD_FAULT_64S@l - and. r0,r5,r0 /* weird error? */ - bne- handle_page_fault /* if not, try to insert a HPTE */ - - /* - * If we are in an "NMI" (e.g., an interrupt when soft-disabled), then - * don't call hash_page, just fail the fault. This is required to - * prevent re-entrancy problems in the hash code, namely perf - * interrupts hitting while something holds H_PAGE_BUSY, and taking a - * hash fault. See the comment in hash_preload(). - */ - ld r11, PACA_THREAD_INFO(r13) - lwz r0,TI_PREEMPT(r11) - andis. r0,r0,NMI_MASK@h - bne 77f - - /* - * r3 contains the trap number - * r4 contains the faulting address - * r5 contains dsisr - * r6 msr - * - * at return r3 = 0 for success, 1 for page fault, negative for error - */ - bl __hash_page /* build HPTE if possible */ - cmpdi r3,0 /* see if __hash_page succeeded */ - - /* Success */ - beq interrupt_return /* Return from exception on success */ - - /* Error */ - blt- 13f - - /* Reload DAR/DSISR into r4/r5 for the DABR check below */ - ld r4,_DAR(r1) - ld r5,_DSISR(r1) -#endif /* CONFIG_PPC_BOOK3S_64 */ - -/* Here we have a page fault that hash_page can't handle. */ -handle_page_fault: -11: andis. r0,r5,DSISR_DABRMATCH@h - bne- handle_dabr_fault - addi r3,r1,STACK_FRAME_OVERHEAD - bl do_page_fault - cmpdi r3,0 - beq+ interrupt_return - mr r5,r3 - addi r3,r1,STACK_FRAME_OVERHEAD - ld r4,_DAR(r1) - bl __bad_page_fault - b interrupt_return - -/* We have a data breakpoint exception - handle it */ -handle_dabr_fault: - ld r4,_DAR(r1) - ld r5,_DSISR(r1) - addi r3,r1,STACK_FRAME_OVERHEAD - bl do_break - /* - * do_break() may have changed the NV GPRS while handling a breakpoint. - * If so, we need to restore them with their updated values. - */ - REST_NVGPRS(r1) - b interrupt_return - - -#ifdef CONFIG_PPC_BOOK3S_64 -/* We have a page fault that hash_page could handle but HV refused - * the PTE insertion - */ -13: mr r5,r3 - addi r3,r1,STACK_FRAME_OVERHEAD - ld r4,_DAR(r1) - bl low_hash_fault - b interrupt_return -#endif - -/* - * We come here as a result of a DSI at a point where we don't want - * to call hash_page, such as when we are accessing memory (possibly - * user memory) inside a PMU interrupt that occurred while interrupts - * were soft-disabled. We want to invoke the exception handler for - * the access, or panic if there isn't a handler. - */ -77: addi r3,r1,STACK_FRAME_OVERHEAD - li r5,SIGSEGV - bl bad_page_fault - b interrupt_return diff --git a/arch/powerpc/kernel/head_32.h b/arch/powerpc/kernel/head_32.h index a2f72c966bafb243d225e33b1cda39e18901682c..5d4706c14572717493c563bd7919c2171fc394f2 100644 --- a/arch/powerpc/kernel/head_32.h +++ b/arch/powerpc/kernel/head_32.h @@ -47,7 +47,7 @@ lwz r1,TASK_STACK-THREAD(r1) addi r1, r1, THREAD_SIZE - INT_FRAME_SIZE 1: - mtcrf 0x7f, r1 + mtcrf 0x3f, r1 bt 32 - THREAD_ALIGN_SHIFT, stack_overflow #else subi r11, r1, INT_FRAME_SIZE /* use r1 if kernel */ @@ -116,114 +116,44 @@ .endm .macro SYSCALL_ENTRY trapno - mfspr r12,SPRN_SPRG_THREAD mfspr r9, SPRN_SRR1 -#ifdef CONFIG_VMAP_STACK - mfspr r11, SPRN_SRR0 - mtctr r11 - andi. r11, r9, MSR_PR + mfspr r10, SPRN_SRR0 + LOAD_REG_IMMEDIATE(r11, MSR_KERNEL) /* can take exceptions */ + lis r12, 1f@h + ori r12, r12, 1f@l + mtspr SPRN_SRR1, r11 + mtspr SPRN_SRR0, r12 + mfspr r12,SPRN_SPRG_THREAD mr r11, r1 lwz r1,TASK_STACK-THREAD(r12) - beq- 99f - addi r1, r1, THREAD_SIZE - INT_FRAME_SIZE - li r10, MSR_KERNEL & ~(MSR_IR | MSR_RI) /* can take DTLB miss */ - mtmsr r10 - isync tovirt(r12, r12) + addi r1, r1, THREAD_SIZE - INT_FRAME_SIZE + rfi +1: stw r11,GPR1(r1) stw r11,0(r1) mr r11, r1 -#else - andi. r11, r9, MSR_PR - lwz r11,TASK_STACK-THREAD(r12) - beq- 99f - addi r11, r11, THREAD_SIZE - INT_FRAME_SIZE - tophys(r11, r11) - stw r1,GPR1(r11) - stw r1,0(r11) - tovirt(r1, r11) /* set new kernel sp */ -#endif + stw r10,_NIP(r11) mflr r10 stw r10, _LINK(r11) -#ifdef CONFIG_VMAP_STACK - mfctr r10 -#else - mfspr r10,SPRN_SRR0 -#endif - stw r10,_NIP(r11) mfcr r10 rlwinm r10,r10,0,4,2 /* Clear SO bit in CR */ stw r10,_CCR(r11) /* save registers */ #ifdef CONFIG_40x rlwinm r9,r9,0,14,12 /* clear MSR_WE (necessary?) */ -#else -#ifdef CONFIG_VMAP_STACK - LOAD_REG_IMMEDIATE(r10, MSR_KERNEL & ~MSR_IR) /* can take exceptions */ -#else - LOAD_REG_IMMEDIATE(r10, MSR_KERNEL & ~(MSR_IR|MSR_DR)) /* can take exceptions */ -#endif - mtmsr r10 /* (except for mach check in rtas) */ #endif lis r10,STACK_FRAME_REGS_MARKER@ha /* exception frame marker */ stw r2,GPR2(r11) addi r10,r10,STACK_FRAME_REGS_MARKER@l stw r9,_MSR(r11) - li r2, \trapno + 1 + li r2, \trapno stw r10,8(r11) stw r2,_TRAP(r11) SAVE_GPR(0, r11) SAVE_4GPRS(3, r11) SAVE_2GPRS(7, r11) - addi r11,r1,STACK_FRAME_OVERHEAD addi r2,r12,-THREAD - stw r11,PT_REGS(r12) -#if defined(CONFIG_40x) - /* Check to see if the dbcr0 register is set up to debug. Use the - internal debug mode bit to do this. */ - lwz r12,THREAD_DBCR0(r12) - andis. r12,r12,DBCR0_IDM@h -#endif - ACCOUNT_CPU_USER_ENTRY(r2, r11, r12) -#if defined(CONFIG_40x) - beq+ 3f - /* From user and task is ptraced - load up global dbcr0 */ - li r12,-1 /* clear all pending debug events */ - mtspr SPRN_DBSR,r12 - lis r11,global_dbcr0@ha - tophys(r11,r11) - addi r11,r11,global_dbcr0@l - lwz r12,0(r11) - mtspr SPRN_DBCR0,r12 - lwz r12,4(r11) - addi r12,r12,-1 - stw r12,4(r11) -#endif - -3: - tovirt_novmstack r2, r2 /* set r2 to current */ - lis r11, transfer_to_syscall@h - ori r11, r11, transfer_to_syscall@l -#ifdef CONFIG_TRACE_IRQFLAGS - /* - * If MSR is changing we need to keep interrupts disabled at this point - * otherwise we might risk taking an interrupt before we tell lockdep - * they are enabled. - */ - LOAD_REG_IMMEDIATE(r10, MSR_KERNEL) - rlwimi r10, r9, 0, MSR_EE -#else - LOAD_REG_IMMEDIATE(r10, MSR_KERNEL | MSR_EE) -#endif -#if defined(CONFIG_PPC_8xx) && defined(CONFIG_PERF_EVENTS) - mtspr SPRN_NRI, r0 -#endif - mtspr SPRN_SRR1,r10 - mtspr SPRN_SRR0,r11 - rfi /* jump to handler, enable MMU */ -#ifdef CONFIG_40x - b . /* Prevent prefetch past rfi */ -#endif -99: b ret_from_kernel_syscall + b transfer_to_syscall /* jump to handler */ .endm .macro save_dar_dsisr_on_stack reg1, reg2, sp diff --git a/arch/powerpc/kernel/head_40x.S b/arch/powerpc/kernel/head_40x.S index a1ae00689e0f4898d17a6782a9e82afc7756edd7..24724a7dad49a678e689aa043b99193f89fa37ef 100644 --- a/arch/powerpc/kernel/head_40x.S +++ b/arch/powerpc/kernel/head_40x.S @@ -179,9 +179,9 @@ _ENTRY(saved_ksp_limit) */ START_EXCEPTION(0x0300, DataStorage) EXCEPTION_PROLOG - mfspr r5, SPRN_ESR /* Grab the ESR, save it, pass arg3 */ + mfspr r5, SPRN_ESR /* Grab the ESR, save it */ stw r5, _ESR(r11) - mfspr r4, SPRN_DEAR /* Grab the DEAR, save it, pass arg2 */ + mfspr r4, SPRN_DEAR /* Grab the DEAR, save it */ stw r4, _DEAR(r11) EXC_XFER_LITE(0x300, handle_page_fault) @@ -191,9 +191,9 @@ _ENTRY(saved_ksp_limit) */ START_EXCEPTION(0x0400, InstructionAccess) EXCEPTION_PROLOG - mr r4,r12 /* Pass SRR0 as arg2 */ - stw r4, _DEAR(r11) - li r5,0 /* Pass zero as arg3 */ + li r5,0 + stw r5, _ESR(r11) /* Zero ESR */ + stw r12, _DEAR(r11) /* SRR0 as DEAR */ EXC_XFER_LITE(0x400, handle_page_fault) /* 0x0500 - External Interrupt Exception */ @@ -476,6 +476,7 @@ _ENTRY(saved_ksp_limit) /* continue normal handling for a critical exception... */ 2: mfspr r4,SPRN_DBSR + stw r4,_ESR(r11) /* DebugException takes DBSR in _ESR */ addi r3,r1,STACK_FRAME_OVERHEAD EXC_XFER_TEMPLATE(DebugException, 0x2002, \ (MSR_KERNEL & ~(MSR_ME|MSR_DE|MSR_CE)), \ diff --git a/arch/powerpc/kernel/head_44x.S b/arch/powerpc/kernel/head_44x.S index 8e36718f31674de7a8f6cc663893512b81b75fc1..813fa305c33bd324ca68f898f4727efd69f96b7a 100644 --- a/arch/powerpc/kernel/head_44x.S +++ b/arch/powerpc/kernel/head_44x.S @@ -376,7 +376,7 @@ interrupt_base: /* Load the next available TLB index */ lwz r13,tlb_44x_index@l(r10) - bne 2f /* Bail if permission mismach */ + bne 2f /* Bail if permission mismatch */ /* Increment, rollover, and store TLB index */ addi r13,r13,1 @@ -471,7 +471,7 @@ interrupt_base: /* Load the next available TLB index */ lwz r13,tlb_44x_index@l(r10) - bne 2f /* Bail if permission mismach */ + bne 2f /* Bail if permission mismatch */ /* Increment, rollover, and store TLB index */ addi r13,r13,1 diff --git a/arch/powerpc/kernel/head_8xx.S b/arch/powerpc/kernel/head_8xx.S index 52702f3db6df6189d77474ade26056840ec52bcb..46dff3f9c31f16ce95ce8d43dd7319ad646f7886 100644 --- a/arch/powerpc/kernel/head_8xx.S +++ b/arch/powerpc/kernel/head_8xx.S @@ -165,7 +165,7 @@ SystemCall: /* On the MPC8xx, this is a software emulation interrupt. It occurs * for all unimplemented and illegal instructions. */ - EXCEPTION(0x1000, SoftEmu, program_check_exception, EXC_XFER_STD) + EXCEPTION(0x1000, SoftEmu, emulation_assist_interrupt, EXC_XFER_STD) . = 0x1100 /* @@ -312,14 +312,14 @@ DataStoreTLBMiss: . = 0x1300 InstructionTLBError: EXCEPTION_PROLOG - mr r4,r12 andis. r5,r9,DSISR_SRR1_MATCH_32S@h /* Filter relevant SRR1 bits */ andis. r10,r9,SRR1_ISI_NOPT@h beq+ .Litlbie - tlbie r4 + tlbie r12 /* 0x400 is InstructionAccess exception, needed by bad_page_fault() */ .Litlbie: - stw r4, _DAR(r11) + stw r12, _DAR(r11) + stw r5, _DSISR(r11) EXC_XFER_LITE(0x400, handle_page_fault) /* This is the data TLB error on the MPC8xx. This could be due to @@ -364,10 +364,9 @@ do_databreakpoint: addi r3,r1,STACK_FRAME_OVERHEAD mfspr r4,SPRN_BAR stw r4,_DAR(r11) -#ifdef CONFIG_VMAP_STACK - lwz r5,_DSISR(r11) -#else +#ifndef CONFIG_VMAP_STACK mfspr r5,SPRN_DSISR + stw r5,_DSISR(r11) #endif EXC_XFER_STD(0x1c00, do_break) diff --git a/arch/powerpc/kernel/head_book3s_32.S b/arch/powerpc/kernel/head_book3s_32.S index 858fbc8b19f325a1f28db35378f3f5f7175f4763..727fdab557c9b70ab6c2dc350b8ba9b786da0adb 100644 --- a/arch/powerpc/kernel/head_book3s_32.S +++ b/arch/powerpc/kernel/head_book3s_32.S @@ -238,8 +238,8 @@ __secondary_hold_acknowledge: /* System reset */ /* core99 pmac starts the seconary here by changing the vector, and - putting it back to what it was (unknown_exception) when done. */ - EXCEPTION(0x100, Reset, unknown_exception, EXC_XFER_STD) + putting it back to what it was (unknown_async_exception) when done. */ + EXCEPTION(0x100, Reset, unknown_async_exception, EXC_XFER_STD) /* Machine check */ /* @@ -278,12 +278,6 @@ MachineCheck: 7: EXCEPTION_PROLOG_2 addi r3,r1,STACK_FRAME_OVERHEAD #ifdef CONFIG_PPC_CHRP -#ifdef CONFIG_VMAP_STACK - mfspr r4, SPRN_SPRG_THREAD - tovirt(r4, r4) - lwz r4, RTAS_SP(r4) - cmpwi cr1, r4, 0 -#endif beq cr1, machine_check_tramp twi 31, 0, 0 #else @@ -295,6 +289,7 @@ MachineCheck: DO_KVM 0x300 DataAccess: #ifdef CONFIG_VMAP_STACK +#ifdef CONFIG_PPC_BOOK3S_604 BEGIN_MMU_FTR_SECTION mtspr SPRN_SPRG_SCRATCH2,r10 mfspr r10, SPRN_SPRG_THREAD @@ -311,12 +306,14 @@ BEGIN_MMU_FTR_SECTION MMU_FTR_SECTION_ELSE b 1f ALT_MMU_FTR_SECTION_END_IFSET(MMU_FTR_HPTE_TABLE) +#endif 1: EXCEPTION_PROLOG_0 handle_dar_dsisr=1 EXCEPTION_PROLOG_1 b handle_page_fault_tramp_1 #else /* CONFIG_VMAP_STACK */ EXCEPTION_PROLOG handle_dar_dsisr=1 get_and_save_dar_dsisr_on_stack r4, r5, r11 +#ifdef CONFIG_PPC_BOOK3S_604 BEGIN_MMU_FTR_SECTION andis. r0, r5, (DSISR_BAD_FAULT_32S | DSISR_DABRMATCH)@h bne handle_page_fault_tramp_2 /* if not, try to put a PTE */ @@ -324,8 +321,11 @@ BEGIN_MMU_FTR_SECTION bl hash_page b handle_page_fault_tramp_1 MMU_FTR_SECTION_ELSE +#endif b handle_page_fault_tramp_2 +#ifdef CONFIG_PPC_BOOK3S_604 ALT_MMU_FTR_SECTION_END_IFSET(MMU_FTR_HPTE_TABLE) +#endif #endif /* CONFIG_VMAP_STACK */ /* Instruction access exception. */ @@ -341,12 +341,14 @@ InstructionAccess: mfspr r11, SPRN_SRR1 /* check whether user or kernel */ stw r11, SRR1(r10) mfcr r10 +#ifdef CONFIG_PPC_BOOK3S_604 BEGIN_MMU_FTR_SECTION andis. r11, r11, SRR1_ISI_NOPT@h /* no pte found? */ bne hash_page_isi .Lhash_page_isi_cont: mfspr r11, SPRN_SRR1 /* check whether user or kernel */ END_MMU_FTR_SECTION_IFSET(MMU_FTR_HPTE_TABLE) +#endif andi. r11, r11, MSR_PR EXCEPTION_PROLOG_1 @@ -357,13 +359,15 @@ END_MMU_FTR_SECTION_IFSET(MMU_FTR_HPTE_TABLE) beq 1f /* if so, try to put a PTE */ li r3,0 /* into the hash table */ mr r4,r12 /* SRR0 is fault address */ +#ifdef CONFIG_PPC_BOOK3S_604 BEGIN_MMU_FTR_SECTION bl hash_page END_MMU_FTR_SECTION_IFSET(MMU_FTR_HPTE_TABLE) +#endif #endif /* CONFIG_VMAP_STACK */ -1: mr r4,r12 andis. r5,r9,DSISR_SRR1_MATCH_32S@h /* Filter relevant SRR1 bits */ - stw r4, _DAR(r11) + stw r5, _DSISR(r11) + stw r12, _DAR(r11) EXC_XFER_LITE(0x400, handle_page_fault) /* External interrupt */ @@ -640,7 +644,7 @@ END_MMU_FTR_SECTION_IFSET(MMU_FTR_NEED_DTLB_SW_LRU) #endif #ifndef CONFIG_TAU_INT -#define TAUException unknown_exception +#define TAUException unknown_async_exception #endif EXCEPTION(0x1300, Trap_13, instruction_breakpoint_exception, EXC_XFER_STD) @@ -685,13 +689,16 @@ handle_page_fault_tramp_1: #ifdef CONFIG_VMAP_STACK EXCEPTION_PROLOG_2 handle_dar_dsisr=1 #endif - lwz r4, _DAR(r11) lwz r5, _DSISR(r11) /* fall through */ handle_page_fault_tramp_2: + andis. r0, r5, DSISR_DABRMATCH@h + bne- 1f EXC_XFER_LITE(0x300, handle_page_fault) +1: EXC_XFER_STD(0x300, do_break) #ifdef CONFIG_VMAP_STACK +#ifdef CONFIG_PPC_BOOK3S_604 .macro save_regs_thread thread stw r0, THR0(\thread) stw r3, THR3(\thread) @@ -763,6 +770,7 @@ fast_hash_page_return: mfspr r11, SPRN_SPRG_SCRATCH1 mfspr r10, SPRN_SPRG_SCRATCH0 rfi +#endif /* CONFIG_PPC_BOOK3S_604 */ stack_overflow: vmap_stack_overflow_exception diff --git a/arch/powerpc/kernel/head_booke.h b/arch/powerpc/kernel/head_booke.h index 74e230c200fbf40bab4443f781c673cfe35b01fa..47857795f50a64c4501406cb344cfb96bacba6dc 100644 --- a/arch/powerpc/kernel/head_booke.h +++ b/arch/powerpc/kernel/head_booke.h @@ -106,10 +106,8 @@ ALT_FTR_SECTION_END_IFSET(CPU_FTR_EMB_HV) #endif mfspr r9, SPRN_SRR1 BOOKE_CLEAR_BTB(r11) - andi. r11, r9, MSR_PR lwz r11, TASK_STACK - THREAD(r10) rlwinm r12,r12,0,4,2 /* Clear SO bit in CR */ - beq- 99f ALLOC_STACK_FRAME(r11, THREAD_SIZE - INT_FRAME_SIZE) stw r12, _CCR(r11) /* save various registers */ mflr r12 @@ -124,60 +122,15 @@ ALT_FTR_SECTION_END_IFSET(CPU_FTR_EMB_HV) stw r2,GPR2(r11) addi r12, r12, STACK_FRAME_REGS_MARKER@l stw r9,_MSR(r11) - li r2, \trapno + 1 + li r2, \trapno stw r12, 8(r11) stw r2,_TRAP(r11) SAVE_GPR(0, r11) SAVE_4GPRS(3, r11) SAVE_2GPRS(7, r11) - addi r11,r1,STACK_FRAME_OVERHEAD addi r2,r10,-THREAD - stw r11,PT_REGS(r10) - /* Check to see if the dbcr0 register is set up to debug. Use the - internal debug mode bit to do this. */ - lwz r12,THREAD_DBCR0(r10) - andis. r12,r12,DBCR0_IDM@h - ACCOUNT_CPU_USER_ENTRY(r2, r11, r12) - beq+ 3f - /* From user and task is ptraced - load up global dbcr0 */ - li r12,-1 /* clear all pending debug events */ - mtspr SPRN_DBSR,r12 - lis r11,global_dbcr0@ha - tophys(r11,r11) - addi r11,r11,global_dbcr0@l -#ifdef CONFIG_SMP - lwz r10, TASK_CPU(r2) - slwi r10, r10, 3 - add r11, r11, r10 -#endif - lwz r12,0(r11) - mtspr SPRN_DBCR0,r12 - lwz r12,4(r11) - addi r12,r12,-1 - stw r12,4(r11) - -3: - tovirt(r2, r2) /* set r2 to current */ - lis r11, transfer_to_syscall@h - ori r11, r11, transfer_to_syscall@l -#ifdef CONFIG_TRACE_IRQFLAGS - /* - * If MSR is changing we need to keep interrupts disabled at this point - * otherwise we might risk taking an interrupt before we tell lockdep - * they are enabled. - */ - lis r10, MSR_KERNEL@h - ori r10, r10, MSR_KERNEL@l - rlwimi r10, r9, 0, MSR_EE -#else - lis r10, (MSR_KERNEL | MSR_EE)@h - ori r10, r10, (MSR_KERNEL | MSR_EE)@l -#endif - mtspr SPRN_SRR1,r10 - mtspr SPRN_SRR0,r11 - rfi /* jump to handler, enable MMU */ -99: b ret_from_kernel_syscall + b transfer_to_syscall /* jump to handler */ .endm /* To handle the additional exception priority levels on 40x and Book-E @@ -406,6 +359,7 @@ ALT_FTR_SECTION_END_IFSET(CPU_FTR_EMB_HV) \ /* continue normal handling for a debug exception... */ \ 2: mfspr r4,SPRN_DBSR; \ + stw r4,_ESR(r11); /* DebugException takes DBSR in _ESR */\ addi r3,r1,STACK_FRAME_OVERHEAD; \ EXC_XFER_TEMPLATE(DebugException, 0x2008, (MSR_KERNEL & ~(MSR_ME|MSR_DE|MSR_CE)), debug_transfer_to_handler, ret_from_debug_exc) @@ -459,6 +413,7 @@ ALT_FTR_SECTION_END_IFSET(CPU_FTR_EMB_HV) \ /* continue normal handling for a critical exception... */ \ 2: mfspr r4,SPRN_DBSR; \ + stw r4,_ESR(r11); /* DebugException takes DBSR in _ESR */\ addi r3,r1,STACK_FRAME_OVERHEAD; \ EXC_XFER_TEMPLATE(DebugException, 0x2002, (MSR_KERNEL & ~(MSR_ME|MSR_DE|MSR_CE)), crit_transfer_to_handler, ret_from_crit_exc) @@ -476,9 +431,7 @@ ALT_FTR_SECTION_END_IFSET(CPU_FTR_EMB_HV) NORMAL_EXCEPTION_PROLOG(INST_STORAGE); \ mfspr r5,SPRN_ESR; /* Grab the ESR and save it */ \ stw r5,_ESR(r11); \ - mr r4,r12; /* Pass SRR0 as arg2 */ \ - stw r4, _DEAR(r11); \ - li r5,0; /* Pass zero as arg3 */ \ + stw r12, _DEAR(r11); /* Pass SRR0 as arg2 */ \ EXC_XFER_LITE(0x0400, handle_page_fault) #define ALIGNMENT_EXCEPTION \ diff --git a/arch/powerpc/kernel/head_fsl_booke.S b/arch/powerpc/kernel/head_fsl_booke.S index fdd4d274c2459dde9afc9824fdc0d9562d3299ab..3f4a40cccef5df87e6deeabc82dffa9d7a39201e 100644 --- a/arch/powerpc/kernel/head_fsl_booke.S +++ b/arch/powerpc/kernel/head_fsl_booke.S @@ -364,12 +364,12 @@ interrupt_base: /* Data Storage Interrupt */ START_EXCEPTION(DataStorage) NORMAL_EXCEPTION_PROLOG(DATA_STORAGE) - mfspr r5,SPRN_ESR /* Grab the ESR, save it, pass arg3 */ + mfspr r5,SPRN_ESR /* Grab the ESR, save it */ stw r5,_ESR(r11) - mfspr r4,SPRN_DEAR /* Grab the DEAR, save it, pass arg2 */ + mfspr r4,SPRN_DEAR /* Grab the DEAR, save it */ + stw r4, _DEAR(r11) andis. r10,r5,(ESR_ILK|ESR_DLK)@h bne 1f - stw r4, _DEAR(r11) EXC_XFER_LITE(0x0300, handle_page_fault) 1: addi r3,r1,STACK_FRAME_OVERHEAD diff --git a/arch/powerpc/kernel/idle_book3s.S b/arch/powerpc/kernel/idle_book3s.S index 22f249b6f58de0bd37187b66fd70aaa039698252..f9e6d83e67207d3638555b14555a055c420a4cd9 100644 --- a/arch/powerpc/kernel/idle_book3s.S +++ b/arch/powerpc/kernel/idle_book3s.S @@ -52,28 +52,32 @@ _GLOBAL(isa300_idle_stop_mayloss) std r1,PACAR1(r13) mflr r4 mfcr r5 - /* use stack red zone rather than a new frame for saving regs */ - std r2,-8*0(r1) - std r14,-8*1(r1) - std r15,-8*2(r1) - std r16,-8*3(r1) - std r17,-8*4(r1) - std r18,-8*5(r1) - std r19,-8*6(r1) - std r20,-8*7(r1) - std r21,-8*8(r1) - std r22,-8*9(r1) - std r23,-8*10(r1) - std r24,-8*11(r1) - std r25,-8*12(r1) - std r26,-8*13(r1) - std r27,-8*14(r1) - std r28,-8*15(r1) - std r29,-8*16(r1) - std r30,-8*17(r1) - std r31,-8*18(r1) - std r4,-8*19(r1) - std r5,-8*20(r1) + /* + * Use the stack red zone rather than a new frame for saving regs since + * in the case of no GPR loss the wakeup code branches directly back to + * the caller without deallocating the stack frame first. + */ + std r2,-8*1(r1) + std r14,-8*2(r1) + std r15,-8*3(r1) + std r16,-8*4(r1) + std r17,-8*5(r1) + std r18,-8*6(r1) + std r19,-8*7(r1) + std r20,-8*8(r1) + std r21,-8*9(r1) + std r22,-8*10(r1) + std r23,-8*11(r1) + std r24,-8*12(r1) + std r25,-8*13(r1) + std r26,-8*14(r1) + std r27,-8*15(r1) + std r28,-8*16(r1) + std r29,-8*17(r1) + std r30,-8*18(r1) + std r31,-8*19(r1) + std r4,-8*20(r1) + std r5,-8*21(r1) /* 168 bytes */ PPC_STOP b . /* catch bugs */ @@ -89,8 +93,8 @@ _GLOBAL(isa300_idle_stop_mayloss) */ _GLOBAL(idle_return_gpr_loss) ld r1,PACAR1(r13) - ld r4,-8*19(r1) - ld r5,-8*20(r1) + ld r4,-8*20(r1) + ld r5,-8*21(r1) mtlr r4 mtcr r5 /* @@ -98,25 +102,25 @@ _GLOBAL(idle_return_gpr_loss) * from PACATOC. This could be avoided for that less common case * if KVM saved its r2. */ - ld r2,-8*0(r1) - ld r14,-8*1(r1) - ld r15,-8*2(r1) - ld r16,-8*3(r1) - ld r17,-8*4(r1) - ld r18,-8*5(r1) - ld r19,-8*6(r1) - ld r20,-8*7(r1) - ld r21,-8*8(r1) - ld r22,-8*9(r1) - ld r23,-8*10(r1) - ld r24,-8*11(r1) - ld r25,-8*12(r1) - ld r26,-8*13(r1) - ld r27,-8*14(r1) - ld r28,-8*15(r1) - ld r29,-8*16(r1) - ld r30,-8*17(r1) - ld r31,-8*18(r1) + ld r2,-8*1(r1) + ld r14,-8*2(r1) + ld r15,-8*3(r1) + ld r16,-8*4(r1) + ld r17,-8*5(r1) + ld r18,-8*6(r1) + ld r19,-8*7(r1) + ld r20,-8*8(r1) + ld r21,-8*9(r1) + ld r22,-8*10(r1) + ld r23,-8*11(r1) + ld r24,-8*12(r1) + ld r25,-8*13(r1) + ld r26,-8*14(r1) + ld r27,-8*15(r1) + ld r28,-8*16(r1) + ld r29,-8*17(r1) + ld r30,-8*18(r1) + ld r31,-8*19(r1) blr /* @@ -154,28 +158,32 @@ _GLOBAL(isa206_idle_insn_mayloss) std r1,PACAR1(r13) mflr r4 mfcr r5 - /* use stack red zone rather than a new frame for saving regs */ - std r2,-8*0(r1) - std r14,-8*1(r1) - std r15,-8*2(r1) - std r16,-8*3(r1) - std r17,-8*4(r1) - std r18,-8*5(r1) - std r19,-8*6(r1) - std r20,-8*7(r1) - std r21,-8*8(r1) - std r22,-8*9(r1) - std r23,-8*10(r1) - std r24,-8*11(r1) - std r25,-8*12(r1) - std r26,-8*13(r1) - std r27,-8*14(r1) - std r28,-8*15(r1) - std r29,-8*16(r1) - std r30,-8*17(r1) - std r31,-8*18(r1) - std r4,-8*19(r1) - std r5,-8*20(r1) + /* + * Use the stack red zone rather than a new frame for saving regs since + * in the case of no GPR loss the wakeup code branches directly back to + * the caller without deallocating the stack frame first. + */ + std r2,-8*1(r1) + std r14,-8*2(r1) + std r15,-8*3(r1) + std r16,-8*4(r1) + std r17,-8*5(r1) + std r18,-8*6(r1) + std r19,-8*7(r1) + std r20,-8*8(r1) + std r21,-8*9(r1) + std r22,-8*10(r1) + std r23,-8*11(r1) + std r24,-8*12(r1) + std r25,-8*13(r1) + std r26,-8*14(r1) + std r27,-8*15(r1) + std r28,-8*16(r1) + std r29,-8*17(r1) + std r30,-8*18(r1) + std r31,-8*19(r1) + std r4,-8*20(r1) + std r5,-8*21(r1) cmpwi r3,PNV_THREAD_NAP bne 1f IDLE_STATE_ENTER_SEQ_NORET(PPC_NAP) diff --git a/arch/powerpc/kernel/syscall_64.c b/arch/powerpc/kernel/interrupt.c similarity index 69% rename from arch/powerpc/kernel/syscall_64.c rename to arch/powerpc/kernel/interrupt.c index 7c85ed04a1641c9f68a93ba3d34bdfca55dd0969..398cd86b6ada15514774666dbcfcebc7bd178a33 100644 --- a/arch/powerpc/kernel/syscall_64.c +++ b/arch/powerpc/kernel/interrupt.c @@ -1,10 +1,15 @@ // SPDX-License-Identifier: GPL-2.0-or-later +#include #include +#include + #include #include #include +#include #include +#include #include #include #include @@ -24,16 +29,21 @@ notrace long system_call_exception(long r3, long r4, long r5, { syscall_fn f; + regs->orig_gpr3 = r3; + if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG)) BUG_ON(irq_soft_mask_return() != IRQS_ALL_DISABLED); + CT_WARN_ON(ct_state() == CONTEXT_KERNEL); + user_exit_irqoff(); + trace_hardirqs_off(); /* finish reconciling */ - if (IS_ENABLED(CONFIG_PPC_BOOK3S)) + if (!IS_ENABLED(CONFIG_BOOKE) && !IS_ENABLED(CONFIG_40x)) BUG_ON(!(regs->msr & MSR_RI)); BUG_ON(!(regs->msr & MSR_PR)); BUG_ON(!FULL_REGS(regs)); - BUG_ON(regs->softe != IRQS_ENABLED); + BUG_ON(arch_irq_disabled_regs(regs)); #ifdef CONFIG_PPC_PKEY if (mmu_has_feature(MMU_FTR_PKEY)) { @@ -59,19 +69,15 @@ notrace long system_call_exception(long r3, long r4, long r5, isync(); } else #endif +#ifdef CONFIG_PPC64 kuap_check_amr(); +#endif - account_cpu_user_entry(); + booke_restore_dbcr0(); -#ifdef CONFIG_PPC_SPLPAR - if (IS_ENABLED(CONFIG_VIRT_CPU_ACCOUNTING_NATIVE) && - firmware_has_feature(FW_FEATURE_SPLPAR)) { - struct lppaca *lp = local_paca->lppaca_ptr; + account_cpu_user_entry(); - if (unlikely(local_paca->dtl_ridx != be64_to_cpu(lp->dtl_idx))) - accumulate_stolen_time(); - } -#endif + account_stolen_time(); /* * This is not required for the syscall exit path, but makes the @@ -79,12 +85,12 @@ notrace long system_call_exception(long r3, long r4, long r5, * frame, or if the unwinder was taught the first stack frame always * returns to user with IRQS_ENABLED, this store could be avoided! */ - regs->softe = IRQS_ENABLED; + irq_soft_mask_regs_set_state(regs, IRQS_ENABLED); local_irq_enable(); if (unlikely(current_thread_info()->flags & _TIF_SYSCALL_DOTRACE)) { - if (unlikely(regs->trap == 0x7ff0)) { + if (unlikely(trap_is_unsupported_scv(regs))) { /* Unsupported scv vector */ _exception(SIGILL, regs, ILL_ILLOPC, regs->nip); return regs->gpr[3]; @@ -107,7 +113,7 @@ notrace long system_call_exception(long r3, long r4, long r5, r8 = regs->gpr[8]; } else if (unlikely(r0 >= NR_syscalls)) { - if (unlikely(regs->trap == 0x7ff0)) { + if (unlikely(trap_is_unsupported_scv(regs))) { /* Unsupported scv vector */ _exception(SIGILL, regs, ILL_ILLOPC, regs->nip); return regs->gpr[3]; @@ -118,7 +124,7 @@ notrace long system_call_exception(long r3, long r4, long r5, /* May be faster to do array_index_nospec? */ barrier_nospec(); - if (unlikely(is_32bit_task())) { + if (unlikely(is_compat_task())) { f = (void *)compat_sys_call_table[r0]; r3 &= 0x00000000ffffffffULL; @@ -138,8 +144,12 @@ notrace long system_call_exception(long r3, long r4, long r5, /* * local irqs must be disabled. Returns false if the caller must re-enable * them, check for new work, and try again. + * + * This should be called with local irqs disabled, but if they were previously + * enabled when the interrupt handler returns (indicating a process-context / + * synchronous interrupt) then irqs_enabled should be true. */ -static notrace inline bool prep_irq_for_enabled_exit(bool clear_ri) +static notrace inline bool __prep_irq_for_enabled_exit(bool clear_ri) { /* This must be done with RI=1 because tracing may touch vmaps */ trace_hardirqs_on(); @@ -149,6 +159,7 @@ static notrace inline bool prep_irq_for_enabled_exit(bool clear_ri) __hard_EE_RI_disable(); else __hard_irq_disable(); +#ifdef CONFIG_PPC64 if (unlikely(lazy_irq_pending_nocheck())) { /* Took an interrupt, may have more exit work to do. */ if (clear_ri) @@ -160,10 +171,63 @@ static notrace inline bool prep_irq_for_enabled_exit(bool clear_ri) } local_paca->irq_happened = 0; irq_soft_mask_set(IRQS_ENABLED); - +#endif return true; } +static notrace inline bool prep_irq_for_enabled_exit(bool clear_ri, bool irqs_enabled) +{ + if (__prep_irq_for_enabled_exit(clear_ri)) + return true; + + /* + * Must replay pending soft-masked interrupts now. Don't just + * local_irq_enabe(); local_irq_disable(); because if we are + * returning from an asynchronous interrupt here, another one + * might hit after irqs are enabled, and it would exit via this + * same path allowing another to fire, and so on unbounded. + * + * If interrupts were enabled when this interrupt exited, + * indicating a process context (synchronous) interrupt, + * local_irq_enable/disable can be used, which will enable + * interrupts rather than keeping them masked (unclear how + * much benefit this is over just replaying for all cases, + * because we immediately disable again, so all we're really + * doing is allowing hard interrupts to execute directly for + * a very small time, rather than being masked and replayed). + */ + if (irqs_enabled) { + local_irq_enable(); + local_irq_disable(); + } else { + replay_soft_interrupts(); + } + + return false; +} + +static notrace void booke_load_dbcr0(void) +{ +#ifdef CONFIG_PPC_ADV_DEBUG_REGS + unsigned long dbcr0 = current->thread.debug.dbcr0; + + if (likely(!(dbcr0 & DBCR0_IDM))) + return; + + /* + * Check to see if the dbcr0 register is set up to debug. + * Use the internal debug mode bit to do this. + */ + mtmsr(mfmsr() & ~MSR_DE); + if (IS_ENABLED(CONFIG_PPC32)) { + isync(); + global_dbcr0[smp_processor_id()] = mfspr(SPRN_DBCR0); + } + mtspr(SPRN_DBCR0, dbcr0); + mtspr(SPRN_DBSR, -1); +#endif +} + /* * This should be called after a syscall returns, with r3 the return value * from the syscall. If this function returns non-zero, the system call @@ -177,20 +241,24 @@ notrace unsigned long syscall_exit_prepare(unsigned long r3, struct pt_regs *regs, long scv) { - unsigned long *ti_flagsp = ¤t_thread_info()->flags; unsigned long ti_flags; unsigned long ret = 0; + bool is_not_scv = !IS_ENABLED(CONFIG_PPC_BOOK3S_64) || !scv; + CT_WARN_ON(ct_state() == CONTEXT_USER); + +#ifdef CONFIG_PPC64 kuap_check_amr(); +#endif regs->result = r3; /* Check whether the syscall is issued inside a restartable sequence */ rseq_syscall(regs); - ti_flags = *ti_flagsp; + ti_flags = current_thread_info()->flags; - if (unlikely(r3 >= (unsigned long)-MAX_ERRNO) && !scv) { + if (unlikely(r3 >= (unsigned long)-MAX_ERRNO) && is_not_scv) { if (likely(!(ti_flags & (_TIF_NOERROR | _TIF_RESTOREALL)))) { r3 = -r3; regs->ccr |= 0x10000000; /* Set SO bit in CR */ @@ -202,7 +270,7 @@ notrace unsigned long syscall_exit_prepare(unsigned long r3, ret = _TIF_RESTOREALL; else regs->gpr[3] = r3; - clear_bits(_TIF_PERSYSCALL_MASK, ti_flagsp); + clear_bits(_TIF_PERSYSCALL_MASK, ¤t_thread_info()->flags); } else { regs->gpr[3] = r3; } @@ -212,9 +280,10 @@ notrace unsigned long syscall_exit_prepare(unsigned long r3, ret |= _TIF_RESTOREALL; } -again: local_irq_disable(); - ti_flags = READ_ONCE(*ti_flagsp); + +again: + ti_flags = READ_ONCE(current_thread_info()->flags); while (unlikely(ti_flags & (_TIF_USER_WORK_MASK & ~_TIF_RESTORE_TM))) { local_irq_enable(); if (ti_flags & _TIF_NEED_RESCHED) { @@ -230,7 +299,7 @@ notrace unsigned long syscall_exit_prepare(unsigned long r3, do_notify_resume(regs, ti_flags); } local_irq_disable(); - ti_flags = READ_ONCE(*ti_flagsp); + ti_flags = READ_ONCE(current_thread_info()->flags); } if (IS_ENABLED(CONFIG_PPC_BOOK3S) && IS_ENABLED(CONFIG_PPC_FPU)) { @@ -257,9 +326,13 @@ notrace unsigned long syscall_exit_prepare(unsigned long r3, } } + user_enter_irqoff(); + /* scv need not set RI=0 because SRRs are not used */ - if (unlikely(!prep_irq_for_enabled_exit(!scv))) { + if (unlikely(!__prep_irq_for_enabled_exit(is_not_scv))) { + user_exit_irqoff(); local_irq_enable(); + local_irq_disable(); goto again; } @@ -267,9 +340,11 @@ notrace unsigned long syscall_exit_prepare(unsigned long r3, local_paca->tm_scratch = regs->msr; #endif + booke_load_dbcr0(); + account_cpu_user_exit(); -#ifdef CONFIG_PPC_BOOK3S /* BOOK3E not yet using this */ +#ifdef CONFIG_PPC_BOOK3S_64 /* BOOK3E and ppc32 not using this */ /* * We do this at the end so that we do context switch with KERNEL AMR */ @@ -278,33 +353,32 @@ notrace unsigned long syscall_exit_prepare(unsigned long r3, return ret; } -#ifdef CONFIG_PPC_BOOK3S /* BOOK3E not yet using this */ +#ifndef CONFIG_PPC_BOOK3E_64 /* BOOK3E not yet using this */ notrace unsigned long interrupt_exit_user_prepare(struct pt_regs *regs, unsigned long msr) { -#ifdef CONFIG_PPC_BOOK3E - struct thread_struct *ts = ¤t->thread; -#endif - unsigned long *ti_flagsp = ¤t_thread_info()->flags; unsigned long ti_flags; unsigned long flags; unsigned long ret = 0; - if (IS_ENABLED(CONFIG_PPC_BOOK3S)) + if (!IS_ENABLED(CONFIG_BOOKE) && !IS_ENABLED(CONFIG_40x)) BUG_ON(!(regs->msr & MSR_RI)); BUG_ON(!(regs->msr & MSR_PR)); BUG_ON(!FULL_REGS(regs)); - BUG_ON(regs->softe != IRQS_ENABLED); + BUG_ON(arch_irq_disabled_regs(regs)); + CT_WARN_ON(ct_state() == CONTEXT_USER); /* * We don't need to restore AMR on the way back to userspace for KUAP. * AMR can only have been unlocked if we interrupted the kernel. */ +#ifdef CONFIG_PPC64 kuap_check_amr(); +#endif local_irq_save(flags); again: - ti_flags = READ_ONCE(*ti_flagsp); + ti_flags = READ_ONCE(current_thread_info()->flags); while (unlikely(ti_flags & (_TIF_USER_WORK_MASK & ~_TIF_RESTORE_TM))) { local_irq_enable(); /* returning to user: may enable */ if (ti_flags & _TIF_NEED_RESCHED) { @@ -315,7 +389,7 @@ notrace unsigned long interrupt_exit_user_prepare(struct pt_regs *regs, unsigned do_notify_resume(regs, ti_flags); } local_irq_disable(); - ti_flags = READ_ONCE(*ti_flagsp); + ti_flags = READ_ONCE(current_thread_info()->flags); } if (IS_ENABLED(CONFIG_PPC_BOOK3S) && IS_ENABLED(CONFIG_PPC_FPU)) { @@ -336,23 +410,16 @@ notrace unsigned long interrupt_exit_user_prepare(struct pt_regs *regs, unsigned } } - if (unlikely(!prep_irq_for_enabled_exit(true))) { + user_enter_irqoff(); + + if (unlikely(!__prep_irq_for_enabled_exit(true))) { + user_exit_irqoff(); local_irq_enable(); local_irq_disable(); goto again; } -#ifdef CONFIG_PPC_BOOK3E - if (unlikely(ts->debug.dbcr0 & DBCR0_IDM)) { - /* - * Check to see if the dbcr0 register is set up to debug. - * Use the internal debug mode bit to do this. - */ - mtmsr(mfmsr() & ~MSR_DE); - mtspr(SPRN_DBCR0, ts->debug.dbcr0); - mtspr(SPRN_DBSR, -1); - } -#endif + booke_load_dbcr0(); #ifdef CONFIG_PPC_TRANSACTIONAL_MEM local_paca->tm_scratch = regs->msr; @@ -363,7 +430,9 @@ notrace unsigned long interrupt_exit_user_prepare(struct pt_regs *regs, unsigned /* * We do this at the end so that we do context switch with KERNEL AMR */ +#ifdef CONFIG_PPC64 kuap_user_restore(regs); +#endif return ret; } @@ -372,56 +441,56 @@ void preempt_schedule_irq(void); notrace unsigned long interrupt_exit_kernel_prepare(struct pt_regs *regs, unsigned long msr) { - unsigned long *ti_flagsp = ¤t_thread_info()->flags; unsigned long flags; unsigned long ret = 0; +#ifdef CONFIG_PPC64 unsigned long amr; +#endif - if (IS_ENABLED(CONFIG_PPC_BOOK3S) && unlikely(!(regs->msr & MSR_RI))) + if (!IS_ENABLED(CONFIG_BOOKE) && !IS_ENABLED(CONFIG_40x) && + unlikely(!(regs->msr & MSR_RI))) unrecoverable_exception(regs); BUG_ON(regs->msr & MSR_PR); BUG_ON(!FULL_REGS(regs)); + /* + * CT_WARN_ON comes here via program_check_exception, + * so avoid recursion. + */ + if (TRAP(regs) != 0x700) + CT_WARN_ON(ct_state() == CONTEXT_USER); +#ifdef CONFIG_PPC64 amr = kuap_get_and_check_amr(); +#endif - if (unlikely(*ti_flagsp & _TIF_EMULATE_STACK_STORE)) { - clear_bits(_TIF_EMULATE_STACK_STORE, ti_flagsp); + if (unlikely(current_thread_info()->flags & _TIF_EMULATE_STACK_STORE)) { + clear_bits(_TIF_EMULATE_STACK_STORE, ¤t_thread_info()->flags); ret = 1; } local_irq_save(flags); - if (regs->softe == IRQS_ENABLED) { + if (!arch_irq_disabled_regs(regs)) { /* Returning to a kernel context with local irqs enabled. */ WARN_ON_ONCE(!(regs->msr & MSR_EE)); again: if (IS_ENABLED(CONFIG_PREEMPT)) { /* Return to preemptible kernel context */ - if (unlikely(*ti_flagsp & _TIF_NEED_RESCHED)) { + if (unlikely(current_thread_info()->flags & _TIF_NEED_RESCHED)) { if (preempt_count() == 0) preempt_schedule_irq(); } } - if (unlikely(!prep_irq_for_enabled_exit(true))) { - /* - * Can't local_irq_restore to replay if we were in - * interrupt context. Must replay directly. - */ - if (irqs_disabled_flags(flags)) { - replay_soft_interrupts(); - } else { - local_irq_restore(flags); - local_irq_save(flags); - } - /* Took an interrupt, may have more exit work to do. */ + if (unlikely(!prep_irq_for_enabled_exit(true, !irqs_disabled_flags(flags)))) goto again; - } } else { /* Returning to a kernel context with local irqs disabled. */ __hard_EE_RI_disable(); +#ifdef CONFIG_PPC64 if (regs->msr & MSR_EE) local_paca->irq_happened &= ~PACA_IRQ_HARD_DIS; +#endif } @@ -434,7 +503,9 @@ notrace unsigned long interrupt_exit_kernel_prepare(struct pt_regs *regs, unsign * which would cause Read-After-Write stalls. Hence, we take the AMR * value from the check above. */ +#ifdef CONFIG_PPC64 kuap_kernel_restore(regs, amr); +#endif return ret; } diff --git a/arch/powerpc/kernel/iommu.c b/arch/powerpc/kernel/iommu.c index 5b69a6a72a0e21d64854ff25699e9ff53a3859bc..c00214a4355c80490bf4347f0bd2f7b2da082f0f 100644 --- a/arch/powerpc/kernel/iommu.c +++ b/arch/powerpc/kernel/iommu.c @@ -25,6 +25,7 @@ #include #include #include +#include #include #include #include @@ -38,6 +39,47 @@ #define DBG(...) +#ifdef CONFIG_IOMMU_DEBUGFS +static int iommu_debugfs_weight_get(void *data, u64 *val) +{ + struct iommu_table *tbl = data; + *val = bitmap_weight(tbl->it_map, tbl->it_size); + return 0; +} +DEFINE_DEBUGFS_ATTRIBUTE(iommu_debugfs_fops_weight, iommu_debugfs_weight_get, NULL, "%llu\n"); + +static void iommu_debugfs_add(struct iommu_table *tbl) +{ + char name[10]; + struct dentry *liobn_entry; + + sprintf(name, "%08lx", tbl->it_index); + liobn_entry = debugfs_create_dir(name, iommu_debugfs_dir); + + debugfs_create_file_unsafe("weight", 0400, liobn_entry, tbl, &iommu_debugfs_fops_weight); + debugfs_create_ulong("it_size", 0400, liobn_entry, &tbl->it_size); + debugfs_create_ulong("it_page_shift", 0400, liobn_entry, &tbl->it_page_shift); + debugfs_create_ulong("it_reserved_start", 0400, liobn_entry, &tbl->it_reserved_start); + debugfs_create_ulong("it_reserved_end", 0400, liobn_entry, &tbl->it_reserved_end); + debugfs_create_ulong("it_indirect_levels", 0400, liobn_entry, &tbl->it_indirect_levels); + debugfs_create_ulong("it_level_size", 0400, liobn_entry, &tbl->it_level_size); +} + +static void iommu_debugfs_del(struct iommu_table *tbl) +{ + char name[10]; + struct dentry *liobn_entry; + + sprintf(name, "%08lx", tbl->it_index); + liobn_entry = debugfs_lookup(name, iommu_debugfs_dir); + if (liobn_entry) + debugfs_remove(liobn_entry); +} +#else +static void iommu_debugfs_add(struct iommu_table *tbl){} +static void iommu_debugfs_del(struct iommu_table *tbl){} +#endif + static int novmerge; static void __iommu_free(struct iommu_table *, dma_addr_t, unsigned int); @@ -725,6 +767,8 @@ struct iommu_table *iommu_init_table(struct iommu_table *tbl, int nid, welcomed = 1; } + iommu_debugfs_add(tbl); + return tbl; } @@ -744,6 +788,8 @@ static void iommu_table_free(struct kref *kref) return; } + iommu_debugfs_del(tbl); + iommu_table_release_pages(tbl); /* verify that table contains no entries */ diff --git a/arch/powerpc/kernel/irq.c b/arch/powerpc/kernel/irq.c index cc7a6271b6b4ec5408bfd2ed07a41432f0cc5a37..d71fd10a1dd46bb9daa025b10d9e51b738ae6f8e 100644 --- a/arch/powerpc/kernel/irq.c +++ b/arch/powerpc/kernel/irq.c @@ -54,6 +54,7 @@ #include #include +#include #include #include #include @@ -65,6 +66,7 @@ #include #include #include +#include #ifdef CONFIG_PPC64 #include @@ -269,6 +271,31 @@ void replay_soft_interrupts(void) } } +#if defined(CONFIG_PPC_BOOK3S_64) && defined(CONFIG_PPC_KUAP) +static inline void replay_soft_interrupts_irqrestore(void) +{ + unsigned long kuap_state = get_kuap(); + + /* + * Check if anything calls local_irq_enable/restore() when KUAP is + * disabled (user access enabled). We handle that case here by saving + * and re-locking AMR but we shouldn't get here in the first place, + * hence the warning. + */ + kuap_check_amr(); + + if (kuap_state != AMR_KUAP_BLOCKED) + set_kuap(AMR_KUAP_BLOCKED); + + replay_soft_interrupts(); + + if (kuap_state != AMR_KUAP_BLOCKED) + set_kuap(kuap_state); +} +#else +#define replay_soft_interrupts_irqrestore() replay_soft_interrupts() +#endif + notrace void arch_local_irq_restore(unsigned long mask) { unsigned char irq_happened; @@ -332,7 +359,7 @@ notrace void arch_local_irq_restore(unsigned long mask) irq_soft_mask_set(IRQS_ALL_DISABLED); trace_hardirqs_off(); - replay_soft_interrupts(); + replay_soft_interrupts_irqrestore(); local_paca->irq_happened = 0; trace_hardirqs_on(); @@ -644,8 +671,6 @@ void __do_irq(struct pt_regs *regs) { unsigned int irq; - irq_enter(); - trace_irq_entry(regs); /* @@ -665,11 +690,9 @@ void __do_irq(struct pt_regs *regs) generic_handle_irq(irq); trace_irq_exit(regs); - - irq_exit(); } -void do_IRQ(struct pt_regs *regs) +DEFINE_INTERRUPT_HANDLER_ASYNC(do_IRQ) { struct pt_regs *old_regs = set_irq_regs(regs); void *cursp, *irqsp, *sirqsp; diff --git a/arch/powerpc/kernel/mce.c b/arch/powerpc/kernel/mce.c index 9f3e133b57b78c0421a338adbf12937de5386db4..11f0cae086edba8b30c02f6ae9c027d57e22aa36 100644 --- a/arch/powerpc/kernel/mce.c +++ b/arch/powerpc/kernel/mce.c @@ -17,22 +17,15 @@ #include #include #include +#include +#include #include #include #include +#include -static DEFINE_PER_CPU(int, mce_nest_count); -static DEFINE_PER_CPU(struct machine_check_event[MAX_MC_EVT], mce_event); - -/* Queue for delayed MCE events. */ -static DEFINE_PER_CPU(int, mce_queue_count); -static DEFINE_PER_CPU(struct machine_check_event[MAX_MC_EVT], mce_event_queue); - -/* Queue for delayed MCE UE events. */ -static DEFINE_PER_CPU(int, mce_ue_count); -static DEFINE_PER_CPU(struct machine_check_event[MAX_MC_EVT], - mce_ue_event_queue); +#include "setup.h" static void machine_check_process_queued_event(struct irq_work *work); static void machine_check_ue_irq_work(struct irq_work *work); @@ -103,9 +96,10 @@ void save_mce_event(struct pt_regs *regs, long handled, struct mce_error_info *mce_err, uint64_t nip, uint64_t addr, uint64_t phys_addr) { - int index = __this_cpu_inc_return(mce_nest_count) - 1; - struct machine_check_event *mce = this_cpu_ptr(&mce_event[index]); + int index = local_paca->mce_info->mce_nest_count++; + struct machine_check_event *mce; + mce = &local_paca->mce_info->mce_event[index]; /* * Return if we don't have enough space to log mce event. * mce_nest_count may go beyond MAX_MC_EVT but that's ok, @@ -191,7 +185,7 @@ void save_mce_event(struct pt_regs *regs, long handled, */ int get_mce_event(struct machine_check_event *mce, bool release) { - int index = __this_cpu_read(mce_nest_count) - 1; + int index = local_paca->mce_info->mce_nest_count - 1; struct machine_check_event *mc_evt; int ret = 0; @@ -201,7 +195,7 @@ int get_mce_event(struct machine_check_event *mce, bool release) /* Check if we have MCE info to process. */ if (index < MAX_MC_EVT) { - mc_evt = this_cpu_ptr(&mce_event[index]); + mc_evt = &local_paca->mce_info->mce_event[index]; /* Copy the event structure and release the original */ if (mce) *mce = *mc_evt; @@ -211,7 +205,7 @@ int get_mce_event(struct machine_check_event *mce, bool release) } /* Decrement the count to free the slot. */ if (release) - __this_cpu_dec(mce_nest_count); + local_paca->mce_info->mce_nest_count--; return ret; } @@ -233,13 +227,14 @@ static void machine_check_ue_event(struct machine_check_event *evt) { int index; - index = __this_cpu_inc_return(mce_ue_count) - 1; + index = local_paca->mce_info->mce_ue_count++; /* If queue is full, just return for now. */ if (index >= MAX_MC_EVT) { - __this_cpu_dec(mce_ue_count); + local_paca->mce_info->mce_ue_count--; return; } - memcpy(this_cpu_ptr(&mce_ue_event_queue[index]), evt, sizeof(*evt)); + memcpy(&local_paca->mce_info->mce_ue_event_queue[index], + evt, sizeof(*evt)); /* Queue work to process this event later. */ irq_work_queue(&mce_ue_event_irq_work); @@ -256,13 +251,14 @@ void machine_check_queue_event(void) if (!get_mce_event(&evt, MCE_EVENT_RELEASE)) return; - index = __this_cpu_inc_return(mce_queue_count) - 1; + index = local_paca->mce_info->mce_queue_count++; /* If queue is full, just return for now. */ if (index >= MAX_MC_EVT) { - __this_cpu_dec(mce_queue_count); + local_paca->mce_info->mce_queue_count--; return; } - memcpy(this_cpu_ptr(&mce_event_queue[index]), &evt, sizeof(evt)); + memcpy(&local_paca->mce_info->mce_event_queue[index], + &evt, sizeof(evt)); /* Queue irq work to process this event later. */ irq_work_queue(&mce_event_process_work); @@ -289,9 +285,9 @@ static void machine_process_ue_event(struct work_struct *work) int index; struct machine_check_event *evt; - while (__this_cpu_read(mce_ue_count) > 0) { - index = __this_cpu_read(mce_ue_count) - 1; - evt = this_cpu_ptr(&mce_ue_event_queue[index]); + while (local_paca->mce_info->mce_ue_count > 0) { + index = local_paca->mce_info->mce_ue_count - 1; + evt = &local_paca->mce_info->mce_ue_event_queue[index]; blocking_notifier_call_chain(&mce_notifier_list, 0, evt); #ifdef CONFIG_MEMORY_FAILURE /* @@ -304,7 +300,7 @@ static void machine_process_ue_event(struct work_struct *work) */ if (evt->error_type == MCE_ERROR_TYPE_UE) { if (evt->u.ue_error.ignore_event) { - __this_cpu_dec(mce_ue_count); + local_paca->mce_info->mce_ue_count--; continue; } @@ -320,7 +316,7 @@ static void machine_process_ue_event(struct work_struct *work) "was generated\n"); } #endif - __this_cpu_dec(mce_ue_count); + local_paca->mce_info->mce_ue_count--; } } /* @@ -338,17 +334,17 @@ static void machine_check_process_queued_event(struct irq_work *work) * For now just print it to console. * TODO: log this error event to FSP or nvram. */ - while (__this_cpu_read(mce_queue_count) > 0) { - index = __this_cpu_read(mce_queue_count) - 1; - evt = this_cpu_ptr(&mce_event_queue[index]); + while (local_paca->mce_info->mce_queue_count > 0) { + index = local_paca->mce_info->mce_queue_count - 1; + evt = &local_paca->mce_info->mce_event_queue[index]; if (evt->error_type == MCE_ERROR_TYPE_UE && evt->u.ue_error.ignore_event) { - __this_cpu_dec(mce_queue_count); + local_paca->mce_info->mce_queue_count--; continue; } machine_check_print_event_info(evt, false, false); - __this_cpu_dec(mce_queue_count); + local_paca->mce_info->mce_queue_count--; } } @@ -588,15 +584,9 @@ EXPORT_SYMBOL_GPL(machine_check_print_event_info); * * regs->nip and regs->msr contains srr0 and ssr1. */ -long notrace machine_check_early(struct pt_regs *regs) +DEFINE_INTERRUPT_HANDLER_NMI(machine_check_early) { long handled = 0; - u8 ftrace_enabled = this_cpu_get_ftrace_enabled(); - - this_cpu_set_ftrace_enabled(0); - /* Do not use nmi_enter/exit for pseries hpte guest */ - if (radix_enabled() || !firmware_has_feature(FW_FEATURE_LPAR)) - nmi_enter(); hv_nmi_check_nonrecoverable(regs); @@ -606,11 +596,6 @@ long notrace machine_check_early(struct pt_regs *regs) if (ppc_md.machine_check_early) handled = ppc_md.machine_check_early(regs); - if (radix_enabled() || !firmware_has_feature(FW_FEATURE_LPAR)) - nmi_exit(); - - this_cpu_set_ftrace_enabled(ftrace_enabled); - return handled; } @@ -722,7 +707,7 @@ long hmi_handle_debugtrig(struct pt_regs *regs) /* * Return values: */ -long hmi_exception_realmode(struct pt_regs *regs) +DEFINE_INTERRUPT_HANDLER_NMI(hmi_exception_realmode) { int ret; @@ -741,3 +726,24 @@ long hmi_exception_realmode(struct pt_regs *regs) return 1; } + +void __init mce_init(void) +{ + struct mce_info *mce_info; + u64 limit; + int i; + + limit = min(ppc64_bolted_size(), ppc64_rma_size); + for_each_possible_cpu(i) { + mce_info = memblock_alloc_try_nid(sizeof(*mce_info), + __alignof__(*mce_info), + MEMBLOCK_LOW_LIMIT, + limit, cpu_to_node(i)); + if (!mce_info) + goto err; + paca_ptrs[i]->mce_info = mce_info; + } + return; +err: + panic("Failed to allocate memory for MCE event data\n"); +} diff --git a/arch/powerpc/kernel/optprobes.c b/arch/powerpc/kernel/optprobes.c index 69bfe96884e2f49b3fd0f713c526f3278841540f..7f7cdbeacd1ac8a1ec17d4f0c5f1194729128fc2 100644 --- a/arch/powerpc/kernel/optprobes.c +++ b/arch/powerpc/kernel/optprobes.c @@ -141,30 +141,11 @@ void arch_remove_optimized_kprobe(struct optimized_kprobe *op) } } -/* - * emulate_step() requires insn to be emulated as - * second parameter. Load register 'r4' with the - * instruction. - */ -void patch_imm32_load_insns(unsigned int val, kprobe_opcode_t *addr) -{ - /* addis r4,0,(insn)@h */ - patch_instruction((struct ppc_inst *)addr, - ppc_inst(PPC_INST_ADDIS | ___PPC_RT(4) | - ((val >> 16) & 0xffff))); - addr++; - - /* ori r4,r4,(insn)@l */ - patch_instruction((struct ppc_inst *)addr, - ppc_inst(PPC_INST_ORI | ___PPC_RA(4) | - ___PPC_RS(4) | (val & 0xffff))); -} - /* * Generate instructions to load provided immediate 64-bit value * to register 'reg' and patch these instructions at 'addr'. */ -void patch_imm64_load_insns(unsigned long val, int reg, kprobe_opcode_t *addr) +static void patch_imm64_load_insns(unsigned long val, int reg, kprobe_opcode_t *addr) { /* lis reg,(op)@highest */ patch_instruction((struct ppc_inst *)addr, diff --git a/arch/powerpc/kernel/pci-common.c b/arch/powerpc/kernel/pci-common.c index 2b555997b295064b255328c24d10c270a87130a2..001e90cd8948bff42891c454777ee7a477e41484 100644 --- a/arch/powerpc/kernel/pci-common.c +++ b/arch/powerpc/kernel/pci-common.c @@ -1699,3 +1699,13 @@ static void fixup_hide_host_resource_fsl(struct pci_dev *dev) } DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_MOTOROLA, PCI_ANY_ID, fixup_hide_host_resource_fsl); DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_FREESCALE, PCI_ANY_ID, fixup_hide_host_resource_fsl); + + +static int __init discover_phbs(void) +{ + if (ppc_md.discover_phbs) + ppc_md.discover_phbs(); + + return 0; +} +core_initcall(discover_phbs); diff --git a/arch/powerpc/kernel/pci_dn.c b/arch/powerpc/kernel/pci_dn.c index e99b7c547d7e973adf840e7595cda8a4dc972e18..61571ae23953063ee5ae4e80589922b8102a8804 100644 --- a/arch/powerpc/kernel/pci_dn.c +++ b/arch/powerpc/kernel/pci_dn.c @@ -443,46 +443,6 @@ void *pci_traverse_device_nodes(struct device_node *start, } EXPORT_SYMBOL_GPL(pci_traverse_device_nodes); -static struct pci_dn *pci_dn_next_one(struct pci_dn *root, - struct pci_dn *pdn) -{ - struct list_head *next = pdn->child_list.next; - - if (next != &pdn->child_list) - return list_entry(next, struct pci_dn, list); - - while (1) { - if (pdn == root) - return NULL; - - next = pdn->list.next; - if (next != &pdn->parent->child_list) - break; - - pdn = pdn->parent; - } - - return list_entry(next, struct pci_dn, list); -} - -void *traverse_pci_dn(struct pci_dn *root, - void *(*fn)(struct pci_dn *, void *), - void *data) -{ - struct pci_dn *pdn = root; - void *ret; - - /* Only scan the child nodes */ - for (pdn = pci_dn_next_one(root, pdn); pdn; - pdn = pci_dn_next_one(root, pdn)) { - ret = fn(pdn, data); - if (ret) - return ret; - } - - return NULL; -} - static void *add_pdn(struct device_node *dn, void *data) { struct pci_controller *hose = data; @@ -521,28 +481,6 @@ void pci_devs_phb_init_dynamic(struct pci_controller *phb) pci_traverse_device_nodes(dn, add_pdn, phb); } -/** - * pci_devs_phb_init - Initialize phbs and pci devs under them. - * - * This routine walks over all phb's (pci-host bridges) on the - * system, and sets up assorted pci-related structures - * (including pci info in the device node structs) for each - * pci device found underneath. This routine runs once, - * early in the boot sequence. - */ -static int __init pci_devs_phb_init(void) -{ - struct pci_controller *phb, *tmp; - - /* This must be done first so the device nodes have valid pci info! */ - list_for_each_entry_safe(phb, tmp, &hose_list, list_node) - pci_devs_phb_init_dynamic(phb); - - return 0; -} - -core_initcall(pci_devs_phb_init); - static void pci_dev_pdn_setup(struct pci_dev *pdev) { struct pci_dn *pdn; diff --git a/arch/powerpc/kernel/process.c b/arch/powerpc/kernel/process.c index a66f435dabbfe93f58bfa5d3eabee06ff04bec06..3231c2df9e261fba57c74792c177256e0dafd678 100644 --- a/arch/powerpc/kernel/process.c +++ b/arch/powerpc/kernel/process.c @@ -41,6 +41,7 @@ #include #include +#include #include #include #include @@ -659,11 +660,10 @@ static void do_break_handler(struct pt_regs *regs) } } -void do_break (struct pt_regs *regs, unsigned long address, - unsigned long error_code) +DEFINE_INTERRUPT_HANDLER(do_break) { current->thread.trap_nr = TRAP_HWBKPT; - if (notify_die(DIE_DABR_MATCH, "dabr_match", regs, error_code, + if (notify_die(DIE_DABR_MATCH, "dabr_match", regs, regs->dsisr, 11, SIGSEGV) == NOTIFY_STOP) return; @@ -681,7 +681,7 @@ void do_break (struct pt_regs *regs, unsigned long address, do_break_handler(regs); /* Deliver the signal to userspace */ - force_sig_fault(SIGTRAP, TRAP_HWBKPT, (void __user *)address); + force_sig_fault(SIGTRAP, TRAP_HWBKPT, (void __user *)regs->dar); } #endif /* CONFIG_PPC_ADV_DEBUG_REGS */ @@ -1670,7 +1670,7 @@ int copy_thread(unsigned long clone_flags, unsigned long usp, /* Copy registers */ sp -= sizeof(struct pt_regs); childregs = (struct pt_regs *) sp; - if (unlikely(p->flags & PF_KTHREAD)) { + if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { /* kernel thread */ memset(childregs, 0, sizeof(struct pt_regs)); childregs->gpr[1] = sp + sizeof(struct pt_regs); @@ -2047,6 +2047,9 @@ static inline int valid_emergency_stack(unsigned long sp, struct task_struct *p, unsigned long stack_page; unsigned long cpu = task_cpu(p); + if (!paca_ptrs) + return 0; + stack_page = (unsigned long)paca_ptrs[cpu]->emergency_sp - THREAD_SIZE; if (sp >= stack_page && sp <= stack_page + THREAD_SIZE - nbytes) return 1; @@ -2176,7 +2179,7 @@ void show_stack(struct task_struct *tsk, unsigned long *stack, * See if this is an exception frame. * We look for the "regshere" marker in the current frame. */ - if (validate_sp(sp, tsk, STACK_INT_FRAME_SIZE) + if (validate_sp(sp, tsk, STACK_FRAME_WITH_PT_REGS) && stack[STACK_FRAME_MARKER] == STACK_FRAME_REGS_MARKER) { struct pt_regs *regs = (struct pt_regs *) (sp + STACK_FRAME_OVERHEAD); diff --git a/arch/powerpc/kernel/prom.c b/arch/powerpc/kernel/prom.c index ae3c41730367913c36df137a6feb130e8a103c8d..9a4797d1d40d5401d80521297f2b0e07edc9fc23 100644 --- a/arch/powerpc/kernel/prom.c +++ b/arch/powerpc/kernel/prom.c @@ -707,7 +707,7 @@ static void __init save_fscr_to_task(void) init_task.thread.fscr = mfspr(SPRN_FSCR); } #else -static inline void save_fscr_to_task(void) {}; +static inline void save_fscr_to_task(void) {} #endif diff --git a/arch/powerpc/kernel/prom_init.c b/arch/powerpc/kernel/prom_init.c index e9d4eb6144e1f4ce9f536a5f319591a23269c7fd..ccf77b985c8f6b3b81a7c171c4c68be3d035c474 100644 --- a/arch/powerpc/kernel/prom_init.c +++ b/arch/powerpc/kernel/prom_init.c @@ -1331,14 +1331,10 @@ static void __init prom_check_platform_support(void) if (prop_len > sizeof(vec)) prom_printf("WARNING: ibm,arch-vec-5-platform-support longer than expected (len: %d)\n", prop_len); - prom_getprop(prom.chosen, "ibm,arch-vec-5-platform-support", - &vec, sizeof(vec)); - for (i = 0; i < sizeof(vec); i += 2) { - prom_debug("%d: index = 0x%x val = 0x%x\n", i / 2 - , vec[i] - , vec[i + 1]); - prom_parse_platform_support(vec[i], vec[i + 1], - &supported); + prom_getprop(prom.chosen, "ibm,arch-vec-5-platform-support", &vec, sizeof(vec)); + for (i = 0; i < prop_len; i += 2) { + prom_debug("%d: index = 0x%x val = 0x%x\n", i / 2, vec[i], vec[i + 1]); + prom_parse_platform_support(vec[i], vec[i + 1], &supported); } } diff --git a/arch/powerpc/kernel/ptrace/ptrace.c b/arch/powerpc/kernel/ptrace/ptrace.c index 3d44b73adb8333bc0828f827efaf6b59d331a7c3..4f3d4ff3728cb05a7d3e713b916fdb2737da02a6 100644 --- a/arch/powerpc/kernel/ptrace/ptrace.c +++ b/arch/powerpc/kernel/ptrace/ptrace.c @@ -262,8 +262,6 @@ long do_syscall_trace_enter(struct pt_regs *regs) { u32 flags; - user_exit(); - flags = READ_ONCE(current_thread_info()->flags) & (_TIF_SYSCALL_EMU | _TIF_SYSCALL_TRACE); @@ -340,8 +338,6 @@ void do_syscall_trace_leave(struct pt_regs *regs) step = test_thread_flag(TIF_SINGLESTEP); if (step || test_thread_flag(TIF_SYSCALL_TRACE)) tracehook_report_syscall_exit(regs, step); - - user_enter(); } void __init pt_regs_check(void); diff --git a/arch/powerpc/kernel/setup-common.c b/arch/powerpc/kernel/setup-common.c index 71f38e9248be15d431b3ef2cf0afa32bdc917bb7..bee984b1887b6c5c837b6f351e5df2827bbc295f 100644 --- a/arch/powerpc/kernel/setup-common.c +++ b/arch/powerpc/kernel/setup-common.c @@ -64,6 +64,7 @@ #include #include #include +#include #include "setup.h" @@ -237,18 +238,17 @@ static int show_cpuinfo(struct seq_file *m, void *v) maj = (pvr >> 8) & 0xFF; min = pvr & 0xFF; - seq_printf(m, "processor\t: %lu\n", cpu_id); - seq_printf(m, "cpu\t\t: "); + seq_printf(m, "processor\t: %lu\ncpu\t\t: ", cpu_id); if (cur_cpu_spec->pvr_mask && cur_cpu_spec->cpu_name) - seq_printf(m, "%s", cur_cpu_spec->cpu_name); + seq_puts(m, cur_cpu_spec->cpu_name); else seq_printf(m, "unknown (%08x)", pvr); if (cpu_has_feature(CPU_FTR_ALTIVEC)) - seq_printf(m, ", altivec supported"); + seq_puts(m, ", altivec supported"); - seq_printf(m, "\n"); + seq_putc(m, '\n'); #ifdef CONFIG_TAU if (cpu_has_feature(CPU_FTR_TAU)) { @@ -327,7 +327,7 @@ static int show_cpuinfo(struct seq_file *m, void *v) seq_printf(m, "bogomips\t: %lu.%02lu\n", loops_per_jiffy / (500000 / HZ), (loops_per_jiffy / (5000 / HZ)) % 100); - seq_printf(m, "\n"); + seq_putc(m, '\n'); /* If this is the last cpu, print the summary */ if (cpumask_next(cpu_id, cpu_online_mask) >= nr_cpu_ids) @@ -938,6 +938,7 @@ void __init setup_arch(char **cmdline_p) exc_lvl_early_init(); emergency_stack_init(); + mce_init(); smp_release_cpus(); initmem_init(); diff --git a/arch/powerpc/kernel/setup.h b/arch/powerpc/kernel/setup.h index 2dd0d9cb5a208601c443004df41d255fe27beb79..84058bbc8fe98b0066921ca27781391690129ec2 100644 --- a/arch/powerpc/kernel/setup.h +++ b/arch/powerpc/kernel/setup.h @@ -14,31 +14,31 @@ void irqstack_early_init(void); #ifdef CONFIG_PPC32 void setup_power_save(void); #else -static inline void setup_power_save(void) { }; +static inline void setup_power_save(void) { } #endif #if defined(CONFIG_PPC64) && defined(CONFIG_SMP) void check_smt_enabled(void); #else -static inline void check_smt_enabled(void) { }; +static inline void check_smt_enabled(void) { } #endif #if defined(CONFIG_PPC_BOOK3E) && defined(CONFIG_SMP) void setup_tlb_core_data(void); #else -static inline void setup_tlb_core_data(void) { }; +static inline void setup_tlb_core_data(void) { } #endif #if defined(CONFIG_PPC_BOOK3E) || defined(CONFIG_BOOKE) || defined(CONFIG_40x) void exc_lvl_early_init(void); #else -static inline void exc_lvl_early_init(void) { }; +static inline void exc_lvl_early_init(void) { } #endif #if defined(CONFIG_PPC64) || defined(CONFIG_VMAP_STACK) void emergency_stack_init(void); #else -static inline void emergency_stack_init(void) { }; +static inline void emergency_stack_init(void) { } #endif #ifdef CONFIG_PPC64 @@ -55,7 +55,7 @@ extern unsigned long spr_default_dscr; #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE void kvm_cma_reserve(void); #else -static inline void kvm_cma_reserve(void) { }; +static inline void kvm_cma_reserve(void) { } #endif #ifdef CONFIG_TAU diff --git a/arch/powerpc/kernel/setup_64.c b/arch/powerpc/kernel/setup_64.c index c28e949cc22294b9cafe016ec3f96c6b295e00d7..560ed8b975e77feee406edacc56e8aafa091dc36 100644 --- a/arch/powerpc/kernel/setup_64.c +++ b/arch/powerpc/kernel/setup_64.c @@ -67,6 +67,7 @@ #include #include #include +#include #include "setup.h" @@ -258,7 +259,7 @@ static void cpu_ready_for_interrupts(void) unsigned long spr_default_dscr = 0; -void __init record_spr_defaults(void) +static void __init record_spr_defaults(void) { if (early_cpu_has_feature(CPU_FTR_DSCR)) spr_default_dscr = mfspr(SPRN_DSCR); @@ -1008,7 +1009,7 @@ void rfi_flush_enable(bool enable) rfi_flush = enable; } -void entry_flush_enable(bool enable) +static void entry_flush_enable(bool enable) { if (enable) { do_entry_flush_fixups(enabled_flush_types); @@ -1020,7 +1021,7 @@ void entry_flush_enable(bool enable) entry_flush = enable; } -void uaccess_flush_enable(bool enable) +static void uaccess_flush_enable(bool enable) { if (enable) { do_uaccess_flush_fixups(enabled_flush_types); diff --git a/arch/powerpc/kernel/signal.c b/arch/powerpc/kernel/signal.c index 53782aa60ade678543231c3483e122a65103472f..9ded046edb0e16aa850c423a084465b8abfa42bb 100644 --- a/arch/powerpc/kernel/signal.c +++ b/arch/powerpc/kernel/signal.c @@ -282,8 +282,6 @@ static void do_signal(struct task_struct *tsk) void do_notify_resume(struct pt_regs *regs, unsigned long thread_info_flags) { - user_exit(); - if (thread_info_flags & _TIF_UPROBE) uprobe_notify_resume(regs); @@ -299,8 +297,6 @@ void do_notify_resume(struct pt_regs *regs, unsigned long thread_info_flags) tracehook_notify_resume(regs); rseq_handle_notify_resume(NULL, regs); } - - user_enter(); } static unsigned long get_tm_stackpointer(struct task_struct *tsk) diff --git a/arch/powerpc/kernel/signal_32.c b/arch/powerpc/kernel/signal_32.c index 934cbdf6dd10e4f966e7c690375ac4ece1cf2a24..75ee918a120a5e67f499e53e0c1f7df5aabeba13 100644 --- a/arch/powerpc/kernel/signal_32.c +++ b/arch/powerpc/kernel/signal_32.c @@ -929,8 +929,9 @@ int handle_signal32(struct ksignal *ksig, sigset_t *oldset, regs->gpr[3] = ksig->sig; regs->gpr[4] = (unsigned long) sc; regs->nip = (unsigned long)ksig->ka.sa.sa_handler; - /* enter the signal handler in big-endian mode */ + /* enter the signal handler in native-endian mode */ regs->msr &= ~MSR_LE; + regs->msr |= (MSR_KERNEL & MSR_LE); return 0; failed: diff --git a/arch/powerpc/kernel/smp.c b/arch/powerpc/kernel/smp.c index 9e2246e80efd6c63b780df63956cdd895b14ca79..5a4d59a1070d5a3b32f65f7ea9de4c436c08527a 100644 --- a/arch/powerpc/kernel/smp.c +++ b/arch/powerpc/kernel/smp.c @@ -34,6 +34,7 @@ #include #include #include +#include #include #include @@ -576,7 +577,7 @@ void tick_broadcast(const struct cpumask *mask) #endif #ifdef CONFIG_DEBUGGER -void debugger_ipi_callback(struct pt_regs *regs) +static void debugger_ipi_callback(struct pt_regs *regs) { debugger_ipi(regs); } diff --git a/arch/powerpc/kernel/sys_ppc32.c b/arch/powerpc/kernel/sys_ppc32.c index d36c6391eaf58844b815a75115bf7fc546c32d8c..16ff0399a257499fee59294c534fbfa97032172d 100644 --- a/arch/powerpc/kernel/sys_ppc32.c +++ b/arch/powerpc/kernel/sys_ppc32.c @@ -59,57 +59,64 @@ unsigned long compat_sys_mmap2(unsigned long addr, size_t len, /* * long long munging: * The 32 bit ABI passes long longs in an odd even register pair. + * High and low parts are swapped depending on endian mode, + * so define a macro (similar to mips linux32) to handle that. */ +#ifdef __LITTLE_ENDIAN__ +#define merge_64(low, high) ((u64)high << 32) | low +#else +#define merge_64(high, low) ((u64)high << 32) | low +#endif compat_ssize_t compat_sys_pread64(unsigned int fd, char __user *ubuf, compat_size_t count, - u32 reg6, u32 poshi, u32 poslo) + u32 reg6, u32 pos1, u32 pos2) { - return ksys_pread64(fd, ubuf, count, ((loff_t)poshi << 32) | poslo); + return ksys_pread64(fd, ubuf, count, merge_64(pos1, pos2)); } compat_ssize_t compat_sys_pwrite64(unsigned int fd, const char __user *ubuf, compat_size_t count, - u32 reg6, u32 poshi, u32 poslo) + u32 reg6, u32 pos1, u32 pos2) { - return ksys_pwrite64(fd, ubuf, count, ((loff_t)poshi << 32) | poslo); + return ksys_pwrite64(fd, ubuf, count, merge_64(pos1, pos2)); } -compat_ssize_t compat_sys_readahead(int fd, u32 r4, u32 offhi, u32 offlo, u32 count) +compat_ssize_t compat_sys_readahead(int fd, u32 r4, u32 offset1, u32 offset2, u32 count) { - return ksys_readahead(fd, ((loff_t)offhi << 32) | offlo, count); + return ksys_readahead(fd, merge_64(offset1, offset2), count); } asmlinkage int compat_sys_truncate64(const char __user * path, u32 reg4, - unsigned long high, unsigned long low) + unsigned long len1, unsigned long len2) { - return ksys_truncate(path, (high << 32) | low); + return ksys_truncate(path, merge_64(len1, len2)); } -asmlinkage long compat_sys_fallocate(int fd, int mode, u32 offhi, u32 offlo, - u32 lenhi, u32 lenlo) +asmlinkage long compat_sys_fallocate(int fd, int mode, u32 offset1, u32 offset2, + u32 len1, u32 len2) { - return ksys_fallocate(fd, mode, ((loff_t)offhi << 32) | offlo, - ((loff_t)lenhi << 32) | lenlo); + return ksys_fallocate(fd, mode, ((loff_t)offset1 << 32) | offset2, + merge_64(len1, len2)); } -asmlinkage int compat_sys_ftruncate64(unsigned int fd, u32 reg4, unsigned long high, - unsigned long low) +asmlinkage int compat_sys_ftruncate64(unsigned int fd, u32 reg4, unsigned long len1, + unsigned long len2) { - return ksys_ftruncate(fd, (high << 32) | low); + return ksys_ftruncate(fd, merge_64(len1, len2)); } -long ppc32_fadvise64(int fd, u32 unused, u32 offset_high, u32 offset_low, +long ppc32_fadvise64(int fd, u32 unused, u32 offset1, u32 offset2, size_t len, int advice) { - return ksys_fadvise64_64(fd, (u64)offset_high << 32 | offset_low, len, + return ksys_fadvise64_64(fd, merge_64(offset1, offset2), len, advice); } asmlinkage long compat_sys_sync_file_range2(int fd, unsigned int flags, - unsigned offset_hi, unsigned offset_lo, - unsigned nbytes_hi, unsigned nbytes_lo) + unsigned offset1, unsigned offset2, + unsigned nbytes1, unsigned nbytes2) { - loff_t offset = ((loff_t)offset_hi << 32) | offset_lo; - loff_t nbytes = ((loff_t)nbytes_hi << 32) | nbytes_lo; + loff_t offset = merge_64(offset1, offset2); + loff_t nbytes = merge_64(nbytes1, nbytes2); return ksys_sync_file_range(fd, offset, nbytes, flags); } diff --git a/arch/powerpc/kernel/syscalls/Makefile b/arch/powerpc/kernel/syscalls/Makefile index 27b48954808dfc583c7af88184b99f04fd7659cf..9e3be295dbba26d2506603f36ff6c54769ce8973 100644 --- a/arch/powerpc/kernel/syscalls/Makefile +++ b/arch/powerpc/kernel/syscalls/Makefile @@ -5,7 +5,7 @@ uapi := arch/$(SRCARCH)/include/generated/uapi/asm _dummy := $(shell [ -d '$(uapi)' ] || mkdir -p '$(uapi)') \ $(shell [ -d '$(kapi)' ] || mkdir -p '$(kapi)') -syscall := $(srctree)/$(src)/syscall.tbl +syscall := $(src)/syscall.tbl syshdr := $(srctree)/$(src)/syscallhdr.sh systbl := $(srctree)/$(src)/syscalltbl.sh @@ -22,31 +22,31 @@ quiet_cmd_systbl = SYSTBL $@ '$(systbl_offset_$(basetarget))' syshdr_abis_unistd_32 := common,nospu,32 -$(uapi)/unistd_32.h: $(syscall) $(syshdr) +$(uapi)/unistd_32.h: $(syscall) $(syshdr) FORCE $(call if_changed,syshdr) syshdr_abis_unistd_64 := common,nospu,64 -$(uapi)/unistd_64.h: $(syscall) $(syshdr) +$(uapi)/unistd_64.h: $(syscall) $(syshdr) FORCE $(call if_changed,syshdr) systbl_abis_syscall_table_32 := common,nospu,32 systbl_abi_syscall_table_32 := 32 -$(kapi)/syscall_table_32.h: $(syscall) $(systbl) +$(kapi)/syscall_table_32.h: $(syscall) $(systbl) FORCE $(call if_changed,systbl) systbl_abis_syscall_table_64 := common,nospu,64 systbl_abi_syscall_table_64 := 64 -$(kapi)/syscall_table_64.h: $(syscall) $(systbl) +$(kapi)/syscall_table_64.h: $(syscall) $(systbl) FORCE $(call if_changed,systbl) systbl_abis_syscall_table_c32 := common,nospu,32 systbl_abi_syscall_table_c32 := c32 -$(kapi)/syscall_table_c32.h: $(syscall) $(systbl) +$(kapi)/syscall_table_c32.h: $(syscall) $(systbl) FORCE $(call if_changed,systbl) systbl_abis_syscall_table_spu := common,spu systbl_abi_syscall_table_spu := spu -$(kapi)/syscall_table_spu.h: $(syscall) $(systbl) +$(kapi)/syscall_table_spu.h: $(syscall) $(systbl) FORCE $(call if_changed,systbl) uapisyshdr-y += unistd_32.h unistd_64.h @@ -55,9 +55,10 @@ kapisyshdr-y += syscall_table_32.h \ syscall_table_c32.h \ syscall_table_spu.h -targets += $(uapisyshdr-y) $(kapisyshdr-y) +uapisyshdr-y := $(addprefix $(uapi)/, $(uapisyshdr-y)) +kapisyshdr-y := $(addprefix $(kapi)/, $(kapisyshdr-y)) +targets += $(addprefix ../../../../, $(uapisyshdr-y) $(kapisyshdr-y)) PHONY += all -all: $(addprefix $(uapi)/,$(uapisyshdr-y)) -all: $(addprefix $(kapi)/,$(kapisyshdr-y)) +all: $(uapisyshdr-y) $(kapisyshdr-y) @: diff --git a/arch/powerpc/kernel/syscalls/syscall.tbl b/arch/powerpc/kernel/syscalls/syscall.tbl index f744eb5cba887d398e150e9ef9fbb1293d5fe08f..0b2480cf3e4793bdedf18942696d8b2f118aa564 100644 --- a/arch/powerpc/kernel/syscalls/syscall.tbl +++ b/arch/powerpc/kernel/syscalls/syscall.tbl @@ -9,9 +9,7 @@ # 0 nospu restart_syscall sys_restart_syscall 1 nospu exit sys_exit -2 32 fork ppc_fork sys_fork -2 64 fork sys_fork -2 spu fork sys_ni_syscall +2 nospu fork sys_fork 3 common read sys_read 4 common write sys_write 5 common open sys_open compat_sys_open @@ -160,9 +158,7 @@ 119 32 sigreturn sys_sigreturn compat_sys_sigreturn 119 64 sigreturn sys_ni_syscall 119 spu sigreturn sys_ni_syscall -120 32 clone ppc_clone sys_clone -120 64 clone sys_clone -120 spu clone sys_ni_syscall +120 nospu clone sys_clone 121 common setdomainname sys_setdomainname 122 common uname sys_newuname 123 common modify_ldt sys_ni_syscall @@ -244,9 +240,7 @@ 186 spu sendfile sys_sendfile64 187 common getpmsg sys_ni_syscall 188 common putpmsg sys_ni_syscall -189 32 vfork ppc_vfork sys_vfork -189 64 vfork sys_vfork -189 spu vfork sys_ni_syscall +189 nospu vfork sys_vfork 190 common ugetrlimit sys_getrlimit compat_sys_getrlimit 191 common readahead sys_readahead compat_sys_readahead 192 32 mmap2 sys_mmap2 compat_sys_mmap2 @@ -322,9 +316,7 @@ 248 32 clock_nanosleep sys_clock_nanosleep_time32 248 64 clock_nanosleep sys_clock_nanosleep 248 spu clock_nanosleep sys_clock_nanosleep -249 32 swapcontext ppc_swapcontext compat_sys_swapcontext -249 64 swapcontext sys_swapcontext -249 spu swapcontext sys_ni_syscall +249 nospu swapcontext sys_swapcontext compat_sys_swapcontext 250 common tgkill sys_tgkill 251 32 utimes sys_utimes_time32 251 64 utimes sys_utimes @@ -522,12 +514,11 @@ 432 common fsmount sys_fsmount 433 common fspick sys_fspick 434 common pidfd_open sys_pidfd_open -435 32 clone3 ppc_clone3 sys_clone3 -435 64 clone3 sys_clone3 -435 spu clone3 sys_ni_syscall +435 nospu clone3 sys_clone3 436 common close_range sys_close_range 437 common openat2 sys_openat2 438 common pidfd_getfd sys_pidfd_getfd 439 common faccessat2 sys_faccessat2 440 common process_madvise sys_process_madvise 441 common epoll_pwait2 sys_epoll_pwait2 compat_sys_epoll_pwait2 +442 common mount_setattr sys_mount_setattr diff --git a/arch/powerpc/kernel/tau_6xx.c b/arch/powerpc/kernel/tau_6xx.c index 0b4694b8d2482567942ca792991dc652c3a53759..6c31af7f4fa82a77fdf1cf87c9e8b3a5a77ea93e 100644 --- a/arch/powerpc/kernel/tau_6xx.c +++ b/arch/powerpc/kernel/tau_6xx.c @@ -22,6 +22,7 @@ #include #include +#include #include #include #include @@ -100,16 +101,13 @@ static void TAUupdate(int cpu) * with interrupts disabled */ -void TAUException(struct pt_regs * regs) +DEFINE_INTERRUPT_HANDLER_ASYNC(TAUException) { int cpu = smp_processor_id(); - irq_enter(); tau[cpu].interrupts++; TAUupdate(cpu); - - irq_exit(); } #endif /* CONFIG_TAU_INT */ diff --git a/arch/powerpc/kernel/time.c b/arch/powerpc/kernel/time.c index 67feb35244606bdfd5df9f189c9fe9739e815f31..b67d93a609a2d6e331b4a06f696e6ee80d5796de 100644 --- a/arch/powerpc/kernel/time.c +++ b/arch/powerpc/kernel/time.c @@ -53,9 +53,11 @@ #include #include #include +#include #include #include +#include #include #include #include @@ -570,7 +572,7 @@ void arch_irq_work_raise(void) * timer_interrupt - gets called when the decrementer overflows, * with interrupts disabled. */ -void timer_interrupt(struct pt_regs *regs) +DEFINE_INTERRUPT_HANDLER_ASYNC(timer_interrupt) { struct clock_event_device *evt = this_cpu_ptr(&decrementers); u64 *next_tb = this_cpu_ptr(&decrementers_next_tb); @@ -609,7 +611,7 @@ void timer_interrupt(struct pt_regs *regs) #endif old_regs = set_irq_regs(regs); - irq_enter(); + trace_timer_interrupt_entry(regs); if (test_irq_work_pending()) { @@ -634,7 +636,7 @@ void timer_interrupt(struct pt_regs *regs) } trace_timer_interrupt_exit(regs); - irq_exit(); + set_irq_regs(old_regs); } EXPORT_SYMBOL(timer_interrupt); @@ -1030,6 +1032,7 @@ void __init time_init(void) tick_setup_hrtimer_broadcast(); of_clk_init(NULL); + enable_sched_clock_irqtime(); } /* diff --git a/arch/powerpc/kernel/traps.c b/arch/powerpc/kernel/traps.c index 3ec7b443fe6bbabdc511ac8c8773dbb40774600b..1583fd1c60103c2d2a230fff481b8d81fa147260 100644 --- a/arch/powerpc/kernel/traps.c +++ b/arch/powerpc/kernel/traps.c @@ -41,6 +41,7 @@ #include #include #include +#include #include #include #include @@ -342,8 +343,8 @@ static bool exception_common(int signr, struct pt_regs *regs, int code, show_signal_msg(signr, regs, code, addr); - if (arch_irqs_disabled() && !arch_irq_disabled_regs(regs)) - local_irq_enable(); + if (arch_irqs_disabled()) + interrupt_cond_local_irq_enable(regs); current->thread.trap_nr = code; @@ -430,16 +431,10 @@ void hv_nmi_check_nonrecoverable(struct pt_regs *regs) regs->msr &= ~MSR_RI; #endif } - -void system_reset_exception(struct pt_regs *regs) +DEFINE_INTERRUPT_HANDLER_NMI(system_reset_exception) { unsigned long hsrr0, hsrr1; bool saved_hsrrs = false; - u8 ftrace_enabled = this_cpu_get_ftrace_enabled(); - - this_cpu_set_ftrace_enabled(0); - - nmi_enter(); /* * System reset can interrupt code where HSRRs are live and MSR[RI]=1. @@ -503,19 +498,20 @@ void system_reset_exception(struct pt_regs *regs) die("Unrecoverable nested System Reset", regs, SIGABRT); #endif /* Must die if the interrupt is not recoverable */ - if (!(regs->msr & MSR_RI)) + if (!(regs->msr & MSR_RI)) { + /* For the reason explained in die_mce, nmi_exit before die */ + nmi_exit(); die("Unrecoverable System Reset", regs, SIGABRT); + } if (saved_hsrrs) { mtspr(SPRN_HSRR0, hsrr0); mtspr(SPRN_HSRR1, hsrr1); } - nmi_exit(); - - this_cpu_set_ftrace_enabled(ftrace_enabled); - /* What should we do here? We could issue a shutdown or hard reset. */ + + return 0; } /* @@ -788,23 +784,33 @@ int machine_check_generic(struct pt_regs *regs) } #endif /* everything else */ -void machine_check_exception(struct pt_regs *regs) +void die_mce(const char *str, struct pt_regs *regs, long err) { - int recover = 0; - /* - * BOOK3S_64 does not call this handler as a non-maskable interrupt - * (it uses its own early real-mode handler to handle the MCE proper - * and then raises irq_work to call this handler when interrupts are - * enabled). - * - * This is silly. The BOOK3S_64 should just call a different function - * rather than expecting semantics to magically change. Something - * like 'non_nmi_machine_check_exception()', perhaps? + * The machine check wants to kill the interrupted context, but + * do_exit() checks for in_interrupt() and panics in that case, so + * exit the irq/nmi before calling die. */ - const bool nmi = !IS_ENABLED(CONFIG_PPC_BOOK3S_64); + if (IS_ENABLED(CONFIG_PPC_BOOK3S_64)) + irq_exit(); + else + nmi_exit(); + die(str, regs, err); +} - if (nmi) nmi_enter(); +/* + * BOOK3S_64 does not call this handler as a non-maskable interrupt + * (it uses its own early real-mode handler to handle the MCE proper + * and then raises irq_work to call this handler when interrupts are + * enabled). + */ +#ifdef CONFIG_PPC_BOOK3S_64 +DEFINE_INTERRUPT_HANDLER_ASYNC(machine_check_exception) +#else +DEFINE_INTERRUPT_HANDLER_NMI(machine_check_exception) +#endif +{ + int recover = 0; __this_cpu_inc(irq_stat.mce_exceptions); @@ -830,21 +836,21 @@ void machine_check_exception(struct pt_regs *regs) if (check_io_access(regs)) goto bail; - if (nmi) nmi_exit(); - - die("Machine check", regs, SIGBUS); + die_mce("Machine check", regs, SIGBUS); +bail: /* Must die if the interrupt is not recoverable */ if (!(regs->msr & MSR_RI)) - die("Unrecoverable Machine check", regs, SIGBUS); + die_mce("Unrecoverable Machine check", regs, SIGBUS); +#ifdef CONFIG_PPC_BOOK3S_64 return; - -bail: - if (nmi) nmi_exit(); +#else + return 0; +#endif } -void SMIException(struct pt_regs *regs) +DEFINE_INTERRUPT_HANDLER(SMIException) /* async? */ { die("System Management Interrupt", regs, SIGABRT); } @@ -1030,12 +1036,11 @@ static void p9_hmi_special_emu(struct pt_regs *regs) } #endif /* CONFIG_VSX */ -void handle_hmi_exception(struct pt_regs *regs) +DEFINE_INTERRUPT_HANDLER_ASYNC(handle_hmi_exception) { struct pt_regs *old_regs; old_regs = set_irq_regs(regs); - irq_enter(); #ifdef CONFIG_VSX /* Real mode flagged P9 special emu is needed */ @@ -1055,46 +1060,42 @@ void handle_hmi_exception(struct pt_regs *regs) if (ppc_md.handle_hmi_exception) ppc_md.handle_hmi_exception(regs); - irq_exit(); set_irq_regs(old_regs); } -void unknown_exception(struct pt_regs *regs) +DEFINE_INTERRUPT_HANDLER(unknown_exception) { - enum ctx_state prev_state = exception_enter(); - printk("Bad trap at PC: %lx, SR: %lx, vector=%lx\n", regs->nip, regs->msr, regs->trap); _exception(SIGTRAP, regs, TRAP_UNK, 0); - - exception_exit(prev_state); } -void instruction_breakpoint_exception(struct pt_regs *regs) +DEFINE_INTERRUPT_HANDLER_ASYNC(unknown_async_exception) { - enum ctx_state prev_state = exception_enter(); + printk("Bad trap at PC: %lx, SR: %lx, vector=%lx\n", + regs->nip, regs->msr, regs->trap); + _exception(SIGTRAP, regs, TRAP_UNK, 0); +} + +DEFINE_INTERRUPT_HANDLER(instruction_breakpoint_exception) +{ if (notify_die(DIE_IABR_MATCH, "iabr_match", regs, 5, 5, SIGTRAP) == NOTIFY_STOP) - goto bail; + return; if (debugger_iabr_match(regs)) - goto bail; + return; _exception(SIGTRAP, regs, TRAP_BRKPT, regs->nip); - -bail: - exception_exit(prev_state); } -void RunModeException(struct pt_regs *regs) +DEFINE_INTERRUPT_HANDLER(RunModeException) { _exception(SIGTRAP, regs, TRAP_UNK, 0); } -void single_step_exception(struct pt_regs *regs) +DEFINE_INTERRUPT_HANDLER(single_step_exception) { - enum ctx_state prev_state = exception_enter(); - clear_single_step(regs); clear_br_trace(regs); @@ -1103,16 +1104,12 @@ void single_step_exception(struct pt_regs *regs) if (notify_die(DIE_SSTEP, "single_step", regs, 5, 5, SIGTRAP) == NOTIFY_STOP) - goto bail; + return; if (debugger_sstep(regs)) - goto bail; + return; _exception(SIGTRAP, regs, TRAP_TRACE, regs->nip); - -bail: - exception_exit(prev_state); } -NOKPROBE_SYMBOL(single_step_exception); /* * After we have successfully emulated an instruction, we have to @@ -1436,9 +1433,8 @@ static int emulate_math(struct pt_regs *regs) static inline int emulate_math(struct pt_regs *regs) { return -1; } #endif -void program_check_exception(struct pt_regs *regs) +static void do_program_check(struct pt_regs *regs) { - enum ctx_state prev_state = exception_enter(); unsigned int reason = get_reason(regs); /* We can now get here via a FP Unavailable exception if the core @@ -1447,22 +1443,22 @@ void program_check_exception(struct pt_regs *regs) if (reason & REASON_FP) { /* IEEE FP exception */ parse_fpe(regs); - goto bail; + return; } if (reason & REASON_TRAP) { unsigned long bugaddr; /* Debugger is first in line to stop recursive faults in * rcu_lock, notify_die, or atomic_notifier_call_chain */ if (debugger_bpt(regs)) - goto bail; + return; if (kprobe_handler(regs)) - goto bail; + return; /* trap exception */ if (notify_die(DIE_BPT, "breakpoint", regs, 5, 5, SIGTRAP) == NOTIFY_STOP) - goto bail; + return; bugaddr = regs->nip; /* @@ -1474,10 +1470,10 @@ void program_check_exception(struct pt_regs *regs) if (!(regs->msr & MSR_PR) && /* not user-mode */ report_bug(bugaddr, regs) == BUG_TRAP_TYPE_WARN) { regs->nip += 4; - goto bail; + return; } _exception(SIGTRAP, regs, TRAP_BRKPT, regs->nip); - goto bail; + return; } #ifdef CONFIG_PPC_TRANSACTIONAL_MEM if (reason & REASON_TM) { @@ -1498,7 +1494,7 @@ void program_check_exception(struct pt_regs *regs) */ if (user_mode(regs)) { _exception(SIGILL, regs, ILL_ILLOPN, regs->nip); - goto bail; + return; } else { printk(KERN_EMERG "Unexpected TM Bad Thing exception " "at %lx (msr 0x%lx) tm_scratch=%llx\n", @@ -1518,9 +1514,7 @@ void program_check_exception(struct pt_regs *regs) if (!user_mode(regs)) goto sigill; - /* We restore the interrupt state now */ - if (!arch_irq_disabled_regs(regs)) - local_irq_enable(); + interrupt_cond_local_irq_enable(regs); /* (reason & REASON_ILLEGAL) would be the obvious thing here, * but there seems to be a hardware bug on the 405GP (RevD) @@ -1531,7 +1525,7 @@ void program_check_exception(struct pt_regs *regs) * pattern to occurrences etc. -dgibson 31/Mar/2003 */ if (!emulate_math(regs)) - goto bail; + return; /* Try to emulate it if we should. */ if (reason & (REASON_ILLEGAL | REASON_PRIVILEGED)) { @@ -1539,10 +1533,10 @@ void program_check_exception(struct pt_regs *regs) case 0: regs->nip += 4; emulate_single_step(regs); - goto bail; + return; case -EFAULT: _exception(SIGSEGV, regs, SEGV_MAPERR, regs->nip); - goto bail; + return; } } @@ -1552,34 +1546,31 @@ void program_check_exception(struct pt_regs *regs) else _exception(SIGILL, regs, ILL_ILLOPC, regs->nip); -bail: - exception_exit(prev_state); } -NOKPROBE_SYMBOL(program_check_exception); + +DEFINE_INTERRUPT_HANDLER(program_check_exception) +{ + do_program_check(regs); +} /* * This occurs when running in hypervisor mode on POWER6 or later * and an illegal instruction is encountered. */ -void emulation_assist_interrupt(struct pt_regs *regs) +DEFINE_INTERRUPT_HANDLER(emulation_assist_interrupt) { regs->msr |= REASON_ILLEGAL; - program_check_exception(regs); + do_program_check(regs); } -NOKPROBE_SYMBOL(emulation_assist_interrupt); -void alignment_exception(struct pt_regs *regs) +DEFINE_INTERRUPT_HANDLER(alignment_exception) { - enum ctx_state prev_state = exception_enter(); int sig, code, fixed = 0; unsigned long reason; - /* We restore the interrupt state now */ - if (!arch_irq_disabled_regs(regs)) - local_irq_enable(); + interrupt_cond_local_irq_enable(regs); reason = get_reason(regs); - if (reason & REASON_BOUNDARY) { sig = SIGBUS; code = BUS_ADRALN; @@ -1587,7 +1578,7 @@ void alignment_exception(struct pt_regs *regs) } if (tm_abort_check(regs, TM_CAUSE_ALIGNMENT | TM_CAUSE_PERSISTENT)) - goto bail; + return; /* we don't implement logging of alignment exceptions */ if (!(current->thread.align_ctl & PR_UNALIGN_SIGBUS)) @@ -1597,7 +1588,7 @@ void alignment_exception(struct pt_regs *regs) /* skip over emulated instruction */ regs->nip += inst_length(reason); emulate_single_step(regs); - goto bail; + return; } /* Operand address was bad */ @@ -1612,13 +1603,10 @@ void alignment_exception(struct pt_regs *regs) if (user_mode(regs)) _exception(sig, regs, code, regs->dar); else - bad_page_fault(regs, regs->dar, sig); - -bail: - exception_exit(prev_state); + bad_page_fault(regs, sig); } -void StackOverflow(struct pt_regs *regs) +DEFINE_INTERRUPT_HANDLER(StackOverflow) { pr_crit("Kernel stack overflow in process %s[%d], r1=%lx\n", current->comm, task_pid_nr(current), regs->gpr[1]); @@ -1627,46 +1615,33 @@ void StackOverflow(struct pt_regs *regs) panic("kernel stack overflow"); } -void stack_overflow_exception(struct pt_regs *regs) +DEFINE_INTERRUPT_HANDLER(stack_overflow_exception) { - enum ctx_state prev_state = exception_enter(); - die("Kernel stack overflow", regs, SIGSEGV); - - exception_exit(prev_state); } -void kernel_fp_unavailable_exception(struct pt_regs *regs) +DEFINE_INTERRUPT_HANDLER(kernel_fp_unavailable_exception) { - enum ctx_state prev_state = exception_enter(); - printk(KERN_EMERG "Unrecoverable FP Unavailable Exception " "%lx at %lx\n", regs->trap, regs->nip); die("Unrecoverable FP Unavailable Exception", regs, SIGABRT); - - exception_exit(prev_state); } -void altivec_unavailable_exception(struct pt_regs *regs) +DEFINE_INTERRUPT_HANDLER(altivec_unavailable_exception) { - enum ctx_state prev_state = exception_enter(); - if (user_mode(regs)) { /* A user program has executed an altivec instruction, but this kernel doesn't support altivec. */ _exception(SIGILL, regs, ILL_ILLOPC, regs->nip); - goto bail; + return; } printk(KERN_EMERG "Unrecoverable VMX/Altivec Unavailable Exception " "%lx at %lx\n", regs->trap, regs->nip); die("Unrecoverable VMX/Altivec Unavailable Exception", regs, SIGABRT); - -bail: - exception_exit(prev_state); } -void vsx_unavailable_exception(struct pt_regs *regs) +DEFINE_INTERRUPT_HANDLER(vsx_unavailable_exception) { if (user_mode(regs)) { /* A user program has executed an vsx instruction, @@ -1697,7 +1672,7 @@ static void tm_unavailable(struct pt_regs *regs) die("Unrecoverable TM Unavailable Exception", regs, SIGABRT); } -void facility_unavailable_exception(struct pt_regs *regs) +DEFINE_INTERRUPT_HANDLER(facility_unavailable_exception) { static char *facility_strings[] = { [FSCR_FP_LG] = "FPU", @@ -1737,9 +1712,7 @@ void facility_unavailable_exception(struct pt_regs *regs) die("Unexpected facility unavailable exception", regs, SIGABRT); } - /* We restore the interrupt state now */ - if (!arch_irq_disabled_regs(regs)) - local_irq_enable(); + interrupt_cond_local_irq_enable(regs); if (status == FSCR_DSCR_LG) { /* @@ -1817,7 +1790,7 @@ void facility_unavailable_exception(struct pt_regs *regs) #ifdef CONFIG_PPC_TRANSACTIONAL_MEM -void fp_unavailable_tm(struct pt_regs *regs) +DEFINE_INTERRUPT_HANDLER(fp_unavailable_tm) { /* Note: This does not handle any kind of FP laziness. */ @@ -1850,7 +1823,7 @@ void fp_unavailable_tm(struct pt_regs *regs) tm_recheckpoint(¤t->thread); } -void altivec_unavailable_tm(struct pt_regs *regs) +DEFINE_INTERRUPT_HANDLER(altivec_unavailable_tm) { /* See the comments in fp_unavailable_tm(). This function operates * the same way. @@ -1865,7 +1838,7 @@ void altivec_unavailable_tm(struct pt_regs *regs) current->thread.used_vr = 1; } -void vsx_unavailable_tm(struct pt_regs *regs) +DEFINE_INTERRUPT_HANDLER(vsx_unavailable_tm) { /* See the comments in fp_unavailable_tm(). This works similarly, * though we're loading both FP and VEC registers in here. @@ -1890,11 +1863,40 @@ void vsx_unavailable_tm(struct pt_regs *regs) } #endif /* CONFIG_PPC_TRANSACTIONAL_MEM */ -void performance_monitor_exception(struct pt_regs *regs) +#ifdef CONFIG_PPC64 +DECLARE_INTERRUPT_HANDLER_NMI(performance_monitor_exception_nmi); +DEFINE_INTERRUPT_HANDLER_NMI(performance_monitor_exception_nmi) { __this_cpu_inc(irq_stat.pmu_irqs); perf_irq(regs); + + return 0; +} +#endif + +DECLARE_INTERRUPT_HANDLER_ASYNC(performance_monitor_exception_async); +DEFINE_INTERRUPT_HANDLER_ASYNC(performance_monitor_exception_async) +{ + __this_cpu_inc(irq_stat.pmu_irqs); + + perf_irq(regs); +} + +DEFINE_INTERRUPT_HANDLER_RAW(performance_monitor_exception) +{ + /* + * On 64-bit, if perf interrupts hit in a local_irq_disable + * (soft-masked) region, we consider them as NMIs. This is required to + * prevent hash faults on user addresses when reading callchains (and + * looks better from an irq tracing perspective). + */ + if (IS_ENABLED(CONFIG_PPC64) && unlikely(arch_irq_disabled_regs(regs))) + performance_monitor_exception_nmi(regs); + else + performance_monitor_exception_async(regs); + + return 0; } #ifdef CONFIG_PPC_ADV_DEBUG_REGS @@ -1957,8 +1959,10 @@ static void handle_debug(struct pt_regs *regs, unsigned long debug_status) mtspr(SPRN_DBCR0, current->thread.debug.dbcr0); } -void DebugException(struct pt_regs *regs, unsigned long debug_status) +DEFINE_INTERRUPT_HANDLER(DebugException) { + unsigned long debug_status = regs->dsisr; + current->thread.debug.dbsr = debug_status; /* Hack alert: On BookE, Branch Taken stops on the branch itself, while @@ -2024,11 +2028,10 @@ void DebugException(struct pt_regs *regs, unsigned long debug_status) } else handle_debug(regs, debug_status); } -NOKPROBE_SYMBOL(DebugException); #endif /* CONFIG_PPC_ADV_DEBUG_REGS */ #ifdef CONFIG_ALTIVEC -void altivec_assist_exception(struct pt_regs *regs) +DEFINE_INTERRUPT_HANDLER(altivec_assist_exception) { int err; @@ -2062,9 +2065,10 @@ void altivec_assist_exception(struct pt_regs *regs) #endif /* CONFIG_ALTIVEC */ #ifdef CONFIG_FSL_BOOKE -void CacheLockingException(struct pt_regs *regs, unsigned long address, - unsigned long error_code) +DEFINE_INTERRUPT_HANDLER(CacheLockingException) { + unsigned long error_code = regs->dsisr; + /* We treat cache locking instructions from the user * as priv ops, in the future we could try to do * something smarter @@ -2076,7 +2080,7 @@ void CacheLockingException(struct pt_regs *regs, unsigned long address, #endif /* CONFIG_FSL_BOOKE */ #ifdef CONFIG_SPE -void SPEFloatingPointException(struct pt_regs *regs) +DEFINE_INTERRUPT_HANDLER(SPEFloatingPointException) { extern int do_spe_mathemu(struct pt_regs *regs); unsigned long spefscr; @@ -2084,9 +2088,7 @@ void SPEFloatingPointException(struct pt_regs *regs) int code = FPE_FLTUNK; int err; - /* We restore the interrupt state now */ - if (!arch_irq_disabled_regs(regs)) - local_irq_enable(); + interrupt_cond_local_irq_enable(regs); flush_spe_to_thread(current); @@ -2128,14 +2130,12 @@ void SPEFloatingPointException(struct pt_regs *regs) return; } -void SPEFloatingPointRoundException(struct pt_regs *regs) +DEFINE_INTERRUPT_HANDLER(SPEFloatingPointRoundException) { extern int speround_handler(struct pt_regs *regs); int err; - /* We restore the interrupt state now */ - if (!arch_irq_disabled_regs(regs)) - local_irq_enable(); + interrupt_cond_local_irq_enable(regs); preempt_disable(); if (regs->msr & MSR_SPE) @@ -2170,13 +2170,12 @@ void SPEFloatingPointRoundException(struct pt_regs *regs) * in the MSR is 0. This indicates that SRR0/1 are live, and that * we therefore lost state by taking this exception. */ -void unrecoverable_exception(struct pt_regs *regs) +DEFINE_INTERRUPT_HANDLER(unrecoverable_exception) { pr_emerg("Unrecoverable exception %lx at %lx (msr=%lx)\n", regs->trap, regs->nip, regs->msr); die("Unrecoverable exception", regs, SIGABRT); } -NOKPROBE_SYMBOL(unrecoverable_exception); #if defined(CONFIG_BOOKE_WDT) || defined(CONFIG_40x) /* @@ -2190,7 +2189,7 @@ void __attribute__ ((weak)) WatchdogHandler(struct pt_regs *regs) return; } -void WatchdogException(struct pt_regs *regs) +DEFINE_INTERRUPT_HANDLER(WatchdogException) /* XXX NMI? async? */ { printk (KERN_EMERG "PowerPC Book-E Watchdog Exception\n"); WatchdogHandler(regs); @@ -2201,13 +2200,12 @@ void WatchdogException(struct pt_regs *regs) * We enter here if we discover during exception entry that we are * running in supervisor mode with a userspace value in the stack pointer. */ -void kernel_bad_stack(struct pt_regs *regs) +DEFINE_INTERRUPT_HANDLER(kernel_bad_stack) { printk(KERN_EMERG "Bad kernel stack pointer %lx at %lx\n", regs->gpr[1], regs->nip); die("Bad kernel stack pointer", regs, SIGABRT); } -NOKPROBE_SYMBOL(kernel_bad_stack); void __init trap_init(void) { diff --git a/arch/powerpc/kernel/watchdog.c b/arch/powerpc/kernel/watchdog.c index af3c15a1d41eb159670be90218c12b9966afa7f1..c9a8f4781a1018afcb4fe15cba3129eb1447b6e5 100644 --- a/arch/powerpc/kernel/watchdog.c +++ b/arch/powerpc/kernel/watchdog.c @@ -26,7 +26,9 @@ #include #include +#include #include +#include /* * The powerpc watchdog ensures that each CPU is able to service timers. @@ -247,16 +249,17 @@ static void watchdog_timer_interrupt(int cpu) watchdog_smp_panic(cpu, tb); } -void soft_nmi_interrupt(struct pt_regs *regs) +DEFINE_INTERRUPT_HANDLER_NMI(soft_nmi_interrupt) { unsigned long flags; int cpu = raw_smp_processor_id(); u64 tb; - if (!cpumask_test_cpu(cpu, &wd_cpus_enabled)) - return; + /* should only arrive from kernel, with irqs disabled */ + WARN_ON_ONCE(!arch_irq_disabled_regs(regs)); - nmi_enter(); + if (!cpumask_test_cpu(cpu, &wd_cpus_enabled)) + return 0; __this_cpu_inc(irq_stat.soft_nmi_irqs); @@ -265,7 +268,7 @@ void soft_nmi_interrupt(struct pt_regs *regs) wd_smp_lock(&flags); if (cpumask_test_cpu(cpu, &wd_smp_cpus_stuck)) { wd_smp_unlock(&flags); - goto out; + return 0; } set_cpu_stuck(cpu, tb); @@ -289,8 +292,7 @@ void soft_nmi_interrupt(struct pt_regs *regs) if (wd_panic_timeout_tb < 0x7fffffff) mtspr(SPRN_DEC, wd_panic_timeout_tb); -out: - nmi_exit(); + return 0; } static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer) diff --git a/arch/powerpc/kexec/elf_64.c b/arch/powerpc/kexec/elf_64.c index d0e459bb2f05a69bb9de7d4cffa3f4b76ed8bed5..9842e33533df154802bec1fa80de4c8c12f22bdd 100644 --- a/arch/powerpc/kexec/elf_64.c +++ b/arch/powerpc/kexec/elf_64.c @@ -102,7 +102,7 @@ static void *elf64_load(struct kimage *image, char *kernel_buf, pr_debug("Loaded initrd at 0x%lx\n", initrd_load_addr); } - fdt_size = fdt_totalsize(initial_boot_params) * 2; + fdt_size = kexec_fdt_totalsize_ppc64(image); fdt = kmalloc(fdt_size, GFP_KERNEL); if (!fdt) { pr_err("Not enough memory for the device tree.\n"); diff --git a/arch/powerpc/kexec/file_load_64.c b/arch/powerpc/kexec/file_load_64.c index c69bcf9b547a8cd97ccf8d2ba9072e4c6fff13fb..02b9e4d0dc40b072ca2fad7d18e6487d9da1d4ac 100644 --- a/arch/powerpc/kexec/file_load_64.c +++ b/arch/powerpc/kexec/file_load_64.c @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -925,6 +926,40 @@ int setup_purgatory_ppc64(struct kimage *image, const void *slave_code, return ret; } +/** + * kexec_fdt_totalsize_ppc64 - Return the estimated size needed to setup FDT + * for kexec/kdump kernel. + * @image: kexec image being loaded. + * + * Returns the estimated size needed for kexec/kdump kernel FDT. + */ +unsigned int kexec_fdt_totalsize_ppc64(struct kimage *image) +{ + unsigned int fdt_size; + u64 usm_entries; + + /* + * The below estimate more than accounts for a typical kexec case where + * the additional space is to accommodate things like kexec cmdline, + * chosen node with properties for initrd start & end addresses and + * a property to indicate kexec boot.. + */ + fdt_size = fdt_totalsize(initial_boot_params) + (2 * COMMAND_LINE_SIZE); + if (image->type != KEXEC_TYPE_CRASH) + return fdt_size; + + /* + * For kdump kernel, also account for linux,usable-memory and + * linux,drconf-usable-memory properties. Get an approximate on the + * number of usable memory entries and use for FDT size estimation. + */ + usm_entries = ((memblock_end_of_DRAM() / drmem_lmb_size()) + + (2 * (resource_size(&crashk_res) / drmem_lmb_size()))); + fdt_size += (unsigned int)(usm_entries * sizeof(u64)); + + return fdt_size; +} + /** * setup_new_fdt_ppc64 - Update the flattend device-tree of the kernel * being loaded. diff --git a/arch/powerpc/kvm/Kconfig b/arch/powerpc/kvm/Kconfig index 549591d9aaa2cfd7c5f945fb34ff3df6f6aa4371..e45644657d49dc617220792aae8f83917956e2a0 100644 --- a/arch/powerpc/kvm/Kconfig +++ b/arch/powerpc/kvm/Kconfig @@ -54,6 +54,7 @@ config KVM_BOOK3S_32 select KVM select KVM_BOOK3S_32_HANDLER select KVM_BOOK3S_PR_POSSIBLE + select PPC_FPU help Support running unmodified book3s_32 guest kernels in virtual machines on book3s_32 host processors. diff --git a/arch/powerpc/kvm/book3s_64_mmu_hv.c b/arch/powerpc/kvm/book3s_64_mmu_hv.c index 38ea396a23d6e7e9e261effa737688c42e024f19..bb6773594cf8280fc275a6269d58f4da0070e689 100644 --- a/arch/powerpc/kvm/book3s_64_mmu_hv.c +++ b/arch/powerpc/kvm/book3s_64_mmu_hv.c @@ -27,6 +27,7 @@ #include #include +#include "book3s.h" #include "trace_hv.h" //#define DEBUG_RESIZE_HPT 1 @@ -590,7 +591,7 @@ int kvmppc_book3s_hv_page_fault(struct kvm_vcpu *vcpu, } else { /* Call KVM generic code to do the slow-path check */ pfn = __gfn_to_pfn_memslot(memslot, gfn, false, NULL, - writing, &write_ok); + writing, &write_ok, NULL); if (is_error_noslot_pfn(pfn)) return -EFAULT; page = NULL; diff --git a/arch/powerpc/kvm/book3s_64_mmu_radix.c b/arch/powerpc/kvm/book3s_64_mmu_radix.c index bb35490400e994f886acf45de5b3432012f4bd86..e603de7ade52ef301d1e04c3574ae8aef6515f90 100644 --- a/arch/powerpc/kvm/book3s_64_mmu_radix.c +++ b/arch/powerpc/kvm/book3s_64_mmu_radix.c @@ -822,7 +822,7 @@ int kvmppc_book3s_instantiate_page(struct kvm_vcpu *vcpu, /* Call KVM generic code to do the slow-path check */ pfn = __gfn_to_pfn_memslot(memslot, gfn, false, NULL, - writing, upgrade_p); + writing, upgrade_p, NULL); if (is_error_noslot_pfn(pfn)) return -EFAULT; page = NULL; diff --git a/arch/powerpc/kvm/book3s_emulate.c b/arch/powerpc/kvm/book3s_emulate.c index b08cc15f31c7b546c238cfc906f049a52275baf6..fdb57be71aa659542d5e4195e3904bc66ec52a92 100644 --- a/arch/powerpc/kvm/book3s_emulate.c +++ b/arch/powerpc/kvm/book3s_emulate.c @@ -61,10 +61,6 @@ #define SPRN_GQR6 918 #define SPRN_GQR7 919 -/* Book3S_32 defines mfsrin(v) - but that messes up our abstract - * function pointers, so let's just disable the define. */ -#undef mfsrin - enum priv_level { PRIV_PROBLEM = 0, PRIV_SUPER = 1, diff --git a/arch/powerpc/kvm/book3s_hv.c b/arch/powerpc/kvm/book3s_hv.c index f09708da216ef5079ea1a0201f0e93b4f962080d..13bad6bf4c9589f827f867663d20a3575d4090c4 100644 --- a/arch/powerpc/kvm/book3s_hv.c +++ b/arch/powerpc/kvm/book3s_hv.c @@ -53,6 +53,7 @@ #include #include #include +#include #include #include #include @@ -3408,8 +3409,9 @@ static noinline void kvmppc_run_core(struct kvmppc_vcore *vc) kvmppc_set_host_core(pcpu); + guest_exit_irqoff(); + local_irq_enable(); - guest_exit(); /* Let secondaries go back to the offline loop */ for (i = 0; i < controlled_threads; ++i) { @@ -4236,8 +4238,9 @@ int kvmhv_run_single_vcpu(struct kvm_vcpu *vcpu, u64 time_limit, kvmppc_set_host_core(pcpu); + guest_exit_irqoff(); + local_irq_enable(); - guest_exit(); cpumask_clear_cpu(pcpu, &kvm->arch.cpu_in_guest); diff --git a/arch/powerpc/kvm/book3s_hv_builtin.c b/arch/powerpc/kvm/book3s_hv_builtin.c index f3d3183249fe88bb365584ef81722e966044e939..158d309b42a387e9adff5eac7861c793136dd76c 100644 --- a/arch/powerpc/kvm/book3s_hv_builtin.c +++ b/arch/powerpc/kvm/book3s_hv_builtin.c @@ -17,6 +17,7 @@ #include #include +#include #include #include #include diff --git a/arch/powerpc/kvm/book3s_xive.c b/arch/powerpc/kvm/book3s_xive.c index 30dfeac731c6d668abe84fca7aa6e9c95679a0d5..e7219b6f5f9a5e98a4e2229d5e66a79792ca53da 100644 --- a/arch/powerpc/kvm/book3s_xive.c +++ b/arch/powerpc/kvm/book3s_xive.c @@ -1813,9 +1813,9 @@ int kvmppc_xive_set_irq(struct kvm *kvm, int irq_source_id, u32 irq, int level, return -EINVAL; if ((level == 1 && state->lsi) || level == KVM_INTERRUPT_SET_LEVEL) - state->asserted = 1; + state->asserted = true; else if (level == 0 || level == KVM_INTERRUPT_UNSET) { - state->asserted = 0; + state->asserted = false; return 0; } diff --git a/arch/powerpc/kvm/booke.c b/arch/powerpc/kvm/booke.c index f38ae3e54b37364fbf34caeaea019c0d936cc4ea..7d5fe43f85c450d441ab52c7908fad0af67d4181 100644 --- a/arch/powerpc/kvm/booke.c +++ b/arch/powerpc/kvm/booke.c @@ -20,6 +20,7 @@ #include #include +#include #include #include #include diff --git a/arch/powerpc/kvm/powerpc.c b/arch/powerpc/kvm/powerpc.c index 6c083a9b354576326fc789b9ac88462bee88da47..a2a68a958fa01b2af5643f0f7468aefd9e9e975c 100644 --- a/arch/powerpc/kvm/powerpc.c +++ b/arch/powerpc/kvm/powerpc.c @@ -1522,7 +1522,7 @@ int kvmppc_handle_vmx_load(struct kvm_vcpu *vcpu, return emulated; } -int kvmppc_get_vmx_dword(struct kvm_vcpu *vcpu, int index, u64 *val) +static int kvmppc_get_vmx_dword(struct kvm_vcpu *vcpu, int index, u64 *val) { union kvmppc_one_reg reg; int vmx_offset = 0; @@ -1540,7 +1540,7 @@ int kvmppc_get_vmx_dword(struct kvm_vcpu *vcpu, int index, u64 *val) return result; } -int kvmppc_get_vmx_word(struct kvm_vcpu *vcpu, int index, u64 *val) +static int kvmppc_get_vmx_word(struct kvm_vcpu *vcpu, int index, u64 *val) { union kvmppc_one_reg reg; int vmx_offset = 0; @@ -1558,7 +1558,7 @@ int kvmppc_get_vmx_word(struct kvm_vcpu *vcpu, int index, u64 *val) return result; } -int kvmppc_get_vmx_hword(struct kvm_vcpu *vcpu, int index, u64 *val) +static int kvmppc_get_vmx_hword(struct kvm_vcpu *vcpu, int index, u64 *val) { union kvmppc_one_reg reg; int vmx_offset = 0; @@ -1576,7 +1576,7 @@ int kvmppc_get_vmx_hword(struct kvm_vcpu *vcpu, int index, u64 *val) return result; } -int kvmppc_get_vmx_byte(struct kvm_vcpu *vcpu, int index, u64 *val) +static int kvmppc_get_vmx_byte(struct kvm_vcpu *vcpu, int index, u64 *val) { union kvmppc_one_reg reg; int vmx_offset = 0; diff --git a/arch/powerpc/lib/Makefile b/arch/powerpc/lib/Makefile index 69a91b571845d7745bc3217629697f975c24e64c..d4efc182662a8cecf0b358fa844bd50c15570d76 100644 --- a/arch/powerpc/lib/Makefile +++ b/arch/powerpc/lib/Makefile @@ -31,7 +31,7 @@ obj-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o # 64-bit linker creates .sfpr on demand for final link (vmlinux), # so it is only needed for modules, and only for older linkers which # do not support --save-restore-funcs -ifeq ($(call ld-ifversion, -lt, 225000000, y),y) +ifeq ($(call ld-ifversion, -lt, 22500, y),y) extra-$(CONFIG_PPC64) += crtsavres.o endif diff --git a/arch/powerpc/lib/pmem.c b/arch/powerpc/lib/pmem.c index 1550e0d2513a0d26a240aefc362eafed1fa02dad..eb2919ddf9b9c9f536e6359403cf10e26d06ca0a 100644 --- a/arch/powerpc/lib/pmem.c +++ b/arch/powerpc/lib/pmem.c @@ -6,6 +6,7 @@ #include #include #include +#include #include diff --git a/arch/powerpc/lib/sstep.c b/arch/powerpc/lib/sstep.c index ede093e9623472f1f9652d98843ef9f48fcc7aaa..bb5c20d4ca91c2423869372306d3d12e95d576fe 100644 --- a/arch/powerpc/lib/sstep.c +++ b/arch/powerpc/lib/sstep.c @@ -1306,9 +1306,11 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs, if ((word & 0xfe2) == 2) op->type = SYSCALL; else if (IS_ENABLED(CONFIG_PPC_BOOK3S_64) && - (word & 0xfe3) == 1) + (word & 0xfe3) == 1) { /* scv */ op->type = SYSCALL_VECTORED_0; - else + if (!cpu_has_feature(CPU_FTR_ARCH_300)) + goto unknown_opcode; + } else op->type = UNKNOWN; return 0; #endif @@ -1412,7 +1414,7 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs, #ifdef __powerpc64__ case 1: if (!cpu_has_feature(CPU_FTR_ARCH_31)) - return -1; + goto unknown_opcode; prefix_r = GET_PREFIX_R(word); ra = GET_PREFIX_RA(suffix); @@ -1445,8 +1447,13 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs, #ifdef __powerpc64__ case 4: + /* + * There are very many instructions with this primary opcode + * introduced in the ISA as early as v2.03. However, the ones + * we currently emulate were all introduced with ISA 3.0 + */ if (!cpu_has_feature(CPU_FTR_ARCH_300)) - return -1; + goto unknown_opcode; switch (word & 0x3f) { case 48: /* maddhd */ @@ -1472,7 +1479,7 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs, * There are other instructions from ISA 3.0 with the same * primary opcode which do not have emulation support yet. */ - return -1; + goto unknown_opcode; #endif case 7: /* mulli */ @@ -1532,6 +1539,8 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs, case 19: if (((word >> 1) & 0x1f) == 2) { /* addpcis */ + if (!cpu_has_feature(CPU_FTR_ARCH_300)) + goto unknown_opcode; imm = (short) (word & 0xffc1); /* d0 + d2 fields */ imm |= (word >> 15) & 0x3e; /* d1 field */ op->val = regs->nip + (imm << 16) + 4; @@ -1844,7 +1853,7 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs, #ifdef __powerpc64__ case 265: /* modud */ if (!cpu_has_feature(CPU_FTR_ARCH_300)) - return -1; + goto unknown_opcode; op->val = regs->gpr[ra] % regs->gpr[rb]; goto compute_done; #endif @@ -1854,7 +1863,7 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs, case 267: /* moduw */ if (!cpu_has_feature(CPU_FTR_ARCH_300)) - return -1; + goto unknown_opcode; op->val = (unsigned int) regs->gpr[ra] % (unsigned int) regs->gpr[rb]; goto compute_done; @@ -1891,7 +1900,7 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs, #endif case 755: /* darn */ if (!cpu_has_feature(CPU_FTR_ARCH_300)) - return -1; + goto unknown_opcode; switch (ra & 0x3) { case 0: /* 32-bit conditioned */ @@ -1909,18 +1918,18 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs, goto compute_done; } - return -1; + goto unknown_opcode; #ifdef __powerpc64__ case 777: /* modsd */ if (!cpu_has_feature(CPU_FTR_ARCH_300)) - return -1; + goto unknown_opcode; op->val = (long int) regs->gpr[ra] % (long int) regs->gpr[rb]; goto compute_done; #endif case 779: /* modsw */ if (!cpu_has_feature(CPU_FTR_ARCH_300)) - return -1; + goto unknown_opcode; op->val = (int) regs->gpr[ra] % (int) regs->gpr[rb]; goto compute_done; @@ -1997,14 +2006,14 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs, #endif case 538: /* cnttzw */ if (!cpu_has_feature(CPU_FTR_ARCH_300)) - return -1; + goto unknown_opcode; val = (unsigned int) regs->gpr[rd]; op->val = (val ? __builtin_ctz(val) : 32); goto logical_done; #ifdef __powerpc64__ case 570: /* cnttzd */ if (!cpu_has_feature(CPU_FTR_ARCH_300)) - return -1; + goto unknown_opcode; val = regs->gpr[rd]; op->val = (val ? __builtin_ctzl(val) : 64); goto logical_done; @@ -2114,7 +2123,7 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs, case 890: /* extswsli with sh_5 = 0 */ case 891: /* extswsli with sh_5 = 1 */ if (!cpu_has_feature(CPU_FTR_ARCH_300)) - return -1; + goto unknown_opcode; op->type = COMPUTE + SETREG; sh = rb | ((word & 2) << 4); val = (signed int) regs->gpr[rd]; @@ -2441,6 +2450,8 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs, break; case 268: /* lxvx */ + if (!cpu_has_feature(CPU_FTR_ARCH_300)) + goto unknown_opcode; op->reg = rd | ((word & 1) << 5); op->type = MKOP(LOAD_VSX, 0, 16); op->element_size = 16; @@ -2450,6 +2461,8 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs, case 269: /* lxvl */ case 301: { /* lxvll */ int nb; + if (!cpu_has_feature(CPU_FTR_ARCH_300)) + goto unknown_opcode; op->reg = rd | ((word & 1) << 5); op->ea = ra ? regs->gpr[ra] : 0; nb = regs->gpr[rb] & 0xff; @@ -2470,13 +2483,15 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs, case 333: /* lxvpx */ if (!cpu_has_feature(CPU_FTR_ARCH_31)) - return -1; + goto unknown_opcode; op->reg = VSX_REGISTER_XTP(rd); op->type = MKOP(LOAD_VSX, 0, 32); op->element_size = 32; break; case 364: /* lxvwsx */ + if (!cpu_has_feature(CPU_FTR_ARCH_300)) + goto unknown_opcode; op->reg = rd | ((word & 1) << 5); op->type = MKOP(LOAD_VSX, 0, 4); op->element_size = 4; @@ -2484,6 +2499,8 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs, break; case 396: /* stxvx */ + if (!cpu_has_feature(CPU_FTR_ARCH_300)) + goto unknown_opcode; op->reg = rd | ((word & 1) << 5); op->type = MKOP(STORE_VSX, 0, 16); op->element_size = 16; @@ -2493,6 +2510,8 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs, case 397: /* stxvl */ case 429: { /* stxvll */ int nb; + if (!cpu_has_feature(CPU_FTR_ARCH_300)) + goto unknown_opcode; op->reg = rd | ((word & 1) << 5); op->ea = ra ? regs->gpr[ra] : 0; nb = regs->gpr[rb] & 0xff; @@ -2506,7 +2525,7 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs, } case 461: /* stxvpx */ if (!cpu_has_feature(CPU_FTR_ARCH_31)) - return -1; + goto unknown_opcode; op->reg = VSX_REGISTER_XTP(rd); op->type = MKOP(STORE_VSX, 0, 32); op->element_size = 32; @@ -2544,6 +2563,8 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs, break; case 781: /* lxsibzx */ + if (!cpu_has_feature(CPU_FTR_ARCH_300)) + goto unknown_opcode; op->reg = rd | ((word & 1) << 5); op->type = MKOP(LOAD_VSX, 0, 1); op->element_size = 8; @@ -2551,6 +2572,8 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs, break; case 812: /* lxvh8x */ + if (!cpu_has_feature(CPU_FTR_ARCH_300)) + goto unknown_opcode; op->reg = rd | ((word & 1) << 5); op->type = MKOP(LOAD_VSX, 0, 16); op->element_size = 2; @@ -2558,6 +2581,8 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs, break; case 813: /* lxsihzx */ + if (!cpu_has_feature(CPU_FTR_ARCH_300)) + goto unknown_opcode; op->reg = rd | ((word & 1) << 5); op->type = MKOP(LOAD_VSX, 0, 2); op->element_size = 8; @@ -2571,6 +2596,8 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs, break; case 876: /* lxvb16x */ + if (!cpu_has_feature(CPU_FTR_ARCH_300)) + goto unknown_opcode; op->reg = rd | ((word & 1) << 5); op->type = MKOP(LOAD_VSX, 0, 16); op->element_size = 1; @@ -2584,6 +2611,8 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs, break; case 909: /* stxsibx */ + if (!cpu_has_feature(CPU_FTR_ARCH_300)) + goto unknown_opcode; op->reg = rd | ((word & 1) << 5); op->type = MKOP(STORE_VSX, 0, 1); op->element_size = 8; @@ -2591,6 +2620,8 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs, break; case 940: /* stxvh8x */ + if (!cpu_has_feature(CPU_FTR_ARCH_300)) + goto unknown_opcode; op->reg = rd | ((word & 1) << 5); op->type = MKOP(STORE_VSX, 0, 16); op->element_size = 2; @@ -2598,6 +2629,8 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs, break; case 941: /* stxsihx */ + if (!cpu_has_feature(CPU_FTR_ARCH_300)) + goto unknown_opcode; op->reg = rd | ((word & 1) << 5); op->type = MKOP(STORE_VSX, 0, 2); op->element_size = 8; @@ -2611,6 +2644,8 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs, break; case 1004: /* stxvb16x */ + if (!cpu_has_feature(CPU_FTR_ARCH_300)) + goto unknown_opcode; op->reg = rd | ((word & 1) << 5); op->type = MKOP(STORE_VSX, 0, 16); op->element_size = 1; @@ -2719,12 +2754,16 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs, op->type = MKOP(LOAD_FP, 0, 16); break; case 2: /* lxsd */ + if (!cpu_has_feature(CPU_FTR_ARCH_300)) + goto unknown_opcode; op->reg = rd + 32; op->type = MKOP(LOAD_VSX, 0, 8); op->element_size = 8; op->vsx_flags = VSX_CHECK_VEC; break; case 3: /* lxssp */ + if (!cpu_has_feature(CPU_FTR_ARCH_300)) + goto unknown_opcode; op->reg = rd + 32; op->type = MKOP(LOAD_VSX, 0, 4); op->element_size = 8; @@ -2754,7 +2793,7 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs, #ifdef CONFIG_VSX case 6: if (!cpu_has_feature(CPU_FTR_ARCH_31)) - return -1; + goto unknown_opcode; op->ea = dqform_ea(word, regs); op->reg = VSX_REGISTER_XTP(rd); op->element_size = 32; @@ -2777,6 +2816,8 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs, break; case 1: /* lxv */ + if (!cpu_has_feature(CPU_FTR_ARCH_300)) + goto unknown_opcode; op->ea = dqform_ea(word, regs); if (word & 8) op->reg = rd + 32; @@ -2787,6 +2828,8 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs, case 2: /* stxsd with LSB of DS field = 0 */ case 6: /* stxsd with LSB of DS field = 1 */ + if (!cpu_has_feature(CPU_FTR_ARCH_300)) + goto unknown_opcode; op->ea = dsform_ea(word, regs); op->reg = rd + 32; op->type = MKOP(STORE_VSX, 0, 8); @@ -2796,6 +2839,8 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs, case 3: /* stxssp with LSB of DS field = 0 */ case 7: /* stxssp with LSB of DS field = 1 */ + if (!cpu_has_feature(CPU_FTR_ARCH_300)) + goto unknown_opcode; op->ea = dsform_ea(word, regs); op->reg = rd + 32; op->type = MKOP(STORE_VSX, 0, 4); @@ -2804,6 +2849,8 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs, break; case 5: /* stxv */ + if (!cpu_has_feature(CPU_FTR_ARCH_300)) + goto unknown_opcode; op->ea = dqform_ea(word, regs); if (word & 8) op->reg = rd + 32; @@ -2833,7 +2880,7 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs, break; case 1: /* Prefixed instructions */ if (!cpu_has_feature(CPU_FTR_ARCH_31)) - return -1; + goto unknown_opcode; prefix_r = GET_PREFIX_R(word); ra = GET_PREFIX_RA(suffix); @@ -2972,6 +3019,20 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs, } + if (OP_IS_LOAD_STORE(op->type) && (op->type & UPDATE)) { + switch (GETTYPE(op->type)) { + case LOAD: + if (ra == rd) + goto unknown_opcode; + fallthrough; + case STORE: + case LOAD_FP: + case STORE_FP: + if (ra == 0) + goto unknown_opcode; + } + } + #ifdef CONFIG_VSX if ((GETTYPE(op->type) == LOAD_VSX || GETTYPE(op->type) == STORE_VSX) && @@ -2982,6 +3043,10 @@ int analyse_instr(struct instruction_op *op, const struct pt_regs *regs, return 0; + unknown_opcode: + op->type = UNKNOWN; + return 0; + logical_done: if (word & 1) set_cr0(regs, op); diff --git a/arch/powerpc/mm/book3s32/Makefile b/arch/powerpc/mm/book3s32/Makefile index 3f972db17761970e49a3584028a46e9cea76685c..446d9de88ce4c1d3a0acd63d7656fbb152ef5f8e 100644 --- a/arch/powerpc/mm/book3s32/Makefile +++ b/arch/powerpc/mm/book3s32/Makefile @@ -6,4 +6,6 @@ ifdef CONFIG_KASAN CFLAGS_mmu.o += -DDISABLE_BRANCH_PROFILING endif -obj-y += mmu.o hash_low.o mmu_context.o tlb.o nohash_low.o +obj-y += mmu.o mmu_context.o +obj-$(CONFIG_PPC_BOOK3S_603) += nohash_low.o +obj-$(CONFIG_PPC_BOOK3S_604) += hash_low.o tlb.o diff --git a/arch/powerpc/mm/book3s32/mmu.c b/arch/powerpc/mm/book3s32/mmu.c index 859e5bd603ac82fe39a9c8fe7ac63fd6eae5b718..d7eb266a3f7ad83840229cc4bfd95810ec1d4550 100644 --- a/arch/powerpc/mm/book3s32/mmu.c +++ b/arch/powerpc/mm/book3s32/mmu.c @@ -234,7 +234,7 @@ void mmu_mark_initmem_nx(void) if (is_module_segment(i << 28)) continue; - mtsrin(mfsrin(i << 28) | 0x10000000, i << 28); + mtsr(mfsr(i << 28) | 0x10000000, i << 28); } } diff --git a/arch/powerpc/mm/book3s64/hash_hugetlbpage.c b/arch/powerpc/mm/book3s64/hash_hugetlbpage.c index b5e9fff8c217dd15068e22582953f2722959b346..a688e1324ae5587e6a428e350fb63ff28557d870 100644 --- a/arch/powerpc/mm/book3s64/hash_hugetlbpage.c +++ b/arch/powerpc/mm/book3s64/hash_hugetlbpage.c @@ -16,10 +16,6 @@ unsigned int hpage_shift; EXPORT_SYMBOL(hpage_shift); -extern long hpte_insert_repeating(unsigned long hash, unsigned long vpn, - unsigned long pa, unsigned long rlags, - unsigned long vflags, int psize, int ssize); - int __hash_page_huge(unsigned long ea, unsigned long access, unsigned long vsid, pte_t *ptep, unsigned long trap, unsigned long flags, int ssize, unsigned int shift, unsigned int mmu_psize) diff --git a/arch/powerpc/mm/book3s64/hash_utils.c b/arch/powerpc/mm/book3s64/hash_utils.c index 73b06adb6eebe8928313a678b94728240425c3c4..581b20a2feaf610af74327f1fabbc2dccb0b080a 100644 --- a/arch/powerpc/mm/book3s64/hash_utils.c +++ b/arch/powerpc/mm/book3s64/hash_utils.c @@ -38,6 +38,7 @@ #include #include +#include #include #include #include @@ -1143,10 +1144,10 @@ unsigned int hash_page_do_lazy_icache(unsigned int pp, pte_t pte, int trap) page = pte_page(pte); /* page is dirty */ - if (!test_bit(PG_arch_1, &page->flags) && !PageReserved(page)) { + if (!test_bit(PG_dcache_clean, &page->flags) && !PageReserved(page)) { if (trap == 0x400) { flush_dcache_icache_page(page); - set_bit(PG_arch_1, &page->flags); + set_bit(PG_dcache_clean, &page->flags); } else pp |= HPTE_R_N; } @@ -1288,7 +1289,6 @@ int hash_page_mm(struct mm_struct *mm, unsigned long ea, unsigned long flags) { bool is_thp; - enum ctx_state prev_state = exception_enter(); pgd_t *pgdir; unsigned long vsid; pte_t *ptep; @@ -1490,7 +1490,6 @@ int hash_page_mm(struct mm_struct *mm, unsigned long ea, DBG_LOW(" -> rc=%d\n", rc); bail: - exception_exit(prev_state); return rc; } EXPORT_SYMBOL_GPL(hash_page_mm); @@ -1512,16 +1511,22 @@ int hash_page(unsigned long ea, unsigned long access, unsigned long trap, } EXPORT_SYMBOL_GPL(hash_page); -int __hash_page(unsigned long trap, unsigned long ea, unsigned long dsisr, - unsigned long msr) +DECLARE_INTERRUPT_HANDLER_RET(__do_hash_fault); +DEFINE_INTERRUPT_HANDLER_RET(__do_hash_fault) { + unsigned long ea = regs->dar; + unsigned long dsisr = regs->dsisr; unsigned long access = _PAGE_PRESENT | _PAGE_READ; unsigned long flags = 0; - struct mm_struct *mm = current->mm; - unsigned int region_id = get_region_id(ea); + struct mm_struct *mm; + unsigned int region_id; + long err; + region_id = get_region_id(ea); if ((region_id == VMALLOC_REGION_ID) || (region_id == IO_REGION_ID)) mm = &init_mm; + else + mm = current->mm; if (dsisr & DSISR_NOHPTE) flags |= HPTE_NOHPTE_UPDATE; @@ -1537,13 +1542,66 @@ int __hash_page(unsigned long trap, unsigned long ea, unsigned long dsisr, * 2) user space access kernel space. */ access |= _PAGE_PRIVILEGED; - if ((msr & MSR_PR) || (region_id == USER_REGION_ID)) + if (user_mode(regs) || (region_id == USER_REGION_ID)) access &= ~_PAGE_PRIVILEGED; - if (trap == 0x400) + if (regs->trap == 0x400) access |= _PAGE_EXEC; - return hash_page_mm(mm, ea, access, trap, flags); + err = hash_page_mm(mm, ea, access, regs->trap, flags); + if (unlikely(err < 0)) { + // failed to instert a hash PTE due to an hypervisor error + if (user_mode(regs)) { + if (IS_ENABLED(CONFIG_PPC_SUBPAGE_PROT) && err == -2) + _exception(SIGSEGV, regs, SEGV_ACCERR, ea); + else + _exception(SIGBUS, regs, BUS_ADRERR, ea); + } else { + bad_page_fault(regs, SIGBUS); + } + err = 0; + } + + return err; +} + +/* + * The _RAW interrupt entry checks for the in_nmi() case before + * running the full handler. + */ +DEFINE_INTERRUPT_HANDLER_RAW(do_hash_fault) +{ + unsigned long dsisr = regs->dsisr; + long err; + + if (unlikely(dsisr & (DSISR_BAD_FAULT_64S | DSISR_KEYFAULT))) + goto page_fault; + + /* + * If we are in an "NMI" (e.g., an interrupt when soft-disabled), then + * don't call hash_page, just fail the fault. This is required to + * prevent re-entrancy problems in the hash code, namely perf + * interrupts hitting while something holds H_PAGE_BUSY, and taking a + * hash fault. See the comment in hash_preload(). + * + * We come here as a result of a DSI at a point where we don't want + * to call hash_page, such as when we are accessing memory (possibly + * user memory) inside a PMU interrupt that occurred while interrupts + * were soft-disabled. We want to invoke the exception handler for + * the access, or panic if there isn't a handler. + */ + if (unlikely(in_nmi())) { + do_bad_page_fault_segv(regs); + return 0; + } + + err = __do_hash_fault(regs); + if (err) { +page_fault: + err = hash__do_page_fault(regs); + } + + return err; } #ifdef CONFIG_PPC_MM_SLICES @@ -1843,27 +1901,6 @@ void flush_hash_range(unsigned long number, int local) } } -/* - * low_hash_fault is called when we the low level hash code failed - * to instert a PTE due to an hypervisor error - */ -void low_hash_fault(struct pt_regs *regs, unsigned long address, int rc) -{ - enum ctx_state prev_state = exception_enter(); - - if (user_mode(regs)) { -#ifdef CONFIG_PPC_SUBPAGE_PROT - if (rc == -2) - _exception(SIGSEGV, regs, SEGV_ACCERR, address); - else -#endif - _exception(SIGBUS, regs, BUS_ADRERR, address); - } else - bad_page_fault(regs, address, SIGBUS); - - exception_exit(prev_state); -} - long hpte_insert_repeating(unsigned long hash, unsigned long vpn, unsigned long pa, unsigned long rflags, unsigned long vflags, int psize, int ssize) diff --git a/arch/powerpc/mm/book3s64/internal.h b/arch/powerpc/mm/book3s64/internal.h index c12d78ee42f57b29f9764211f6c69137bb07fd32..5045048ce244da88439a469fe766b4e7577ca02a 100644 --- a/arch/powerpc/mm/book3s64/internal.h +++ b/arch/powerpc/mm/book3s64/internal.h @@ -15,4 +15,6 @@ static inline bool stress_slb(void) void slb_setup_new_exec(void); +void exit_lazy_flush_tlb(struct mm_struct *mm, bool always_flush); + #endif /* ARCH_POWERPC_MM_BOOK3S64_INTERNAL_H */ diff --git a/arch/powerpc/mm/book3s64/iommu_api.c b/arch/powerpc/mm/book3s64/iommu_api.c index 685d7bb3d26fa81419ba36189a692b15cc407f5c..cd18e94d0843f5d5a28ffc13d637201e83a27fb1 100644 --- a/arch/powerpc/mm/book3s64/iommu_api.c +++ b/arch/powerpc/mm/book3s64/iommu_api.c @@ -129,7 +129,8 @@ static long mm_iommu_do_alloc(struct mm_struct *mm, unsigned long ua, mutex_lock(&mem_list_mutex); - list_for_each_entry_rcu(mem2, &mm->context.iommu_group_mem_list, next) { + list_for_each_entry_rcu(mem2, &mm->context.iommu_group_mem_list, next, + lockdep_is_held(&mem_list_mutex)) { /* Overlap? */ if ((mem2->ua < (ua + (entries << PAGE_SHIFT))) && (ua < (mem2->ua + @@ -289,6 +290,7 @@ struct mm_iommu_table_group_mem_t *mm_iommu_lookup(struct mm_struct *mm, { struct mm_iommu_table_group_mem_t *mem, *ret = NULL; + rcu_read_lock(); list_for_each_entry_rcu(mem, &mm->context.iommu_group_mem_list, next) { if ((mem->ua <= ua) && (ua + size <= mem->ua + @@ -297,6 +299,7 @@ struct mm_iommu_table_group_mem_t *mm_iommu_lookup(struct mm_struct *mm, break; } } + rcu_read_unlock(); return ret; } @@ -327,7 +330,8 @@ struct mm_iommu_table_group_mem_t *mm_iommu_get(struct mm_struct *mm, mutex_lock(&mem_list_mutex); - list_for_each_entry_rcu(mem, &mm->context.iommu_group_mem_list, next) { + list_for_each_entry_rcu(mem, &mm->context.iommu_group_mem_list, next, + lockdep_is_held(&mem_list_mutex)) { if ((mem->ua == ua) && (mem->entries == entries)) { ret = mem; ++mem->used; @@ -421,6 +425,7 @@ bool mm_iommu_is_devmem(struct mm_struct *mm, unsigned long hpa, struct mm_iommu_table_group_mem_t *mem; unsigned long end; + rcu_read_lock(); list_for_each_entry_rcu(mem, &mm->context.iommu_group_mem_list, next) { if (mem->dev_hpa == MM_IOMMU_TABLE_INVALID_HPA) continue; @@ -437,6 +442,7 @@ bool mm_iommu_is_devmem(struct mm_struct *mm, unsigned long hpa, return true; } } + rcu_read_unlock(); return false; } diff --git a/arch/powerpc/mm/book3s64/pgtable.c b/arch/powerpc/mm/book3s64/pgtable.c index 5b3a3bae21aa6a16dded5a7c9c79d1110cba51c7..9ffa65074cb00810d9bc145bbd345804d661cc4a 100644 --- a/arch/powerpc/mm/book3s64/pgtable.c +++ b/arch/powerpc/mm/book3s64/pgtable.c @@ -20,6 +20,8 @@ #include #include +#include "internal.h" + unsigned long __pmd_frag_nr; EXPORT_SYMBOL(__pmd_frag_nr); unsigned long __pmd_frag_size_shift; @@ -79,10 +81,15 @@ void set_pmd_at(struct mm_struct *mm, unsigned long addr, return set_pte_at(mm, addr, pmdp_ptep(pmdp), pmd_pte(pmd)); } -static void do_nothing(void *unused) +static void do_serialize(void *arg) { - + /* We've taken the IPI, so try to trim the mask while here */ + if (radix_enabled()) { + struct mm_struct *mm = arg; + exit_lazy_flush_tlb(mm, false); + } } + /* * Serialize against find_current_mm_pte which does lock-less * lookup in page tables with local interrupts disabled. For huge pages @@ -96,7 +103,7 @@ static void do_nothing(void *unused) void serialize_against_pte_lookup(struct mm_struct *mm) { smp_mb(); - smp_call_function_many(mm_cpumask(mm), do_nothing, NULL, 1); + smp_call_function_many(mm_cpumask(mm), do_serialize, mm, 1); } /* diff --git a/arch/powerpc/mm/book3s64/radix_tlb.c b/arch/powerpc/mm/book3s64/radix_tlb.c index fb66d154b26cf90f3a6d7093116021f1402f3387..409e612107892c64061ce7ecfa3dfa830b5c5f65 100644 --- a/arch/powerpc/mm/book3s64/radix_tlb.c +++ b/arch/powerpc/mm/book3s64/radix_tlb.c @@ -18,6 +18,8 @@ #include #include +#include "internal.h" + #define RIC_FLUSH_TLB 0 #define RIC_FLUSH_PWC 1 #define RIC_FLUSH_ALL 2 @@ -627,15 +629,6 @@ void radix__local_flush_tlb_page(struct vm_area_struct *vma, unsigned long vmadd } EXPORT_SYMBOL(radix__local_flush_tlb_page); -static bool mm_is_singlethreaded(struct mm_struct *mm) -{ - if (atomic_read(&mm->context.copros) > 0) - return false; - if (atomic_read(&mm->mm_users) <= 1 && current->mm == mm) - return true; - return false; -} - static bool mm_needs_flush_escalation(struct mm_struct *mm) { /* @@ -648,21 +641,24 @@ static bool mm_needs_flush_escalation(struct mm_struct *mm) return false; } -#ifdef CONFIG_SMP -static void do_exit_flush_lazy_tlb(void *arg) +/* + * If always_flush is true, then flush even if this CPU can't be removed + * from mm_cpumask. + */ +void exit_lazy_flush_tlb(struct mm_struct *mm, bool always_flush) { - struct mm_struct *mm = arg; unsigned long pid = mm->context.id; + int cpu = smp_processor_id(); /* * A kthread could have done a mmget_not_zero() after the flushing CPU - * checked mm_is_singlethreaded, and be in the process of - * kthread_use_mm when interrupted here. In that case, current->mm will - * be set to mm, because kthread_use_mm() setting ->mm and switching to - * the mm is done with interrupts off. + * checked mm_cpumask, and be in the process of kthread_use_mm when + * interrupted here. In that case, current->mm will be set to mm, + * because kthread_use_mm() setting ->mm and switching to the mm is + * done with interrupts off. */ if (current->mm == mm) - goto out_flush; + goto out; if (current->active_mm == mm) { WARN_ON_ONCE(current->mm != NULL); @@ -673,11 +669,30 @@ static void do_exit_flush_lazy_tlb(void *arg) mmdrop(mm); } - atomic_dec(&mm->context.active_cpus); - cpumask_clear_cpu(smp_processor_id(), mm_cpumask(mm)); + /* + * This IPI may be initiated from any source including those not + * running the mm, so there may be a racing IPI that comes after + * this one which finds the cpumask already clear. Check and avoid + * underflowing the active_cpus count in that case. The race should + * not otherwise be a problem, but the TLB must be flushed because + * that's what the caller expects. + */ + if (cpumask_test_cpu(cpu, mm_cpumask(mm))) { + atomic_dec(&mm->context.active_cpus); + cpumask_clear_cpu(cpu, mm_cpumask(mm)); + always_flush = true; + } -out_flush: - _tlbiel_pid(pid, RIC_FLUSH_ALL); +out: + if (always_flush) + _tlbiel_pid(pid, RIC_FLUSH_ALL); +} + +#ifdef CONFIG_SMP +static void do_exit_flush_lazy_tlb(void *arg) +{ + struct mm_struct *mm = arg; + exit_lazy_flush_tlb(mm, true); } static void exit_flush_lazy_tlbs(struct mm_struct *mm) @@ -693,9 +708,110 @@ static void exit_flush_lazy_tlbs(struct mm_struct *mm) (void *)mm, 1); } +#else /* CONFIG_SMP */ +static inline void exit_flush_lazy_tlbs(struct mm_struct *mm) { } +#endif /* CONFIG_SMP */ + +static DEFINE_PER_CPU(unsigned int, mm_cpumask_trim_clock); + +/* + * Interval between flushes at which we send out IPIs to check whether the + * mm_cpumask can be trimmed for the case where it's not a single-threaded + * process flushing its own mm. The intent is to reduce the cost of later + * flushes. Don't want this to be so low that it adds noticable cost to TLB + * flushing, or so high that it doesn't help reduce global TLBIEs. + */ +static unsigned long tlb_mm_cpumask_trim_timer = 1073; + +static bool tick_and_test_trim_clock(void) +{ + if (__this_cpu_inc_return(mm_cpumask_trim_clock) == + tlb_mm_cpumask_trim_timer) { + __this_cpu_write(mm_cpumask_trim_clock, 0); + return true; + } + return false; +} + +enum tlb_flush_type { + FLUSH_TYPE_NONE, + FLUSH_TYPE_LOCAL, + FLUSH_TYPE_GLOBAL, +}; + +static enum tlb_flush_type flush_type_needed(struct mm_struct *mm, bool fullmm) +{ + int active_cpus = atomic_read(&mm->context.active_cpus); + int cpu = smp_processor_id(); + + if (active_cpus == 0) + return FLUSH_TYPE_NONE; + if (active_cpus == 1 && cpumask_test_cpu(cpu, mm_cpumask(mm))) { + if (current->mm != mm) { + /* + * Asynchronous flush sources may trim down to nothing + * if the process is not running, so occasionally try + * to trim. + */ + if (tick_and_test_trim_clock()) { + exit_lazy_flush_tlb(mm, true); + return FLUSH_TYPE_NONE; + } + } + return FLUSH_TYPE_LOCAL; + } + + /* Coprocessors require TLBIE to invalidate nMMU. */ + if (atomic_read(&mm->context.copros) > 0) + return FLUSH_TYPE_GLOBAL; + + /* + * In the fullmm case there's no point doing the exit_flush_lazy_tlbs + * because the mm is being taken down anyway, and a TLBIE tends to + * be faster than an IPI+TLBIEL. + */ + if (fullmm) + return FLUSH_TYPE_GLOBAL; + + /* + * If we are running the only thread of a single-threaded process, + * then we should almost always be able to trim off the rest of the + * CPU mask (except in the case of use_mm() races), so always try + * trimming the mask. + */ + if (atomic_read(&mm->mm_users) <= 1 && current->mm == mm) { + exit_flush_lazy_tlbs(mm); + /* + * use_mm() race could prevent IPIs from being able to clear + * the cpumask here, however those users are established + * after our first check (and so after the PTEs are removed), + * and the TLB still gets flushed by the IPI, so this CPU + * will only require a local flush. + */ + return FLUSH_TYPE_LOCAL; + } + + /* + * Occasionally try to trim down the cpumask. It's possible this can + * bring the mask to zero, which results in no flush. + */ + if (tick_and_test_trim_clock()) { + exit_flush_lazy_tlbs(mm); + if (current->mm == mm) + return FLUSH_TYPE_LOCAL; + if (cpumask_test_cpu(cpu, mm_cpumask(mm))) + exit_lazy_flush_tlb(mm, true); + return FLUSH_TYPE_NONE; + } + + return FLUSH_TYPE_GLOBAL; +} + +#ifdef CONFIG_SMP void radix__flush_tlb_mm(struct mm_struct *mm) { unsigned long pid; + enum tlb_flush_type type; pid = mm->context.id; if (unlikely(pid == MMU_NO_CONTEXT)) @@ -703,16 +819,15 @@ void radix__flush_tlb_mm(struct mm_struct *mm) preempt_disable(); /* - * Order loads of mm_cpumask vs previous stores to clear ptes before - * the invalidate. See barrier in switch_mm_irqs_off + * Order loads of mm_cpumask (in flush_type_needed) vs previous + * stores to clear ptes before the invalidate. See barrier in + * switch_mm_irqs_off */ smp_mb(); - if (!mm_is_thread_local(mm)) { - if (unlikely(mm_is_singlethreaded(mm))) { - exit_flush_lazy_tlbs(mm); - goto local; - } - + type = flush_type_needed(mm, false); + if (type == FLUSH_TYPE_LOCAL) { + _tlbiel_pid(pid, RIC_FLUSH_TLB); + } else if (type == FLUSH_TYPE_GLOBAL) { if (!mmu_has_feature(MMU_FTR_GTSE)) { unsigned long tgt = H_RPTI_TARGET_CMMU; @@ -728,9 +843,6 @@ void radix__flush_tlb_mm(struct mm_struct *mm) } else { _tlbiel_pid_multicast(mm, pid, RIC_FLUSH_TLB); } - } else { -local: - _tlbiel_pid(pid, RIC_FLUSH_TLB); } preempt_enable(); } @@ -739,6 +851,7 @@ EXPORT_SYMBOL(radix__flush_tlb_mm); static void __flush_all_mm(struct mm_struct *mm, bool fullmm) { unsigned long pid; + enum tlb_flush_type type; pid = mm->context.id; if (unlikely(pid == MMU_NO_CONTEXT)) @@ -746,13 +859,10 @@ static void __flush_all_mm(struct mm_struct *mm, bool fullmm) preempt_disable(); smp_mb(); /* see radix__flush_tlb_mm */ - if (!mm_is_thread_local(mm)) { - if (unlikely(mm_is_singlethreaded(mm))) { - if (!fullmm) { - exit_flush_lazy_tlbs(mm); - goto local; - } - } + type = flush_type_needed(mm, fullmm); + if (type == FLUSH_TYPE_LOCAL) { + _tlbiel_pid(pid, RIC_FLUSH_ALL); + } else if (type == FLUSH_TYPE_GLOBAL) { if (!mmu_has_feature(MMU_FTR_GTSE)) { unsigned long tgt = H_RPTI_TARGET_CMMU; unsigned long type = H_RPTI_TYPE_TLB | H_RPTI_TYPE_PWC | @@ -766,9 +876,6 @@ static void __flush_all_mm(struct mm_struct *mm, bool fullmm) _tlbie_pid(pid, RIC_FLUSH_ALL); else _tlbiel_pid_multicast(mm, pid, RIC_FLUSH_ALL); - } else { -local: - _tlbiel_pid(pid, RIC_FLUSH_ALL); } preempt_enable(); } @@ -783,6 +890,7 @@ void radix__flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr, int psize) { unsigned long pid; + enum tlb_flush_type type; pid = mm->context.id; if (unlikely(pid == MMU_NO_CONTEXT)) @@ -790,11 +898,10 @@ void radix__flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr, preempt_disable(); smp_mb(); /* see radix__flush_tlb_mm */ - if (!mm_is_thread_local(mm)) { - if (unlikely(mm_is_singlethreaded(mm))) { - exit_flush_lazy_tlbs(mm); - goto local; - } + type = flush_type_needed(mm, false); + if (type == FLUSH_TYPE_LOCAL) { + _tlbiel_va(vmaddr, pid, psize, RIC_FLUSH_TLB); + } else if (type == FLUSH_TYPE_GLOBAL) { if (!mmu_has_feature(MMU_FTR_GTSE)) { unsigned long tgt, pg_sizes, size; @@ -811,9 +918,6 @@ void radix__flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr, _tlbie_va(vmaddr, pid, psize, RIC_FLUSH_TLB); else _tlbiel_va_multicast(mm, vmaddr, pid, psize, RIC_FLUSH_TLB); - } else { -local: - _tlbiel_va(vmaddr, pid, psize, RIC_FLUSH_TLB); } preempt_enable(); } @@ -828,8 +932,6 @@ void radix__flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr) } EXPORT_SYMBOL(radix__flush_tlb_page); -#else /* CONFIG_SMP */ -static inline void exit_flush_lazy_tlbs(struct mm_struct *mm) { } #endif /* CONFIG_SMP */ static void do_tlbiel_kernel(void *info) @@ -893,7 +995,9 @@ static inline void __radix__flush_tlb_range(struct mm_struct *mm, unsigned int page_shift = mmu_psize_defs[mmu_virtual_psize].shift; unsigned long page_size = 1UL << page_shift; unsigned long nr_pages = (end - start) >> page_shift; - bool local, full; + bool fullmm = (end == TLB_FLUSH_ALL); + bool flush_pid; + enum tlb_flush_type type; pid = mm->context.id; if (unlikely(pid == MMU_NO_CONTEXT)) @@ -901,24 +1005,18 @@ static inline void __radix__flush_tlb_range(struct mm_struct *mm, preempt_disable(); smp_mb(); /* see radix__flush_tlb_mm */ - if (!mm_is_thread_local(mm)) { - if (unlikely(mm_is_singlethreaded(mm))) { - if (end != TLB_FLUSH_ALL) { - exit_flush_lazy_tlbs(mm); - goto is_local; - } - } - local = false; - full = (end == TLB_FLUSH_ALL || - nr_pages > tlb_single_page_flush_ceiling); - } else { -is_local: - local = true; - full = (end == TLB_FLUSH_ALL || - nr_pages > tlb_local_single_page_flush_ceiling); - } + type = flush_type_needed(mm, fullmm); + if (type == FLUSH_TYPE_NONE) + goto out; + + if (fullmm) + flush_pid = true; + else if (type == FLUSH_TYPE_GLOBAL) + flush_pid = nr_pages > tlb_single_page_flush_ceiling; + else + flush_pid = nr_pages > tlb_local_single_page_flush_ceiling; - if (!mmu_has_feature(MMU_FTR_GTSE) && !local) { + if (!mmu_has_feature(MMU_FTR_GTSE) && type == FLUSH_TYPE_GLOBAL) { unsigned long tgt = H_RPTI_TARGET_CMMU; unsigned long pg_sizes = psize_to_rpti_pgsize(mmu_virtual_psize); @@ -928,8 +1026,8 @@ static inline void __radix__flush_tlb_range(struct mm_struct *mm, tgt |= H_RPTI_TARGET_NMMU; pseries_rpt_invalidate(pid, tgt, H_RPTI_TYPE_TLB, pg_sizes, start, end); - } else if (full) { - if (local) { + } else if (flush_pid) { + if (type == FLUSH_TYPE_LOCAL) { _tlbiel_pid(pid, RIC_FLUSH_TLB); } else { if (cputlb_use_tlbie()) { @@ -952,7 +1050,7 @@ static inline void __radix__flush_tlb_range(struct mm_struct *mm, hflush = true; } - if (local) { + if (type == FLUSH_TYPE_LOCAL) { asm volatile("ptesync": : :"memory"); __tlbiel_va_range(start, end, pid, page_size, mmu_virtual_psize); if (hflush) @@ -974,6 +1072,7 @@ static inline void __radix__flush_tlb_range(struct mm_struct *mm, hstart, hend, pid, PMD_SIZE, MMU_PAGE_2M, false); } } +out: preempt_enable(); } @@ -1085,32 +1184,30 @@ static __always_inline void __radix__flush_tlb_range_psize(struct mm_struct *mm, unsigned int page_shift = mmu_psize_defs[psize].shift; unsigned long page_size = 1UL << page_shift; unsigned long nr_pages = (end - start) >> page_shift; - bool local, full; + bool fullmm = (end == TLB_FLUSH_ALL); + bool flush_pid; + enum tlb_flush_type type; pid = mm->context.id; if (unlikely(pid == MMU_NO_CONTEXT)) return; + fullmm = (end == TLB_FLUSH_ALL); + preempt_disable(); smp_mb(); /* see radix__flush_tlb_mm */ - if (!mm_is_thread_local(mm)) { - if (unlikely(mm_is_singlethreaded(mm))) { - if (end != TLB_FLUSH_ALL) { - exit_flush_lazy_tlbs(mm); - goto is_local; - } - } - local = false; - full = (end == TLB_FLUSH_ALL || - nr_pages > tlb_single_page_flush_ceiling); - } else { -is_local: - local = true; - full = (end == TLB_FLUSH_ALL || - nr_pages > tlb_local_single_page_flush_ceiling); - } + type = flush_type_needed(mm, fullmm); + if (type == FLUSH_TYPE_NONE) + goto out; + + if (fullmm) + flush_pid = true; + else if (type == FLUSH_TYPE_GLOBAL) + flush_pid = nr_pages > tlb_single_page_flush_ceiling; + else + flush_pid = nr_pages > tlb_local_single_page_flush_ceiling; - if (!mmu_has_feature(MMU_FTR_GTSE) && !local) { + if (!mmu_has_feature(MMU_FTR_GTSE) && type == FLUSH_TYPE_GLOBAL) { unsigned long tgt = H_RPTI_TARGET_CMMU; unsigned long type = H_RPTI_TYPE_TLB; unsigned long pg_sizes = psize_to_rpti_pgsize(psize); @@ -1120,8 +1217,8 @@ static __always_inline void __radix__flush_tlb_range_psize(struct mm_struct *mm, if (atomic_read(&mm->context.copros) > 0) tgt |= H_RPTI_TARGET_NMMU; pseries_rpt_invalidate(pid, tgt, type, pg_sizes, start, end); - } else if (full) { - if (local) { + } else if (flush_pid) { + if (type == FLUSH_TYPE_LOCAL) { _tlbiel_pid(pid, also_pwc ? RIC_FLUSH_ALL : RIC_FLUSH_TLB); } else { if (cputlb_use_tlbie()) { @@ -1137,7 +1234,7 @@ static __always_inline void __radix__flush_tlb_range_psize(struct mm_struct *mm, } } else { - if (local) + if (type == FLUSH_TYPE_LOCAL) _tlbiel_va_range(start, end, pid, page_size, psize, also_pwc); else if (cputlb_use_tlbie()) _tlbie_va_range(start, end, pid, page_size, psize, also_pwc); @@ -1145,6 +1242,7 @@ static __always_inline void __radix__flush_tlb_range_psize(struct mm_struct *mm, _tlbiel_va_range_multicast(mm, start, end, pid, page_size, psize, also_pwc); } +out: preempt_enable(); } @@ -1164,6 +1262,7 @@ static void radix__flush_tlb_pwc_range_psize(struct mm_struct *mm, unsigned long void radix__flush_tlb_collapsed_pmd(struct mm_struct *mm, unsigned long addr) { unsigned long pid, end; + enum tlb_flush_type type; pid = mm->context.id; if (unlikely(pid == MMU_NO_CONTEXT)) @@ -1180,11 +1279,10 @@ void radix__flush_tlb_collapsed_pmd(struct mm_struct *mm, unsigned long addr) /* Otherwise first do the PWC, then iterate the pages. */ preempt_disable(); smp_mb(); /* see radix__flush_tlb_mm */ - if (!mm_is_thread_local(mm)) { - if (unlikely(mm_is_singlethreaded(mm))) { - exit_flush_lazy_tlbs(mm); - goto local; - } + type = flush_type_needed(mm, false); + if (type == FLUSH_TYPE_LOCAL) { + _tlbiel_va_range(addr, end, pid, PAGE_SIZE, mmu_virtual_psize, true); + } else if (type == FLUSH_TYPE_GLOBAL) { if (!mmu_has_feature(MMU_FTR_GTSE)) { unsigned long tgt, type, pg_sizes; @@ -1202,9 +1300,6 @@ void radix__flush_tlb_collapsed_pmd(struct mm_struct *mm, unsigned long addr) else _tlbiel_va_range_multicast(mm, addr, end, pid, PAGE_SIZE, mmu_virtual_psize, true); - } else { -local: - _tlbiel_va_range(addr, end, pid, PAGE_SIZE, mmu_virtual_psize, true); } preempt_enable(); diff --git a/arch/powerpc/mm/book3s64/slb.c b/arch/powerpc/mm/book3s64/slb.c index 584567970c116e3e82ede9b4f08edc008eb3283e..c91bd85eb90e3b3a2f876c5528c3241dde4241e4 100644 --- a/arch/powerpc/mm/book3s64/slb.c +++ b/arch/powerpc/mm/book3s64/slb.c @@ -10,6 +10,7 @@ */ #include +#include #include #include #include @@ -813,8 +814,9 @@ static long slb_allocate_user(struct mm_struct *mm, unsigned long ea) return slb_insert_entry(ea, context, flags, ssize, false); } -long do_slb_fault(struct pt_regs *regs, unsigned long ea) +DEFINE_INTERRUPT_HANDLER_RAW(do_slb_fault) { + unsigned long ea = regs->dar; unsigned long id = get_region_id(ea); /* IRQs are not reconciled here, so can't check irqs_disabled */ @@ -824,19 +826,21 @@ long do_slb_fault(struct pt_regs *regs, unsigned long ea) return -EINVAL; /* - * SLB kernel faults must be very careful not to touch anything - * that is not bolted. E.g., PACA and global variables are okay, - * mm->context stuff is not. - * - * SLB user faults can access all of kernel memory, but must be - * careful not to touch things like IRQ state because it is not - * "reconciled" here. The difficulty is that we must use - * fast_exception_return to return from kernel SLB faults without - * looking at possible non-bolted memory. We could test user vs - * kernel faults in the interrupt handler asm and do a full fault, - * reconcile, ret_from_except for user faults which would make them - * first class kernel code. But for performance it's probably nicer - * if they go via fast_exception_return too. + * SLB kernel faults must be very careful not to touch anything that is + * not bolted. E.g., PACA and global variables are okay, mm->context + * stuff is not. SLB user faults may access all of memory (and induce + * one recursive SLB kernel fault), so the kernel fault must not + * trample on the user fault state at those points. + */ + + /* + * This is a raw interrupt handler, for performance, so that + * fast_interrupt_return can be used. The handler must not touch local + * irq state, or schedule. We could test for usermode and upgrade to a + * normal process context (synchronous) interrupt for those, which + * would make them first-class kernel code and able to be traced and + * instrumented, although performance would suffer a bit, it would + * probably be a good tradeoff. */ if (id >= LINEAR_MAP_REGION_ID) { long err; @@ -865,13 +869,15 @@ long do_slb_fault(struct pt_regs *regs, unsigned long ea) } } -void do_bad_slb_fault(struct pt_regs *regs, unsigned long ea, long err) +DEFINE_INTERRUPT_HANDLER(do_bad_slb_fault) { + int err = regs->result; + if (err == -EFAULT) { if (user_mode(regs)) - _exception(SIGSEGV, regs, SEGV_BNDERR, ea); + _exception(SIGSEGV, regs, SEGV_BNDERR, regs->dar); else - bad_page_fault(regs, ea, SIGSEGV); + bad_page_fault(regs, SIGSEGV); } else if (err == -EINVAL) { unrecoverable_exception(regs); } else { diff --git a/arch/powerpc/mm/fault.c b/arch/powerpc/mm/fault.c index 8961b44f350cd66480905e33dcc753337cf467f6..bb368257b55cb432eedc1486c2ab0b60f2fc1e90 100644 --- a/arch/powerpc/mm/fault.c +++ b/arch/powerpc/mm/fault.c @@ -34,6 +34,7 @@ #include #include +#include #include #include #include @@ -377,18 +378,16 @@ static void sanity_check_fault(bool is_write, bool is_user, /* * For 600- and 800-family processors, the error_code parameter is DSISR - * for a data fault, SRR1 for an instruction fault. For 400-family processors - * the error_code parameter is ESR for a data fault, 0 for an instruction - * fault. - * For 64-bit processors, the error_code parameter is - * - DSISR for a non-SLB data access fault, - * - SRR1 & 0x08000000 for a non-SLB instruction access fault - * - 0 any SLB fault. + * for a data fault, SRR1 for an instruction fault. + * For 400-family processors the error_code parameter is ESR for a data fault, + * 0 for an instruction fault. + * For 64-bit processors, the error_code parameter is DSISR for a data access + * fault, SRR1 & 0x08000000 for an instruction access fault. * * The return value is 0 if the fault was handled, or the signal * number if this is a kernel fault that can't be handled here. */ -static int __do_page_fault(struct pt_regs *regs, unsigned long address, +static int ___do_page_fault(struct pt_regs *regs, unsigned long address, unsigned long error_code) { struct vm_area_struct * vma; @@ -435,9 +434,7 @@ static int __do_page_fault(struct pt_regs *regs, unsigned long address, return bad_area_nosemaphore(regs, address); } - /* We restore the interrupt state now */ - if (!arch_irq_disabled_regs(regs)) - local_irq_enable(); + interrupt_cond_local_irq_enable(regs); perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); @@ -540,34 +537,51 @@ static int __do_page_fault(struct pt_regs *regs, unsigned long address, return 0; } -NOKPROBE_SYMBOL(__do_page_fault); +NOKPROBE_SYMBOL(___do_page_fault); -int do_page_fault(struct pt_regs *regs, unsigned long address, - unsigned long error_code) +static long __do_page_fault(struct pt_regs *regs) { const struct exception_table_entry *entry; - enum ctx_state prev_state = exception_enter(); - int rc = __do_page_fault(regs, address, error_code); - exception_exit(prev_state); - if (likely(!rc)) - return 0; + long err; + + err = ___do_page_fault(regs, regs->dar, regs->dsisr); + if (likely(!err)) + return err; entry = search_exception_tables(regs->nip); - if (unlikely(!entry)) - return rc; + if (likely(entry)) { + instruction_pointer_set(regs, extable_fixup(entry)); + return 0; + } else if (IS_ENABLED(CONFIG_PPC_BOOK3S_64)) { + __bad_page_fault(regs, err); + return 0; + } else { + /* 32 and 64e handle the bad page fault in asm */ + return err; + } +} +NOKPROBE_SYMBOL(__do_page_fault); - instruction_pointer_set(regs, extable_fixup(entry)); +DEFINE_INTERRUPT_HANDLER_RET(do_page_fault) +{ + return __do_page_fault(regs); +} - return 0; +#ifdef CONFIG_PPC_BOOK3S_64 +/* Same as do_page_fault but interrupt entry has already run in do_hash_fault */ +long hash__do_page_fault(struct pt_regs *regs) +{ + return __do_page_fault(regs); } -NOKPROBE_SYMBOL(do_page_fault); +NOKPROBE_SYMBOL(hash__do_page_fault); +#endif /* * bad_page_fault is called when we have a bad access from the kernel. * It is called from the DSI and ISI handlers in head.S and from some * of the procedures in traps.c. */ -void __bad_page_fault(struct pt_regs *regs, unsigned long address, int sig) +void __bad_page_fault(struct pt_regs *regs, int sig) { int is_write = page_fault_is_write(regs->dsisr); @@ -605,7 +619,7 @@ void __bad_page_fault(struct pt_regs *regs, unsigned long address, int sig) die("Kernel access of bad area", regs, sig); } -void bad_page_fault(struct pt_regs *regs, unsigned long address, int sig) +void bad_page_fault(struct pt_regs *regs, int sig) { const struct exception_table_entry *entry; @@ -614,5 +628,12 @@ void bad_page_fault(struct pt_regs *regs, unsigned long address, int sig) if (entry) instruction_pointer_set(regs, extable_fixup(entry)); else - __bad_page_fault(regs, address, sig); + __bad_page_fault(regs, sig); } + +#ifdef CONFIG_PPC_BOOK3S_64 +DEFINE_INTERRUPT_HANDLER(do_bad_page_fault_segv) +{ + bad_page_fault(regs, SIGSEGV); +} +#endif diff --git a/arch/powerpc/mm/hugetlbpage.c b/arch/powerpc/mm/hugetlbpage.c index 8b3cc4d688e8b4c0cc4b76245b0e2912617746fd..d142b76d507d5c765b23df5f79a5734715519d2a 100644 --- a/arch/powerpc/mm/hugetlbpage.c +++ b/arch/powerpc/mm/hugetlbpage.c @@ -217,7 +217,7 @@ void __init pseries_add_gpage(u64 addr, u64 page_size, unsigned long number_of_p } } -int __init pseries_alloc_bootmem_huge_page(struct hstate *hstate) +static int __init pseries_alloc_bootmem_huge_page(struct hstate *hstate) { struct huge_bootmem_page *m; if (nr_gpages == 0) @@ -663,24 +663,6 @@ static int __init hugetlbpage_init(void) arch_initcall(hugetlbpage_init); -void flush_dcache_icache_hugepage(struct page *page) -{ - int i; - void *start; - - BUG_ON(!PageCompound(page)); - - for (i = 0; i < compound_nr(page); i++) { - if (!PageHighMem(page)) { - __flush_dcache_icache(page_address(page+i)); - } else { - start = kmap_atomic(page+i); - __flush_dcache_icache(start); - kunmap_atomic(start); - } - } -} - void __init gigantic_hugetlb_cma_reserve(void) { unsigned long order = 0; diff --git a/arch/powerpc/mm/mem.c b/arch/powerpc/mm/mem.c index afab328d088740fdbe454d4f9d53737fea1371c0..4e8ce6d85232375c09fdf18b401cb19cdf858605 100644 --- a/arch/powerpc/mm/mem.c +++ b/arch/powerpc/mm/mem.c @@ -91,27 +91,6 @@ int __weak remove_section_mapping(unsigned long start, unsigned long end) return -ENODEV; } -#define FLUSH_CHUNK_SIZE SZ_1G -/** - * flush_dcache_range_chunked(): Write any modified data cache blocks out to - * memory and invalidate them, in chunks of up to FLUSH_CHUNK_SIZE - * Does not invalidate the corresponding instruction cache blocks. - * - * @start: the start address - * @stop: the stop address (exclusive) - * @chunk: the max size of the chunks - */ -static void flush_dcache_range_chunked(unsigned long start, unsigned long stop, - unsigned long chunk) -{ - unsigned long i; - - for (i = start; i < stop; i += chunk) { - flush_dcache_range(i, min(stop, i + chunk)); - cond_resched(); - } -} - int __ref arch_create_linear_mapping(int nid, u64 start, u64 size, struct mhp_params *params) { @@ -136,7 +115,6 @@ void __ref arch_remove_linear_mapping(u64 start, u64 size) /* Remove htab bolted mappings for this section of memory */ start = (unsigned long)__va(start); - flush_dcache_range_chunked(start, start + size, FLUSH_CHUNK_SIZE); mutex_lock(&linear_mapping_mutex); ret = remove_section_mapping(start, start + size); @@ -489,19 +467,35 @@ void flush_dcache_page(struct page *page) if (cpu_has_feature(CPU_FTR_COHERENT_ICACHE)) return; /* avoid an atomic op if possible */ - if (test_bit(PG_arch_1, &page->flags)) - clear_bit(PG_arch_1, &page->flags); + if (test_bit(PG_dcache_clean, &page->flags)) + clear_bit(PG_dcache_clean, &page->flags); } EXPORT_SYMBOL(flush_dcache_page); -void flush_dcache_icache_page(struct page *page) +static void flush_dcache_icache_hugepage(struct page *page) { -#ifdef CONFIG_HUGETLB_PAGE - if (PageCompound(page)) { - flush_dcache_icache_hugepage(page); - return; + int i; + void *start; + + BUG_ON(!PageCompound(page)); + + for (i = 0; i < compound_nr(page); i++) { + if (!PageHighMem(page)) { + __flush_dcache_icache(page_address(page+i)); + } else { + start = kmap_atomic(page+i); + __flush_dcache_icache(start); + kunmap_atomic(start); + } } -#endif +} + +void flush_dcache_icache_page(struct page *page) +{ + + if (PageCompound(page)) + return flush_dcache_icache_hugepage(page); + #if defined(CONFIG_PPC_8xx) || defined(CONFIG_PPC64) /* On 8xx there is no need to kmap since highmem is not supported */ __flush_dcache_icache(page_address(page)); diff --git a/arch/powerpc/mm/pgtable.c b/arch/powerpc/mm/pgtable.c index 15555c95cebc70ea708e41753148f73142af0367..3546119401183ed02cc09b77b5fec21d4902fad9 100644 --- a/arch/powerpc/mm/pgtable.c +++ b/arch/powerpc/mm/pgtable.c @@ -26,6 +26,7 @@ #include #include #include +#include static inline int is_exec_fault(void) { @@ -81,9 +82,9 @@ static pte_t set_pte_filter_hash(pte_t pte) struct page *pg = maybe_pte_to_page(pte); if (!pg) return pte; - if (!test_bit(PG_arch_1, &pg->flags)) { + if (!test_bit(PG_dcache_clean, &pg->flags)) { flush_dcache_icache_page(pg); - set_bit(PG_arch_1, &pg->flags); + set_bit(PG_dcache_clean, &pg->flags); } } return pte; @@ -116,13 +117,13 @@ static inline pte_t set_pte_filter(pte_t pte) return pte; /* If the page clean, we move on */ - if (test_bit(PG_arch_1, &pg->flags)) + if (test_bit(PG_dcache_clean, &pg->flags)) return pte; /* If it's an exec fault, we flush the cache and make it clean */ if (is_exec_fault()) { flush_dcache_icache_page(pg); - set_bit(PG_arch_1, &pg->flags); + set_bit(PG_dcache_clean, &pg->flags); return pte; } @@ -161,12 +162,12 @@ static pte_t set_access_flags_filter(pte_t pte, struct vm_area_struct *vma, goto bail; /* If the page is already clean, we move on */ - if (test_bit(PG_arch_1, &pg->flags)) + if (test_bit(PG_dcache_clean, &pg->flags)) goto bail; - /* Clean the page and set PG_arch_1 */ + /* Clean the page and set PG_dcache_clean */ flush_dcache_icache_page(pg); - set_bit(PG_arch_1, &pg->flags); + set_bit(PG_dcache_clean, &pg->flags); bail: return pte_mkexec(pte); diff --git a/arch/powerpc/mm/ptdump/segment_regs.c b/arch/powerpc/mm/ptdump/segment_regs.c index dde2fe8de4b2a867093158981442dab4ac41ef51..565048a0c9be3b45e4c2283ea6e906a191cee9e1 100644 --- a/arch/powerpc/mm/ptdump/segment_regs.c +++ b/arch/powerpc/mm/ptdump/segment_regs.c @@ -10,7 +10,7 @@ static void seg_show(struct seq_file *m, int i) { - u32 val = mfsrin(i << 28); + u32 val = mfsr(i << 28); seq_printf(m, "0x%01x0000000-0x%01xfffffff ", i, i); seq_printf(m, "Kern key %d ", (val >> 30) & 1); diff --git a/arch/powerpc/perf/core-book3s.c b/arch/powerpc/perf/core-book3s.c index 869d999a836ede16d015af8ad8ee93012e1c82b1..6817331e22ffcbea822fa293fbe122c73e5fbd02 100644 --- a/arch/powerpc/perf/core-book3s.c +++ b/arch/powerpc/perf/core-book3s.c @@ -54,6 +54,9 @@ struct cpu_hw_events { struct perf_branch_stack bhrb_stack; struct perf_branch_entry bhrb_entries[BHRB_MAX_ENTRIES]; u64 ic_init; + + /* Store the PMC values */ + unsigned long pmcs[MAX_HWEVENTS]; }; static DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events); @@ -110,10 +113,6 @@ static inline void perf_read_regs(struct pt_regs *regs) { regs->result = 0; } -static inline int perf_intr_is_nmi(struct pt_regs *regs) -{ - return 0; -} static inline int siar_valid(struct pt_regs *regs) { @@ -147,6 +146,17 @@ bool is_sier_available(void) return false; } +/* + * Return PMC value corresponding to the + * index passed. + */ +unsigned long get_pmcs_ext_regs(int idx) +{ + struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events); + + return cpuhw->pmcs[idx]; +} + static bool regs_use_siar(struct pt_regs *regs) { /* @@ -353,15 +363,6 @@ static inline void perf_read_regs(struct pt_regs *regs) regs->result = use_siar; } -/* - * If interrupts were soft-disabled when a PMU interrupt occurs, treat - * it as an NMI. - */ -static inline int perf_intr_is_nmi(struct pt_regs *regs) -{ - return (regs->softe & IRQS_DISABLED); -} - /* * On processors like P7+ that have the SIAR-Valid bit, marked instructions * must be sampled only if the SIAR-valid bit is set. @@ -915,7 +916,7 @@ void perf_event_print_debug(void) */ static int power_check_constraints(struct cpu_hw_events *cpuhw, u64 event_id[], unsigned int cflags[], - int n_ev) + int n_ev, struct perf_event **event) { unsigned long mask, value, nv; unsigned long smasks[MAX_HWEVENTS], svalues[MAX_HWEVENTS]; @@ -938,7 +939,7 @@ static int power_check_constraints(struct cpu_hw_events *cpuhw, event_id[i] = cpuhw->alternatives[i][0]; } if (ppmu->get_constraint(event_id[i], &cpuhw->amasks[i][0], - &cpuhw->avalues[i][0])) + &cpuhw->avalues[i][0], event[i]->attr.config1)) return -1; } value = mask = 0; @@ -973,7 +974,8 @@ static int power_check_constraints(struct cpu_hw_events *cpuhw, for (j = 1; j < n_alt[i]; ++j) ppmu->get_constraint(cpuhw->alternatives[i][j], &cpuhw->amasks[i][j], - &cpuhw->avalues[i][j]); + &cpuhw->avalues[i][j], + event[i]->attr.config1); } /* enumerate all possibilities and see if any will work */ @@ -1391,7 +1393,7 @@ static void power_pmu_enable(struct pmu *pmu) memset(&cpuhw->mmcr, 0, sizeof(cpuhw->mmcr)); if (ppmu->compute_mmcr(cpuhw->events, cpuhw->n_events, hwc_index, - &cpuhw->mmcr, cpuhw->event)) { + &cpuhw->mmcr, cpuhw->event, ppmu->flags)) { /* shouldn't ever get here */ printk(KERN_ERR "oops compute_mmcr failed\n"); goto out; @@ -1579,7 +1581,7 @@ static int power_pmu_add(struct perf_event *event, int ef_flags) if (check_excludes(cpuhw->event, cpuhw->flags, n0, 1)) goto out; - if (power_check_constraints(cpuhw, cpuhw->events, cpuhw->flags, n0 + 1)) + if (power_check_constraints(cpuhw, cpuhw->events, cpuhw->flags, n0 + 1, cpuhw->event)) goto out; event->hw.config = cpuhw->events[n0]; @@ -1789,7 +1791,7 @@ static int power_pmu_commit_txn(struct pmu *pmu) n = cpuhw->n_events; if (check_excludes(cpuhw->event, cpuhw->flags, 0, n)) return -EAGAIN; - i = power_check_constraints(cpuhw, cpuhw->events, cpuhw->flags, n); + i = power_check_constraints(cpuhw, cpuhw->events, cpuhw->flags, n, cpuhw->event); if (i < 0) return -EAGAIN; @@ -2027,7 +2029,7 @@ static int power_pmu_event_init(struct perf_event *event) local_irq_save(irq_flags); cpuhw = this_cpu_ptr(&cpu_hw_events); - err = power_check_constraints(cpuhw, events, cflags, n + 1); + err = power_check_constraints(cpuhw, events, cflags, n + 1, ctrs); if (has_branch_stack(event)) { u64 bhrb_filter = -1; @@ -2149,7 +2151,17 @@ static void record_and_restart(struct perf_event *event, unsigned long val, left += period; if (left <= 0) left = period; - record = siar_valid(regs); + + /* + * If address is not requested in the sample via + * PERF_SAMPLE_IP, just record that sample irrespective + * of SIAR valid check. + */ + if (event->attr.sample_type & PERF_SAMPLE_IP) + record = siar_valid(regs); + else + record = 1; + event->hw.last_period = event->hw.sample_period; } if (left < 0x80000000LL) @@ -2167,9 +2179,10 @@ static void record_and_restart(struct perf_event *event, unsigned long val, * MMCR2. Check attr.exclude_kernel and address to drop the sample in * these cases. */ - if (event->attr.exclude_kernel && record) - if (is_kernel_addr(mfspr(SPRN_SIAR))) - record = 0; + if (event->attr.exclude_kernel && + (event->attr.sample_type & PERF_SAMPLE_IP) && + is_kernel_addr(mfspr(SPRN_SIAR))) + record = 0; /* * Finally record data if requested. @@ -2277,9 +2290,7 @@ static void __perf_event_interrupt(struct pt_regs *regs) int i, j; struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events); struct perf_event *event; - unsigned long val[8]; int found, active; - int nmi; if (cpuhw->n_limited) freeze_limited_counters(cpuhw, mfspr(SPRN_PMC5), @@ -2287,26 +2298,14 @@ static void __perf_event_interrupt(struct pt_regs *regs) perf_read_regs(regs); - /* - * If perf interrupts hit in a local_irq_disable (soft-masked) region, - * we consider them as NMIs. This is required to prevent hash faults on - * user addresses when reading callchains. See the NMI test in - * do_hash_page. - */ - nmi = perf_intr_is_nmi(regs); - if (nmi) - nmi_enter(); - else - irq_enter(); - /* Read all the PMCs since we'll need them a bunch of times */ for (i = 0; i < ppmu->n_counter; ++i) - val[i] = read_pmc(i + 1); + cpuhw->pmcs[i] = read_pmc(i + 1); /* Try to find what caused the IRQ */ found = 0; for (i = 0; i < ppmu->n_counter; ++i) { - if (!pmc_overflow(val[i])) + if (!pmc_overflow(cpuhw->pmcs[i])) continue; if (is_limited_pmc(i + 1)) continue; /* these won't generate IRQs */ @@ -2321,7 +2320,7 @@ static void __perf_event_interrupt(struct pt_regs *regs) event = cpuhw->event[j]; if (event->hw.idx == (i + 1)) { active = 1; - record_and_restart(event, val[i], regs); + record_and_restart(event, cpuhw->pmcs[i], regs); break; } } @@ -2335,17 +2334,17 @@ static void __perf_event_interrupt(struct pt_regs *regs) event = cpuhw->event[i]; if (!event->hw.idx || is_limited_pmc(event->hw.idx)) continue; - if (pmc_overflow_power7(val[event->hw.idx - 1])) { + if (pmc_overflow_power7(cpuhw->pmcs[event->hw.idx - 1])) { /* event has overflowed in a buggy way*/ found = 1; record_and_restart(event, - val[event->hw.idx - 1], + cpuhw->pmcs[event->hw.idx - 1], regs); } } } - if (!found && !nmi && printk_ratelimit()) - printk(KERN_WARNING "Can't find PMC that caused IRQ\n"); + if (unlikely(!found) && !arch_irq_disabled_regs(regs)) + printk_ratelimited(KERN_WARNING "Can't find PMC that caused IRQ\n"); /* * Reset MMCR0 to its normal value. This will set PMXE and @@ -2356,10 +2355,9 @@ static void __perf_event_interrupt(struct pt_regs *regs) */ write_mmcr0(cpuhw, cpuhw->mmcr.mmcr0); - if (nmi) - nmi_exit(); - else - irq_exit(); + /* Clear the cpuhw->pmcs */ + memset(&cpuhw->pmcs, 0, sizeof(cpuhw->pmcs)); + } static void perf_event_interrupt(struct pt_regs *regs) diff --git a/arch/powerpc/perf/core-fsl-emb.c b/arch/powerpc/perf/core-fsl-emb.c index e0e7e276bfd25aeecf709ee49399ec87692bcb16..ee721f420a7bac495c3363b74bf78a8c6e0b044e 100644 --- a/arch/powerpc/perf/core-fsl-emb.c +++ b/arch/powerpc/perf/core-fsl-emb.c @@ -31,19 +31,6 @@ static atomic_t num_events; /* Used to avoid races in calling reserve/release_pmc_hardware */ static DEFINE_MUTEX(pmc_reserve_mutex); -/* - * If interrupts were soft-disabled when a PMU interrupt occurs, treat - * it as an NMI. - */ -static inline int perf_intr_is_nmi(struct pt_regs *regs) -{ -#ifdef __powerpc64__ - return (regs->softe & IRQS_DISABLED); -#else - return 0; -#endif -} - static void perf_event_interrupt(struct pt_regs *regs); /* @@ -659,13 +646,6 @@ static void perf_event_interrupt(struct pt_regs *regs) struct perf_event *event; unsigned long val; int found = 0; - int nmi; - - nmi = perf_intr_is_nmi(regs); - if (nmi) - nmi_enter(); - else - irq_enter(); for (i = 0; i < ppmu->n_counter; ++i) { event = cpuhw->event[i]; @@ -690,11 +670,6 @@ static void perf_event_interrupt(struct pt_regs *regs) mtmsr(mfmsr() | MSR_PMM); mtpmr(PMRN_PMGC0, PMGC0_PMIE | PMGC0_FCECE); isync(); - - if (nmi) - nmi_exit(); - else - irq_exit(); } void hw_perf_event_setup(int cpu) diff --git a/arch/powerpc/perf/hv-24x7.c b/arch/powerpc/perf/hv-24x7.c index 6e7e820508df7c071f7ef38ecf2eba6c461a2944..e5eb33255066c4bc1e8638fc9597aa3db6845e28 100644 --- a/arch/powerpc/perf/hv-24x7.c +++ b/arch/powerpc/perf/hv-24x7.c @@ -764,6 +764,14 @@ static ssize_t catalog_event_len_validate(struct hv_24x7_event_data *event, return ev_len; } +/* + * Return true incase of invalid or dummy events with names like RESERVED* + */ +static bool ignore_event(const char *name) +{ + return strncmp(name, "RESERVED", 8) == 0; +} + #define MAX_4K (SIZE_MAX / 4096) static int create_events_from_catalog(struct attribute ***events_, @@ -894,6 +902,10 @@ static int create_events_from_catalog(struct attribute ***events_, name = event_name(event, &nl); + if (ignore_event(name)) { + junk_events++; + continue; + } if (event->event_group_record_len == 0) { pr_devel("invalid event %zu (%.*s): group_record_len == 0, skipping\n", event_idx, nl, name); @@ -955,6 +967,9 @@ static int create_events_from_catalog(struct attribute ***events_, continue; name = event_name(event, &nl); + if (ignore_event(name)) + continue; + nonce = event_uniq_add(&ev_uniq, name, nl, event->domain); ct = event_data_to_attrs(event_idx, events + event_attr_ct, event, nonce); diff --git a/arch/powerpc/perf/isa207-common.c b/arch/powerpc/perf/isa207-common.c index 6ab5b272090a7eed9b2f3f7a45dd0bf83c99cb11..e4f577da33d8b15b3886571efb62bb53278565f3 100644 --- a/arch/powerpc/perf/isa207-common.c +++ b/arch/powerpc/perf/isa207-common.c @@ -108,12 +108,57 @@ static void mmcra_sdar_mode(u64 event, unsigned long *mmcra) *mmcra |= MMCRA_SDAR_MODE_TLB; } +static u64 p10_thresh_cmp_val(u64 value) +{ + int exp = 0; + u64 result = value; + + if (!value) + return value; + + /* + * Incase of P10, thresh_cmp value is not part of raw event code + * and provided via attr.config1 parameter. To program threshold in MMCRA, + * take a 18 bit number N and shift right 2 places and increment + * the exponent E by 1 until the upper 10 bits of N are zero. + * Write E to the threshold exponent and write the lower 8 bits of N + * to the threshold mantissa. + * The max threshold that can be written is 261120. + */ + if (cpu_has_feature(CPU_FTR_ARCH_31)) { + if (value > 261120) + value = 261120; + while ((64 - __builtin_clzl(value)) > 8) { + exp++; + value >>= 2; + } + + /* + * Note that it is invalid to write a mantissa with the + * upper 2 bits of mantissa being zero, unless the + * exponent is also zero. + */ + if (!(value & 0xC0) && exp) + result = 0; + else + result = (exp << 8) | value; + } + return result; +} + static u64 thresh_cmp_val(u64 value) { + if (cpu_has_feature(CPU_FTR_ARCH_31)) + value = p10_thresh_cmp_val(value); + + /* + * Since location of threshold compare bits in MMCRA + * is different for p8, using different shift value. + */ if (cpu_has_feature(CPU_FTR_ARCH_300)) return value << p9_MMCRA_THR_CMP_SHIFT; - - return value << MMCRA_THR_CMP_SHIFT; + else + return value << MMCRA_THR_CMP_SHIFT; } static unsigned long combine_from_event(u64 event) @@ -141,13 +186,13 @@ static bool is_thresh_cmp_valid(u64 event) { unsigned int cmp, exp; + if (cpu_has_feature(CPU_FTR_ARCH_31)) + return p10_thresh_cmp_val(event) != 0; + /* * Check the mantissa upper two bits are not zero, unless the * exponent is also zero. See the THRESH_CMP_MANTISSA doc. - * Power10: thresh_cmp is replaced by l2_l3 event select. */ - if (cpu_has_feature(CPU_FTR_ARCH_31)) - return false; cmp = (event >> EVENT_THR_CMP_SHIFT) & EVENT_THR_CMP_MASK; exp = cmp >> 7; @@ -256,7 +301,7 @@ void isa207_get_mem_weight(u64 *weight) *weight = mantissa << (2 * exp); } -int isa207_get_constraint(u64 event, unsigned long *maskp, unsigned long *valp) +int isa207_get_constraint(u64 event, unsigned long *maskp, unsigned long *valp, u64 event_config1) { unsigned int unit, pmc, cache, ebb; unsigned long mask, value; @@ -355,9 +400,11 @@ int isa207_get_constraint(u64 event, unsigned long *maskp, unsigned long *valp) } if (cpu_has_feature(CPU_FTR_ARCH_31)) { - if (event_is_threshold(event)) { + if (event_is_threshold(event) && is_thresh_cmp_valid(event_config1)) { mask |= CNST_THRESH_CTL_SEL_MASK; value |= CNST_THRESH_CTL_SEL_VAL(event >> EVENT_THRESH_SHIFT); + mask |= p10_CNST_THRESH_CMP_MASK; + value |= p10_CNST_THRESH_CMP_VAL(p10_thresh_cmp_val(event_config1)); } } else if (cpu_has_feature(CPU_FTR_ARCH_300)) { if (event_is_threshold(event) && is_thresh_cmp_valid(event)) { @@ -411,7 +458,7 @@ int isa207_get_constraint(u64 event, unsigned long *maskp, unsigned long *valp) int isa207_compute_mmcr(u64 event[], int n_ev, unsigned int hwc[], struct mmcr_regs *mmcr, - struct perf_event *pevents[]) + struct perf_event *pevents[], u32 flags) { unsigned long mmcra, mmcr1, mmcr2, unit, combine, psel, cache, val; unsigned long mmcr3; @@ -504,6 +551,10 @@ int isa207_compute_mmcr(u64 event[], int n_ev, val = (event[i] >> EVENT_THR_CMP_SHIFT) & EVENT_THR_CMP_MASK; mmcra |= thresh_cmp_val(val); + } else if (flags & PPMU_HAS_ATTR_CONFIG1) { + val = (pevents[i]->attr.config1 >> p10_EVENT_THR_CMP_SHIFT) & + p10_EVENT_THR_CMP_MASK; + mmcra |= thresh_cmp_val(val); } } diff --git a/arch/powerpc/perf/isa207-common.h b/arch/powerpc/perf/isa207-common.h index 454b32c3144065af989a342194e121358f22b917..1af0e8c97ac7bf7b473ed51a3b14e314b00649ff 100644 --- a/arch/powerpc/perf/isa207-common.h +++ b/arch/powerpc/perf/isa207-common.h @@ -105,6 +105,10 @@ #define p10_EVENT_RADIX_SCOPE_QUAL_MASK 0x1 #define p10_MMCR1_RADIX_SCOPE_QUAL_SHIFT 45 +/* Event Threshold Compare bit constant for power10 in config1 attribute */ +#define p10_EVENT_THR_CMP_SHIFT 0 +#define p10_EVENT_THR_CMP_MASK 0x3FFFFull + #define p10_EVENT_VALID_MASK \ ((p10_SDAR_MODE_MASK << p10_SDAR_MODE_SHIFT | \ (p10_EVENT_THRESH_MASK << EVENT_THRESH_SHIFT) | \ @@ -124,8 +128,8 @@ * 60 56 52 48 44 40 36 32 * | - - - - | - - - - | - - - - | - - - - | - - - - | - - - - | - - - - | - - - - | * [ fab_match ] [ thresh_cmp ] [ thresh_ctl ] [ ] - * | - * thresh_sel -* + * | | + * [ thresh_cmp bits for p10] thresh_sel -* * * 28 24 20 16 12 8 4 0 * | - - - - | - - - - | - - - - | - - - - | - - - - | - - - - | - - - - | - - - - | @@ -152,6 +156,9 @@ #define CNST_THRESH_CTL_SEL_VAL(v) (((v) & 0x7ffull) << 32) #define CNST_THRESH_CTL_SEL_MASK CNST_THRESH_CTL_SEL_VAL(0x7ff) +#define p10_CNST_THRESH_CMP_VAL(v) (((v) & 0x7ffull) << 43) +#define p10_CNST_THRESH_CMP_MASK p10_CNST_THRESH_CMP_VAL(0x7ff) + #define CNST_EBB_VAL(v) (((v) & EVENT_EBB_MASK) << 24) #define CNST_EBB_MASK CNST_EBB_VAL(EVENT_EBB_MASK) @@ -262,10 +269,10 @@ #define PH(a, b) (P(LVL, HIT) | P(a, b)) #define PM(a, b) (P(LVL, MISS) | P(a, b)) -int isa207_get_constraint(u64 event, unsigned long *maskp, unsigned long *valp); +int isa207_get_constraint(u64 event, unsigned long *maskp, unsigned long *valp, u64 event_config1); int isa207_compute_mmcr(u64 event[], int n_ev, unsigned int hwc[], struct mmcr_regs *mmcr, - struct perf_event *pevents[]); + struct perf_event *pevents[], u32 flags); void isa207_disable_pmc(unsigned int pmc, struct mmcr_regs *mmcr); int isa207_get_alternatives(u64 event, u64 alt[], int size, unsigned int flags, const unsigned int ev_alt[][MAX_ALT]); diff --git a/arch/powerpc/perf/mpc7450-pmu.c b/arch/powerpc/perf/mpc7450-pmu.c index 1919e9df9165abfccc9ea38835b6605e47a8a302..e39b15b79a8340cc8c93ec950f0dee74e4a4d751 100644 --- a/arch/powerpc/perf/mpc7450-pmu.c +++ b/arch/powerpc/perf/mpc7450-pmu.c @@ -148,7 +148,7 @@ static u32 classbits[N_CLASSES - 1][2] = { }; static int mpc7450_get_constraint(u64 event, unsigned long *maskp, - unsigned long *valp) + unsigned long *valp, u64 event_config1 __maybe_unused) { int pmc, class; u32 mask, value; @@ -258,7 +258,8 @@ static const u32 pmcsel_mask[N_COUNTER] = { */ static int mpc7450_compute_mmcr(u64 event[], int n_ev, unsigned int hwc[], struct mmcr_regs *mmcr, - struct perf_event *pevents[]) + struct perf_event *pevents[], + u32 flags __maybe_unused) { u8 event_index[N_CLASSES][N_COUNTER]; int n_classevent[N_CLASSES]; diff --git a/arch/powerpc/perf/perf_regs.c b/arch/powerpc/perf/perf_regs.c index 6f681b105eec21bb1d0375f136aaea0e6a71da1a..b931eed482c9341ac1cf6c8272b37b85698f9cd9 100644 --- a/arch/powerpc/perf/perf_regs.c +++ b/arch/powerpc/perf/perf_regs.c @@ -75,6 +75,8 @@ static unsigned int pt_regs_offset[PERF_REG_POWERPC_MAX] = { static u64 get_ext_regs_value(int idx) { switch (idx) { + case PERF_REG_POWERPC_PMC1 ... PERF_REG_POWERPC_PMC6: + return get_pmcs_ext_regs(idx - PERF_REG_POWERPC_PMC1); case PERF_REG_POWERPC_MMCR0: return mfspr(SPRN_MMCR0); case PERF_REG_POWERPC_MMCR1: @@ -95,13 +97,6 @@ static u64 get_ext_regs_value(int idx) u64 perf_reg_value(struct pt_regs *regs, int idx) { - u64 perf_reg_extended_max = PERF_REG_POWERPC_MAX; - - if (cpu_has_feature(CPU_FTR_ARCH_31)) - perf_reg_extended_max = PERF_REG_MAX_ISA_31; - else if (cpu_has_feature(CPU_FTR_ARCH_300)) - perf_reg_extended_max = PERF_REG_MAX_ISA_300; - if (idx == PERF_REG_POWERPC_SIER && (IS_ENABLED(CONFIG_FSL_EMB_PERF_EVENT) || IS_ENABLED(CONFIG_PPC32) || @@ -113,14 +108,14 @@ u64 perf_reg_value(struct pt_regs *regs, int idx) IS_ENABLED(CONFIG_PPC32))) return 0; - if (idx >= PERF_REG_POWERPC_MAX && idx < perf_reg_extended_max) + if (idx >= PERF_REG_POWERPC_MAX && idx < PERF_REG_EXTENDED_MAX) return get_ext_regs_value(idx); /* * If the idx is referring to value beyond the * supported registers, return 0 with a warning */ - if (WARN_ON_ONCE(idx >= perf_reg_extended_max)) + if (WARN_ON_ONCE(idx >= PERF_REG_EXTENDED_MAX)) return 0; return regs_get_register(regs, pt_regs_offset[idx]); diff --git a/arch/powerpc/perf/power10-pmu.c b/arch/powerpc/perf/power10-pmu.c index 79e0206ca4546d599304c45634f710655600dd50..a901c1348cad0ddf571a4b2f27357bf6fa99d1c0 100644 --- a/arch/powerpc/perf/power10-pmu.c +++ b/arch/powerpc/perf/power10-pmu.c @@ -216,6 +216,7 @@ PMU_FORMAT_ATTR(invert_bit, "config:47"); PMU_FORMAT_ATTR(src_mask, "config:48-53"); PMU_FORMAT_ATTR(src_match, "config:54-59"); PMU_FORMAT_ATTR(radix_scope, "config:9"); +PMU_FORMAT_ATTR(thresh_cmp, "config1:0-17"); static struct attribute *power10_pmu_format_attr[] = { &format_attr_event.attr, @@ -236,6 +237,7 @@ static struct attribute *power10_pmu_format_attr[] = { &format_attr_src_mask.attr, &format_attr_src_match.attr, &format_attr_radix_scope.attr, + &format_attr_thresh_cmp.attr, NULL, }; @@ -550,7 +552,7 @@ static struct power_pmu power10_pmu = { .get_mem_weight = isa207_get_mem_weight, .disable_pmc = isa207_disable_pmc, .flags = PPMU_HAS_SIER | PPMU_ARCH_207S | - PPMU_ARCH_31, + PPMU_ARCH_31 | PPMU_HAS_ATTR_CONFIG1, .n_generic = ARRAY_SIZE(power10_generic_events), .generic_events = power10_generic_events, .cache_events = &power10_cache_events, diff --git a/arch/powerpc/perf/power5+-pmu.c b/arch/powerpc/perf/power5+-pmu.c index 3e64b4a1511f66c5f967443d67872d154599d5ad..18732267993a184a3bb98c57561a38782bc60839 100644 --- a/arch/powerpc/perf/power5+-pmu.c +++ b/arch/powerpc/perf/power5+-pmu.c @@ -132,7 +132,7 @@ static unsigned long unit_cons[PM_LASTUNIT+1][2] = { }; static int power5p_get_constraint(u64 event, unsigned long *maskp, - unsigned long *valp) + unsigned long *valp, u64 event_config1 __maybe_unused) { int pmc, byte, unit, sh; int bit, fmask; @@ -451,7 +451,8 @@ static int power5p_marked_instr_event(u64 event) static int power5p_compute_mmcr(u64 event[], int n_ev, unsigned int hwc[], struct mmcr_regs *mmcr, - struct perf_event *pevents[]) + struct perf_event *pevents[], + u32 flags __maybe_unused) { unsigned long mmcr1 = 0; unsigned long mmcra = 0; diff --git a/arch/powerpc/perf/power5-pmu.c b/arch/powerpc/perf/power5-pmu.c index 017bb19b73fbbc1bcfa8f511ffc23a81a1b70924..cb611c1e7abefe99e6d1d6caf7082f895cd9b9ac 100644 --- a/arch/powerpc/perf/power5-pmu.c +++ b/arch/powerpc/perf/power5-pmu.c @@ -136,7 +136,7 @@ static unsigned long unit_cons[PM_LASTUNIT+1][2] = { }; static int power5_get_constraint(u64 event, unsigned long *maskp, - unsigned long *valp) + unsigned long *valp, u64 event_config1 __maybe_unused) { int pmc, byte, unit, sh; int bit, fmask; @@ -382,7 +382,8 @@ static int power5_marked_instr_event(u64 event) static int power5_compute_mmcr(u64 event[], int n_ev, unsigned int hwc[], struct mmcr_regs *mmcr, - struct perf_event *pevents[]) + struct perf_event *pevents[], + u32 flags __maybe_unused) { unsigned long mmcr1 = 0; unsigned long mmcra = MMCRA_SDAR_DCACHE_MISS | MMCRA_SDAR_ERAT_MISS; diff --git a/arch/powerpc/perf/power6-pmu.c b/arch/powerpc/perf/power6-pmu.c index 189974478e9fcce8d59d7d9c89aebd1c039028a3..69ef382164186f118ce7bb66ae7bcc07a42c8869 100644 --- a/arch/powerpc/perf/power6-pmu.c +++ b/arch/powerpc/perf/power6-pmu.c @@ -173,7 +173,8 @@ static int power6_marked_instr_event(u64 event) * Assign PMC numbers and compute MMCR1 value for a set of events */ static int p6_compute_mmcr(u64 event[], int n_ev, - unsigned int hwc[], struct mmcr_regs *mmcr, struct perf_event *pevents[]) + unsigned int hwc[], struct mmcr_regs *mmcr, struct perf_event *pevents[], + u32 flags __maybe_unused) { unsigned long mmcr1 = 0; unsigned long mmcra = MMCRA_SDAR_DCACHE_MISS | MMCRA_SDAR_ERAT_MISS; @@ -266,7 +267,7 @@ static int p6_compute_mmcr(u64 event[], int n_ev, * 32-34 select field: nest (subunit) event selector */ static int p6_get_constraint(u64 event, unsigned long *maskp, - unsigned long *valp) + unsigned long *valp, u64 event_config1 __maybe_unused) { int pmc, byte, sh, subunit; unsigned long mask = 0, value = 0; diff --git a/arch/powerpc/perf/power7-pmu.c b/arch/powerpc/perf/power7-pmu.c index bacfab104a1a15128ea991303100471f3612bd6d..894c17f9a762b4c054da5e6149b94415994160a6 100644 --- a/arch/powerpc/perf/power7-pmu.c +++ b/arch/powerpc/perf/power7-pmu.c @@ -81,7 +81,7 @@ enum { */ static int power7_get_constraint(u64 event, unsigned long *maskp, - unsigned long *valp) + unsigned long *valp, u64 event_config1 __maybe_unused) { int pmc, sh, unit; unsigned long mask = 0, value = 0; @@ -245,7 +245,8 @@ static int power7_marked_instr_event(u64 event) static int power7_compute_mmcr(u64 event[], int n_ev, unsigned int hwc[], struct mmcr_regs *mmcr, - struct perf_event *pevents[]) + struct perf_event *pevents[], + u32 flags __maybe_unused) { unsigned long mmcr1 = 0; unsigned long mmcra = MMCRA_SDAR_DCACHE_MISS | MMCRA_SDAR_ERAT_MISS; diff --git a/arch/powerpc/perf/ppc970-pmu.c b/arch/powerpc/perf/ppc970-pmu.c index 7d78df97f2721c1da9bb84eadd5128b2f0056b88..1f82637852862ac3a0ef4e240f48dca98134fc8a 100644 --- a/arch/powerpc/perf/ppc970-pmu.c +++ b/arch/powerpc/perf/ppc970-pmu.c @@ -190,7 +190,7 @@ static unsigned long unit_cons[PM_LASTUNIT+1][2] = { }; static int p970_get_constraint(u64 event, unsigned long *maskp, - unsigned long *valp) + unsigned long *valp, u64 event_config1 __maybe_unused) { int pmc, byte, unit, sh, spcsel; unsigned long mask = 0, value = 0; @@ -256,7 +256,8 @@ static int p970_get_alternatives(u64 event, unsigned int flags, u64 alt[]) static int p970_compute_mmcr(u64 event[], int n_ev, unsigned int hwc[], struct mmcr_regs *mmcr, - struct perf_event *pevents[]) + struct perf_event *pevents[], + u32 flags __maybe_unused) { unsigned long mmcr0 = 0, mmcr1 = 0, mmcra = 0; unsigned int pmc, unit, byte, psel; diff --git a/arch/powerpc/platforms/44x/Kconfig b/arch/powerpc/platforms/44x/Kconfig index 78ac6d67a9356fc210c1e68d5a69229334e6cf43..7d41e9264510e5a276a422374728bfa655678553 100644 --- a/arch/powerpc/platforms/44x/Kconfig +++ b/arch/powerpc/platforms/44x/Kconfig @@ -206,17 +206,10 @@ config AKEBONO select PPC4xx_HSTA_MSI select I2C select I2C_IBM_IIC - select NETDEVICES - select ETHERNET - select NET_VENDOR_IBM select IBM_EMAC_EMAC4 if IBM_EMAC select USB if USB_SUPPORT select USB_OHCI_HCD_PLATFORM if USB_OHCI_HCD select USB_EHCI_HCD_PLATFORM if USB_EHCI_HCD - select MMC_SDHCI - select MMC_SDHCI_PLTFM - select ATA - select SATA_AHCI_PLATFORM help This option enables support for the IBM Akebono (476gtr) evaluation board diff --git a/arch/powerpc/platforms/512x/mpc5121_ads.c b/arch/powerpc/platforms/512x/mpc5121_ads.c index 6303fbfc4e4f7382b87ef1e79fd6ec23b3fc9511..9d030c2e000470c0f377833a5d7c9f974f60ae89 100644 --- a/arch/powerpc/platforms/512x/mpc5121_ads.c +++ b/arch/powerpc/platforms/512x/mpc5121_ads.c @@ -24,21 +24,23 @@ static void __init mpc5121_ads_setup_arch(void) { -#ifdef CONFIG_PCI - struct device_node *np; -#endif printk(KERN_INFO "MPC5121 ADS board from Freescale Semiconductor\n"); /* * cpld regs are needed early */ mpc5121_ads_cpld_map(); + mpc512x_setup_arch(); +} + +static void __init mpc5121_ads_setup_pci(void) +{ #ifdef CONFIG_PCI + struct device_node *np; + for_each_compatible_node(np, "pci", "fsl,mpc5121-pci") mpc83xx_add_bridge(np); #endif - - mpc512x_setup_arch(); } static void __init mpc5121_ads_init_IRQ(void) @@ -64,6 +66,7 @@ define_machine(mpc5121_ads) { .name = "MPC5121 ADS", .probe = mpc5121_ads_probe, .setup_arch = mpc5121_ads_setup_arch, + .discover_phbs = mpc5121_ads_setup_pci, .init = mpc512x_init, .init_IRQ = mpc5121_ads_init_IRQ, .get_irq = ipic_get_irq, diff --git a/arch/powerpc/platforms/52xx/efika.c b/arch/powerpc/platforms/52xx/efika.c index 4514a6f7458a274b78a599f59b539c72c4456428..3b7d70d716925ffdde300245b99991efd9d759c5 100644 --- a/arch/powerpc/platforms/52xx/efika.c +++ b/arch/powerpc/platforms/52xx/efika.c @@ -185,8 +185,6 @@ static void __init efika_setup_arch(void) /* Map important registers from the internal memory map */ mpc52xx_map_common_devices(); - efika_pcisetup(); - #ifdef CONFIG_PM mpc52xx_suspend.board_suspend_prepare = efika_suspend_prepare; mpc52xx_pm_init(); @@ -218,6 +216,7 @@ define_machine(efika) .name = EFIKA_PLATFORM_NAME, .probe = efika_probe, .setup_arch = efika_setup_arch, + .discover_phbs = efika_pcisetup, .init = mpc52xx_declare_of_platform_devices, .show_cpuinfo = efika_show_cpuinfo, .init_IRQ = mpc52xx_init_irq, diff --git a/arch/powerpc/platforms/52xx/lite5200.c b/arch/powerpc/platforms/52xx/lite5200.c index 3181aac0822532e5870c28ff32c4afa48faceca7..04cc97397095a29fa4f3e5741766dfbbe5928399 100644 --- a/arch/powerpc/platforms/52xx/lite5200.c +++ b/arch/powerpc/platforms/52xx/lite5200.c @@ -165,8 +165,6 @@ static void __init lite5200_setup_arch(void) mpc52xx_suspend.board_resume_finish = lite5200_resume_finish; lite5200_pm_init(); #endif - - mpc52xx_setup_pci(); } static const char * const board[] __initconst = { @@ -187,6 +185,7 @@ define_machine(lite5200) { .name = "lite5200", .probe = lite5200_probe, .setup_arch = lite5200_setup_arch, + .discover_phbs = mpc52xx_setup_pci, .init = mpc52xx_declare_of_platform_devices, .init_IRQ = mpc52xx_init_irq, .get_irq = mpc52xx_get_irq, diff --git a/arch/powerpc/platforms/52xx/media5200.c b/arch/powerpc/platforms/52xx/media5200.c index 07c5bc4ed0b5273e0e8c69c987851eac91591e9e..efb8bdecbcc7190fc95f34dc40f555221e3aca14 100644 --- a/arch/powerpc/platforms/52xx/media5200.c +++ b/arch/powerpc/platforms/52xx/media5200.c @@ -202,8 +202,6 @@ static void __init media5200_setup_arch(void) /* Some mpc5200 & mpc5200b related configuration */ mpc5200_setup_xlb_arbiter(); - mpc52xx_setup_pci(); - np = of_find_matching_node(NULL, mpc5200_gpio_ids); gpio = of_iomap(np, 0); of_node_put(np); @@ -244,6 +242,7 @@ define_machine(media5200_platform) { .name = "media5200-platform", .probe = media5200_probe, .setup_arch = media5200_setup_arch, + .discover_phbs = mpc52xx_setup_pci, .init = mpc52xx_declare_of_platform_devices, .init_IRQ = media5200_init_irq, .get_irq = mpc52xx_get_irq, diff --git a/arch/powerpc/platforms/52xx/mpc5200_simple.c b/arch/powerpc/platforms/52xx/mpc5200_simple.c index 2d01e9b2e779b9ff2105648bafde19219c6d8def..b9f5675b0a1d01fb98a07392a91ccb94ba8f4c32 100644 --- a/arch/powerpc/platforms/52xx/mpc5200_simple.c +++ b/arch/powerpc/platforms/52xx/mpc5200_simple.c @@ -40,8 +40,6 @@ static void __init mpc5200_simple_setup_arch(void) /* Some mpc5200 & mpc5200b related configuration */ mpc5200_setup_xlb_arbiter(); - - mpc52xx_setup_pci(); } /* list of the supported boards */ @@ -73,6 +71,7 @@ define_machine(mpc5200_simple_platform) { .name = "mpc5200-simple-platform", .probe = mpc5200_simple_probe, .setup_arch = mpc5200_simple_setup_arch, + .discover_phbs = mpc52xx_setup_pci, .init = mpc52xx_declare_of_platform_devices, .init_IRQ = mpc52xx_init_irq, .get_irq = mpc52xx_get_irq, diff --git a/arch/powerpc/platforms/52xx/mpc52xx_lpbfifo.c b/arch/powerpc/platforms/52xx/mpc52xx_lpbfifo.c index 05e19470d523c17f2bbea7c3a645bd186e712701..b91ebebd9ff20e70da9a38f1c3477215ec80ae0b 100644 --- a/arch/powerpc/platforms/52xx/mpc52xx_lpbfifo.c +++ b/arch/powerpc/platforms/52xx/mpc52xx_lpbfifo.c @@ -229,7 +229,7 @@ static irqreturn_t mpc52xx_lpbfifo_irq(int irq, void *dev_id) int dma, write, poll_dma; spin_lock_irqsave(&lpbfifo.lock, flags); - ts = get_tbl(); + ts = mftb(); req = lpbfifo.req; if (!req) { @@ -307,7 +307,7 @@ static irqreturn_t mpc52xx_lpbfifo_irq(int irq, void *dev_id) if (irq != 0) /* don't increment on polled case */ req->irq_count++; - req->irq_ticks += get_tbl() - ts; + req->irq_ticks += mftb() - ts; spin_unlock_irqrestore(&lpbfifo.lock, flags); /* Spinlock is released; it is now safe to call the callback */ @@ -330,7 +330,7 @@ static irqreturn_t mpc52xx_lpbfifo_bcom_irq(int irq, void *dev_id) u32 ts; spin_lock_irqsave(&lpbfifo.lock, flags); - ts = get_tbl(); + ts = mftb(); req = lpbfifo.req; if (!req || (req->flags & MPC52XX_LPBFIFO_FLAG_NO_DMA)) { @@ -361,7 +361,7 @@ static irqreturn_t mpc52xx_lpbfifo_bcom_irq(int irq, void *dev_id) lpbfifo.req = NULL; /* Release the lock before calling out to the callback. */ - req->irq_ticks += get_tbl() - ts; + req->irq_ticks += mftb() - ts; spin_unlock_irqrestore(&lpbfifo.lock, flags); if (req->callback) diff --git a/arch/powerpc/platforms/82xx/mpc8272_ads.c b/arch/powerpc/platforms/82xx/mpc8272_ads.c index 3fe1a65932807f044b5c91626e14f96b23726d8e..0b5b9dec16d5569ccdb469b4aad516df44abc583 100644 --- a/arch/powerpc/platforms/82xx/mpc8272_ads.c +++ b/arch/powerpc/platforms/82xx/mpc8272_ads.c @@ -171,7 +171,6 @@ static void __init mpc8272_ads_setup_arch(void) iounmap(bcsr); init_ioports(); - pq2_init_pci(); if (ppc_md.progress) ppc_md.progress("mpc8272_ads_setup_arch(), finish", 0); @@ -205,6 +204,7 @@ define_machine(mpc8272_ads) .name = "Freescale MPC8272 ADS", .probe = mpc8272_ads_probe, .setup_arch = mpc8272_ads_setup_arch, + .discover_phbs = pq2_init_pci, .init_IRQ = mpc8272_ads_pic_init, .get_irq = cpm2_get_irq, .calibrate_decr = generic_calibrate_decr, diff --git a/arch/powerpc/platforms/82xx/pq2ads-pci-pic.c b/arch/powerpc/platforms/82xx/pq2ads-pci-pic.c index 096cc0d59fd8813dbef0ce9756e67c3ecaa24dc7..f82f75a6085cf8b0a419351221bb5e54ef092442 100644 --- a/arch/powerpc/platforms/82xx/pq2ads-pci-pic.c +++ b/arch/powerpc/platforms/82xx/pq2ads-pci-pic.c @@ -123,20 +123,17 @@ int __init pq2ads_pci_init_irq(void) np = of_find_compatible_node(NULL, NULL, "fsl,pq2ads-pci-pic"); if (!np) { printk(KERN_ERR "No pci pic node in device tree.\n"); - of_node_put(np); goto out; } irq = irq_of_parse_and_map(np, 0); if (!irq) { printk(KERN_ERR "No interrupt in pci pic node.\n"); - of_node_put(np); - goto out; + goto out_put_node; } priv = kzalloc(sizeof(*priv), GFP_KERNEL); if (!priv) { - of_node_put(np); ret = -ENOMEM; goto out_unmap_irq; } @@ -161,17 +158,17 @@ int __init pq2ads_pci_init_irq(void) priv->host = host; irq_set_handler_data(irq, priv); irq_set_chained_handler(irq, pq2ads_pci_irq_demux); - - of_node_put(np); - return 0; + ret = 0; + goto out_put_node; out_unmap_regs: iounmap(priv->regs); out_free_kmalloc: kfree(priv); - of_node_put(np); out_unmap_irq: irq_dispose_mapping(irq); +out_put_node: + of_node_put(np); out: return ret; } diff --git a/arch/powerpc/platforms/82xx/pq2fads.c b/arch/powerpc/platforms/82xx/pq2fads.c index a74082140718e6fad2ec4ecb2572d9944d46e0e0..ac9113d524af1a9ed6bab873c489b766f794efa6 100644 --- a/arch/powerpc/platforms/82xx/pq2fads.c +++ b/arch/powerpc/platforms/82xx/pq2fads.c @@ -150,8 +150,6 @@ static void __init pq2fads_setup_arch(void) /* Enable external IRQs */ clrbits32(&cpm2_immr->im_siu_conf.siu_82xx.sc_siumcr, 0x0c000000); - pq2_init_pci(); - if (ppc_md.progress) ppc_md.progress("pq2fads_setup_arch(), finish", 0); } @@ -184,6 +182,7 @@ define_machine(pq2fads) .name = "Freescale PQ2FADS", .probe = pq2fads_probe, .setup_arch = pq2fads_setup_arch, + .discover_phbs = pq2_init_pci, .init_IRQ = pq2fads_pic_init, .get_irq = cpm2_get_irq, .calibrate_decr = generic_calibrate_decr, diff --git a/arch/powerpc/platforms/83xx/asp834x.c b/arch/powerpc/platforms/83xx/asp834x.c index 28474876f41b7acc03e3d88d1449ac3cfaecbd5c..68061c2a57c14d1fd0a492d126365abbf1affdf6 100644 --- a/arch/powerpc/platforms/83xx/asp834x.c +++ b/arch/powerpc/platforms/83xx/asp834x.c @@ -44,6 +44,7 @@ define_machine(asp834x) { .name = "ASP8347E", .probe = asp834x_probe, .setup_arch = asp834x_setup_arch, + .discover_phbs = mpc83xx_setup_pci, .init_IRQ = mpc83xx_ipic_init_IRQ, .get_irq = ipic_get_irq, .restart = mpc83xx_restart, diff --git a/arch/powerpc/platforms/83xx/km83xx.c b/arch/powerpc/platforms/83xx/km83xx.c index bcdc2c203ec9391ad3b011a06e2ff31ca042d8ab..108e1e4d2683efdb418edb24c5c2272634fe56eb 100644 --- a/arch/powerpc/platforms/83xx/km83xx.c +++ b/arch/powerpc/platforms/83xx/km83xx.c @@ -180,6 +180,7 @@ define_machine(mpc83xx_km) { .name = "mpc83xx-km-platform", .probe = mpc83xx_km_probe, .setup_arch = mpc83xx_km_setup_arch, + .discover_phbs = mpc83xx_setup_pci, .init_IRQ = mpc83xx_ipic_init_IRQ, .get_irq = ipic_get_irq, .restart = mpc83xx_restart, diff --git a/arch/powerpc/platforms/83xx/misc.c b/arch/powerpc/platforms/83xx/misc.c index a952e91db3eeec2e1ab0040189c00cb32f8d6441..3285dabcf923bbb71282f06a7adfa3354a0b6563 100644 --- a/arch/powerpc/platforms/83xx/misc.c +++ b/arch/powerpc/platforms/83xx/misc.c @@ -132,8 +132,6 @@ void __init mpc83xx_setup_arch(void) setbat(-1, va, immrbase, immrsize, PAGE_KERNEL_NCG); update_bats(); } - - mpc83xx_setup_pci(); } int machine_check_83xx(struct pt_regs *regs) diff --git a/arch/powerpc/platforms/83xx/mpc830x_rdb.c b/arch/powerpc/platforms/83xx/mpc830x_rdb.c index 51426e88ec67989a2329aff6b5e5e3d32fab61a1..956d4389effa23d0392cad0c485f7ef36c3e3171 100644 --- a/arch/powerpc/platforms/83xx/mpc830x_rdb.c +++ b/arch/powerpc/platforms/83xx/mpc830x_rdb.c @@ -48,6 +48,7 @@ define_machine(mpc830x_rdb) { .name = "MPC830x RDB", .probe = mpc830x_rdb_probe, .setup_arch = mpc830x_rdb_setup_arch, + .discover_phbs = mpc83xx_setup_pci, .init_IRQ = mpc83xx_ipic_init_IRQ, .get_irq = ipic_get_irq, .restart = mpc83xx_restart, diff --git a/arch/powerpc/platforms/83xx/mpc831x_rdb.c b/arch/powerpc/platforms/83xx/mpc831x_rdb.c index 5ccd57a4849273c5ca6988c22e8495e4eb713e9d..3b578f080e3b781dcd2862b9fbf2b36e994461cd 100644 --- a/arch/powerpc/platforms/83xx/mpc831x_rdb.c +++ b/arch/powerpc/platforms/83xx/mpc831x_rdb.c @@ -48,6 +48,7 @@ define_machine(mpc831x_rdb) { .name = "MPC831x RDB", .probe = mpc831x_rdb_probe, .setup_arch = mpc831x_rdb_setup_arch, + .discover_phbs = mpc83xx_setup_pci, .init_IRQ = mpc83xx_ipic_init_IRQ, .get_irq = ipic_get_irq, .restart = mpc83xx_restart, diff --git a/arch/powerpc/platforms/83xx/mpc832x_mds.c b/arch/powerpc/platforms/83xx/mpc832x_mds.c index 6fa5402ebf2037b6a7d86fbfc4495b21c9f54ad6..850d566ef900bde97821140f2b6476ed64101b73 100644 --- a/arch/powerpc/platforms/83xx/mpc832x_mds.c +++ b/arch/powerpc/platforms/83xx/mpc832x_mds.c @@ -101,6 +101,7 @@ define_machine(mpc832x_mds) { .name = "MPC832x MDS", .probe = mpc832x_sys_probe, .setup_arch = mpc832x_sys_setup_arch, + .discover_phbs = mpc83xx_setup_pci, .init_IRQ = mpc83xx_ipic_init_IRQ, .get_irq = ipic_get_irq, .restart = mpc83xx_restart, diff --git a/arch/powerpc/platforms/83xx/mpc832x_rdb.c b/arch/powerpc/platforms/83xx/mpc832x_rdb.c index 622c625d5ce4be17bd31b776b92af09079a3d244..b6133a237a70953e867ccd0a39c5f5bab1aeecc2 100644 --- a/arch/powerpc/platforms/83xx/mpc832x_rdb.c +++ b/arch/powerpc/platforms/83xx/mpc832x_rdb.c @@ -219,6 +219,7 @@ define_machine(mpc832x_rdb) { .name = "MPC832x RDB", .probe = mpc832x_rdb_probe, .setup_arch = mpc832x_rdb_setup_arch, + .discover_phbs = mpc83xx_setup_pci, .init_IRQ = mpc83xx_ipic_init_IRQ, .get_irq = ipic_get_irq, .restart = mpc83xx_restart, diff --git a/arch/powerpc/platforms/83xx/mpc834x_itx.c b/arch/powerpc/platforms/83xx/mpc834x_itx.c index ebfd139bca20d644ee2547c1c9ce985d79d6f73b..9630f3aa4d9c08fd9611930fe3e5513c3924aaf7 100644 --- a/arch/powerpc/platforms/83xx/mpc834x_itx.c +++ b/arch/powerpc/platforms/83xx/mpc834x_itx.c @@ -70,6 +70,7 @@ define_machine(mpc834x_itx) { .name = "MPC834x ITX", .probe = mpc834x_itx_probe, .setup_arch = mpc834x_itx_setup_arch, + .discover_phbs = mpc83xx_setup_pci, .init_IRQ = mpc83xx_ipic_init_IRQ, .get_irq = ipic_get_irq, .restart = mpc83xx_restart, diff --git a/arch/powerpc/platforms/83xx/mpc834x_mds.c b/arch/powerpc/platforms/83xx/mpc834x_mds.c index 356228e35279968e913a8b139e8a93452bb0154c..6d91bdce0a18d871baadf7cf72189c884685ed0f 100644 --- a/arch/powerpc/platforms/83xx/mpc834x_mds.c +++ b/arch/powerpc/platforms/83xx/mpc834x_mds.c @@ -91,6 +91,7 @@ define_machine(mpc834x_mds) { .name = "MPC834x MDS", .probe = mpc834x_mds_probe, .setup_arch = mpc834x_mds_setup_arch, + .discover_phbs = mpc83xx_setup_pci, .init_IRQ = mpc83xx_ipic_init_IRQ, .get_irq = ipic_get_irq, .restart = mpc83xx_restart, diff --git a/arch/powerpc/platforms/83xx/mpc836x_mds.c b/arch/powerpc/platforms/83xx/mpc836x_mds.c index 90d9cbfae6591f0eda1b3f38a5ac3e3eb7cb9781..da4cf52cb55b5ba0b943841c53670b4cad7d6c35 100644 --- a/arch/powerpc/platforms/83xx/mpc836x_mds.c +++ b/arch/powerpc/platforms/83xx/mpc836x_mds.c @@ -201,6 +201,7 @@ define_machine(mpc836x_mds) { .name = "MPC836x MDS", .probe = mpc836x_mds_probe, .setup_arch = mpc836x_mds_setup_arch, + .discover_phbs = mpc83xx_setup_pci, .init_IRQ = mpc83xx_ipic_init_IRQ, .get_irq = ipic_get_irq, .restart = mpc83xx_restart, diff --git a/arch/powerpc/platforms/83xx/mpc836x_rdk.c b/arch/powerpc/platforms/83xx/mpc836x_rdk.c index b4aac2cde849ee2b801d7ac21cb67eb4ccc0d9ff..3427ad0d9d387a9bc1639486e8901ed497df54f5 100644 --- a/arch/powerpc/platforms/83xx/mpc836x_rdk.c +++ b/arch/powerpc/platforms/83xx/mpc836x_rdk.c @@ -41,6 +41,7 @@ define_machine(mpc836x_rdk) { .name = "MPC836x RDK", .probe = mpc836x_rdk_probe, .setup_arch = mpc836x_rdk_setup_arch, + .discover_phbs = mpc83xx_setup_pci, .init_IRQ = mpc83xx_ipic_init_IRQ, .get_irq = ipic_get_irq, .restart = mpc83xx_restart, diff --git a/arch/powerpc/platforms/83xx/mpc837x_mds.c b/arch/powerpc/platforms/83xx/mpc837x_mds.c index 9d3721c965be42238b451e69a520432b26b5552f..f28d166ea7db869b939bb49bb4673fa0fffba119 100644 --- a/arch/powerpc/platforms/83xx/mpc837x_mds.c +++ b/arch/powerpc/platforms/83xx/mpc837x_mds.c @@ -93,6 +93,7 @@ define_machine(mpc837x_mds) { .name = "MPC837x MDS", .probe = mpc837x_mds_probe, .setup_arch = mpc837x_mds_setup_arch, + .discover_phbs = mpc83xx_setup_pci, .init_IRQ = mpc83xx_ipic_init_IRQ, .get_irq = ipic_get_irq, .restart = mpc83xx_restart, diff --git a/arch/powerpc/platforms/83xx/mpc837x_rdb.c b/arch/powerpc/platforms/83xx/mpc837x_rdb.c index 7c45f7ac26072838c965a229b54518619c5b1724..7fb7684c256b031334e9240bbfa0fb574703e2e9 100644 --- a/arch/powerpc/platforms/83xx/mpc837x_rdb.c +++ b/arch/powerpc/platforms/83xx/mpc837x_rdb.c @@ -73,6 +73,7 @@ define_machine(mpc837x_rdb) { .name = "MPC837x RDB/WLAN", .probe = mpc837x_rdb_probe, .setup_arch = mpc837x_rdb_setup_arch, + .discover_phbs = mpc83xx_setup_pci, .init_IRQ = mpc83xx_ipic_init_IRQ, .get_irq = ipic_get_irq, .restart = mpc83xx_restart, diff --git a/arch/powerpc/platforms/83xx/mpc83xx.h b/arch/powerpc/platforms/83xx/mpc83xx.h index f37d04332fc7c200a6294ae65a018f981d40792c..a30d30588cf6979c086a7f542d91b8f2f31ffbb6 100644 --- a/arch/powerpc/platforms/83xx/mpc83xx.h +++ b/arch/powerpc/platforms/83xx/mpc83xx.h @@ -76,7 +76,7 @@ extern void mpc83xx_ipic_init_IRQ(void); #ifdef CONFIG_PCI extern void mpc83xx_setup_pci(void); #else -#define mpc83xx_setup_pci() do {} while (0) +#define mpc83xx_setup_pci NULL #endif extern int mpc83xx_declare_of_platform_devices(void); diff --git a/arch/powerpc/platforms/8xx/machine_check.c b/arch/powerpc/platforms/8xx/machine_check.c index 88dedf38eccd175c5a821a7f720faa572f4e7ab4..65636597589519f6a8d16061fa3283f5758d08f1 100644 --- a/arch/powerpc/platforms/8xx/machine_check.c +++ b/arch/powerpc/platforms/8xx/machine_check.c @@ -26,7 +26,7 @@ int machine_check_8xx(struct pt_regs *regs) * to deal with that than having a wart in the mcheck handler. * -- BenH */ - bad_page_fault(regs, regs->dar, SIGBUS); + bad_page_fault(regs, SIGBUS); return 1; #else return 0; diff --git a/arch/powerpc/platforms/amigaone/setup.c b/arch/powerpc/platforms/amigaone/setup.c index f5d0bf9997596ea11f2055785a196bb39fdd7236..9d252c554f7f84685b0935415d0e4c7acdf7df5d 100644 --- a/arch/powerpc/platforms/amigaone/setup.c +++ b/arch/powerpc/platforms/amigaone/setup.c @@ -65,6 +65,12 @@ static int __init amigaone_add_bridge(struct device_node *dev) } void __init amigaone_setup_arch(void) +{ + if (ppc_md.progress) + ppc_md.progress("Linux/PPC "UTS_RELEASE"\n", 0); +} + +static void __init amigaone_discover_phbs(void) { struct device_node *np; int phb = -ENODEV; @@ -74,9 +80,6 @@ void __init amigaone_setup_arch(void) phb = amigaone_add_bridge(np); BUG_ON(phb != 0); - - if (ppc_md.progress) - ppc_md.progress("Linux/PPC "UTS_RELEASE"\n", 0); } void __init amigaone_init_IRQ(void) @@ -159,6 +162,7 @@ define_machine(amigaone) { .name = "AmigaOne", .probe = amigaone_probe, .setup_arch = amigaone_setup_arch, + .discover_phbs = amigaone_discover_phbs, .show_cpuinfo = amigaone_show_cpuinfo, .init_IRQ = amigaone_init_IRQ, .restart = amigaone_restart, diff --git a/arch/powerpc/platforms/cell/pervasive.c b/arch/powerpc/platforms/cell/pervasive.c index 9068edef71f7832c375be25cfb0754e92553b146..5b9a7e9f144b356c27154d1af4c3204c3c941c9f 100644 --- a/arch/powerpc/platforms/cell/pervasive.c +++ b/arch/powerpc/platforms/cell/pervasive.c @@ -25,6 +25,7 @@ #include #include "pervasive.h" +#include "ras.h" static void cbe_power_save(void) { diff --git a/arch/powerpc/platforms/cell/pervasive.h b/arch/powerpc/platforms/cell/pervasive.h index c6fccad6caeef5cb32154ae52cdf1f17ac0a709b..0da74ab10716461177d16018df85f8c2756ca8d7 100644 --- a/arch/powerpc/platforms/cell/pervasive.h +++ b/arch/powerpc/platforms/cell/pervasive.h @@ -13,9 +13,6 @@ #define PERVASIVE_H extern void cbe_pervasive_init(void); -extern void cbe_system_error_exception(struct pt_regs *regs); -extern void cbe_maintenance_exception(struct pt_regs *regs); -extern void cbe_thermal_exception(struct pt_regs *regs); #ifdef CONFIG_PPC_IBM_CELL_RESETBUTTON extern int cbe_sysreset_hack(void); diff --git a/arch/powerpc/platforms/cell/ras.c b/arch/powerpc/platforms/cell/ras.c index 6ea4805394198ed24f148943d4f71898b6fc67c9..4325c05bedd97306f4f7541d2992d248676a6946 100644 --- a/arch/powerpc/platforms/cell/ras.c +++ b/arch/powerpc/platforms/cell/ras.c @@ -49,7 +49,7 @@ static void dump_fir(int cpu) } -void cbe_system_error_exception(struct pt_regs *regs) +DEFINE_INTERRUPT_HANDLER(cbe_system_error_exception) { int cpu = smp_processor_id(); @@ -58,7 +58,7 @@ void cbe_system_error_exception(struct pt_regs *regs) dump_stack(); } -void cbe_maintenance_exception(struct pt_regs *regs) +DEFINE_INTERRUPT_HANDLER(cbe_maintenance_exception) { int cpu = smp_processor_id(); @@ -70,7 +70,7 @@ void cbe_maintenance_exception(struct pt_regs *regs) dump_stack(); } -void cbe_thermal_exception(struct pt_regs *regs) +DEFINE_INTERRUPT_HANDLER(cbe_thermal_exception) { int cpu = smp_processor_id(); diff --git a/arch/powerpc/platforms/cell/ras.h b/arch/powerpc/platforms/cell/ras.h index 6c2e6bc0062e10571dc7b00f8df518d3c655bcac..226dbd48efad0f5e2c89b03ea82fc6a7a86f4e4b 100644 --- a/arch/powerpc/platforms/cell/ras.h +++ b/arch/powerpc/platforms/cell/ras.h @@ -2,9 +2,12 @@ #ifndef RAS_H #define RAS_H -extern void cbe_system_error_exception(struct pt_regs *regs); -extern void cbe_maintenance_exception(struct pt_regs *regs); -extern void cbe_thermal_exception(struct pt_regs *regs); +#include + +DECLARE_INTERRUPT_HANDLER(cbe_system_error_exception); +DECLARE_INTERRUPT_HANDLER(cbe_maintenance_exception); +DECLARE_INTERRUPT_HANDLER(cbe_thermal_exception); + extern void cbe_ras_init(void); #endif /* RAS_H */ diff --git a/arch/powerpc/platforms/cell/spufs/inode.c b/arch/powerpc/platforms/cell/spufs/inode.c index 25390569e24cd8bf9e11a7d8202bc572670002fd..b83a3670bd74e5c0c1295bf28b147a0e9a05caba 100644 --- a/arch/powerpc/platforms/cell/spufs/inode.c +++ b/arch/powerpc/platforms/cell/spufs/inode.c @@ -91,14 +91,15 @@ spufs_new_inode(struct super_block *sb, umode_t mode) } static int -spufs_setattr(struct dentry *dentry, struct iattr *attr) +spufs_setattr(struct user_namespace *mnt_userns, struct dentry *dentry, + struct iattr *attr) { struct inode *inode = d_inode(dentry); if ((attr->ia_valid & ATTR_SIZE) && (attr->ia_size != inode->i_size)) return -EINVAL; - setattr_copy(inode, attr); + setattr_copy(&init_user_ns, inode, attr); mark_inode_dirty(inode); return 0; } diff --git a/arch/powerpc/platforms/chrp/pci.c b/arch/powerpc/platforms/chrp/pci.c index b2c2bf35b76c2b475ee2e279303107398a418157..8c421dc78b28334bef99bfb13969c6f8dd719096 100644 --- a/arch/powerpc/platforms/chrp/pci.c +++ b/arch/powerpc/platforms/chrp/pci.c @@ -314,6 +314,14 @@ chrp_find_bridges(void) } } of_node_put(root); + + /* + * "Temporary" fixes for PCI devices. + * -- Geert + */ + hydra_init(); /* Mac I/O */ + + pci_create_OF_bus_map(); } /* SL82C105 IDE Control/Status Register */ diff --git a/arch/powerpc/platforms/chrp/setup.c b/arch/powerpc/platforms/chrp/setup.c index c45435aa5e36f4bf8e69684e3a5c6d5823d6c89d..3cfc382841e52522fd15a0cc00a758aadab38e84 100644 --- a/arch/powerpc/platforms/chrp/setup.c +++ b/arch/powerpc/platforms/chrp/setup.c @@ -334,22 +334,11 @@ static void __init chrp_setup_arch(void) /* On pegasos, enable the L2 cache if not already done by OF */ pegasos_set_l2cr(); - /* Lookup PCI host bridges */ - chrp_find_bridges(); - - /* - * Temporary fixes for PCI devices. - * -- Geert - */ - hydra_init(); /* Mac I/O */ - /* * Fix the Super I/O configuration */ sio_init(); - pci_create_OF_bus_map(); - /* * Print the banner, then scroll down so boot progress * can be printed. -- Cort @@ -582,6 +571,7 @@ define_machine(chrp) { .name = "CHRP", .probe = chrp_probe, .setup_arch = chrp_setup_arch, + .discover_phbs = chrp_find_bridges, .init = chrp_init2, .show_cpuinfo = chrp_show_cpuinfo, .init_IRQ = chrp_init_IRQ, diff --git a/arch/powerpc/platforms/embedded6xx/holly.c b/arch/powerpc/platforms/embedded6xx/holly.c index d8f2e2c737bb65f9a11e013dab1a0ff9af4ee7bf..53065d564161d7b44076a8de5b6320e2fcadbd86 100644 --- a/arch/powerpc/platforms/embedded6xx/holly.c +++ b/arch/powerpc/platforms/embedded6xx/holly.c @@ -108,15 +108,13 @@ static void holly_remap_bridge(void) tsi108_write_reg(TSI108_PCI_P2O_BAR2, 0x0); } -static void __init holly_setup_arch(void) +static void __init holly_init_pci(void) { struct device_node *np; if (ppc_md.progress) ppc_md.progress("holly_setup_arch():set_bridge", 0); - tsi108_csr_vir_base = get_vir_csrbase(); - /* setup PCI host bridge */ holly_remap_bridge(); @@ -127,6 +125,11 @@ static void __init holly_setup_arch(void) ppc_md.pci_exclude_device = holly_exclude_device; if (ppc_md.progress) ppc_md.progress("tsi108: resources set", 0x100); +} + +static void __init holly_setup_arch(void) +{ + tsi108_csr_vir_base = get_vir_csrbase(); printk(KERN_INFO "PPC750GX/CL Platform\n"); } @@ -259,6 +262,7 @@ define_machine(holly){ .name = "PPC750 GX/CL TSI", .probe = holly_probe, .setup_arch = holly_setup_arch, + .discover_phbs = holly_init_pci, .init_IRQ = holly_init_IRQ, .show_cpuinfo = holly_show_cpuinfo, .get_irq = mpic_get_irq, diff --git a/arch/powerpc/platforms/embedded6xx/linkstation.c b/arch/powerpc/platforms/embedded6xx/linkstation.c index f514d5d28cd4fc7041a700bce0e699172932c6c8..eb8342e7f84e9ec07149753d390e3ff5c491d973 100644 --- a/arch/powerpc/platforms/embedded6xx/linkstation.c +++ b/arch/powerpc/platforms/embedded6xx/linkstation.c @@ -63,15 +63,18 @@ static int __init linkstation_add_bridge(struct device_node *dev) } static void __init linkstation_setup_arch(void) +{ + printk(KERN_INFO "BUFFALO Network Attached Storage Series\n"); + printk(KERN_INFO "(C) 2002-2005 BUFFALO INC.\n"); +} + +static void __init linkstation_setup_pci(void) { struct device_node *np; /* Lookup PCI host bridges */ for_each_compatible_node(np, "pci", "mpc10x-pci") linkstation_add_bridge(np); - - printk(KERN_INFO "BUFFALO Network Attached Storage Series\n"); - printk(KERN_INFO "(C) 2002-2005 BUFFALO INC.\n"); } /* @@ -153,6 +156,7 @@ define_machine(linkstation){ .name = "Buffalo Linkstation", .probe = linkstation_probe, .setup_arch = linkstation_setup_arch, + .discover_phbs = linkstation_setup_pci, .init_IRQ = linkstation_init_IRQ, .show_cpuinfo = linkstation_show_cpuinfo, .get_irq = mpic_get_irq, diff --git a/arch/powerpc/platforms/embedded6xx/mpc7448_hpc2.c b/arch/powerpc/platforms/embedded6xx/mpc7448_hpc2.c index b95c3380d2b5854269b6ca6353305a55bfb6bc3f..5565647dc879e8ad1b103b93ab8f5537e7dc6ed2 100644 --- a/arch/powerpc/platforms/embedded6xx/mpc7448_hpc2.c +++ b/arch/powerpc/platforms/embedded6xx/mpc7448_hpc2.c @@ -58,16 +58,14 @@ int mpc7448_hpc2_exclude_device(struct pci_controller *hose, return PCIBIOS_SUCCESSFUL; } -static void __init mpc7448_hpc2_setup_arch(void) +static void __init mpc7448_hpc2_setup_pci(void) { +#ifdef CONFIG_PCI struct device_node *np; if (ppc_md.progress) - ppc_md.progress("mpc7448_hpc2_setup_arch():set_bridge", 0); - - tsi108_csr_vir_base = get_vir_csrbase(); + ppc_md.progress("mpc7448_hpc2_setup_pci():set_bridge", 0); /* setup PCI host bridge */ -#ifdef CONFIG_PCI for_each_compatible_node(np, "pci", "tsi108-pci") tsi108_setup_pci(np, MPC7448HPC2_PCI_CFG_PHYS, 0); @@ -75,6 +73,11 @@ static void __init mpc7448_hpc2_setup_arch(void) if (ppc_md.progress) ppc_md.progress("tsi108: resources set", 0x100); #endif +} + +static void __init mpc7448_hpc2_setup_arch(void) +{ + tsi108_csr_vir_base = get_vir_csrbase(); printk(KERN_INFO "MPC7448HPC2 (TAIGA) Platform\n"); printk(KERN_INFO @@ -181,6 +184,7 @@ define_machine(mpc7448_hpc2){ .name = "MPC7448 HPC2", .probe = mpc7448_hpc2_probe, .setup_arch = mpc7448_hpc2_setup_arch, + .discover_phbs = mpc7448_hpc2_setup_pci, .init_IRQ = mpc7448_hpc2_init_IRQ, .show_cpuinfo = mpc7448_hpc2_show_cpuinfo, .get_irq = mpic_get_irq, diff --git a/arch/powerpc/platforms/embedded6xx/mvme5100.c b/arch/powerpc/platforms/embedded6xx/mvme5100.c index 1cd488daa0bf959c6cfcb8637a50ca13c3fed609..c06a0490d157f0e61acd3337b3ecae8ec75f8c9e 100644 --- a/arch/powerpc/platforms/embedded6xx/mvme5100.c +++ b/arch/powerpc/platforms/embedded6xx/mvme5100.c @@ -154,17 +154,19 @@ static const struct of_device_id mvme5100_of_bus_ids[] __initconst = { */ static void __init mvme5100_setup_arch(void) { - struct device_node *np; - if (ppc_md.progress) ppc_md.progress("mvme5100_setup_arch()", 0); - for_each_compatible_node(np, "pci", "hawk-pci") - mvme5100_add_bridge(np); - restart = ioremap(BOARD_MODRST_REG, 4); } +static void __init mvme5100_setup_pci(void) +{ + struct device_node *np; + + for_each_compatible_node(np, "pci", "hawk-pci") + mvme5100_add_bridge(np); +} static void mvme5100_show_cpuinfo(struct seq_file *m) { @@ -205,6 +207,7 @@ define_machine(mvme5100) { .name = "MVME5100", .probe = mvme5100_probe, .setup_arch = mvme5100_setup_arch, + .discover_phbs = mvme5100_setup_pci, .init_IRQ = mvme5100_pic_init, .show_cpuinfo = mvme5100_show_cpuinfo, .get_irq = mpic_get_irq, diff --git a/arch/powerpc/platforms/embedded6xx/storcenter.c b/arch/powerpc/platforms/embedded6xx/storcenter.c index e346ddcef45e6916405012760af4edf00ce731d9..e188b90f7016e7f545db8dc57eca9b9639d0be98 100644 --- a/arch/powerpc/platforms/embedded6xx/storcenter.c +++ b/arch/powerpc/platforms/embedded6xx/storcenter.c @@ -65,14 +65,17 @@ static int __init storcenter_add_bridge(struct device_node *dev) } static void __init storcenter_setup_arch(void) +{ + printk(KERN_INFO "IOMEGA StorCenter\n"); +} + +static void __init storcenter_setup_pci(void) { struct device_node *np; /* Lookup PCI host bridges */ for_each_compatible_node(np, "pci", "mpc10x-pci") storcenter_add_bridge(np); - - printk(KERN_INFO "IOMEGA StorCenter\n"); } /* @@ -117,6 +120,7 @@ define_machine(storcenter){ .name = "IOMEGA StorCenter", .probe = storcenter_probe, .setup_arch = storcenter_setup_arch, + .discover_phbs = storcenter_setup_pci, .init_IRQ = storcenter_init_IRQ, .get_irq = mpic_get_irq, .restart = storcenter_restart, diff --git a/arch/powerpc/platforms/maple/pci.c b/arch/powerpc/platforms/maple/pci.c index c86a66d5e998ac1e21a58876fc0de891c8e804b5..a20b9576de225d71cdaf76ad708570261150754a 100644 --- a/arch/powerpc/platforms/maple/pci.c +++ b/arch/powerpc/platforms/maple/pci.c @@ -536,6 +536,9 @@ static int __init maple_add_bridge(struct device_node *dev) /* Check for legacy IOs */ isa_bridge_find_early(hose); + /* create pci_dn's for DT nodes under this PHB */ + pci_devs_phb_init_dynamic(hose); + return 0; } diff --git a/arch/powerpc/platforms/maple/setup.c b/arch/powerpc/platforms/maple/setup.c index f7e66a2005b405c670cdc77bc246b8603a6bf09d..4e9ad5bf3efb4f7218965e7fd3efaddb6bd0649f 100644 --- a/arch/powerpc/platforms/maple/setup.c +++ b/arch/powerpc/platforms/maple/setup.c @@ -179,9 +179,6 @@ static void __init maple_setup_arch(void) #ifdef CONFIG_SMP smp_ops = &maple_smp_ops; #endif - /* Lookup PCI hosts */ - maple_pci_init(); - maple_use_rtas_reboot_and_halt_if_present(); printk(KERN_DEBUG "Using native/NAP idle loop\n"); @@ -351,6 +348,7 @@ define_machine(maple) { .name = "Maple", .probe = maple_probe, .setup_arch = maple_setup_arch, + .discover_phbs = maple_pci_init, .init_IRQ = maple_init_IRQ, .pci_irq_fixup = maple_pci_irq_fixup, .pci_get_legacy_ide_irq = maple_pci_get_legacy_ide_irq, diff --git a/arch/powerpc/platforms/pasemi/setup.c b/arch/powerpc/platforms/pasemi/setup.c index b612474f8f8e6f576f3f6fe2b76fab725a374374..376797eb78940dbd210baec16149caff126f8f2e 100644 --- a/arch/powerpc/platforms/pasemi/setup.c +++ b/arch/powerpc/platforms/pasemi/setup.c @@ -144,8 +144,6 @@ static void __init pas_setup_arch(void) /* Setup SMP callback */ smp_ops = &pas_smp_ops; #endif - /* Lookup PCI hosts */ - pas_pci_init(); /* Remap SDC register for doing reset */ /* XXXOJN This should maybe come out of the device tree */ @@ -446,6 +444,7 @@ define_machine(pasemi) { .name = "PA Semi PWRficient", .probe = pas_probe, .setup_arch = pas_setup_arch, + .discover_phbs = pas_pci_init, .init_IRQ = pas_init_IRQ, .get_irq = mpic_get_irq, .restart = pas_restart, diff --git a/arch/powerpc/platforms/powermac/pci.c b/arch/powerpc/platforms/powermac/pci.c index e35eaa9cf9386f3b0c92bde3eb36eeb421895945..e9abe0f2e7f0a7b9bd14f32df45157500694b756 100644 --- a/arch/powerpc/platforms/powermac/pci.c +++ b/arch/powerpc/platforms/powermac/pci.c @@ -850,6 +850,10 @@ static int __init pmac_add_bridge(struct device_node *dev) /* Fixup "bus-range" OF property */ fixup_bus_range(dev); + /* create pci_dn's for DT nodes under this PHB */ + if (IS_ENABLED(CONFIG_PPC64)) + pci_devs_phb_init_dynamic(hose); + return 0; } diff --git a/arch/powerpc/platforms/powermac/setup.c b/arch/powerpc/platforms/powermac/setup.c index 2e2cc0c75d87f05404376a8ad6223c702edc47e3..86aee3f2483f4013e9db478da3fc4db50b8af851 100644 --- a/arch/powerpc/platforms/powermac/setup.c +++ b/arch/powerpc/platforms/powermac/setup.c @@ -298,9 +298,6 @@ static void __init pmac_setup_arch(void) of_node_put(ic); } - /* Lookup PCI hosts */ - pmac_pci_init(); - #ifdef CONFIG_PPC32 ohare_init(); l2cr_init(); @@ -600,6 +597,7 @@ define_machine(powermac) { .name = "PowerMac", .probe = pmac_probe, .setup_arch = pmac_setup_arch, + .discover_phbs = pmac_pci_init, .show_cpuinfo = pmac_show_cpuinfo, .init_IRQ = pmac_pic_init, .get_irq = NULL, /* changed later */ diff --git a/arch/powerpc/platforms/powernv/idle.c b/arch/powerpc/platforms/powernv/idle.c index e6f461812856c1b2cc5a167c5f4ebf148b3a6d76..999997d9e9a9fcd5a10bb9dbbaaf71c4dbd7e81e 100644 --- a/arch/powerpc/platforms/powernv/idle.c +++ b/arch/powerpc/platforms/powernv/idle.c @@ -14,6 +14,7 @@ #include #include +#include #include #include #include diff --git a/arch/powerpc/platforms/powernv/memtrace.c b/arch/powerpc/platforms/powernv/memtrace.c index 5fc9408bb0b3f22edff5d0cf1d3041e0bd4cb71e..019669eb21d271e51668878246461e582928146c 100644 --- a/arch/powerpc/platforms/powernv/memtrace.c +++ b/arch/powerpc/platforms/powernv/memtrace.c @@ -19,6 +19,7 @@ #include #include #include +#include /* This enables us to keep track of the memory removed from each node. */ struct memtrace_entry { @@ -51,6 +52,27 @@ static const struct file_operations memtrace_fops = { .open = simple_open, }; +#define FLUSH_CHUNK_SIZE SZ_1G +/** + * flush_dcache_range_chunked(): Write any modified data cache blocks out to + * memory and invalidate them, in chunks of up to FLUSH_CHUNK_SIZE + * Does not invalidate the corresponding instruction cache blocks. + * + * @start: the start address + * @stop: the stop address (exclusive) + * @chunk: the max size of the chunks + */ +static void flush_dcache_range_chunked(unsigned long start, unsigned long stop, + unsigned long chunk) +{ + unsigned long i; + + for (i = start; i < stop; i += chunk) { + flush_dcache_range(i, min(stop, i + chunk)); + cond_resched(); + } +} + static void memtrace_clear_range(unsigned long start_pfn, unsigned long nr_pages) { @@ -62,6 +84,13 @@ static void memtrace_clear_range(unsigned long start_pfn, cond_resched(); clear_page(__va(PFN_PHYS(pfn))); } + /* + * Before we go ahead and use this range as cache inhibited range + * flush the cache. + */ + flush_dcache_range_chunked(PFN_PHYS(start_pfn), + PFN_PHYS(start_pfn + nr_pages), + FLUSH_CHUNK_SIZE); } static u64 memtrace_alloc_node(u32 nid, u64 size) diff --git a/arch/powerpc/platforms/powernv/opal.c b/arch/powerpc/platforms/powernv/opal.c index c61c3b62c8c628c290c9bc1846b35378d480ccfd..303d7c775740308af7651e2036c2545569106ebd 100644 --- a/arch/powerpc/platforms/powernv/opal.c +++ b/arch/powerpc/platforms/powernv/opal.c @@ -624,7 +624,7 @@ static int opal_recover_mce(struct pt_regs *regs, */ recovered = 0; } else { - die("Machine check", regs, SIGBUS); + die_mce("Machine check", regs, SIGBUS); recovered = 1; } } diff --git a/arch/powerpc/platforms/powernv/pci-cxl.c b/arch/powerpc/platforms/powernv/pci-cxl.c index 8c739c94ed28d66744ca9436699fc3764b054fb2..53172862d23bd3dbda6780334e6fb609979025b4 100644 --- a/arch/powerpc/platforms/powernv/pci-cxl.c +++ b/arch/powerpc/platforms/powernv/pci-cxl.c @@ -150,25 +150,3 @@ int pnv_cxl_ioda_msi_setup(struct pci_dev *dev, unsigned int hwirq, return 0; } EXPORT_SYMBOL(pnv_cxl_ioda_msi_setup); - -#if IS_MODULE(CONFIG_CXL) -static inline int get_cxl_module(void) -{ - struct module *cxl_module; - - mutex_lock(&module_mutex); - - cxl_module = find_module("cxl"); - if (cxl_module) - __module_get(cxl_module); - - mutex_unlock(&module_mutex); - - if (!cxl_module) - return -ENODEV; - - return 0; -} -#else -static inline int get_cxl_module(void) { return 0; } -#endif diff --git a/arch/powerpc/platforms/powernv/pci-ioda-tce.c b/arch/powerpc/platforms/powernv/pci-ioda-tce.c index 5218f5da27370b29b8d1a98aaaf8753596236ef1..30551bbd7988aca459748df92ffe7300c3bd68c4 100644 --- a/arch/powerpc/platforms/powernv/pci-ioda-tce.c +++ b/arch/powerpc/platforms/powernv/pci-ioda-tce.c @@ -380,6 +380,8 @@ void pnv_pci_unlink_table_and_group(struct iommu_table *tbl, /* Remove link to a group from table's list of attached groups */ found = false; + + rcu_read_lock(); list_for_each_entry_rcu(tgl, &tbl->it_group_list, next) { if (tgl->table_group == table_group) { list_del_rcu(&tgl->next); @@ -388,6 +390,8 @@ void pnv_pci_unlink_table_and_group(struct iommu_table *tbl, break; } } + rcu_read_unlock(); + if (WARN_ON(!found)) return; diff --git a/arch/powerpc/platforms/powernv/pci-ioda.c b/arch/powerpc/platforms/powernv/pci-ioda.c index c4f72cdc9b516cdb69a9bd8968a1f7ae65a9b296..f0f901683a2fe1056cc67190cccc09c76732640d 100644 --- a/arch/powerpc/platforms/powernv/pci-ioda.c +++ b/arch/powerpc/platforms/powernv/pci-ioda.c @@ -2402,9 +2402,6 @@ static void pnv_pci_ioda_create_dbgfs(void) list_for_each_entry_safe(hose, tmp, &hose_list, list_node) { phb = hose->private_data; - /* Notify initialization of PHB done */ - phb->initialized = 1; - sprintf(name, "PCI%04x", hose->global_number); phb->dbgfs = debugfs_create_dir(name, powerpc_debugfs_root); @@ -2601,17 +2598,8 @@ static resource_size_t pnv_pci_default_alignment(void) */ static bool pnv_pci_enable_device_hook(struct pci_dev *dev) { - struct pnv_phb *phb = pci_bus_to_pnvhb(dev->bus); struct pci_dn *pdn; - /* The function is probably called while the PEs have - * not be created yet. For example, resource reassignment - * during PCI probe period. We just skip the check if - * PEs isn't ready. - */ - if (!phb->initialized) - return true; - pdn = pci_get_pdn(dev); if (!pdn || pdn->pe_number == IODA_INVALID_PE) { pci_err(dev, "pci_enable_device() blocked, no PE assigned.\n"); @@ -2623,14 +2611,9 @@ static bool pnv_pci_enable_device_hook(struct pci_dev *dev) static bool pnv_ocapi_enable_device_hook(struct pci_dev *dev) { - struct pci_controller *hose = pci_bus_to_host(dev->bus); - struct pnv_phb *phb = hose->private_data; struct pci_dn *pdn; struct pnv_ioda_pe *pe; - if (!phb->initialized) - return true; - pdn = pci_get_pdn(dev); if (!pdn) return false; @@ -2938,7 +2921,7 @@ static void __init pnv_pci_init_ioda_phb(struct device_node *np, phb_id = be64_to_cpup(prop64); pr_debug(" PHB-ID : 0x%016llx\n", phb_id); - phb = memblock_alloc(sizeof(*phb), SMP_CACHE_BYTES); + phb = kzalloc(sizeof(*phb), GFP_KERNEL); if (!phb) panic("%s: Failed to allocate %zu bytes\n", __func__, sizeof(*phb)); @@ -2987,7 +2970,7 @@ static void __init pnv_pci_init_ioda_phb(struct device_node *np, else phb->diag_data_size = PNV_PCI_DIAG_BUF_SIZE; - phb->diag_data = memblock_alloc(phb->diag_data_size, SMP_CACHE_BYTES); + phb->diag_data = kzalloc(phb->diag_data_size, GFP_KERNEL); if (!phb->diag_data) panic("%s: Failed to allocate %u bytes\n", __func__, phb->diag_data_size); @@ -3049,9 +3032,10 @@ static void __init pnv_pci_init_ioda_phb(struct device_node *np, } pemap_off = size; size += phb->ioda.total_pe_num * sizeof(struct pnv_ioda_pe); - aux = memblock_alloc(size, SMP_CACHE_BYTES); + aux = kzalloc(size, GFP_KERNEL); if (!aux) panic("%s: Failed to allocate %lu bytes\n", __func__, size); + phb->ioda.pe_alloc = aux; phb->ioda.m64_segmap = aux + m64map_off; phb->ioda.m32_segmap = aux + m32map_off; @@ -3178,6 +3162,9 @@ static void __init pnv_pci_init_ioda_phb(struct device_node *np, /* Remove M64 resource if we can't configure it successfully */ if (!phb->init_m64 || phb->init_m64(phb)) hose->mem_resources[1].flags = 0; + + /* create pci_dn's for DT nodes under this PHB */ + pci_devs_phb_init_dynamic(hose); } void __init pnv_pci_init_ioda2_phb(struct device_node *np) diff --git a/arch/powerpc/platforms/powernv/pci.h b/arch/powerpc/platforms/powernv/pci.h index 739a0b3b72e1563d8dad167a2db8cc47177e218f..36d22920f5a3cb06ed8c2b61f442cdd7957ebb0c 100644 --- a/arch/powerpc/platforms/powernv/pci.h +++ b/arch/powerpc/platforms/powernv/pci.h @@ -119,7 +119,6 @@ struct pnv_phb { int flags; void __iomem *regs; u64 regs_phys; - int initialized; spinlock_t lock; #ifdef CONFIG_DEBUG_FS diff --git a/arch/powerpc/platforms/powernv/setup.c b/arch/powerpc/platforms/powernv/setup.c index 4426a109ec2f4390044e74a62218166bd5805292..aadf932c4e61bb358a073921415914596100ca7f 100644 --- a/arch/powerpc/platforms/powernv/setup.c +++ b/arch/powerpc/platforms/powernv/setup.c @@ -180,9 +180,6 @@ static void __init pnv_setup_arch(void) /* Initialize SMP */ pnv_smp_init(); - /* Setup PCI */ - pnv_pci_init(); - /* Setup RTC and NVRAM callbacks */ if (firmware_has_feature(FW_FEATURE_OPAL)) opal_nvram_init(); @@ -547,6 +544,7 @@ define_machine(powernv) { .init_IRQ = pnv_init_IRQ, .show_cpuinfo = pnv_show_cpuinfo, .get_proc_freq = pnv_get_proc_freq, + .discover_phbs = pnv_pci_init, .progress = pnv_progress, .machine_shutdown = pnv_shutdown, .power_save = NULL, diff --git a/arch/powerpc/platforms/powernv/subcore.h b/arch/powerpc/platforms/powernv/subcore.h index c8f574d1c04a7a32ed231fa7720cc1828e301a4f..77feee8436d48cb6cc84c2f9c51df05f2320ee72 100644 --- a/arch/powerpc/platforms/powernv/subcore.h +++ b/arch/powerpc/platforms/powernv/subcore.h @@ -15,7 +15,7 @@ void split_core_secondary_loop(u8 *state); extern void update_subcore_sibling_mask(void); #else -static inline void update_subcore_sibling_mask(void) { }; +static inline void update_subcore_sibling_mask(void) { } #endif /* CONFIG_SMP */ #endif /* __ASSEMBLY__ */ diff --git a/arch/powerpc/platforms/powernv/vas.c b/arch/powerpc/platforms/powernv/vas.c index 598e4cd563fb19647e630c95e9231b4e938d58b5..b65256a63e871022ca8d61adb89c88b94cdaca58 100644 --- a/arch/powerpc/platforms/powernv/vas.c +++ b/arch/powerpc/platforms/powernv/vas.c @@ -28,12 +28,10 @@ static DEFINE_PER_CPU(int, cpu_vas_id); static int vas_irq_fault_window_setup(struct vas_instance *vinst) { - char devname[64]; int rc = 0; - snprintf(devname, sizeof(devname), "vas-%d", vinst->vas_id); rc = request_threaded_irq(vinst->virq, vas_fault_handler, - vas_fault_thread_fn, 0, devname, vinst); + vas_fault_thread_fn, 0, vinst->name, vinst); if (rc) { pr_err("VAS[%d]: Request IRQ(%d) failed with %d\n", @@ -80,6 +78,12 @@ static int init_vas_instance(struct platform_device *pdev) if (!vinst) return -ENOMEM; + vinst->name = kasprintf(GFP_KERNEL, "vas-%d", vasid); + if (!vinst->name) { + kfree(vinst); + return -ENOMEM; + } + INIT_LIST_HEAD(&vinst->node); ida_init(&vinst->ida); mutex_init(&vinst->mutex); @@ -162,6 +166,7 @@ static int init_vas_instance(struct platform_device *pdev) return 0; free_vinst: + kfree(vinst->name); kfree(vinst); return -ENODEV; diff --git a/arch/powerpc/platforms/powernv/vas.h b/arch/powerpc/platforms/powernv/vas.h index 70f793e8f6cc6756618903e080eb76c15b5e2f50..c7db3190baca4976e0053de0b8c09f44168c828f 100644 --- a/arch/powerpc/platforms/powernv/vas.h +++ b/arch/powerpc/platforms/powernv/vas.h @@ -340,6 +340,7 @@ struct vas_instance { struct vas_window *rxwin[VAS_COP_TYPE_MAX]; struct vas_window *windows[VAS_WINDOWS_PER_CHIP]; + char *name; char *dbgname; struct dentry *dbgdir; }; diff --git a/arch/powerpc/platforms/pseries/dlpar.c b/arch/powerpc/platforms/pseries/dlpar.c index 16e86ba8aa2099d1356f941c8e1549f14a152dbf..233503fcf8f08fb1747633b8f4f41907dab9025b 100644 --- a/arch/powerpc/platforms/pseries/dlpar.c +++ b/arch/powerpc/platforms/pseries/dlpar.c @@ -127,7 +127,6 @@ void dlpar_free_cc_nodes(struct device_node *dn) #define NEXT_PROPERTY 3 #define PREV_PARENT 4 #define MORE_MEMORY 5 -#define CALL_AGAIN -2 #define ERR_CFG_USE -9003 struct device_node *dlpar_configure_connector(__be32 drc_index, @@ -168,6 +167,9 @@ struct device_node *dlpar_configure_connector(__be32 drc_index, spin_unlock(&rtas_data_buf_lock); + if (rtas_busy_delay(rc)) + continue; + switch (rc) { case COMPLETE: break; @@ -216,9 +218,6 @@ struct device_node *dlpar_configure_connector(__be32 drc_index, last_dn = last_dn->parent; break; - case CALL_AGAIN: - break; - case MORE_MEMORY: case ERR_CFG_USE: default: @@ -521,11 +520,8 @@ static ssize_t dlpar_store(struct class *class, struct class_attribute *attr, int rc; args = argbuf = kstrdup(buf, GFP_KERNEL); - if (!argbuf) { - pr_info("Could not allocate resources for DLPAR operation\n"); - kfree(argbuf); + if (!argbuf) return -ENOMEM; - } /* * Parse out the request from the user, this will be in the form: diff --git a/arch/powerpc/platforms/pseries/eeh_pseries.c b/arch/powerpc/platforms/pseries/eeh_pseries.c index cf024fa37bda09f277fdad5e69e46d5b45d41da2..bc15200852b7c3059dd44060a7ebc87e860f4c4f 100644 --- a/arch/powerpc/platforms/pseries/eeh_pseries.c +++ b/arch/powerpc/platforms/pseries/eeh_pseries.c @@ -43,7 +43,7 @@ static int ibm_get_config_addr_info; static int ibm_get_config_addr_info2; static int ibm_configure_pe; -void pseries_pcibios_bus_add_device(struct pci_dev *pdev) +static void pseries_pcibios_bus_add_device(struct pci_dev *pdev) { struct pci_dn *pdn = pci_get_pdn(pdev); @@ -694,8 +694,7 @@ static int pseries_eeh_write_config(struct eeh_dev *edev, int where, int size, u } #ifdef CONFIG_PCI_IOV -int pseries_send_allow_unfreeze(struct pci_dn *pdn, - u16 *vf_pe_array, int cur_vfs) +static int pseries_send_allow_unfreeze(struct pci_dn *pdn, u16 *vf_pe_array, int cur_vfs) { int rc; int ibm_allow_unfreeze = rtas_token("ibm,open-sriov-allow-unfreeze"); diff --git a/arch/powerpc/platforms/pseries/ibmebus.c b/arch/powerpc/platforms/pseries/ibmebus.c index 8c6e509f696756e5a877af8a4e9161263bcc1671..a15ab33646b30ed90f4d0e06f8ae719ed55a4e59 100644 --- a/arch/powerpc/platforms/pseries/ibmebus.c +++ b/arch/powerpc/platforms/pseries/ibmebus.c @@ -355,12 +355,12 @@ static int ibmebus_bus_device_probe(struct device *dev) if (!drv->probe) return error; - of_dev_get(of_dev); + get_device(dev); if (of_driver_match_device(dev, dev->driver)) error = drv->probe(of_dev); if (error) - of_dev_put(of_dev); + put_device(dev); return error; } diff --git a/arch/powerpc/platforms/pseries/pci.c b/arch/powerpc/platforms/pseries/pci.c index 72a4d416784943c3f2db6662bed4e52aefb5f08a..1bffbd1c9a94b1aacd28abfcfbefe5b40ed9724a 100644 --- a/arch/powerpc/platforms/pseries/pci.c +++ b/arch/powerpc/platforms/pseries/pci.c @@ -55,9 +55,8 @@ struct pe_map_bar_entry { __be32 reserved; /* Reserved Space */ }; -int pseries_send_map_pe(struct pci_dev *pdev, - u16 num_vfs, - struct pe_map_bar_entry *vf_pe_array) +static int pseries_send_map_pe(struct pci_dev *pdev, u16 num_vfs, + struct pe_map_bar_entry *vf_pe_array) { struct pci_dn *pdn; int rc; @@ -88,7 +87,7 @@ int pseries_send_map_pe(struct pci_dev *pdev, return rc; } -void pseries_set_pe_num(struct pci_dev *pdev, u16 vf_index, __be16 pe_num) +static void pseries_set_pe_num(struct pci_dev *pdev, u16 vf_index, __be16 pe_num) { struct pci_dn *pdn; @@ -102,7 +101,7 @@ void pseries_set_pe_num(struct pci_dev *pdev, u16 vf_index, __be16 pe_num) pdn->pe_num_map[vf_index]); } -int pseries_associate_pes(struct pci_dev *pdev, u16 num_vfs) +static int pseries_associate_pes(struct pci_dev *pdev, u16 num_vfs) { struct pci_dn *pdn; int i, rc, vf_index; @@ -146,7 +145,7 @@ int pseries_associate_pes(struct pci_dev *pdev, u16 num_vfs) return rc; } -int pseries_pci_sriov_enable(struct pci_dev *pdev, u16 num_vfs) +static int pseries_pci_sriov_enable(struct pci_dev *pdev, u16 num_vfs) { struct pci_dn *pdn; int rc; @@ -189,14 +188,14 @@ int pseries_pci_sriov_enable(struct pci_dev *pdev, u16 num_vfs) return rc; } -int pseries_pcibios_sriov_enable(struct pci_dev *pdev, u16 num_vfs) +static int pseries_pcibios_sriov_enable(struct pci_dev *pdev, u16 num_vfs) { /* Allocate PCI data */ add_sriov_vf_pdns(pdev); return pseries_pci_sriov_enable(pdev, num_vfs); } -int pseries_pcibios_sriov_disable(struct pci_dev *pdev) +static int pseries_pcibios_sriov_disable(struct pci_dev *pdev) { struct pci_dn *pdn; diff --git a/arch/powerpc/platforms/pseries/pseries.h b/arch/powerpc/platforms/pseries/pseries.h index 593840847cd3dc83324ee4c157c1afaeff3efe16..4fe48c04c6c20abea5879f51f873bca7f35aa509 100644 --- a/arch/powerpc/platforms/pseries/pseries.h +++ b/arch/powerpc/platforms/pseries/pseries.h @@ -33,7 +33,7 @@ int smp_query_cpu_stopped(unsigned int pcpu); #define QCSS_HARDWARE_ERROR -1 #define QCSS_HARDWARE_BUSY -2 #else -static inline void smp_init_pseries(void) { }; +static inline void smp_init_pseries(void) { } #endif extern void pseries_kexec_cpu_down(int crash_shutdown, int secondary); diff --git a/arch/powerpc/platforms/pseries/ras.c b/arch/powerpc/platforms/pseries/ras.c index 149cec2212e6aad4954a173500648249acf956f7..f8b390a9d9fb3169c587b3a625a00ac474f55ac1 100644 --- a/arch/powerpc/platforms/pseries/ras.c +++ b/arch/powerpc/platforms/pseries/ras.c @@ -122,7 +122,7 @@ static inline u8 rtas_mc_error_sub_type(const struct pseries_mc_errorlog *mlog) * devices or systems (e.g. hugepages) that have not been initialized at the * subsys stage. */ -int __init init_ras_hotplug_IRQ(void) +static int __init init_ras_hotplug_IRQ(void) { struct device_node *np; @@ -315,12 +315,10 @@ static irqreturn_t ras_hotplug_interrupt(int irq, void *dev_id) /* Handle environmental and power warning (EPOW) interrupts. */ static irqreturn_t ras_epow_interrupt(int irq, void *dev_id) { - int status; int state; int critical; - status = rtas_get_sensor_fast(EPOW_SENSOR_TOKEN, EPOW_SENSOR_INDEX, - &state); + rtas_get_sensor_fast(EPOW_SENSOR_TOKEN, EPOW_SENSOR_INDEX, &state); if (state > 3) critical = 1; /* Time Critical */ @@ -329,12 +327,9 @@ static irqreturn_t ras_epow_interrupt(int irq, void *dev_id) spin_lock(&ras_log_buf_lock); - status = rtas_call(ras_check_exception_token, 6, 1, NULL, - RTAS_VECTOR_EXTERNAL_INTERRUPT, - virq_to_hw(irq), - RTAS_EPOW_WARNING, - critical, __pa(&ras_log_buf), - rtas_get_error_log_max()); + rtas_call(ras_check_exception_token, 6, 1, NULL, RTAS_VECTOR_EXTERNAL_INTERRUPT, + virq_to_hw(irq), RTAS_EPOW_WARNING, critical, __pa(&ras_log_buf), + rtas_get_error_log_max()); log_error(ras_log_buf, ERR_TYPE_RTAS_LOG, 0); @@ -722,6 +717,7 @@ static int mce_handle_error(struct pt_regs *regs, struct rtas_error_log *errp) struct pseries_errorlog *pseries_log; struct pseries_mc_errorlog *mce_log = NULL; int disposition = rtas_error_disposition(errp); + unsigned long msr; u8 error_type; if (!rtas_error_extended(errp)) @@ -747,9 +743,21 @@ static int mce_handle_error(struct pt_regs *regs, struct rtas_error_log *errp) * SLB multihit is done by now. */ out: - mtmsr(mfmsr() | MSR_IR | MSR_DR); + msr = mfmsr(); + mtmsr(msr | MSR_IR | MSR_DR); + disposition = mce_handle_err_virtmode(regs, errp, mce_log, disposition); + + /* + * Queue irq work to log this rtas event later. + * irq_work_queue uses per-cpu variables, so do this in virt + * mode as well. + */ + irq_work_queue(&mce_errlog_process_work); + + mtmsr(msr); + return disposition; } @@ -813,7 +821,7 @@ static int recover_mce(struct pt_regs *regs, struct machine_check_event *evt) */ recovered = 0; } else { - die("Machine check", regs, SIGBUS); + die_mce("Machine check", regs, SIGBUS); recovered = 1; } } @@ -865,10 +873,8 @@ long pseries_machine_check_realmode(struct pt_regs *regs) * virtual mode. */ disposition = mce_handle_error(regs, errp); - fwnmi_release_errinfo(); - /* Queue irq work to log this rtas event later. */ - irq_work_queue(&mce_errlog_process_work); + fwnmi_release_errinfo(); if (disposition == RTAS_DISP_FULLY_RECOVERED) return 1; diff --git a/arch/powerpc/platforms/pseries/setup.c b/arch/powerpc/platforms/pseries/setup.c index 090c13f6c8815c67992f8f55dec929faac3bbfb2..46e1540abc229720a2fe4c5e219249aeb48dc96d 100644 --- a/arch/powerpc/platforms/pseries/setup.c +++ b/arch/powerpc/platforms/pseries/setup.c @@ -463,7 +463,7 @@ void pseries_little_endian_exceptions(void) } #endif -static void __init find_and_init_phbs(void) +static void __init pSeries_discover_phbs(void) { struct device_node *node; struct pci_controller *phb; @@ -481,6 +481,9 @@ static void __init find_and_init_phbs(void) pci_process_bridge_OF_ranges(phb, node, 0); isa_bridge_find_early(phb); phb->controller_ops = pseries_pci_controller_ops; + + /* create pci_dn's for DT nodes under this PHB */ + pci_devs_phb_init_dynamic(phb); } of_node_put(root); @@ -607,8 +610,8 @@ enum get_iov_fw_value_index { WDW_SIZE = 3 /* Get Window Size */ }; -resource_size_t pseries_get_iov_fw_value(struct pci_dev *dev, int resno, - enum get_iov_fw_value_index value) +static resource_size_t pseries_get_iov_fw_value(struct pci_dev *dev, int resno, + enum get_iov_fw_value_index value) { const int *indexes; struct device_node *dn = pci_device_to_OF_node(dev); @@ -643,7 +646,7 @@ resource_size_t pseries_get_iov_fw_value(struct pci_dev *dev, int resno, return ret; } -void of_pci_set_vf_bar_size(struct pci_dev *dev, const int *indexes) +static void of_pci_set_vf_bar_size(struct pci_dev *dev, const int *indexes) { struct resource *res; resource_size_t base, size; @@ -665,7 +668,7 @@ void of_pci_set_vf_bar_size(struct pci_dev *dev, const int *indexes) } } -void of_pci_parse_iov_addrs(struct pci_dev *dev, const int *indexes) +static void of_pci_parse_iov_addrs(struct pci_dev *dev, const int *indexes) { struct resource *res, *root, *conflict; resource_size_t base, size; @@ -786,7 +789,6 @@ static void __init pSeries_setup_arch(void) /* Find and initialize PCI host bridges */ init_pci_config_tokens(); - find_and_init_phbs(); of_reconfig_notifier_register(&pci_dn_reconfig_nb); pSeries_nvram_init(); @@ -1050,6 +1052,7 @@ define_machine(pseries) { .init_IRQ = pseries_init_irq, .show_cpuinfo = pSeries_show_cpuinfo, .log_error = pSeries_log_error, + .discover_phbs = pSeries_discover_phbs, .pcibios_fixup = pSeries_final_fixup, .restart = rtas_restart, .halt = rtas_halt, diff --git a/arch/powerpc/xmon/xmon.c b/arch/powerpc/xmon/xmon.c index dcd817ca2edfdc321fc15d54895a31de149999d6..3fe37495f63dc7ecf2e5550c98d07953caed10df 100644 --- a/arch/powerpc/xmon/xmon.c +++ b/arch/powerpc/xmon/xmon.c @@ -1383,7 +1383,6 @@ static long check_bp_loc(unsigned long addr) return 1; } -#ifndef CONFIG_PPC_8xx static int find_free_data_bpt(void) { int i; @@ -1395,7 +1394,6 @@ static int find_free_data_bpt(void) printf("Couldn't find free breakpoint register\n"); return -1; } -#endif static void print_data_bpts(void) { @@ -1435,7 +1433,6 @@ bpt_cmds(void) cmd = inchar(); switch (cmd) { -#ifndef CONFIG_PPC_8xx static const char badaddr[] = "Only kernel addresses are permitted for breakpoints\n"; int mode; case 'd': /* bd - hardware data breakpoint */ @@ -1497,7 +1494,6 @@ bpt_cmds(void) force_enable_xmon(); } break; -#endif case 'c': if (!scanhex(&a)) { @@ -3723,7 +3719,7 @@ void dump_segments(void) printf("sr0-15 ="); for (i = 0; i < 16; ++i) - printf(" %x", mfsrin(i << 28)); + printf(" %x", mfsr(i << 28)); printf("\n"); } #endif diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index e0a34eb5ed3b37a199d44fd661a4dabd57be603b..85d626b8ce5e07667bac85e76dd3c1a2eaaf9453 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -57,6 +57,7 @@ config RISCV select HAVE_ARCH_JUMP_LABEL select HAVE_ARCH_JUMP_LABEL_RELATIVE select HAVE_ARCH_KASAN if MMU && 64BIT + select HAVE_ARCH_KASAN_VMALLOC if MMU && 64BIT select HAVE_ARCH_KGDB select HAVE_ARCH_KGDB_QXFER_PKT select HAVE_ARCH_MMAP_RND_BITS if MMU @@ -67,14 +68,19 @@ config RISCV select HAVE_DEBUG_KMEMLEAK select HAVE_DMA_CONTIGUOUS if MMU select HAVE_EBPF_JIT if MMU + select HAVE_FUNCTION_ERROR_INJECTION select HAVE_FUTEX_CMPXCHG if FUTEX select HAVE_GCC_PLUGINS select HAVE_GENERIC_VDSO if MMU && 64BIT select HAVE_IRQ_TIME_ACCOUNTING + select HAVE_KPROBES + select HAVE_KPROBES_ON_FTRACE + select HAVE_KRETPROBES select HAVE_PCI select HAVE_PERF_EVENTS select HAVE_PERF_REGS select HAVE_PERF_USER_STACK_DUMP + select HAVE_REGS_AND_STACK_ACCESS_API select HAVE_STACKPROTECTOR select HAVE_SYSCALL_TRACEPOINTS select IRQ_DOMAIN @@ -143,7 +149,7 @@ config PAGE_OFFSET default 0xffffffe000000000 if 64BIT && MAXPHYSMEM_128GB config ARCH_FLATMEM_ENABLE - def_bool y + def_bool !NUMA config ARCH_SPARSEMEM_ENABLE def_bool y @@ -156,6 +162,9 @@ config ARCH_SELECT_MEMORY_MODEL config ARCH_WANT_GENERAL_HUGETLB def_bool y +config ARCH_SUPPORTS_UPROBES + def_bool y + config SYS_SUPPORTS_HUGETLBFS depends on MMU def_bool y @@ -302,6 +311,36 @@ config TUNE_GENERIC endchoice +# Common NUMA Features +config NUMA + bool "NUMA Memory Allocation and Scheduler Support" + depends on SMP + select GENERIC_ARCH_NUMA + select OF_NUMA + select ARCH_SUPPORTS_NUMA_BALANCING + help + Enable NUMA (Non-Uniform Memory Access) support. + + The kernel will try to allocate memory used by a CPU on the + local memory of the CPU and add some more NUMA awareness to the kernel. + +config NODES_SHIFT + int "Maximum NUMA Nodes (as a power of 2)" + range 1 10 + default "2" + depends on NEED_MULTIPLE_NODES + help + Specify the maximum number of NUMA Nodes available on the target + system. Increases memory reserved to accommodate various tables. + +config USE_PERCPU_NUMA_NODE_ID + def_bool y + depends on NUMA + +config NEED_PER_CPU_EMBED_FIRST_CHUNK + def_bool y + depends on NUMA + config RISCV_ISA_C bool "Emit compressed instructions when building Linux" default y @@ -416,11 +455,17 @@ config EFI allow the kernel to be booted as an EFI application. This is only useful on systems that have UEFI firmware. +config CC_HAVE_STACKPROTECTOR_TLS + def_bool $(cc-option,-mstack-protector-guard=tls -mstack-protector-guard-reg=tp -mstack-protector-guard-offset=0) + +config STACKPROTECTOR_PER_TASK + def_bool y + depends on STACKPROTECTOR && CC_HAVE_STACKPROTECTOR_TLS + endmenu config BUILTIN_DTB def_bool n - depends on RISCV_M_MODE depends on OF menu "Power management options" diff --git a/arch/riscv/Kconfig.socs b/arch/riscv/Kconfig.socs index 3284d5c291be541f0cc7d7316b9938b66e4c4ec3..7efcece8896cfe7dd361f97b9850c1da9f4665cb 100644 --- a/arch/riscv/Kconfig.socs +++ b/arch/riscv/Kconfig.socs @@ -22,30 +22,41 @@ config SOC_VIRT help This enables support for QEMU Virt Machine. -config SOC_KENDRYTE - bool "Kendryte K210 SoC" +config SOC_CANAAN + bool "Canaan Kendryte K210 SoC" depends on !MMU select CLINT_TIMER if RISCV_M_MODE select SERIAL_SIFIVE if TTY select SERIAL_SIFIVE_CONSOLE if TTY select SIFIVE_PLIC + select ARCH_HAS_RESET_CONTROLLER + select PINCTRL help - This enables support for Kendryte K210 SoC platform hardware. + This enables support for Canaan Kendryte K210 SoC platform hardware. -config SOC_KENDRYTE_K210_DTB - def_bool y - depends on SOC_KENDRYTE_K210_DTB_BUILTIN +if SOC_CANAAN -config SOC_KENDRYTE_K210_DTB_BUILTIN - bool "Builtin device tree for the Kendryte K210" - depends on SOC_KENDRYTE +config SOC_CANAAN_K210_DTB_BUILTIN + bool "Builtin device tree for the Canaan Kendryte K210" + depends on SOC_CANAAN default y select OF select BUILTIN_DTB - select SOC_KENDRYTE_K210_DTB help - Builds a device tree for the Kendryte K210 into the Linux image. + Build a device tree for the Kendryte K210 into the Linux image. This option should be selected if no bootloader is being used. If unsure, say Y. +config SOC_CANAAN_K210_DTB_SOURCE + string "Source file for the Canaan Kendryte K210 builtin DTB" + depends on SOC_CANAAN + depends on SOC_CANAAN_K210_DTB_BUILTIN + default "k210_generic" + help + Base name (without suffix, relative to arch/riscv/boot/dts/canaan) + for the DTS file that will be used to produce the DTB linked into the + kernel. + +endif + endmenu diff --git a/arch/riscv/Makefile b/arch/riscv/Makefile index 8c29e553ef7fd945a13b80a11e69b9f98d4ab932..1368d943f1f398089b2b7a1501f2f76671a41da7 100644 --- a/arch/riscv/Makefile +++ b/arch/riscv/Makefile @@ -12,6 +12,8 @@ OBJCOPYFLAGS := -O binary LDFLAGS_vmlinux := ifeq ($(CONFIG_DYNAMIC_FTRACE),y) LDFLAGS_vmlinux := --no-relax + KBUILD_CPPFLAGS += -DCC_USING_PATCHABLE_FUNCTION_ENTRY + CC_FLAGS_FTRACE := -fpatchable-function-entry=8 endif ifeq ($(CONFIG_64BIT)$(CONFIG_CMODEL_MEDLOW),yy) @@ -65,6 +67,16 @@ KBUILD_CFLAGS_MODULE += $(call cc-option,-mno-relax) # architectures. It's faster to have GCC emit only aligned accesses. KBUILD_CFLAGS += $(call cc-option,-mstrict-align) +ifeq ($(CONFIG_STACKPROTECTOR_PER_TASK),y) +prepare: stack_protector_prepare +stack_protector_prepare: prepare0 + $(eval KBUILD_CFLAGS += -mstack-protector-guard=tls \ + -mstack-protector-guard-reg=tp \ + -mstack-protector-guard-offset=$(shell \ + awk '{if ($$2 == "TSK_STACK_CANARY") print $$3;}' \ + include/generated/asm-offsets.h)) +endif + # arch specific predefines for sparse CHECKFLAGS += -D__riscv -D__riscv_xlen=$(BITS) @@ -83,7 +95,7 @@ PHONY += vdso_install vdso_install: $(Q)$(MAKE) $(build)=arch/riscv/kernel/vdso $@ -ifeq ($(CONFIG_RISCV_M_MODE)$(CONFIG_SOC_KENDRYTE),yy) +ifeq ($(CONFIG_RISCV_M_MODE)$(CONFIG_SOC_CANAAN),yy) KBUILD_IMAGE := $(boot)/loader.bin else KBUILD_IMAGE := $(boot)/Image.gz diff --git a/arch/riscv/boot/dts/Makefile b/arch/riscv/boot/dts/Makefile index ca1f8cbd78c0e3a44bf17753237d7eb517256148..7ffd502e3e7b9d5ea5a1a78dc796748155c8f49f 100644 --- a/arch/riscv/boot/dts/Makefile +++ b/arch/riscv/boot/dts/Makefile @@ -1,5 +1,5 @@ # SPDX-License-Identifier: GPL-2.0 subdir-y += sifive -subdir-y += kendryte +subdir-$(CONFIG_SOC_CANAAN_K210_DTB_BUILTIN) += canaan obj-$(CONFIG_BUILTIN_DTB) := $(addsuffix /, $(subdir-y)) diff --git a/arch/riscv/boot/dts/canaan/Makefile b/arch/riscv/boot/dts/canaan/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..9ee7156c0c31b03cd65dbaa61919dd8a4299aa58 --- /dev/null +++ b/arch/riscv/boot/dts/canaan/Makefile @@ -0,0 +1,5 @@ +# SPDX-License-Identifier: GPL-2.0 +ifneq ($(CONFIG_SOC_CANAAN_K210_DTB_SOURCE),"") +dtb-y += $(strip $(shell echo $(CONFIG_SOC_CANAAN_K210_DTB_SOURCE))).dtb +obj-$(CONFIG_SOC_CANAAN_K210_DTB_BUILTIN) += $(addsuffix .o, $(dtb-y)) +endif diff --git a/arch/riscv/boot/dts/canaan/canaan_kd233.dts b/arch/riscv/boot/dts/canaan/canaan_kd233.dts new file mode 100644 index 0000000000000000000000000000000000000000..039b92abf046c29c70eec876494ae0f8ffdfaebb --- /dev/null +++ b/arch/riscv/boot/dts/canaan/canaan_kd233.dts @@ -0,0 +1,152 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Copyright (C) 2019-20 Sean Anderson + * Copyright (C) 2020 Western Digital Corporation or its affiliates. + */ + +/dts-v1/; + +#include "k210.dtsi" + +#include +#include + +/ { + model = "Kendryte KD233"; + compatible = "canaan,kendryte-kd233", "canaan,kendryte-k210"; + + chosen { + bootargs = "earlycon console=ttySIF0"; + stdout-path = "serial0:115200n8"; + }; + + gpio-leds { + compatible = "gpio-leds"; + + led0 { + gpios = <&gpio0 8 GPIO_ACTIVE_LOW>; + }; + + led1 { + gpios = <&gpio0 9 GPIO_ACTIVE_LOW>; + }; + }; + + gpio-keys { + compatible = "gpio-keys"; + + key0 { + label = "KEY0"; + linux,code = ; + gpios = <&gpio0 10 GPIO_ACTIVE_LOW>; + }; + }; +}; + +&fpioa { + pinctrl-0 = <&jtag_pinctrl>; + pinctrl-names = "default"; + status = "okay"; + + jtag_pinctrl: jtag-pinmux { + pinmux = , + , + , + ; + }; + + uarths_pinctrl: uarths-pinmux { + pinmux = , + ; + }; + + spi0_pinctrl: spi0-pinmux { + pinmux = , /* cs */ + , /* wr */ + ; /* dc */ + }; + + dvp_pinctrl: dvp-pinmux { + pinmux = , + , + , + , + , + , + , + ; + }; + + gpiohs_pinctrl: gpiohs-pinmux { + pinmux = , + , /* Rot. dip sw line 8 */ + , /* Rot. dip sw line 4 */ + , /* Rot. dip sw line 2 */ + , /* Rot. dip sw line 1 */ + , + , + ; + }; + + spi1_pinctrl: spi1-pinmux { + pinmux = , + , + , + ; /* cs */ + }; + + i2s0_pinctrl: i2s0-pinmux { + pinmux = , + , + ; + }; +}; + +&uarths0 { + pinctrl-0 = <&uarths_pinctrl>; + pinctrl-names = "default"; + status = "okay"; +}; + +&gpio0 { + pinctrl-0 = <&gpiohs_pinctrl>; + pinctrl-names = "default"; + status = "okay"; +}; + +&i2s0 { + #sound-dai-cells = <1>; + pinctrl-0 = <&i2s0_pinctrl>; + pinctrl-names = "default"; +}; + +&spi0 { + pinctrl-0 = <&spi0_pinctrl>; + pinctrl-names = "default"; + num-cs = <1>; + cs-gpios = <&gpio0 20 GPIO_ACTIVE_HIGH>; + + panel@0 { + compatible = "ilitek,ili9341"; + reg = <0>; + dc-gpios = <&gpio0 21 GPIO_ACTIVE_HIGH>; + spi-max-frequency = <15000000>; + status = "disabled"; + }; +}; + +&spi1 { + pinctrl-0 = <&spi1_pinctrl>; + pinctrl-names = "default"; + num-cs = <1>; + cs-gpios = <&gpio0 16 GPIO_ACTIVE_LOW>; + status = "okay"; + + slot@0 { + compatible = "mmc-spi-slot"; + reg = <0>; + voltage-ranges = <3300 3300>; + spi-max-frequency = <25000000>; + broken-cd; + }; +}; diff --git a/arch/riscv/boot/dts/canaan/k210.dtsi b/arch/riscv/boot/dts/canaan/k210.dtsi new file mode 100644 index 0000000000000000000000000000000000000000..5e8ca8142482153bb3888ff0b5039778958a8bfe --- /dev/null +++ b/arch/riscv/boot/dts/canaan/k210.dtsi @@ -0,0 +1,459 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Copyright (C) 2019-20 Sean Anderson + * Copyright (C) 2020 Western Digital Corporation or its affiliates. + */ +#include +#include +#include + +/ { + /* + * Although the K210 is a 64-bit CPU, the address bus is only 32-bits + * wide, and the upper half of all addresses is ignored. + */ + #address-cells = <1>; + #size-cells = <1>; + compatible = "canaan,kendryte-k210"; + + aliases { + serial0 = &uarths0; + serial1 = &uart1; + serial2 = &uart2; + serial3 = &uart3; + }; + + /* + * The K210 has an sv39 MMU following the privileged specification v1.9. + * Since this is a non-ratified draft specification, the kernel does not + * support it and the K210 support enabled only for the !MMU case. + * Be consistent with this by setting the CPUs MMU type to "none". + */ + cpus { + #address-cells = <1>; + #size-cells = <0>; + timebase-frequency = <7800000>; + cpu0: cpu@0 { + device_type = "cpu"; + compatible = "canaan,k210", "riscv"; + reg = <0>; + riscv,isa = "rv64imafdc"; + mmu-type = "riscv,none"; + i-cache-block-size = <64>; + i-cache-size = <0x8000>; + d-cache-block-size = <64>; + d-cache-size = <0x8000>; + cpu0_intc: interrupt-controller { + #interrupt-cells = <1>; + interrupt-controller; + compatible = "riscv,cpu-intc"; + }; + }; + cpu1: cpu@1 { + device_type = "cpu"; + compatible = "canaan,k210", "riscv"; + reg = <1>; + riscv,isa = "rv64imafdc"; + mmu-type = "riscv,none"; + i-cache-block-size = <64>; + i-cache-size = <0x8000>; + d-cache-block-size = <64>; + d-cache-size = <0x8000>; + cpu1_intc: interrupt-controller { + #interrupt-cells = <1>; + interrupt-controller; + compatible = "riscv,cpu-intc"; + }; + }; + }; + + sram: memory@80000000 { + device_type = "memory"; + compatible = "canaan,k210-sram"; + reg = <0x80000000 0x400000>, + <0x80400000 0x200000>, + <0x80600000 0x200000>; + reg-names = "sram0", "sram1", "aisram"; + clocks = <&sysclk K210_CLK_SRAM0>, + <&sysclk K210_CLK_SRAM1>, + <&sysclk K210_CLK_AI>; + clock-names = "sram0", "sram1", "aisram"; + }; + + clocks { + in0: oscillator { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <26000000>; + }; + }; + + soc { + #address-cells = <1>; + #size-cells = <1>; + compatible = "simple-bus"; + ranges; + interrupt-parent = <&plic0>; + + rom0: nvmem@1000 { + reg = <0x1000 0x1000>; + read-only; + }; + + clint0: timer@2000000 { + compatible = "canaan,k210-clint", "sifive,clint0"; + reg = <0x2000000 0xC000>; + interrupts-extended = <&cpu0_intc 3 &cpu0_intc 7 + &cpu1_intc 3 &cpu1_intc 7>; + }; + + plic0: interrupt-controller@c000000 { + #interrupt-cells = <1>; + #address-cells = <0>; + compatible = "canaan,k210-plic", "sifive,plic-1.0.0"; + reg = <0xC000000 0x4000000>; + interrupt-controller; + interrupts-extended = <&cpu0_intc 11 &cpu1_intc 11>; + riscv,ndev = <65>; + }; + + uarths0: serial@38000000 { + compatible = "canaan,k210-uarths", "sifive,uart0"; + reg = <0x38000000 0x1000>; + interrupts = <33>; + clocks = <&sysclk K210_CLK_CPU>; + }; + + gpio0: gpio-controller@38001000 { + #interrupt-cells = <2>; + #gpio-cells = <2>; + compatible = "canaan,k210-gpiohs", "sifive,gpio0"; + reg = <0x38001000 0x1000>; + interrupt-controller; + interrupts = <34 35 36 37 38 39 40 41 + 42 43 44 45 46 47 48 49 + 50 51 52 53 54 55 56 57 + 58 59 60 61 62 63 64 65>; + gpio-controller; + ngpios = <32>; + }; + + dmac0: dma-controller@50000000 { + compatible = "snps,axi-dma-1.01a"; + reg = <0x50000000 0x1000>; + interrupts = <27 28 29 30 31 32>; + #dma-cells = <1>; + clocks = <&sysclk K210_CLK_DMA>, <&sysclk K210_CLK_DMA>; + clock-names = "core-clk", "cfgr-clk"; + resets = <&sysrst K210_RST_DMA>; + dma-channels = <6>; + snps,dma-masters = <2>; + snps,priority = <0 1 2 3 4 5>; + snps,data-width = <5>; + snps,block-size = <0x200000 0x200000 0x200000 + 0x200000 0x200000 0x200000>; + snps,axi-max-burst-len = <256>; + }; + + apb0: bus@50200000 { + #address-cells = <1>; + #size-cells = <1>; + compatible = "simple-pm-bus"; + ranges; + clocks = <&sysclk K210_CLK_APB0>; + + gpio1: gpio@50200000 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "snps,dw-apb-gpio"; + reg = <0x50200000 0x80>; + clocks = <&sysclk K210_CLK_APB0>, + <&sysclk K210_CLK_GPIO>; + clock-names = "bus", "db"; + resets = <&sysrst K210_RST_GPIO>; + + gpio1_0: gpio-port@0 { + #gpio-cells = <2>; + #interrupt-cells = <2>; + compatible = "snps,dw-apb-gpio-port"; + reg = <0>; + interrupt-controller; + interrupts = <23>; + gpio-controller; + ngpios = <8>; + }; + }; + + uart1: serial@50210000 { + compatible = "snps,dw-apb-uart"; + reg = <0x50210000 0x100>; + interrupts = <11>; + clocks = <&sysclk K210_CLK_UART1>, + <&sysclk K210_CLK_APB0>; + clock-names = "baudclk", "apb_pclk"; + resets = <&sysrst K210_RST_UART1>; + reg-io-width = <4>; + reg-shift = <2>; + dcd-override; + dsr-override; + cts-override; + ri-override; + }; + + uart2: serial@50220000 { + compatible = "snps,dw-apb-uart"; + reg = <0x50220000 0x100>; + interrupts = <12>; + clocks = <&sysclk K210_CLK_UART2>, + <&sysclk K210_CLK_APB0>; + clock-names = "baudclk", "apb_pclk"; + resets = <&sysrst K210_RST_UART2>; + reg-io-width = <4>; + reg-shift = <2>; + dcd-override; + dsr-override; + cts-override; + ri-override; + }; + + uart3: serial@50230000 { + compatible = "snps,dw-apb-uart"; + reg = <0x50230000 0x100>; + interrupts = <13>; + clocks = <&sysclk K210_CLK_UART3>, + <&sysclk K210_CLK_APB0>; + clock-names = "baudclk", "apb_pclk"; + resets = <&sysrst K210_RST_UART3>; + reg-io-width = <4>; + reg-shift = <2>; + dcd-override; + dsr-override; + cts-override; + ri-override; + }; + + spi2: spi@50240000 { + compatible = "canaan,k210-spi"; + spi-slave; + reg = <0x50240000 0x100>; + #address-cells = <0>; + #size-cells = <0>; + interrupts = <3>; + clocks = <&sysclk K210_CLK_SPI2>, + <&sysclk K210_CLK_APB0>; + clock-names = "ssi_clk", "pclk"; + resets = <&sysrst K210_RST_SPI2>; + spi-max-frequency = <25000000>; + }; + + i2s0: i2s@50250000 { + compatible = "snps,designware-i2s"; + reg = <0x50250000 0x200>; + interrupts = <5>; + clocks = <&sysclk K210_CLK_I2S0>; + clock-names = "i2sclk"; + resets = <&sysrst K210_RST_I2S0>; + }; + + i2s1: i2s@50260000 { + compatible = "snps,designware-i2s"; + reg = <0x50260000 0x200>; + interrupts = <6>; + clocks = <&sysclk K210_CLK_I2S1>; + clock-names = "i2sclk"; + resets = <&sysrst K210_RST_I2S1>; + }; + + i2s2: i2s@50270000 { + compatible = "snps,designware-i2s"; + reg = <0x50270000 0x200>; + interrupts = <7>; + clocks = <&sysclk K210_CLK_I2S2>; + clock-names = "i2sclk"; + resets = <&sysrst K210_RST_I2S2>; + }; + + i2c0: i2c@50280000 { + compatible = "snps,designware-i2c"; + reg = <0x50280000 0x100>; + interrupts = <8>; + clocks = <&sysclk K210_CLK_I2C0>, + <&sysclk K210_CLK_APB0>; + clock-names = "ref", "pclk"; + resets = <&sysrst K210_RST_I2C0>; + }; + + i2c1: i2c@50290000 { + compatible = "snps,designware-i2c"; + reg = <0x50290000 0x100>; + interrupts = <9>; + clocks = <&sysclk K210_CLK_I2C1>, + <&sysclk K210_CLK_APB0>; + clock-names = "ref", "pclk"; + resets = <&sysrst K210_RST_I2C1>; + }; + + i2c2: i2c@502a0000 { + compatible = "snps,designware-i2c"; + reg = <0x502A0000 0x100>; + interrupts = <10>; + clocks = <&sysclk K210_CLK_I2C2>, + <&sysclk K210_CLK_APB0>; + clock-names = "ref", "pclk"; + resets = <&sysrst K210_RST_I2C2>; + }; + + fpioa: pinmux@502b0000 { + compatible = "canaan,k210-fpioa"; + reg = <0x502B0000 0x100>; + clocks = <&sysclk K210_CLK_FPIOA>, + <&sysclk K210_CLK_APB0>; + clock-names = "ref", "pclk"; + resets = <&sysrst K210_RST_FPIOA>; + canaan,k210-sysctl-power = <&sysctl 108>; + }; + + timer0: timer@502d0000 { + compatible = "snps,dw-apb-timer"; + reg = <0x502D0000 0x100>; + interrupts = <14 15>; + clocks = <&sysclk K210_CLK_TIMER0>, + <&sysclk K210_CLK_APB0>; + clock-names = "timer", "pclk"; + resets = <&sysrst K210_RST_TIMER0>; + }; + + timer1: timer@502e0000 { + compatible = "snps,dw-apb-timer"; + reg = <0x502E0000 0x100>; + interrupts = <16 17>; + clocks = <&sysclk K210_CLK_TIMER1>, + <&sysclk K210_CLK_APB0>; + clock-names = "timer", "pclk"; + resets = <&sysrst K210_RST_TIMER1>; + }; + + timer2: timer@502f0000 { + compatible = "snps,dw-apb-timer"; + reg = <0x502F0000 0x100>; + interrupts = <18 19>; + clocks = <&sysclk K210_CLK_TIMER2>, + <&sysclk K210_CLK_APB0>; + clock-names = "timer", "pclk"; + resets = <&sysrst K210_RST_TIMER2>; + }; + }; + + apb1: bus@50400000 { + #address-cells = <1>; + #size-cells = <1>; + compatible = "simple-pm-bus"; + ranges; + clocks = <&sysclk K210_CLK_APB1>; + + wdt0: watchdog@50400000 { + compatible = "snps,dw-wdt"; + reg = <0x50400000 0x100>; + interrupts = <21>; + clocks = <&sysclk K210_CLK_WDT0>, + <&sysclk K210_CLK_APB1>; + clock-names = "tclk", "pclk"; + resets = <&sysrst K210_RST_WDT0>; + }; + + wdt1: watchdog@50410000 { + compatible = "snps,dw-wdt"; + reg = <0x50410000 0x100>; + interrupts = <22>; + clocks = <&sysclk K210_CLK_WDT1>, + <&sysclk K210_CLK_APB1>; + clock-names = "tclk", "pclk"; + resets = <&sysrst K210_RST_WDT1>; + }; + + sysctl: syscon@50440000 { + compatible = "canaan,k210-sysctl", + "syscon", "simple-mfd"; + reg = <0x50440000 0x100>; + clocks = <&sysclk K210_CLK_APB1>; + clock-names = "pclk"; + + sysclk: clock-controller { + #clock-cells = <1>; + compatible = "canaan,k210-clk"; + clocks = <&in0>; + }; + + sysrst: reset-controller { + compatible = "canaan,k210-rst"; + #reset-cells = <1>; + }; + + reboot: syscon-reboot { + compatible = "syscon-reboot"; + regmap = <&sysctl>; + offset = <48>; + mask = <1>; + value = <1>; + }; + }; + }; + + apb2: bus@52000000 { + #address-cells = <1>; + #size-cells = <1>; + compatible = "simple-pm-bus"; + ranges; + clocks = <&sysclk K210_CLK_APB2>; + + spi0: spi@52000000 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "canaan,k210-spi"; + reg = <0x52000000 0x100>; + interrupts = <1>; + clocks = <&sysclk K210_CLK_SPI0>, + <&sysclk K210_CLK_APB2>; + clock-names = "ssi_clk", "pclk"; + resets = <&sysrst K210_RST_SPI0>; + reset-names = "spi"; + spi-max-frequency = <25000000>; + num-cs = <4>; + reg-io-width = <4>; + }; + + spi1: spi@53000000 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "canaan,k210-spi"; + reg = <0x53000000 0x100>; + interrupts = <2>; + clocks = <&sysclk K210_CLK_SPI1>, + <&sysclk K210_CLK_APB2>; + clock-names = "ssi_clk", "pclk"; + resets = <&sysrst K210_RST_SPI1>; + reset-names = "spi"; + spi-max-frequency = <25000000>; + num-cs = <4>; + reg-io-width = <4>; + }; + + spi3: spi@54000000 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "snps,dwc-ssi-1.01a"; + reg = <0x54000000 0x200>; + interrupts = <4>; + clocks = <&sysclk K210_CLK_SPI3>, + <&sysclk K210_CLK_APB2>; + clock-names = "ssi_clk", "pclk"; + resets = <&sysrst K210_RST_SPI3>; + reset-names = "spi"; + /* Could possibly go up to 200 MHz */ + spi-max-frequency = <100000000>; + num-cs = <4>; + reg-io-width = <4>; + }; + }; + }; +}; diff --git a/arch/riscv/boot/dts/canaan/k210_generic.dts b/arch/riscv/boot/dts/canaan/k210_generic.dts new file mode 100644 index 0000000000000000000000000000000000000000..396c8ca4d24db79ae9748ff7f263fa1b514f0da5 --- /dev/null +++ b/arch/riscv/boot/dts/canaan/k210_generic.dts @@ -0,0 +1,46 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Copyright (C) 2019-20 Sean Anderson + * Copyright (C) 2020 Western Digital Corporation or its affiliates. + */ + +/dts-v1/; + +#include "k210.dtsi" + +#include +#include + +/ { + model = "Kendryte K210 generic"; + compatible = "canaan,kendryte-k210"; + + chosen { + bootargs = "earlycon console=ttySIF0"; + stdout-path = "serial0:115200n8"; + }; +}; + +&fpioa { + pinctrl-0 = <&jtag_pins>; + pinctrl-names = "default"; + status = "okay"; + + jtag_pins: jtag-pinmux { + pinmux = , + , + , + ; + }; + + uarths_pins: uarths-pinmux { + pinmux = , + ; + }; +}; + +&uarths0 { + pinctrl-0 = <&uarths_pins>; + pinctrl-names = "default"; + status = "okay"; +}; diff --git a/arch/riscv/boot/dts/canaan/sipeed_maix_bit.dts b/arch/riscv/boot/dts/canaan/sipeed_maix_bit.dts new file mode 100644 index 0000000000000000000000000000000000000000..0bcaf35045e795ed2c7c518004730a763b42ad36 --- /dev/null +++ b/arch/riscv/boot/dts/canaan/sipeed_maix_bit.dts @@ -0,0 +1,209 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Copyright (C) 2019-20 Sean Anderson + * Copyright (C) 2020 Western Digital Corporation or its affiliates. + */ + +/dts-v1/; + +#include "k210.dtsi" + +#include +#include +#include + +/ { + model = "SiPeed MAIX BiT"; + compatible = "sipeed,maix-bit", "sipeed,maix-bitm", + "canaan,kendryte-k210"; + + chosen { + bootargs = "earlycon console=ttySIF0"; + stdout-path = "serial0:115200n8"; + }; + + gpio-leds { + compatible = "gpio-leds"; + + led0 { + color = ; + label = "green"; + gpios = <&gpio1_0 4 GPIO_ACTIVE_LOW>; + }; + + led1 { + color = ; + label = "red"; + gpios = <&gpio1_0 5 GPIO_ACTIVE_LOW>; + }; + + led2 { + color = ; + label = "blue"; + gpios = <&gpio1_0 6 GPIO_ACTIVE_LOW>; + }; + }; + + gpio-keys { + compatible = "gpio-keys"; + + boot { + label = "BOOT"; + linux,code = ; + gpios = <&gpio0 0 GPIO_ACTIVE_LOW>; + }; + }; +}; + +&fpioa { + pinctrl-names = "default"; + pinctrl-0 = <&jtag_pinctrl>; + status = "okay"; + + jtag_pinctrl: jtag-pinmux { + pinmux = , + , + , + ; + }; + + uarths_pinctrl: uarths-pinmux { + pinmux = , + ; + }; + + gpio_pinctrl: gpio-pinmux { + pinmux = , + , + , + , + , + , + , + ; + }; + + gpiohs_pinctrl: gpiohs-pinmux { + pinmux = , + , + , + , + , + , + , + , + , + , + ; + }; + + i2s0_pinctrl: i2s0-pinmux { + pinmux = , + , + ; + }; + + dvp_pinctrl: dvp-pinmux { + pinmux = , + , + , + , + , + , + , + ; + }; + + spi0_pinctrl: spi0-pinmux { + pinmux = , /* cs */ + , /* rst */ + , /* dc */ + ; /* wr */ + }; + + spi1_pinctrl: spi1-pinmux { + pinmux = , + , + , + ; /* cs */ + }; + + i2c1_pinctrl: i2c1-pinmux { + pinmux = , + ; + }; +}; + +&uarths0 { + pinctrl-0 = <&uarths_pinctrl>; + pinctrl-names = "default"; + status = "okay"; +}; + +&gpio0 { + pinctrl-0 = <&gpiohs_pinctrl>; + pinctrl-names = "default"; + status = "okay"; +}; + +&gpio1 { + pinctrl-0 = <&gpio_pinctrl>; + pinctrl-names = "default"; + status = "okay"; +}; + +&i2s0 { + #sound-dai-cells = <1>; + pinctrl-0 = <&i2s0_pinctrl>; + pinctrl-names = "default"; +}; + +&i2c1 { + pinctrl-0 = <&i2c1_pinctrl>; + pinctrl-names = "default"; + clock-frequency = <400000>; + status = "okay"; +}; + +&spi0 { + pinctrl-0 = <&spi0_pinctrl>; + pinctrl-names = "default"; + num-cs = <1>; + cs-gpios = <&gpio0 20 GPIO_ACTIVE_HIGH>; + + panel@0 { + compatible = "sitronix,st7789v"; + reg = <0>; + reset-gpios = <&gpio0 21 GPIO_ACTIVE_LOW>; + dc-gpios = <&gpio0 22 GPIO_ACTIVE_HIGH>; + spi-max-frequency = <15000000>; + spi-cs-high; + status = "disabled"; + }; +}; + +&spi1 { + pinctrl-0 = <&spi1_pinctrl>; + pinctrl-names = "default"; + num-cs = <1>; + cs-gpios = <&gpio0 13 GPIO_ACTIVE_LOW>; + status = "okay"; + + slot@0 { + compatible = "mmc-spi-slot"; + reg = <0>; + voltage-ranges = <3300 3300>; + spi-max-frequency = <25000000>; + broken-cd; + }; +}; + +&spi3 { + spi-flash@0 { + compatible = "jedec,spi-nor"; + reg = <0>; + spi-max-frequency = <50000000>; + m25p,fast-read; + broken-flash-reset; + }; +}; diff --git a/arch/riscv/boot/dts/canaan/sipeed_maix_dock.dts b/arch/riscv/boot/dts/canaan/sipeed_maix_dock.dts new file mode 100644 index 0000000000000000000000000000000000000000..ac8a03f5867adbd03d33361ebfb3c979c4e0bec9 --- /dev/null +++ b/arch/riscv/boot/dts/canaan/sipeed_maix_dock.dts @@ -0,0 +1,211 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Copyright (C) 2019-20 Sean Anderson + * Copyright (C) 2020 Western Digital Corporation or its affiliates. + */ + +/dts-v1/; + +#include "k210.dtsi" + +#include +#include +#include + +/ { + model = "SiPeed MAIX Dock"; + compatible = "sipeed,maix-dock-m1", "sipeed,maix-dock-m1w", + "canaan,kendryte-k210"; + + chosen { + bootargs = "earlycon console=ttySIF0"; + stdout-path = "serial0:115200n8"; + }; + + gpio-leds { + compatible = "gpio-leds"; + + /* + * Note: the board wiring drawing documents green on + * gpio #4, red on gpio #5 and blue on gpio #6. However, + * the board is actually wired differently as defined here. + */ + led0 { + color = ; + label = "blue"; + gpios = <&gpio1_0 4 GPIO_ACTIVE_LOW>; + }; + + led1 { + color = ; + label = "green"; + gpios = <&gpio1_0 5 GPIO_ACTIVE_LOW>; + }; + + led2 { + color = ; + label = "red"; + gpios = <&gpio1_0 6 GPIO_ACTIVE_LOW>; + }; + }; + + gpio-keys { + compatible = "gpio-keys"; + + boot { + label = "BOOT"; + linux,code = ; + gpios = <&gpio0 0 GPIO_ACTIVE_LOW>; + }; + }; +}; + +&fpioa { + pinctrl-0 = <&jtag_pinctrl>; + pinctrl-names = "default"; + status = "okay"; + + jtag_pinctrl: jtag-pinmux { + pinmux = , + , + , + ; + }; + + uarths_pinctrl: uarths-pinmux { + pinmux = , + ; + }; + + gpio_pinctrl: gpio-pinmux { + pinmux = , + , + , + , + , + ; + }; + + gpiohs_pinctrl: gpiohs-pinmux { + pinmux = , + , + , + , + , + , + , + , + , + , + ; + }; + + i2s0_pinctrl: i2s0-pinmux { + pinmux = , + , + ; + }; + + dvp_pinctrl: dvp-pinmux { + pinmux = , + , + , + , + , + , + , + ; + }; + + spi0_pinctrl: spi0-pinmux { + pinmux = , /* cs */ + , /* rst */ + , /* dc */ + ; /* wr */ + }; + + spi1_pinctrl: spi1-pinmux { + pinmux = , + , + , + ; /* cs */ + }; + + i2c1_pinctrl: i2c1-pinmux { + pinmux = , + ; + }; +}; + +&uarths0 { + pinctrl-0 = <&uarths_pinctrl>; + pinctrl-names = "default"; + status = "okay"; +}; + +&gpio0 { + pinctrl-0 = <&gpiohs_pinctrl>; + pinctrl-names = "default"; + status = "okay"; +}; + +&gpio1 { + pinctrl-0 = <&gpio_pinctrl>; + pinctrl-names = "default"; + status = "okay"; +}; + +&i2s0 { + #sound-dai-cells = <1>; + pinctrl-0 = <&i2s0_pinctrl>; + pinctrl-names = "default"; +}; + +&i2c1 { + pinctrl-0 = <&i2c1_pinctrl>; + pinctrl-names = "default"; + clock-frequency = <400000>; + status = "okay"; +}; + +&spi0 { + pinctrl-0 = <&spi0_pinctrl>; + pinctrl-names = "default"; + num-cs = <1>; + cs-gpios = <&gpio0 20 GPIO_ACTIVE_HIGH>; + + panel@0 { + compatible = "sitronix,st7789v"; + reg = <0>; + reset-gpios = <&gpio0 21 GPIO_ACTIVE_LOW>; + dc-gpios = <&gpio0 22 0>; + spi-max-frequency = <15000000>; + status = "disabled"; + }; +}; + +&spi1 { + pinctrl-0 = <&spi1_pinctrl>; + pinctrl-names = "default"; + num-cs = <1>; + cs-gpios = <&gpio0 13 GPIO_ACTIVE_LOW>; + status = "okay"; + + slot@0 { + compatible = "mmc-spi-slot"; + reg = <0>; + voltage-ranges = <3300 3300>; + spi-max-frequency = <25000000>; + broken-cd; + }; +}; + +&spi3 { + spi-flash@0 { + compatible = "jedec,spi-nor"; + reg = <0>; + spi-max-frequency = <50000000>; + m25p,fast-read; + broken-flash-reset; + }; +}; diff --git a/arch/riscv/boot/dts/canaan/sipeed_maix_go.dts b/arch/riscv/boot/dts/canaan/sipeed_maix_go.dts new file mode 100644 index 0000000000000000000000000000000000000000..623998194bc18aabb0aee79bad26a71665c64c8d --- /dev/null +++ b/arch/riscv/boot/dts/canaan/sipeed_maix_go.dts @@ -0,0 +1,219 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Copyright (C) 2019-20 Sean Anderson + * Copyright (C) 2020 Western Digital Corporation or its affiliates. + */ + +/dts-v1/; + +#include "k210.dtsi" + +#include +#include +#include + +/ { + model = "SiPeed MAIX GO"; + compatible = "sipeed,maix-go", "canaan,kendryte-k210"; + + chosen { + bootargs = "earlycon console=ttySIF0"; + stdout-path = "serial0:115200n8"; + }; + + gpio-leds { + compatible = "gpio-leds"; + + led0 { + color = ; + label = "green"; + gpios = <&gpio1_0 4 GPIO_ACTIVE_LOW>; + }; + + led1 { + color = ; + label = "red"; + gpios = <&gpio1_0 5 GPIO_ACTIVE_LOW>; + }; + + led2 { + color = ; + label = "blue"; + gpios = <&gpio1_0 6 GPIO_ACTIVE_LOW>; + }; + }; + + gpio-keys { + compatible = "gpio-keys"; + + up { + label = "UP"; + linux,code = ; + gpios = <&gpio1_0 7 GPIO_ACTIVE_LOW>; + }; + + press { + label = "PRESS"; + linux,code = ; + gpios = <&gpio0 0 GPIO_ACTIVE_LOW>; + }; + + down { + label = "DOWN"; + linux,code = ; + gpios = <&gpio0 1 GPIO_ACTIVE_LOW>; + }; + }; +}; + +&fpioa { + pinctrl-0 = <&jtag_pinctrl>; + pinctrl-names = "default"; + status = "okay"; + + jtag_pinctrl: jtag-pinmux { + pinmux = , + , + , + ; + }; + + uarths_pinctrl: uarths-pinmux { + pinmux = , + ; + }; + + gpio_pinctrl: gpio-pinmux { + pinmux = , + , + , + , + , + , + , + ; + }; + + gpiohs_pinctrl: gpiohs-pinmux { + pinmux = , + , + , + , + , + , + , + , + , + , + ; + }; + + i2s0_pinctrl: i2s0-pinmux { + pinmux = , + , + ; + }; + + dvp_pinctrl: dvp-pinmux { + pinmux = , + , + , + , + , + , + , + ; + }; + + spi0_pinctrl: spi0-pinmux { + pinmux = , /* cs */ + , /* rst */ + , /* dc */ + ; /* wr */ + }; + + spi1_pinctrl: spi1-pinmux { + pinmux = , + , + , + ; /* cs */ + }; + + i2c1_pinctrl: i2c1-pinmux { + pinmux = , + ; + }; +}; + +&uarths0 { + pinctrl-0 = <&uarths_pinctrl>; + pinctrl-names = "default"; + status = "okay"; +}; + +&gpio0 { + pinctrl-0 = <&gpiohs_pinctrl>; + pinctrl-names = "default"; + status = "okay"; +}; + +&gpio1 { + pinctrl-0 = <&gpio_pinctrl>; + pinctrl-names = "default"; + status = "okay"; +}; + +&i2s0 { + #sound-dai-cells = <1>; + pinctrl-0 = <&i2s0_pinctrl>; + pinctrl-names = "default"; +}; + +&i2c1 { + pinctrl-0 = <&i2c1_pinctrl>; + pinctrl-names = "default"; + clock-frequency = <400000>; + status = "okay"; +}; + +&spi0 { + pinctrl-0 = <&spi0_pinctrl>; + pinctrl-names = "default"; + num-cs = <1>; + cs-gpios = <&gpio0 20 GPIO_ACTIVE_HIGH>; + + panel@0 { + compatible = "sitronix,st7789v"; + reg = <0>; + reset-gpios = <&gpio0 21 GPIO_ACTIVE_LOW>; + dc-gpios = <&gpio0 22 GPIO_ACTIVE_HIGH>; + spi-max-frequency = <15000000>; + status = "disabled"; + }; +}; + +&spi1 { + pinctrl-0 = <&spi1_pinctrl>; + pinctrl-names = "default"; + num-cs = <1>; + cs-gpios = <&gpio0 13 GPIO_ACTIVE_LOW>; + status = "okay"; + + slot@0 { + compatible = "mmc-spi-slot"; + reg = <0>; + voltage-ranges = <3300 3300>; + spi-max-frequency = <25000000>; + broken-cd; + }; +}; + +&spi3 { + spi-flash@0 { + compatible = "jedec,spi-nor"; + reg = <0>; + spi-max-frequency = <50000000>; + m25p,fast-read; + broken-flash-reset; + }; +}; diff --git a/arch/riscv/boot/dts/canaan/sipeed_maixduino.dts b/arch/riscv/boot/dts/canaan/sipeed_maixduino.dts new file mode 100644 index 0000000000000000000000000000000000000000..cf605ba0d67e43cd3303c135ae695d9580ebe119 --- /dev/null +++ b/arch/riscv/boot/dts/canaan/sipeed_maixduino.dts @@ -0,0 +1,184 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Copyright (C) 2019-20 Sean Anderson + * Copyright (C) 2020 Western Digital Corporation or its affiliates. + */ + +/dts-v1/; + +#include "k210.dtsi" + +#include +#include + +/ { + model = "SiPeed MAIXDUINO"; + compatible = "sipeed,maixduino", "canaan,kendryte-k210"; + + chosen { + bootargs = "earlycon console=ttySIF0"; + stdout-path = "serial0:115200n8"; + }; + + gpio-keys { + compatible = "gpio-keys"; + + boot { + label = "BOOT"; + linux,code = ; + gpios = <&gpio0 0 GPIO_ACTIVE_LOW>; + }; + }; + + vcc_3v3: regulator-3v3 { + compatible = "regulator-fixed"; + regulator-name = "3v3"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + }; +}; + +&fpioa { + status = "okay"; + + uarths_pinctrl: uarths-pinmux { + pinmux = , /* Header "0" */ + ; /* Header "1" */ + }; + + gpio_pinctrl: gpio-pinmux { + pinmux = , + ; + }; + + gpiohs_pinctrl: gpiohs-pinmux { + pinmux = , /* BOOT */ + , /* Header "2" */ + , /* Header "3" */ + , /* Header "4" */ + , /* Header "5" */ + , /* Header "6" */ + , /* Header "7" */ + , /* Header "8" */ + , /* Header "9" */ + , /* Header "10" */ + , /* Header "11" */ + , /* Header "12" */ + ; /* Header "13" */ + }; + + i2s0_pinctrl: i2s0-pinmux { + pinmux = , + , + ; + }; + + spi1_pinctrl: spi1-pinmux { + pinmux = , + , + , + ; /* cs */ + }; + + i2c1_pinctrl: i2c1-pinmux { + pinmux = , /* Header "scl" */ + ; /* Header "sda" */ + }; + + i2s1_pinctrl: i2s1-pinmux { + pinmux = , + , + ; + }; + + spi0_pinctrl: spi0-pinmux { + pinmux = , /* cs */ + , /* rst */ + , /* dc */ + ; /* wr */ + }; + + dvp_pinctrl: dvp-pinmux { + pinmux = , + , + , + , + , + , + , + ; + }; +}; + +&uarths0 { + pinctrl-0 = <&uarths_pinctrl>; + pinctrl-names = "default"; + status = "okay"; +}; + +&gpio0 { + pinctrl-0 = <&gpiohs_pinctrl>; + pinctrl-names = "default"; + status = "okay"; +}; + +&gpio1 { + pinctrl-0 = <&gpio_pinctrl>; + pinctrl-names = "default"; + status = "okay"; +}; + +&i2s0 { + #sound-dai-cells = <1>; + pinctrl-0 = <&i2s0_pinctrl>; + pinctrl-names = "default"; +}; + +&i2c1 { + pinctrl-0 = <&i2c1_pinctrl>; + pinctrl-names = "default"; + clock-frequency = <400000>; + status = "okay"; +}; + +&spi0 { + pinctrl-0 = <&spi0_pinctrl>; + pinctrl-names = "default"; + num-cs = <1>; + cs-gpios = <&gpio0 20 GPIO_ACTIVE_HIGH>; + + panel@0 { + compatible = "sitronix,st7789v"; + reg = <0>; + reset-gpios = <&gpio0 21 GPIO_ACTIVE_LOW>; + dc-gpios = <&gpio0 22 0>; + spi-max-frequency = <15000000>; + power-supply = <&vcc_3v3>; + }; +}; + +&spi1 { + pinctrl-0 = <&spi1_pinctrl>; + pinctrl-names = "default"; + num-cs = <1>; + cs-gpios = <&gpio1_0 2 GPIO_ACTIVE_LOW>; + status = "okay"; + + slot@0 { + compatible = "mmc-spi-slot"; + reg = <0>; + voltage-ranges = <3300 3300>; + spi-max-frequency = <25000000>; + broken-cd; + }; +}; + +&spi3 { + spi-flash@0 { + compatible = "jedec,spi-nor"; + reg = <0>; + spi-max-frequency = <50000000>; + m25p,fast-read; + broken-flash-reset; + }; +}; diff --git a/arch/riscv/boot/dts/kendryte/Makefile b/arch/riscv/boot/dts/kendryte/Makefile deleted file mode 100644 index 1a88e616f18e0dafa7eb54842018fde2de6aaa29..0000000000000000000000000000000000000000 --- a/arch/riscv/boot/dts/kendryte/Makefile +++ /dev/null @@ -1,4 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 -dtb-$(CONFIG_SOC_KENDRYTE_K210_DTB) += k210.dtb - -obj-$(CONFIG_SOC_KENDRYTE_K210_DTB_BUILTIN) += $(addsuffix .o, $(dtb-y)) diff --git a/arch/riscv/boot/dts/kendryte/k210.dts b/arch/riscv/boot/dts/kendryte/k210.dts deleted file mode 100644 index 0d1f28fce6b2bd6c72a41dec8d595e32c5cc0498..0000000000000000000000000000000000000000 --- a/arch/riscv/boot/dts/kendryte/k210.dts +++ /dev/null @@ -1,23 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0+ -/* - * Copyright (C) 2020 Western Digital Corporation or its affiliates. - */ - -/dts-v1/; - -#include "k210.dtsi" - -/ { - model = "Kendryte K210 generic"; - compatible = "kendryte,k210"; - - chosen { - bootargs = "earlycon console=ttySIF0"; - stdout-path = "serial0"; - }; -}; - -&uarths0 { - status = "okay"; -}; - diff --git a/arch/riscv/boot/dts/kendryte/k210.dtsi b/arch/riscv/boot/dts/kendryte/k210.dtsi deleted file mode 100644 index d2d0ff6456325bc0276b4bac867644ea761fb967..0000000000000000000000000000000000000000 --- a/arch/riscv/boot/dts/kendryte/k210.dtsi +++ /dev/null @@ -1,125 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0+ -/* - * Copyright (C) 2019 Sean Anderson - * Copyright (C) 2020 Western Digital Corporation or its affiliates. - */ -#include - -/ { - /* - * Although the K210 is a 64-bit CPU, the address bus is only 32-bits - * wide, and the upper half of all addresses is ignored. - */ - #address-cells = <1>; - #size-cells = <1>; - compatible = "kendryte,k210"; - - aliases { - serial0 = &uarths0; - }; - - /* - * The K210 has an sv39 MMU following the priviledge specification v1.9. - * Since this is a non-ratified draft specification, the kernel does not - * support it and the K210 support enabled only for the !MMU case. - * Be consistent with this by setting the CPUs MMU type to "none". - */ - cpus { - #address-cells = <1>; - #size-cells = <0>; - timebase-frequency = <7800000>; - cpu0: cpu@0 { - device_type = "cpu"; - reg = <0>; - compatible = "kendryte,k210", "sifive,rocket0", "riscv"; - riscv,isa = "rv64imafdc"; - mmu-type = "none"; - i-cache-size = <0x8000>; - i-cache-block-size = <64>; - d-cache-size = <0x8000>; - d-cache-block-size = <64>; - clocks = <&sysctl K210_CLK_CPU>; - clock-frequency = <390000000>; - cpu0_intc: interrupt-controller { - #interrupt-cells = <1>; - interrupt-controller; - compatible = "riscv,cpu-intc"; - }; - }; - cpu1: cpu@1 { - device_type = "cpu"; - reg = <1>; - compatible = "kendryte,k210", "sifive,rocket0", "riscv"; - riscv,isa = "rv64imafdc"; - mmu-type = "none"; - i-cache-size = <0x8000>; - i-cache-block-size = <64>; - d-cache-size = <0x8000>; - d-cache-block-size = <64>; - clocks = <&sysctl K210_CLK_CPU>; - clock-frequency = <390000000>; - cpu1_intc: interrupt-controller { - #interrupt-cells = <1>; - interrupt-controller; - compatible = "riscv,cpu-intc"; - }; - }; - }; - - sram: memory@80000000 { - device_type = "memory"; - reg = <0x80000000 0x400000>, - <0x80400000 0x200000>, - <0x80600000 0x200000>; - reg-names = "sram0", "sram1", "aisram"; - }; - - clocks { - in0: oscillator { - compatible = "fixed-clock"; - #clock-cells = <0>; - clock-frequency = <26000000>; - }; - }; - - soc { - #address-cells = <1>; - #size-cells = <1>; - compatible = "kendryte,k210-soc", "simple-bus"; - ranges; - interrupt-parent = <&plic0>; - - sysctl: sysctl@50440000 { - compatible = "kendryte,k210-sysctl", "simple-mfd"; - reg = <0x50440000 0x1000>; - #clock-cells = <1>; - }; - - clint0: clint@2000000 { - #interrupt-cells = <1>; - compatible = "riscv,clint0"; - reg = <0x2000000 0xC000>; - interrupts-extended = <&cpu0_intc 3 &cpu0_intc 7 - &cpu1_intc 3 &cpu1_intc 7>; - clocks = <&sysctl K210_CLK_ACLK>; - }; - - plic0: interrupt-controller@c000000 { - #interrupt-cells = <1>; - interrupt-controller; - compatible = "kendryte,k210-plic0", "riscv,plic0"; - reg = <0xC000000 0x4000000>; - interrupts-extended = <&cpu0_intc 11>, <&cpu0_intc 0xffffffff>, - <&cpu1_intc 11>, <&cpu1_intc 0xffffffff>; - riscv,ndev = <65>; - riscv,max-priority = <7>; - }; - - uarths0: serial@38000000 { - compatible = "kendryte,k210-uarths", "sifive,uart0"; - reg = <0x38000000 0x1000>; - interrupts = <33>; - clocks = <&sysctl K210_CLK_CPU>; - }; - }; -}; diff --git a/arch/riscv/boot/dts/sifive/Makefile b/arch/riscv/boot/dts/sifive/Makefile index 6d6189e6e4af4d8e906e573677e1b4b6728cbf8b..74c47fe9fc22e037c6eb441855fa030fa89a7f4f 100644 --- a/arch/riscv/boot/dts/sifive/Makefile +++ b/arch/riscv/boot/dts/sifive/Makefile @@ -1,2 +1,3 @@ # SPDX-License-Identifier: GPL-2.0 -dtb-$(CONFIG_SOC_SIFIVE) += hifive-unleashed-a00.dtb +dtb-$(CONFIG_SOC_SIFIVE) += hifive-unleashed-a00.dtb \ + hifive-unmatched-a00.dtb diff --git a/arch/riscv/boot/dts/sifive/fu740-c000.dtsi b/arch/riscv/boot/dts/sifive/fu740-c000.dtsi new file mode 100644 index 0000000000000000000000000000000000000000..eeb4f8c3e0e72340372b8ff745cabdebf65982e1 --- /dev/null +++ b/arch/riscv/boot/dts/sifive/fu740-c000.dtsi @@ -0,0 +1,293 @@ +// SPDX-License-Identifier: (GPL-2.0 OR MIT) +/* Copyright (c) 2020 SiFive, Inc */ + +/dts-v1/; + +#include + +/ { + #address-cells = <2>; + #size-cells = <2>; + compatible = "sifive,fu740-c000", "sifive,fu740"; + + aliases { + serial0 = &uart0; + serial1 = &uart1; + ethernet0 = ð0; + }; + + chosen { + }; + + cpus { + #address-cells = <1>; + #size-cells = <0>; + cpu0: cpu@0 { + compatible = "sifive,bullet0", "riscv"; + device_type = "cpu"; + i-cache-block-size = <64>; + i-cache-sets = <128>; + i-cache-size = <16384>; + next-level-cache = <&ccache>; + reg = <0x0>; + riscv,isa = "rv64imac"; + status = "disabled"; + cpu0_intc: interrupt-controller { + #interrupt-cells = <1>; + compatible = "riscv,cpu-intc"; + interrupt-controller; + }; + }; + cpu1: cpu@1 { + compatible = "sifive,bullet0", "riscv"; + d-cache-block-size = <64>; + d-cache-sets = <64>; + d-cache-size = <32768>; + d-tlb-sets = <1>; + d-tlb-size = <40>; + device_type = "cpu"; + i-cache-block-size = <64>; + i-cache-sets = <128>; + i-cache-size = <32768>; + i-tlb-sets = <1>; + i-tlb-size = <40>; + mmu-type = "riscv,sv39"; + next-level-cache = <&ccache>; + reg = <0x1>; + riscv,isa = "rv64imafdc"; + tlb-split; + cpu1_intc: interrupt-controller { + #interrupt-cells = <1>; + compatible = "riscv,cpu-intc"; + interrupt-controller; + }; + }; + cpu2: cpu@2 { + compatible = "sifive,bullet0", "riscv"; + d-cache-block-size = <64>; + d-cache-sets = <64>; + d-cache-size = <32768>; + d-tlb-sets = <1>; + d-tlb-size = <40>; + device_type = "cpu"; + i-cache-block-size = <64>; + i-cache-sets = <128>; + i-cache-size = <32768>; + i-tlb-sets = <1>; + i-tlb-size = <40>; + mmu-type = "riscv,sv39"; + next-level-cache = <&ccache>; + reg = <0x2>; + riscv,isa = "rv64imafdc"; + tlb-split; + cpu2_intc: interrupt-controller { + #interrupt-cells = <1>; + compatible = "riscv,cpu-intc"; + interrupt-controller; + }; + }; + cpu3: cpu@3 { + compatible = "sifive,bullet0", "riscv"; + d-cache-block-size = <64>; + d-cache-sets = <64>; + d-cache-size = <32768>; + d-tlb-sets = <1>; + d-tlb-size = <40>; + device_type = "cpu"; + i-cache-block-size = <64>; + i-cache-sets = <128>; + i-cache-size = <32768>; + i-tlb-sets = <1>; + i-tlb-size = <40>; + mmu-type = "riscv,sv39"; + next-level-cache = <&ccache>; + reg = <0x3>; + riscv,isa = "rv64imafdc"; + tlb-split; + cpu3_intc: interrupt-controller { + #interrupt-cells = <1>; + compatible = "riscv,cpu-intc"; + interrupt-controller; + }; + }; + cpu4: cpu@4 { + compatible = "sifive,bullet0", "riscv"; + d-cache-block-size = <64>; + d-cache-sets = <64>; + d-cache-size = <32768>; + d-tlb-sets = <1>; + d-tlb-size = <40>; + device_type = "cpu"; + i-cache-block-size = <64>; + i-cache-sets = <128>; + i-cache-size = <32768>; + i-tlb-sets = <1>; + i-tlb-size = <40>; + mmu-type = "riscv,sv39"; + next-level-cache = <&ccache>; + reg = <0x4>; + riscv,isa = "rv64imafdc"; + tlb-split; + cpu4_intc: interrupt-controller { + #interrupt-cells = <1>; + compatible = "riscv,cpu-intc"; + interrupt-controller; + }; + }; + }; + soc { + #address-cells = <2>; + #size-cells = <2>; + compatible = "simple-bus"; + ranges; + plic0: interrupt-controller@c000000 { + #interrupt-cells = <1>; + #address-cells = <0>; + compatible = "sifive,fu540-c000-plic", "sifive,plic-1.0.0"; + reg = <0x0 0xc000000 0x0 0x4000000>; + riscv,ndev = <69>; + interrupt-controller; + interrupts-extended = < + &cpu0_intc 0xffffffff + &cpu1_intc 0xffffffff &cpu1_intc 9 + &cpu2_intc 0xffffffff &cpu2_intc 9 + &cpu3_intc 0xffffffff &cpu3_intc 9 + &cpu4_intc 0xffffffff &cpu4_intc 9>; + }; + prci: clock-controller@10000000 { + compatible = "sifive,fu740-c000-prci"; + reg = <0x0 0x10000000 0x0 0x1000>; + clocks = <&hfclk>, <&rtcclk>; + #clock-cells = <1>; + }; + uart0: serial@10010000 { + compatible = "sifive,fu740-c000-uart", "sifive,uart0"; + reg = <0x0 0x10010000 0x0 0x1000>; + interrupt-parent = <&plic0>; + interrupts = <39>; + clocks = <&prci PRCI_CLK_PCLK>; + status = "disabled"; + }; + uart1: serial@10011000 { + compatible = "sifive,fu740-c000-uart", "sifive,uart0"; + reg = <0x0 0x10011000 0x0 0x1000>; + interrupt-parent = <&plic0>; + interrupts = <40>; + clocks = <&prci PRCI_CLK_PCLK>; + status = "disabled"; + }; + i2c0: i2c@10030000 { + compatible = "sifive,fu740-c000-i2c", "sifive,i2c0"; + reg = <0x0 0x10030000 0x0 0x1000>; + interrupt-parent = <&plic0>; + interrupts = <52>; + clocks = <&prci PRCI_CLK_PCLK>; + reg-shift = <2>; + reg-io-width = <1>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + i2c1: i2c@10031000 { + compatible = "sifive,fu740-c000-i2c", "sifive,i2c0"; + reg = <0x0 0x10031000 0x0 0x1000>; + interrupt-parent = <&plic0>; + interrupts = <53>; + clocks = <&prci PRCI_CLK_PCLK>; + reg-shift = <2>; + reg-io-width = <1>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + qspi0: spi@10040000 { + compatible = "sifive,fu740-c000-spi", "sifive,spi0"; + reg = <0x0 0x10040000 0x0 0x1000>, + <0x0 0x20000000 0x0 0x10000000>; + interrupt-parent = <&plic0>; + interrupts = <41>; + clocks = <&prci PRCI_CLK_PCLK>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + qspi1: spi@10041000 { + compatible = "sifive,fu740-c000-spi", "sifive,spi0"; + reg = <0x0 0x10041000 0x0 0x1000>, + <0x0 0x30000000 0x0 0x10000000>; + interrupt-parent = <&plic0>; + interrupts = <42>; + clocks = <&prci PRCI_CLK_PCLK>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + spi0: spi@10050000 { + compatible = "sifive,fu740-c000-spi", "sifive,spi0"; + reg = <0x0 0x10050000 0x0 0x1000>; + interrupt-parent = <&plic0>; + interrupts = <43>; + clocks = <&prci PRCI_CLK_PCLK>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + eth0: ethernet@10090000 { + compatible = "sifive,fu540-c000-gem"; + interrupt-parent = <&plic0>; + interrupts = <55>; + reg = <0x0 0x10090000 0x0 0x2000>, + <0x0 0x100a0000 0x0 0x1000>; + local-mac-address = [00 00 00 00 00 00]; + clock-names = "pclk", "hclk"; + clocks = <&prci PRCI_CLK_GEMGXLPLL>, + <&prci PRCI_CLK_GEMGXLPLL>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + pwm0: pwm@10020000 { + compatible = "sifive,fu740-c000-pwm", "sifive,pwm0"; + reg = <0x0 0x10020000 0x0 0x1000>; + interrupt-parent = <&plic0>; + interrupts = <44>, <45>, <46>, <47>; + clocks = <&prci PRCI_CLK_PCLK>; + #pwm-cells = <3>; + status = "disabled"; + }; + pwm1: pwm@10021000 { + compatible = "sifive,fu740-c000-pwm", "sifive,pwm0"; + reg = <0x0 0x10021000 0x0 0x1000>; + interrupt-parent = <&plic0>; + interrupts = <48>, <49>, <50>, <51>; + clocks = <&prci PRCI_CLK_PCLK>; + #pwm-cells = <3>; + status = "disabled"; + }; + ccache: cache-controller@2010000 { + compatible = "sifive,fu740-c000-ccache", "cache"; + cache-block-size = <64>; + cache-level = <2>; + cache-sets = <2048>; + cache-size = <2097152>; + cache-unified; + interrupt-parent = <&plic0>; + interrupts = <19 20 21 22>; + reg = <0x0 0x2010000 0x0 0x1000>; + }; + gpio: gpio@10060000 { + compatible = "sifive,fu740-c000-gpio", "sifive,gpio0"; + interrupt-parent = <&plic0>; + interrupts = <23>, <24>, <25>, <26>, <27>, <28>, <29>, + <30>, <31>, <32>, <33>, <34>, <35>, <36>, + <37>, <38>; + reg = <0x0 0x10060000 0x0 0x1000>; + gpio-controller; + #gpio-cells = <2>; + interrupt-controller; + #interrupt-cells = <2>; + clocks = <&prci PRCI_CLK_PCLK>; + status = "disabled"; + }; + }; +}; diff --git a/arch/riscv/boot/dts/sifive/hifive-unmatched-a00.dts b/arch/riscv/boot/dts/sifive/hifive-unmatched-a00.dts new file mode 100644 index 0000000000000000000000000000000000000000..b1c3c596578f1f9d43e1edcba5e0580d9e4d4faf --- /dev/null +++ b/arch/riscv/boot/dts/sifive/hifive-unmatched-a00.dts @@ -0,0 +1,253 @@ +// SPDX-License-Identifier: (GPL-2.0 OR MIT) +/* Copyright (c) 2020 SiFive, Inc */ + +#include "fu740-c000.dtsi" +#include + +/* Clock frequency (in Hz) of the PCB crystal for rtcclk */ +#define RTCCLK_FREQ 1000000 + +/ { + #address-cells = <2>; + #size-cells = <2>; + model = "SiFive HiFive Unmatched A00"; + compatible = "sifive,hifive-unmatched-a00", "sifive,fu740-c000", + "sifive,fu740"; + + chosen { + stdout-path = "serial0"; + }; + + cpus { + timebase-frequency = ; + }; + + memory@80000000 { + device_type = "memory"; + reg = <0x0 0x80000000 0x2 0x00000000>; + }; + + soc { + }; + + hfclk: hfclk { + #clock-cells = <0>; + compatible = "fixed-clock"; + clock-frequency = <26000000>; + clock-output-names = "hfclk"; + }; + + rtcclk: rtcclk { + #clock-cells = <0>; + compatible = "fixed-clock"; + clock-frequency = ; + clock-output-names = "rtcclk"; + }; +}; + +&uart0 { + status = "okay"; +}; + +&uart1 { + status = "okay"; +}; + +&i2c0 { + status = "okay"; + + temperature-sensor@4c { + compatible = "ti,tmp451"; + reg = <0x4c>; + interrupt-parent = <&gpio>; + interrupts = <6 IRQ_TYPE_LEVEL_LOW>; + }; + + pmic@58 { + compatible = "dlg,da9063"; + reg = <0x58>; + interrupt-parent = <&gpio>; + interrupts = <1 IRQ_TYPE_LEVEL_LOW>; + interrupt-controller; + + regulators { + vdd_bcore1: bcore1 { + regulator-min-microvolt = <900000>; + regulator-max-microvolt = <900000>; + regulator-min-microamp = <5000000>; + regulator-max-microamp = <5000000>; + regulator-always-on; + }; + + vdd_bcore2: bcore2 { + regulator-min-microvolt = <900000>; + regulator-max-microvolt = <900000>; + regulator-min-microamp = <5000000>; + regulator-max-microamp = <5000000>; + regulator-always-on; + }; + + vdd_bpro: bpro { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-min-microamp = <2500000>; + regulator-max-microamp = <2500000>; + regulator-always-on; + }; + + vdd_bperi: bperi { + regulator-min-microvolt = <1050000>; + regulator-max-microvolt = <1050000>; + regulator-min-microamp = <1500000>; + regulator-max-microamp = <1500000>; + regulator-always-on; + }; + + vdd_bmem: bmem { + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <1200000>; + regulator-min-microamp = <3000000>; + regulator-max-microamp = <3000000>; + regulator-always-on; + }; + + vdd_bio: bio { + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <1200000>; + regulator-min-microamp = <3000000>; + regulator-max-microamp = <3000000>; + regulator-always-on; + }; + + vdd_ldo1: ldo1 { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-min-microamp = <100000>; + regulator-max-microamp = <100000>; + regulator-always-on; + }; + + vdd_ldo2: ldo2 { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-min-microamp = <200000>; + regulator-max-microamp = <200000>; + regulator-always-on; + }; + + vdd_ldo3: ldo3 { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-min-microamp = <200000>; + regulator-max-microamp = <200000>; + regulator-always-on; + }; + + vdd_ldo4: ldo4 { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-min-microamp = <200000>; + regulator-max-microamp = <200000>; + regulator-always-on; + }; + + vdd_ldo5: ldo5 { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-min-microamp = <100000>; + regulator-max-microamp = <100000>; + regulator-always-on; + }; + + vdd_ldo6: ldo6 { + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-min-microamp = <200000>; + regulator-max-microamp = <200000>; + regulator-always-on; + }; + + vdd_ldo7: ldo7 { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-min-microamp = <200000>; + regulator-max-microamp = <200000>; + regulator-always-on; + }; + + vdd_ldo8: ldo8 { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-min-microamp = <200000>; + regulator-max-microamp = <200000>; + regulator-always-on; + }; + + vdd_ld09: ldo9 { + regulator-min-microvolt = <1050000>; + regulator-max-microvolt = <1050000>; + regulator-min-microamp = <200000>; + regulator-max-microamp = <200000>; + }; + + vdd_ldo10: ldo10 { + regulator-min-microvolt = <1000000>; + regulator-max-microvolt = <1000000>; + regulator-min-microamp = <300000>; + regulator-max-microamp = <300000>; + }; + + vdd_ldo11: ldo11 { + regulator-min-microvolt = <2500000>; + regulator-max-microvolt = <2500000>; + regulator-min-microamp = <300000>; + regulator-max-microamp = <300000>; + regulator-always-on; + }; + }; + }; +}; + +&qspi0 { + status = "okay"; + flash@0 { + compatible = "issi,is25wp256", "jedec,spi-nor"; + reg = <0>; + spi-max-frequency = <50000000>; + m25p,fast-read; + spi-tx-bus-width = <4>; + spi-rx-bus-width = <4>; + }; +}; + +&spi0 { + status = "okay"; + mmc@0 { + compatible = "mmc-spi-slot"; + reg = <0>; + spi-max-frequency = <20000000>; + voltage-ranges = <3300 3300>; + disable-wp; + }; +}; + +ð0 { + status = "okay"; + phy-mode = "gmii"; + phy-handle = <&phy0>; + phy0: ethernet-phy@0 { + reg = <0>; + }; +}; + +&pwm0 { + status = "okay"; +}; + +&pwm1 { + status = "okay"; +}; + +&gpio { + status = "okay"; +}; diff --git a/arch/riscv/configs/defconfig b/arch/riscv/configs/defconfig index 8c3d1e4517031ac56b07c1c1c393b0d7b64cee79..6c0625aa96c7a4fd83b01dd4afbe5439e2a9f585 100644 --- a/arch/riscv/configs/defconfig +++ b/arch/riscv/configs/defconfig @@ -17,6 +17,7 @@ CONFIG_BPF_SYSCALL=y CONFIG_SOC_SIFIVE=y CONFIG_SOC_VIRT=y CONFIG_SMP=y +CONFIG_HOTPLUG_CPU=y CONFIG_JUMP_LABEL=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y diff --git a/arch/riscv/configs/nommu_k210_defconfig b/arch/riscv/configs/nommu_k210_defconfig index cd1df62b13c7271789d917fa3159d8a4ebe7721e..b16a2a12c82a8b06ae8ecf8384ee46af181ffb3e 100644 --- a/arch/riscv/configs/nommu_k210_defconfig +++ b/arch/riscv/configs/nommu_k210_defconfig @@ -1,17 +1,19 @@ # CONFIG_CPU_ISOLATION is not set -CONFIG_LOG_BUF_SHIFT=15 +CONFIG_LOG_BUF_SHIFT=13 CONFIG_PRINTK_SAFE_LOG_BUF_SHIFT=12 CONFIG_BLK_DEV_INITRD=y -CONFIG_INITRAMFS_FORCE=y +# CONFIG_RD_GZIP is not set # CONFIG_RD_BZIP2 is not set # CONFIG_RD_LZMA is not set # CONFIG_RD_XZ is not set # CONFIG_RD_LZO is not set # CONFIG_RD_LZ4 is not set +# CONFIG_RD_ZSTD is not set CONFIG_CC_OPTIMIZE_FOR_SIZE=y # CONFIG_SYSFS_SYSCALL is not set # CONFIG_FHANDLE is not set # CONFIG_BASE_FULL is not set +# CONFIG_FUTEX is not set # CONFIG_EPOLL is not set # CONFIG_SIGNALFD is not set # CONFIG_TIMERFD is not set @@ -25,15 +27,17 @@ CONFIG_EMBEDDED=y # CONFIG_VM_EVENT_COUNTERS is not set # CONFIG_COMPAT_BRK is not set CONFIG_SLOB=y -# CONFIG_SLAB_MERGE_DEFAULT is not set # CONFIG_MMU is not set -CONFIG_SOC_KENDRYTE=y +CONFIG_SOC_CANAAN=y +CONFIG_SOC_CANAAN_K210_DTB_SOURCE="k210_generic" CONFIG_MAXPHYSMEM_2GB=y CONFIG_SMP=y CONFIG_NR_CPUS=2 CONFIG_CMDLINE="earlycon console=ttySIF0" CONFIG_CMDLINE_FORCE=y -CONFIG_JUMP_LABEL=y +# CONFIG_SECCOMP is not set +# CONFIG_STACKPROTECTOR is not set +# CONFIG_GCC_PLUGINS is not set # CONFIG_BLOCK is not set CONFIG_BINFMT_FLAT=y # CONFIG_COREDUMP is not set @@ -41,23 +45,47 @@ CONFIG_DEVTMPFS=y CONFIG_DEVTMPFS_MOUNT=y # CONFIG_FW_LOADER is not set # CONFIG_ALLOW_DEV_COREDUMP is not set -# CONFIG_INPUT_KEYBOARD is not set -# CONFIG_INPUT_MOUSE is not set +# CONFIG_INPUT is not set # CONFIG_SERIO is not set +# CONFIG_VT is not set +# CONFIG_UNIX98_PTYS is not set # CONFIG_LEGACY_PTYS is not set # CONFIG_LDISC_AUTOLOAD is not set # CONFIG_HW_RANDOM is not set # CONFIG_DEVMEM is not set +CONFIG_I2C=y +# CONFIG_I2C_COMPAT is not set +CONFIG_I2C_CHARDEV=y +# CONFIG_I2C_HELPER_AUTO is not set +CONFIG_I2C_DESIGNWARE_PLATFORM=y +CONFIG_SPI=y +# CONFIG_SPI_MEM is not set +CONFIG_SPI_DESIGNWARE=y +CONFIG_SPI_DW_MMIO=y +# CONFIG_GPIO_CDEV_V1 is not set +CONFIG_GPIO_DWAPB=y +CONFIG_GPIO_SIFIVE=y +CONFIG_POWER_RESET=y +CONFIG_POWER_RESET_SYSCON=y # CONFIG_HWMON is not set -# CONFIG_VGA_CONSOLE is not set -# CONFIG_HID is not set # CONFIG_USB_SUPPORT is not set +CONFIG_NEW_LEDS=y +CONFIG_LEDS_CLASS=y +CONFIG_LEDS_GPIO=y +CONFIG_LEDS_USER=y # CONFIG_VIRTIO_MENU is not set +# CONFIG_VHOST_MENU is not set +# CONFIG_SURFACE_PLATFORMS is not set +# CONFIG_FILE_LOCKING is not set # CONFIG_DNOTIFY is not set # CONFIG_INOTIFY_USER is not set # CONFIG_MISC_FILESYSTEMS is not set CONFIG_LSM="[]" CONFIG_PRINTK_TIME=y +# CONFIG_SYMBOLIC_ERRNAME is not set +# CONFIG_DEBUG_BUGVERBOSE is not set +# CONFIG_SECTION_MISMATCH_WARN_ONLY is not set +# CONFIG_FRAME_POINTER is not set # CONFIG_DEBUG_MISC is not set CONFIG_PANIC_ON_OOPS=y # CONFIG_SCHED_DEBUG is not set diff --git a/arch/riscv/configs/nommu_k210_sdcard_defconfig b/arch/riscv/configs/nommu_k210_sdcard_defconfig new file mode 100644 index 0000000000000000000000000000000000000000..61f887f65419950c826d69c8826901565ad6f3be --- /dev/null +++ b/arch/riscv/configs/nommu_k210_sdcard_defconfig @@ -0,0 +1,92 @@ +# CONFIG_CPU_ISOLATION is not set +CONFIG_LOG_BUF_SHIFT=13 +CONFIG_PRINTK_SAFE_LOG_BUF_SHIFT=12 +CONFIG_CC_OPTIMIZE_FOR_SIZE=y +# CONFIG_SYSFS_SYSCALL is not set +# CONFIG_FHANDLE is not set +# CONFIG_BASE_FULL is not set +# CONFIG_FUTEX is not set +# CONFIG_EPOLL is not set +# CONFIG_SIGNALFD is not set +# CONFIG_TIMERFD is not set +# CONFIG_EVENTFD is not set +# CONFIG_AIO is not set +# CONFIG_IO_URING is not set +# CONFIG_ADVISE_SYSCALLS is not set +# CONFIG_MEMBARRIER is not set +# CONFIG_KALLSYMS is not set +CONFIG_EMBEDDED=y +# CONFIG_VM_EVENT_COUNTERS is not set +# CONFIG_COMPAT_BRK is not set +CONFIG_SLOB=y +# CONFIG_MMU is not set +CONFIG_SOC_CANAAN=y +CONFIG_SOC_CANAAN_K210_DTB_SOURCE="k210_generic" +CONFIG_MAXPHYSMEM_2GB=y +CONFIG_SMP=y +CONFIG_NR_CPUS=2 +CONFIG_CMDLINE="earlycon console=ttySIF0 rootdelay=2 root=/dev/mmcblk0p1 ro" +CONFIG_CMDLINE_FORCE=y +# CONFIG_SECCOMP is not set +# CONFIG_STACKPROTECTOR is not set +# CONFIG_GCC_PLUGINS is not set +# CONFIG_BLK_DEV_BSG is not set +# CONFIG_MQ_IOSCHED_DEADLINE is not set +# CONFIG_MQ_IOSCHED_KYBER is not set +CONFIG_BINFMT_FLAT=y +# CONFIG_COREDUMP is not set +CONFIG_DEVTMPFS=y +CONFIG_DEVTMPFS_MOUNT=y +# CONFIG_FW_LOADER is not set +# CONFIG_ALLOW_DEV_COREDUMP is not set +# CONFIG_BLK_DEV is not set +# CONFIG_INPUT is not set +# CONFIG_SERIO is not set +# CONFIG_VT is not set +# CONFIG_LEGACY_PTYS is not set +# CONFIG_LDISC_AUTOLOAD is not set +# CONFIG_HW_RANDOM is not set +# CONFIG_DEVMEM is not set +CONFIG_I2C=y +CONFIG_I2C_CHARDEV=y +# CONFIG_I2C_HELPER_AUTO is not set +CONFIG_I2C_DESIGNWARE_PLATFORM=y +CONFIG_SPI=y +# CONFIG_SPI_MEM is not set +CONFIG_SPI_DESIGNWARE=y +CONFIG_SPI_DW_MMIO=y +# CONFIG_GPIO_CDEV_V1 is not set +CONFIG_GPIO_DWAPB=y +CONFIG_GPIO_SIFIVE=y +CONFIG_POWER_RESET=y +CONFIG_POWER_RESET_SYSCON=y +# CONFIG_HWMON is not set +# CONFIG_USB_SUPPORT is not set +CONFIG_MMC=y +# CONFIG_PWRSEQ_EMMC is not set +# CONFIG_PWRSEQ_SIMPLE is not set +CONFIG_MMC_SPI=y +CONFIG_NEW_LEDS=y +CONFIG_LEDS_CLASS=y +CONFIG_LEDS_GPIO=y +CONFIG_LEDS_USER=y +# CONFIG_VIRTIO_MENU is not set +# CONFIG_VHOST_MENU is not set +# CONFIG_SURFACE_PLATFORMS is not set +CONFIG_EXT2_FS=y +# CONFIG_FILE_LOCKING is not set +# CONFIG_DNOTIFY is not set +# CONFIG_INOTIFY_USER is not set +# CONFIG_MISC_FILESYSTEMS is not set +CONFIG_LSM="[]" +CONFIG_PRINTK_TIME=y +# CONFIG_SYMBOLIC_ERRNAME is not set +# CONFIG_DEBUG_BUGVERBOSE is not set +# CONFIG_SECTION_MISMATCH_WARN_ONLY is not set +# CONFIG_FRAME_POINTER is not set +# CONFIG_DEBUG_MISC is not set +CONFIG_PANIC_ON_OOPS=y +# CONFIG_SCHED_DEBUG is not set +# CONFIG_RCU_TRACE is not set +# CONFIG_FTRACE is not set +# CONFIG_RUNTIME_TESTING_MENU is not set diff --git a/arch/riscv/configs/rv32_defconfig b/arch/riscv/configs/rv32_defconfig index 2c2cda6cc1c5352ef794e0db7d941c72f6229383..8dd02b842fef8cb2f3152b383270c7da020cea3e 100644 --- a/arch/riscv/configs/rv32_defconfig +++ b/arch/riscv/configs/rv32_defconfig @@ -18,6 +18,7 @@ CONFIG_SOC_SIFIVE=y CONFIG_SOC_VIRT=y CONFIG_ARCH_RV32I=y CONFIG_SMP=y +CONFIG_HOTPLUG_CPU=y CONFIG_JUMP_LABEL=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y diff --git a/arch/riscv/include/asm/bug.h b/arch/riscv/include/asm/bug.h index d6f1ec08d97b0875ee4f222fc86f448471c92a4d..d3804a2f9aad33fb8089691ca5f5ccd847452b51 100644 --- a/arch/riscv/include/asm/bug.h +++ b/arch/riscv/include/asm/bug.h @@ -85,6 +85,7 @@ do { \ struct pt_regs; struct task_struct; +void __show_regs(struct pt_regs *regs); void die(struct pt_regs *regs, const char *str); void do_trap(struct pt_regs *regs, int signo, int code, unsigned long addr); diff --git a/arch/riscv/include/asm/csr.h b/arch/riscv/include/asm/csr.h index cec462e198ced58c82ecf530115fb079d06b03b1..caadfc1d7487e6969877234bdbce864f06d263fd 100644 --- a/arch/riscv/include/asm/csr.h +++ b/arch/riscv/include/asm/csr.h @@ -41,10 +41,16 @@ #define SATP_PPN _AC(0x003FFFFF, UL) #define SATP_MODE_32 _AC(0x80000000, UL) #define SATP_MODE SATP_MODE_32 +#define SATP_ASID_BITS 9 +#define SATP_ASID_SHIFT 22 +#define SATP_ASID_MASK _AC(0x1FF, UL) #else #define SATP_PPN _AC(0x00000FFFFFFFFFFF, UL) #define SATP_MODE_39 _AC(0x8000000000000000, UL) #define SATP_MODE SATP_MODE_39 +#define SATP_ASID_BITS 16 +#define SATP_ASID_SHIFT 44 +#define SATP_ASID_MASK _AC(0xFFFF, UL) #endif /* Exception cause high bit - is an interrupt if set */ diff --git a/arch/riscv/include/asm/kasan.h b/arch/riscv/include/asm/kasan.h index b04028c6218c9f39fc086fe5a40f94459d58f0cd..a2b3d9cdbc8687ee5f20b547e33684013026bd42 100644 --- a/arch/riscv/include/asm/kasan.h +++ b/arch/riscv/include/asm/kasan.h @@ -8,12 +8,28 @@ #ifdef CONFIG_KASAN +/* + * The following comment was copied from arm64: + * KASAN_SHADOW_START: beginning of the kernel virtual addresses. + * KASAN_SHADOW_END: KASAN_SHADOW_START + 1/N of kernel virtual addresses, + * where N = (1 << KASAN_SHADOW_SCALE_SHIFT). + * + * KASAN_SHADOW_OFFSET: + * This value is used to map an address to the corresponding shadow + * address by the following formula: + * shadow_addr = (address >> KASAN_SHADOW_SCALE_SHIFT) + KASAN_SHADOW_OFFSET + * + * (1 << (64 - KASAN_SHADOW_SCALE_SHIFT)) shadow addresses that lie in range + * [KASAN_SHADOW_OFFSET, KASAN_SHADOW_END) cover all 64-bits of virtual + * addresses. So KASAN_SHADOW_OFFSET should satisfy the following equation: + * KASAN_SHADOW_OFFSET = KASAN_SHADOW_END - + * (1ULL << (64 - KASAN_SHADOW_SCALE_SHIFT)) + */ #define KASAN_SHADOW_SCALE_SHIFT 3 -#define KASAN_SHADOW_SIZE (UL(1) << (38 - KASAN_SHADOW_SCALE_SHIFT)) -#define KASAN_SHADOW_START KERN_VIRT_START /* 2^64 - 2^38 */ +#define KASAN_SHADOW_SIZE (UL(1) << ((CONFIG_VA_BITS - 1) - KASAN_SHADOW_SCALE_SHIFT)) +#define KASAN_SHADOW_START KERN_VIRT_START #define KASAN_SHADOW_END (KASAN_SHADOW_START + KASAN_SHADOW_SIZE) - #define KASAN_SHADOW_OFFSET (KASAN_SHADOW_END - (1ULL << \ (64 - KASAN_SHADOW_SCALE_SHIFT))) diff --git a/arch/riscv/include/asm/kprobes.h b/arch/riscv/include/asm/kprobes.h index 56a98ea3073145dec12a7d9a16fff7bb1a757d30..4647d38018f6a6a310a34bc2c74d1c13fbbcb406 100644 --- a/arch/riscv/include/asm/kprobes.h +++ b/arch/riscv/include/asm/kprobes.h @@ -11,4 +11,44 @@ #include +#ifdef CONFIG_KPROBES +#include +#include +#include + +#define __ARCH_WANT_KPROBES_INSN_SLOT +#define MAX_INSN_SIZE 2 + +#define flush_insn_slot(p) do { } while (0) +#define kretprobe_blacklist_size 0 + +#include + +struct prev_kprobe { + struct kprobe *kp; + unsigned int status; +}; + +/* Single step context for kprobe */ +struct kprobe_step_ctx { + unsigned long ss_pending; + unsigned long match_addr; +}; + +/* per-cpu kprobe control block */ +struct kprobe_ctlblk { + unsigned int kprobe_status; + unsigned long saved_status; + struct prev_kprobe prev_kprobe; + struct kprobe_step_ctx ss_ctx; +}; + +void arch_remove_kprobe(struct kprobe *p); +int kprobe_fault_handler(struct pt_regs *regs, unsigned int trapnr); +bool kprobe_breakpoint_handler(struct pt_regs *regs); +bool kprobe_single_step_handler(struct pt_regs *regs); +void kretprobe_trampoline(void); +void __kprobes *trampoline_probe_handler(struct pt_regs *regs); + +#endif /* CONFIG_KPROBES */ #endif /* _ASM_RISCV_KPROBES_H */ diff --git a/arch/riscv/include/asm/mmu.h b/arch/riscv/include/asm/mmu.h index dabcf2cfb3dc762011662c31c3b36d61db8dc859..0099dc1161683ddd1e3c45460309e331b7e6b0a7 100644 --- a/arch/riscv/include/asm/mmu.h +++ b/arch/riscv/include/asm/mmu.h @@ -12,6 +12,8 @@ typedef struct { #ifndef CONFIG_MMU unsigned long end_brk; +#else + atomic_long_t id; #endif void *vdso; #ifdef CONFIG_SMP diff --git a/arch/riscv/include/asm/mmu_context.h b/arch/riscv/include/asm/mmu_context.h index 250defa06f3ad156de74cf1d73afd12d6c32159e..b0659413a080fc4f5702866b744957fc515437fd 100644 --- a/arch/riscv/include/asm/mmu_context.h +++ b/arch/riscv/include/asm/mmu_context.h @@ -23,6 +23,16 @@ static inline void activate_mm(struct mm_struct *prev, switch_mm(prev, next, NULL); } +#define init_new_context init_new_context +static inline int init_new_context(struct task_struct *tsk, + struct mm_struct *mm) +{ +#ifdef CONFIG_MMU + atomic_long_set(&mm->context.id, 0); +#endif + return 0; +} + #include #endif /* _ASM_RISCV_MMU_CONTEXT_H */ diff --git a/arch/riscv/include/asm/mmzone.h b/arch/riscv/include/asm/mmzone.h new file mode 100644 index 0000000000000000000000000000000000000000..fa17e01d9ab27894a0c8ce443a0a8078cd5e8b24 --- /dev/null +++ b/arch/riscv/include/asm/mmzone.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_MMZONE_H +#define __ASM_MMZONE_H + +#ifdef CONFIG_NUMA + +#include + +extern struct pglist_data *node_data[]; +#define NODE_DATA(nid) (node_data[(nid)]) + +#endif /* CONFIG_NUMA */ +#endif /* __ASM_MMZONE_H */ diff --git a/arch/riscv/include/asm/numa.h b/arch/riscv/include/asm/numa.h new file mode 100644 index 0000000000000000000000000000000000000000..8c8cf4297cc321ace630d4d1a78049d75b70e524 --- /dev/null +++ b/arch/riscv/include/asm/numa.h @@ -0,0 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_NUMA_H +#define __ASM_NUMA_H + +#include +#include + +#endif /* __ASM_NUMA_H */ diff --git a/arch/riscv/include/asm/page.h b/arch/riscv/include/asm/page.h index 64a675c5c30acb2c9806645ba9f3f5e7297cef21..adc9d26f3d752dd02e434f0be04958a871459a0c 100644 --- a/arch/riscv/include/asm/page.h +++ b/arch/riscv/include/asm/page.h @@ -97,9 +97,6 @@ extern unsigned long pfn_base; #define ARCH_PFN_OFFSET (PAGE_OFFSET >> PAGE_SHIFT) #endif /* CONFIG_MMU */ -extern unsigned long max_low_pfn; -extern unsigned long min_low_pfn; - #define __pa_to_va_nodebug(x) ((void *)((unsigned long) (x) + va_pa_offset)) #define __va_to_pa_nodebug(x) ((unsigned long)(x) - va_pa_offset) diff --git a/arch/riscv/include/asm/pci.h b/arch/riscv/include/asm/pci.h index 1c473a1bd9862b9a75d27e8c59292f28ff419bbc..658e112c3ce73bcbce10e627400426ec41c8daec 100644 --- a/arch/riscv/include/asm/pci.h +++ b/arch/riscv/include/asm/pci.h @@ -32,6 +32,20 @@ static inline int pci_proc_domain(struct pci_bus *bus) /* always show the domain in /proc */ return 1; } + +#ifdef CONFIG_NUMA + +static inline int pcibus_to_node(struct pci_bus *bus) +{ + return dev_to_node(&bus->dev); +} +#ifndef cpumask_of_pcibus +#define cpumask_of_pcibus(bus) (pcibus_to_node(bus) == -1 ? \ + cpu_all_mask : \ + cpumask_of_node(pcibus_to_node(bus))) +#endif +#endif /* CONFIG_NUMA */ + #endif /* CONFIG_PCI */ #endif /* _ASM_RISCV_PCI_H */ diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h index 251e1db088fa2d8508762bcc5e459f2e7b120684..ebf817c1bdf4b3d955875670494d1828bd042d26 100644 --- a/arch/riscv/include/asm/pgtable.h +++ b/arch/riscv/include/asm/pgtable.h @@ -186,6 +186,11 @@ static inline unsigned long pmd_page_vaddr(pmd_t pmd) return (unsigned long)pfn_to_virt(pmd_val(pmd) >> _PAGE_PFN_SHIFT); } +static inline pte_t pmd_pte(pmd_t pmd) +{ + return __pte(pmd_val(pmd)); +} + /* Yields the page frame number (PFN) of a page table entry */ static inline unsigned long pte_pfn(pte_t pte) { @@ -289,6 +294,21 @@ static inline pte_t pte_mkhuge(pte_t pte) return pte; } +#ifdef CONFIG_NUMA_BALANCING +/* + * See the comment in include/asm-generic/pgtable.h + */ +static inline int pte_protnone(pte_t pte) +{ + return (pte_val(pte) & (_PAGE_PRESENT | _PAGE_PROT_NONE)) == _PAGE_PROT_NONE; +} + +static inline int pmd_protnone(pmd_t pmd) +{ + return pte_protnone(pmd_pte(pmd)); +} +#endif + /* Modify page protection bits */ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) { @@ -468,6 +488,7 @@ extern void *dtb_early_va; extern uintptr_t dtb_early_pa; void setup_bootmem(void); void paging_init(void); +void misc_mem_init(void); #define FIRST_USER_ADDRESS 0 diff --git a/arch/riscv/include/asm/probes.h b/arch/riscv/include/asm/probes.h new file mode 100644 index 0000000000000000000000000000000000000000..a787e6d537b924bbe39d591cea42f988cd59eff5 --- /dev/null +++ b/arch/riscv/include/asm/probes.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef _ASM_RISCV_PROBES_H +#define _ASM_RISCV_PROBES_H + +typedef u32 probe_opcode_t; +typedef bool (probes_handler_t) (u32 opcode, unsigned long addr, struct pt_regs *); + +/* architecture specific copy of original instruction */ +struct arch_probe_insn { + probe_opcode_t *insn; + probes_handler_t *handler; + /* restore address after simulation */ + unsigned long restore; +}; + +#ifdef CONFIG_KPROBES +typedef u32 kprobe_opcode_t; +struct arch_specific_insn { + struct arch_probe_insn api; +}; +#endif + +#endif /* _ASM_RISCV_PROBES_H */ diff --git a/arch/riscv/include/asm/processor.h b/arch/riscv/include/asm/processor.h index bdddcd5c1b71d905536f2774c0506ed8efb4ec3c..3a240037bde26e1c3a135f373d17805d0b471679 100644 --- a/arch/riscv/include/asm/processor.h +++ b/arch/riscv/include/asm/processor.h @@ -34,6 +34,7 @@ struct thread_struct { unsigned long sp; /* Kernel mode stack */ unsigned long s[12]; /* s[0]: frame pointer */ struct __riscv_d_ext_state fstate; + unsigned long bad_cause; }; #define INIT_THREAD { \ diff --git a/arch/riscv/include/asm/ptrace.h b/arch/riscv/include/asm/ptrace.h index ee49f80c95337f948a3e6fbbb3b42fc20b8709a1..cb4abb639e8d6876ca77b2d255e9875f1946945b 100644 --- a/arch/riscv/include/asm/ptrace.h +++ b/arch/riscv/include/asm/ptrace.h @@ -8,6 +8,7 @@ #include #include +#include #ifndef __ASSEMBLY__ @@ -60,6 +61,7 @@ struct pt_regs { #define user_mode(regs) (((regs)->status & SR_PP) == 0) +#define MAX_REG_OFFSET offsetof(struct pt_regs, orig_a0) /* Helpers for working with the instruction pointer */ static inline unsigned long instruction_pointer(struct pt_regs *regs) @@ -85,6 +87,12 @@ static inline void user_stack_pointer_set(struct pt_regs *regs, regs->sp = val; } +/* Valid only for Kernel mode traps. */ +static inline unsigned long kernel_stack_pointer(struct pt_regs *regs) +{ + return regs->sp; +} + /* Helpers for working with the frame pointer */ static inline unsigned long frame_pointer(struct pt_regs *regs) { @@ -101,6 +109,33 @@ static inline unsigned long regs_return_value(struct pt_regs *regs) return regs->a0; } +static inline void regs_set_return_value(struct pt_regs *regs, + unsigned long val) +{ + regs->a0 = val; +} + +extern int regs_query_register_offset(const char *name); +extern unsigned long regs_get_kernel_stack_nth(struct pt_regs *regs, + unsigned int n); + +/** + * regs_get_register() - get register value from its offset + * @regs: pt_regs from which register value is gotten + * @offset: offset of the register. + * + * regs_get_register returns the value of a register whose offset from @regs. + * The @offset is the offset of the register in struct pt_regs. + * If @offset is bigger than MAX_REG_OFFSET, this returns 0. + */ +static inline unsigned long regs_get_register(struct pt_regs *regs, + unsigned int offset) +{ + if (unlikely(offset > MAX_REG_OFFSET)) + return 0; + + return *(unsigned long *)((unsigned long)regs + offset); +} #endif /* __ASSEMBLY__ */ #endif /* _ASM_RISCV_PTRACE_H */ diff --git a/arch/riscv/include/asm/sbi.h b/arch/riscv/include/asm/sbi.h index 653edb25d4957db90da638159508a58dccb8a619..99895d9c3bddb06469b690b062cdb76631cb0547 100644 --- a/arch/riscv/include/asm/sbi.h +++ b/arch/riscv/include/asm/sbi.h @@ -89,7 +89,7 @@ struct sbiret { long value; }; -int sbi_init(void); +void sbi_init(void); struct sbiret sbi_ecall(int ext, int fid, unsigned long arg0, unsigned long arg1, unsigned long arg2, unsigned long arg3, unsigned long arg4, @@ -100,13 +100,13 @@ int sbi_console_getchar(void); void sbi_set_timer(uint64_t stime_value); void sbi_shutdown(void); void sbi_clear_ipi(void); -void sbi_send_ipi(const unsigned long *hart_mask); -void sbi_remote_fence_i(const unsigned long *hart_mask); -void sbi_remote_sfence_vma(const unsigned long *hart_mask, +int sbi_send_ipi(const unsigned long *hart_mask); +int sbi_remote_fence_i(const unsigned long *hart_mask); +int sbi_remote_sfence_vma(const unsigned long *hart_mask, unsigned long start, unsigned long size); -void sbi_remote_sfence_vma_asid(const unsigned long *hart_mask, +int sbi_remote_sfence_vma_asid(const unsigned long *hart_mask, unsigned long start, unsigned long size, unsigned long asid); @@ -147,11 +147,7 @@ static inline unsigned long sbi_minor_version(void) int sbi_err_map_linux_errno(int err); #else /* CONFIG_RISCV_SBI */ -/* stubs for code that is only reachable under IS_ENABLED(CONFIG_RISCV_SBI): */ -void sbi_set_timer(uint64_t stime_value); -void sbi_clear_ipi(void); -void sbi_send_ipi(const unsigned long *hart_mask); -void sbi_remote_fence_i(const unsigned long *hart_mask); -void sbi_init(void); +static inline int sbi_remote_fence_i(const unsigned long *hart_mask) { return -1; } +static inline void sbi_init(void) {} #endif /* CONFIG_RISCV_SBI */ #endif /* _ASM_RISCV_SBI_H */ diff --git a/arch/riscv/include/asm/set_memory.h b/arch/riscv/include/asm/set_memory.h index 8b80c80c7f1acad9d9e1b3cb29d28a005fd83cd3..6887b3d9f3712bb5f4558b5c40b91b9559ad7335 100644 --- a/arch/riscv/include/asm/set_memory.h +++ b/arch/riscv/include/asm/set_memory.h @@ -22,7 +22,7 @@ static inline int set_memory_ro(unsigned long addr, int numpages) { return 0; } static inline int set_memory_rw(unsigned long addr, int numpages) { return 0; } static inline int set_memory_x(unsigned long addr, int numpages) { return 0; } static inline int set_memory_nx(unsigned long addr, int numpages) { return 0; } -static inline void protect_kernel_text_data(void) {}; +static inline void protect_kernel_text_data(void) {} static inline int set_memory_rw_nx(unsigned long addr, int numpages) { return 0; } #endif diff --git a/arch/riscv/include/asm/soc.h b/arch/riscv/include/asm/soc.h index 6c8363b1f3272ea669ce9761783dc30436e6656f..f494066051a2c10a122c17ecf855f230571c6f98 100644 --- a/arch/riscv/include/asm/soc.h +++ b/arch/riscv/include/asm/soc.h @@ -21,42 +21,4 @@ void soc_early_init(void); extern unsigned long __soc_early_init_table_start; extern unsigned long __soc_early_init_table_end; -/* - * Allows Linux to provide a device tree, which is necessary for SOCs that - * don't provide a useful one on their own. - */ -struct soc_builtin_dtb { - unsigned long vendor_id; - unsigned long arch_id; - unsigned long imp_id; - void *(*dtb_func)(void); -}; - -/* - * The argument name must specify a valid DTS file name without the dts - * extension. - */ -#define SOC_BUILTIN_DTB_DECLARE(name, vendor, arch, impl) \ - extern void *__dtb_##name##_begin; \ - \ - static __init __used \ - void *__soc_builtin_dtb_f__##name(void) \ - { \ - return (void *)&__dtb_##name##_begin; \ - } \ - \ - static const struct soc_builtin_dtb __soc_builtin_dtb__##name \ - __used __section("__soc_builtin_dtb_table") = \ - { \ - .vendor_id = vendor, \ - .arch_id = arch, \ - .imp_id = impl, \ - .dtb_func = __soc_builtin_dtb_f__##name, \ - } - -extern unsigned long __soc_builtin_dtb_table_start; -extern unsigned long __soc_builtin_dtb_table_end; - -void *soc_lookup_builtin_dtb(void); - #endif diff --git a/arch/riscv/include/asm/stackprotector.h b/arch/riscv/include/asm/stackprotector.h index 5962f8891f06f92ebe6223861942a408e0257232..09093af46565e5674a34e21e59808d4c3f36e5fc 100644 --- a/arch/riscv/include/asm/stackprotector.h +++ b/arch/riscv/include/asm/stackprotector.h @@ -24,6 +24,7 @@ static __always_inline void boot_init_stack_canary(void) canary &= CANARY_MASK; current->stack_canary = canary; - __stack_chk_guard = current->stack_canary; + if (!IS_ENABLED(CONFIG_STACKPROTECTOR_PER_TASK)) + __stack_chk_guard = current->stack_canary; } #endif /* _ASM_RISCV_STACKPROTECTOR_H */ diff --git a/arch/riscv/include/asm/stacktrace.h b/arch/riscv/include/asm/stacktrace.h index 470a65c4ccdccddb868d4692d85398e0990dcd39..3450c1912afdf2111543ad5d83973fba5ea7538a 100644 --- a/arch/riscv/include/asm/stacktrace.h +++ b/arch/riscv/include/asm/stacktrace.h @@ -13,5 +13,7 @@ struct stackframe { extern void notrace walk_stackframe(struct task_struct *task, struct pt_regs *regs, bool (*fn)(void *, unsigned long), void *arg); +extern void dump_backtrace(struct pt_regs *regs, struct task_struct *task, + const char *loglvl); #endif /* _ASM_RISCV_STACKTRACE_H */ diff --git a/arch/riscv/include/asm/thread_info.h b/arch/riscv/include/asm/thread_info.h index 97bf5a1575d2ce191e22acf6ea87def9ee014164..0e549a3089b3512c7337508c7ff891d8f38bbf64 100644 --- a/arch/riscv/include/asm/thread_info.h +++ b/arch/riscv/include/asm/thread_info.h @@ -75,6 +75,7 @@ struct thread_info { #define TIF_SYSCALL_AUDIT 7 /* syscall auditing */ #define TIF_SECCOMP 8 /* syscall secure computing */ #define TIF_NOTIFY_SIGNAL 9 /* signal notifications exist */ +#define TIF_UPROBE 10 /* uprobe breakpoint or singlestep */ #define _TIF_SYSCALL_TRACE (1 << TIF_SYSCALL_TRACE) #define _TIF_NOTIFY_RESUME (1 << TIF_NOTIFY_RESUME) @@ -84,10 +85,11 @@ struct thread_info { #define _TIF_SYSCALL_AUDIT (1 << TIF_SYSCALL_AUDIT) #define _TIF_SECCOMP (1 << TIF_SECCOMP) #define _TIF_NOTIFY_SIGNAL (1 << TIF_NOTIFY_SIGNAL) +#define _TIF_UPROBE (1 << TIF_UPROBE) #define _TIF_WORK_MASK \ (_TIF_NOTIFY_RESUME | _TIF_SIGPENDING | _TIF_NEED_RESCHED | \ - _TIF_NOTIFY_SIGNAL) + _TIF_NOTIFY_SIGNAL | _TIF_UPROBE) #define _TIF_SYSCALL_WORK \ (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_TRACEPOINT | _TIF_SYSCALL_AUDIT | \ diff --git a/arch/riscv/include/asm/uprobes.h b/arch/riscv/include/asm/uprobes.h new file mode 100644 index 0000000000000000000000000000000000000000..f2183e00fdd20946052009f05d857e99191bcb40 --- /dev/null +++ b/arch/riscv/include/asm/uprobes.h @@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#ifndef _ASM_RISCV_UPROBES_H +#define _ASM_RISCV_UPROBES_H + +#include +#include +#include + +#define MAX_UINSN_BYTES 8 + +#ifdef CONFIG_RISCV_ISA_C +#define UPROBE_SWBP_INSN __BUG_INSN_16 +#define UPROBE_SWBP_INSN_SIZE 2 +#else +#define UPROBE_SWBP_INSN __BUG_INSN_32 +#define UPROBE_SWBP_INSN_SIZE 4 +#endif +#define UPROBE_XOL_SLOT_BYTES MAX_UINSN_BYTES + +typedef u32 uprobe_opcode_t; + +struct arch_uprobe_task { + unsigned long saved_cause; +}; + +struct arch_uprobe { + union { + u8 insn[MAX_UINSN_BYTES]; + u8 ixol[MAX_UINSN_BYTES]; + }; + struct arch_probe_insn api; + unsigned long insn_size; + bool simulate; +}; + +bool uprobe_breakpoint_handler(struct pt_regs *regs); +bool uprobe_single_step_handler(struct pt_regs *regs); + +#endif /* _ASM_RISCV_UPROBES_H */ diff --git a/arch/riscv/kernel/Makefile b/arch/riscv/kernel/Makefile index f6caf4d9ca150c7a134e4d0b964dc210bfaf6d63..3dc0abde988a27cb89f55b68d2e0d00207b346d7 100644 --- a/arch/riscv/kernel/Makefile +++ b/arch/riscv/kernel/Makefile @@ -4,8 +4,9 @@ # ifdef CONFIG_FTRACE -CFLAGS_REMOVE_ftrace.o = -pg -CFLAGS_REMOVE_patch.o = -pg +CFLAGS_REMOVE_ftrace.o = $(CC_FLAGS_FTRACE) +CFLAGS_REMOVE_patch.o = $(CC_FLAGS_FTRACE) +CFLAGS_REMOVE_sbi.o = $(CC_FLAGS_FTRACE) endif extra-y += head.o @@ -29,6 +30,7 @@ obj-y += riscv_ksyms.o obj-y += stacktrace.o obj-y += cacheinfo.o obj-y += patch.o +obj-y += probes/ obj-$(CONFIG_MMU) += vdso.o vdso/ obj-$(CONFIG_RISCV_M_MODE) += traps_misaligned.o diff --git a/arch/riscv/kernel/asm-offsets.c b/arch/riscv/kernel/asm-offsets.c index b79ffa3561fd7245eacec78a0277e5555b80bbe9..9ef33346853c283f878b4fbf1a81a2bbb5e2511a 100644 --- a/arch/riscv/kernel/asm-offsets.c +++ b/arch/riscv/kernel/asm-offsets.c @@ -68,6 +68,9 @@ void asm_offsets(void) OFFSET(TASK_THREAD_F30, task_struct, thread.fstate.f[30]); OFFSET(TASK_THREAD_F31, task_struct, thread.fstate.f[31]); OFFSET(TASK_THREAD_FCSR, task_struct, thread.fstate.fcsr); +#ifdef CONFIG_STACKPROTECTOR + OFFSET(TSK_STACK_CANARY, task_struct, stack_canary); +#endif DEFINE(PT_SIZE, sizeof(struct pt_regs)); OFFSET(PT_EPC, pt_regs, epc); diff --git a/arch/riscv/kernel/ftrace.c b/arch/riscv/kernel/ftrace.c index 765b62434f303d2c99f63b82be3799725b82e5dc..7f1e5203de886bdf85873159eb4bb9be95fc7e40 100644 --- a/arch/riscv/kernel/ftrace.c +++ b/arch/riscv/kernel/ftrace.c @@ -72,29 +72,56 @@ static int __ftrace_modify_call(unsigned long hook_pos, unsigned long target, return 0; } +/* + * Put 5 instructions with 16 bytes at the front of function within + * patchable function entry nops' area. + * + * 0: REG_S ra, -SZREG(sp) + * 1: auipc ra, 0x? + * 2: jalr -?(ra) + * 3: REG_L ra, -SZREG(sp) + * + * So the opcodes is: + * 0: 0xfe113c23 (sd)/0xfe112e23 (sw) + * 1: 0x???????? -> auipc + * 2: 0x???????? -> jalr + * 3: 0xff813083 (ld)/0xffc12083 (lw) + */ +#if __riscv_xlen == 64 +#define INSN0 0xfe113c23 +#define INSN3 0xff813083 +#elif __riscv_xlen == 32 +#define INSN0 0xfe112e23 +#define INSN3 0xffc12083 +#endif + +#define FUNC_ENTRY_SIZE 16 +#define FUNC_ENTRY_JMP 4 + int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) { - int ret = ftrace_check_current_call(rec->ip, NULL); + unsigned int call[4] = {INSN0, 0, 0, INSN3}; + unsigned long target = addr; + unsigned long caller = rec->ip + FUNC_ENTRY_JMP; - if (ret) - return ret; + call[1] = to_auipc_insn((unsigned int)(target - caller)); + call[2] = to_jalr_insn((unsigned int)(target - caller)); - return __ftrace_modify_call(rec->ip, addr, true); + if (patch_text_nosync((void *)rec->ip, call, FUNC_ENTRY_SIZE)) + return -EPERM; + + return 0; } int ftrace_make_nop(struct module *mod, struct dyn_ftrace *rec, unsigned long addr) { - unsigned int call[2]; - int ret; + unsigned int nops[4] = {NOP4, NOP4, NOP4, NOP4}; - make_call(rec->ip, addr, call); - ret = ftrace_check_current_call(rec->ip, call); - - if (ret) - return ret; + if (patch_text_nosync((void *)rec->ip, nops, FUNC_ENTRY_SIZE)) + return -EPERM; - return __ftrace_modify_call(rec->ip, addr, false); + return 0; } @@ -139,15 +166,16 @@ int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr, unsigned long addr) { unsigned int call[2]; + unsigned long caller = rec->ip + FUNC_ENTRY_JMP; int ret; - make_call(rec->ip, old_addr, call); - ret = ftrace_check_current_call(rec->ip, call); + make_call(caller, old_addr, call); + ret = ftrace_check_current_call(caller, call); if (ret) return ret; - return __ftrace_modify_call(rec->ip, addr, true); + return __ftrace_modify_call(caller, addr, true); } #endif @@ -176,53 +204,30 @@ void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr, #ifdef CONFIG_DYNAMIC_FTRACE extern void ftrace_graph_call(void); +extern void ftrace_graph_regs_call(void); int ftrace_enable_ftrace_graph_caller(void) { - unsigned int call[2]; - static int init_graph = 1; int ret; - make_call(&ftrace_graph_call, &ftrace_stub, call); - - /* - * When enabling graph tracer for the first time, ftrace_graph_call - * should contains a call to ftrace_stub. Once it has been disabled, - * the 8-bytes at the position becomes NOPs. - */ - if (init_graph) { - ret = ftrace_check_current_call((unsigned long)&ftrace_graph_call, - call); - init_graph = 0; - } else { - ret = ftrace_check_current_call((unsigned long)&ftrace_graph_call, - NULL); - } - + ret = __ftrace_modify_call((unsigned long)&ftrace_graph_call, + (unsigned long)&prepare_ftrace_return, true); if (ret) return ret; - return __ftrace_modify_call((unsigned long)&ftrace_graph_call, + return __ftrace_modify_call((unsigned long)&ftrace_graph_regs_call, (unsigned long)&prepare_ftrace_return, true); } int ftrace_disable_ftrace_graph_caller(void) { - unsigned int call[2]; int ret; - make_call(&ftrace_graph_call, &prepare_ftrace_return, call); - - /* - * This is to make sure that ftrace_enable_ftrace_graph_caller - * did the right thing. - */ - ret = ftrace_check_current_call((unsigned long)&ftrace_graph_call, - call); - + ret = __ftrace_modify_call((unsigned long)&ftrace_graph_call, + (unsigned long)&prepare_ftrace_return, false); if (ret) return ret; - return __ftrace_modify_call((unsigned long)&ftrace_graph_call, + return __ftrace_modify_call((unsigned long)&ftrace_graph_regs_call, (unsigned long)&prepare_ftrace_return, false); } #endif /* CONFIG_DYNAMIC_FTRACE */ diff --git a/arch/riscv/kernel/head.S b/arch/riscv/kernel/head.S index 16e9941900c4d0c9a9acca6e4209d9c615711dee..f5a9bad86e5831890441e9c7c216d406664b2540 100644 --- a/arch/riscv/kernel/head.S +++ b/arch/riscv/kernel/head.S @@ -260,7 +260,11 @@ clear_bss_done: /* Initialize page tables and relocate to virtual addresses */ la sp, init_thread_union + THREAD_SIZE +#ifdef CONFIG_BUILTIN_DTB + la a0, __dtb_start +#else mv a0, s1 +#endif /* CONFIG_BUILTIN_DTB */ call setup_vm #ifdef CONFIG_MMU la a0, early_pg_dir diff --git a/arch/riscv/kernel/image-vars.h b/arch/riscv/kernel/image-vars.h index 8c212efb37a64cf0ff33a4fa98a2ac74f625174a..71a76a62325778c4e030b83f24e80d18c49f62bb 100644 --- a/arch/riscv/kernel/image-vars.h +++ b/arch/riscv/kernel/image-vars.h @@ -3,7 +3,7 @@ * Copyright (C) 2020 Western Digital Corporation or its affiliates. * Linker script variables to be set after section resolution, as * ld.lld does not like variables assigned before SECTIONS is processed. - * Based on arch/arm64/kerne/image-vars.h + * Based on arch/arm64/kernel/image-vars.h */ #ifndef __RISCV_KERNEL_IMAGE_VARS_H #define __RISCV_KERNEL_IMAGE_VARS_H diff --git a/arch/riscv/kernel/mcount-dyn.S b/arch/riscv/kernel/mcount-dyn.S index 35a6ed76cb8b74e7806d211173fdb77dc25f9298..d171eca623b6fc744cb5f349dd563d6e583dd788 100644 --- a/arch/riscv/kernel/mcount-dyn.S +++ b/arch/riscv/kernel/mcount-dyn.S @@ -13,224 +13,186 @@ .text - .macro SAVE_ABI_STATE -#ifdef CONFIG_FUNCTION_GRAPH_TRACER - addi sp, sp, -48 - sd s0, 32(sp) - sd ra, 40(sp) - addi s0, sp, 48 - sd t0, 24(sp) - sd t1, 16(sp) -#ifdef HAVE_FUNCTION_GRAPH_FP_TEST - sd t2, 8(sp) -#endif -#else - addi sp, sp, -16 - sd s0, 0(sp) - sd ra, 8(sp) - addi s0, sp, 16 -#endif +#define FENTRY_RA_OFFSET 12 +#define ABI_SIZE_ON_STACK 72 +#define ABI_A0 0 +#define ABI_A1 8 +#define ABI_A2 16 +#define ABI_A3 24 +#define ABI_A4 32 +#define ABI_A5 40 +#define ABI_A6 48 +#define ABI_A7 56 +#define ABI_RA 64 + + .macro SAVE_ABI + addi sp, sp, -SZREG + addi sp, sp, -ABI_SIZE_ON_STACK + + REG_S a0, ABI_A0(sp) + REG_S a1, ABI_A1(sp) + REG_S a2, ABI_A2(sp) + REG_S a3, ABI_A3(sp) + REG_S a4, ABI_A4(sp) + REG_S a5, ABI_A5(sp) + REG_S a6, ABI_A6(sp) + REG_S a7, ABI_A7(sp) + REG_S ra, ABI_RA(sp) .endm - .macro RESTORE_ABI_STATE -#ifdef CONFIG_FUNCTION_GRAPH_TRACER - ld s0, 32(sp) - ld ra, 40(sp) - addi sp, sp, 48 -#else - ld ra, 8(sp) - ld s0, 0(sp) - addi sp, sp, 16 -#endif + .macro RESTORE_ABI + REG_L a0, ABI_A0(sp) + REG_L a1, ABI_A1(sp) + REG_L a2, ABI_A2(sp) + REG_L a3, ABI_A3(sp) + REG_L a4, ABI_A4(sp) + REG_L a5, ABI_A5(sp) + REG_L a6, ABI_A6(sp) + REG_L a7, ABI_A7(sp) + REG_L ra, ABI_RA(sp) + + addi sp, sp, ABI_SIZE_ON_STACK + addi sp, sp, SZREG .endm - .macro RESTORE_GRAPH_ARGS - ld a0, 24(sp) - ld a1, 16(sp) -#ifdef HAVE_FUNCTION_GRAPH_FP_TEST - ld a2, 8(sp) -#endif +#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS + .macro SAVE_ALL + addi sp, sp, -SZREG + addi sp, sp, -PT_SIZE_ON_STACK + + REG_S x1, PT_EPC(sp) + addi sp, sp, PT_SIZE_ON_STACK + REG_L x1, (sp) + addi sp, sp, -PT_SIZE_ON_STACK + REG_S x1, PT_RA(sp) + REG_L x1, PT_EPC(sp) + + REG_S x2, PT_SP(sp) + REG_S x3, PT_GP(sp) + REG_S x4, PT_TP(sp) + REG_S x5, PT_T0(sp) + REG_S x6, PT_T1(sp) + REG_S x7, PT_T2(sp) + REG_S x8, PT_S0(sp) + REG_S x9, PT_S1(sp) + REG_S x10, PT_A0(sp) + REG_S x11, PT_A1(sp) + REG_S x12, PT_A2(sp) + REG_S x13, PT_A3(sp) + REG_S x14, PT_A4(sp) + REG_S x15, PT_A5(sp) + REG_S x16, PT_A6(sp) + REG_S x17, PT_A7(sp) + REG_S x18, PT_S2(sp) + REG_S x19, PT_S3(sp) + REG_S x20, PT_S4(sp) + REG_S x21, PT_S5(sp) + REG_S x22, PT_S6(sp) + REG_S x23, PT_S7(sp) + REG_S x24, PT_S8(sp) + REG_S x25, PT_S9(sp) + REG_S x26, PT_S10(sp) + REG_S x27, PT_S11(sp) + REG_S x28, PT_T3(sp) + REG_S x29, PT_T4(sp) + REG_S x30, PT_T5(sp) + REG_S x31, PT_T6(sp) .endm -ENTRY(ftrace_graph_caller) - addi sp, sp, -16 - sd s0, 0(sp) - sd ra, 8(sp) - addi s0, sp, 16 -ftrace_graph_call: - .global ftrace_graph_call - /* - * Calling ftrace_enable/disable_ftrace_graph_caller would overwrite the - * call below. Check ftrace_modify_all_code for details. - */ - call ftrace_stub - ld ra, 8(sp) - ld s0, 0(sp) - addi sp, sp, 16 - ret -ENDPROC(ftrace_graph_caller) + .macro RESTORE_ALL + REG_L x1, PT_RA(sp) + addi sp, sp, PT_SIZE_ON_STACK + REG_S x1, (sp) + addi sp, sp, -PT_SIZE_ON_STACK + REG_L x1, PT_EPC(sp) + REG_L x2, PT_SP(sp) + REG_L x3, PT_GP(sp) + REG_L x4, PT_TP(sp) + REG_L x5, PT_T0(sp) + REG_L x6, PT_T1(sp) + REG_L x7, PT_T2(sp) + REG_L x8, PT_S0(sp) + REG_L x9, PT_S1(sp) + REG_L x10, PT_A0(sp) + REG_L x11, PT_A1(sp) + REG_L x12, PT_A2(sp) + REG_L x13, PT_A3(sp) + REG_L x14, PT_A4(sp) + REG_L x15, PT_A5(sp) + REG_L x16, PT_A6(sp) + REG_L x17, PT_A7(sp) + REG_L x18, PT_S2(sp) + REG_L x19, PT_S3(sp) + REG_L x20, PT_S4(sp) + REG_L x21, PT_S5(sp) + REG_L x22, PT_S6(sp) + REG_L x23, PT_S7(sp) + REG_L x24, PT_S8(sp) + REG_L x25, PT_S9(sp) + REG_L x26, PT_S10(sp) + REG_L x27, PT_S11(sp) + REG_L x28, PT_T3(sp) + REG_L x29, PT_T4(sp) + REG_L x30, PT_T5(sp) + REG_L x31, PT_T6(sp) + + addi sp, sp, PT_SIZE_ON_STACK + addi sp, sp, SZREG + .endm +#endif /* CONFIG_DYNAMIC_FTRACE_WITH_REGS */ ENTRY(ftrace_caller) - /* - * a0: the address in the caller when calling ftrace_caller - * a1: the caller's return address - * a2: the address of global variable function_trace_op - */ - ld a1, -8(s0) - addi a0, ra, -MCOUNT_INSN_SIZE - la t5, function_trace_op - ld a2, 0(t5) + SAVE_ABI -#ifdef CONFIG_FUNCTION_GRAPH_TRACER - /* - * the graph tracer (specifically, prepare_ftrace_return) needs these - * arguments but for now the function tracer occupies the regs, so we - * save them in temporary regs to recover later. - */ - addi t0, s0, -8 - mv t1, a0 -#ifdef HAVE_FUNCTION_GRAPH_FP_TEST - ld t2, -16(s0) -#endif -#endif + addi a0, ra, -FENTRY_RA_OFFSET + la a1, function_trace_op + REG_L a2, 0(a1) + REG_L a1, ABI_SIZE_ON_STACK(sp) + mv a3, sp - SAVE_ABI_STATE ftrace_call: .global ftrace_call - /* - * For the dynamic ftrace to work, here we should reserve at least - * 8 bytes for a functional auipc-jalr pair. The following call - * serves this purpose. - * - * Calling ftrace_update_ftrace_func would overwrite the nops below. - * Check ftrace_modify_all_code for details. - */ call ftrace_stub #ifdef CONFIG_FUNCTION_GRAPH_TRACER - RESTORE_GRAPH_ARGS - call ftrace_graph_caller + addi a0, sp, ABI_SIZE_ON_STACK + REG_L a1, ABI_RA(sp) + addi a1, a1, -FENTRY_RA_OFFSET +#ifdef HAVE_FUNCTION_GRAPH_FP_TEST + mv a2, s0 #endif - - RESTORE_ABI_STATE +ftrace_graph_call: + .global ftrace_graph_call + call ftrace_stub +#endif + RESTORE_ABI ret ENDPROC(ftrace_caller) #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS - .macro SAVE_ALL - addi sp, sp, -(PT_SIZE_ON_STACK+16) - sd s0, (PT_SIZE_ON_STACK)(sp) - sd ra, (PT_SIZE_ON_STACK+8)(sp) - addi s0, sp, (PT_SIZE_ON_STACK+16) - - sd x1, PT_RA(sp) - sd x2, PT_SP(sp) - sd x3, PT_GP(sp) - sd x4, PT_TP(sp) - sd x5, PT_T0(sp) - sd x6, PT_T1(sp) - sd x7, PT_T2(sp) - sd x8, PT_S0(sp) - sd x9, PT_S1(sp) - sd x10, PT_A0(sp) - sd x11, PT_A1(sp) - sd x12, PT_A2(sp) - sd x13, PT_A3(sp) - sd x14, PT_A4(sp) - sd x15, PT_A5(sp) - sd x16, PT_A6(sp) - sd x17, PT_A7(sp) - sd x18, PT_S2(sp) - sd x19, PT_S3(sp) - sd x20, PT_S4(sp) - sd x21, PT_S5(sp) - sd x22, PT_S6(sp) - sd x23, PT_S7(sp) - sd x24, PT_S8(sp) - sd x25, PT_S9(sp) - sd x26, PT_S10(sp) - sd x27, PT_S11(sp) - sd x28, PT_T3(sp) - sd x29, PT_T4(sp) - sd x30, PT_T5(sp) - sd x31, PT_T6(sp) - .endm - - .macro RESTORE_ALL - ld x1, PT_RA(sp) - ld x2, PT_SP(sp) - ld x3, PT_GP(sp) - ld x4, PT_TP(sp) - ld x5, PT_T0(sp) - ld x6, PT_T1(sp) - ld x7, PT_T2(sp) - ld x8, PT_S0(sp) - ld x9, PT_S1(sp) - ld x10, PT_A0(sp) - ld x11, PT_A1(sp) - ld x12, PT_A2(sp) - ld x13, PT_A3(sp) - ld x14, PT_A4(sp) - ld x15, PT_A5(sp) - ld x16, PT_A6(sp) - ld x17, PT_A7(sp) - ld x18, PT_S2(sp) - ld x19, PT_S3(sp) - ld x20, PT_S4(sp) - ld x21, PT_S5(sp) - ld x22, PT_S6(sp) - ld x23, PT_S7(sp) - ld x24, PT_S8(sp) - ld x25, PT_S9(sp) - ld x26, PT_S10(sp) - ld x27, PT_S11(sp) - ld x28, PT_T3(sp) - ld x29, PT_T4(sp) - ld x30, PT_T5(sp) - ld x31, PT_T6(sp) - - ld s0, (PT_SIZE_ON_STACK)(sp) - ld ra, (PT_SIZE_ON_STACK+8)(sp) - addi sp, sp, (PT_SIZE_ON_STACK+16) - .endm - - .macro RESTORE_GRAPH_REG_ARGS - ld a0, PT_T0(sp) - ld a1, PT_T1(sp) -#ifdef HAVE_FUNCTION_GRAPH_FP_TEST - ld a2, PT_T2(sp) -#endif - .endm - -/* - * Most of the contents are the same as ftrace_caller. - */ ENTRY(ftrace_regs_caller) - /* - * a3: the address of all registers in the stack - */ - ld a1, -8(s0) - addi a0, ra, -MCOUNT_INSN_SIZE - la t5, function_trace_op - ld a2, 0(t5) - addi a3, sp, -(PT_SIZE_ON_STACK+16) - -#ifdef CONFIG_FUNCTION_GRAPH_TRACER - addi t0, s0, -8 - mv t1, a0 -#ifdef HAVE_FUNCTION_GRAPH_FP_TEST - ld t2, -16(s0) -#endif -#endif SAVE_ALL + addi a0, ra, -FENTRY_RA_OFFSET + la a1, function_trace_op + REG_L a2, 0(a1) + REG_L a1, PT_SIZE_ON_STACK(sp) + mv a3, sp + ftrace_regs_call: .global ftrace_regs_call call ftrace_stub #ifdef CONFIG_FUNCTION_GRAPH_TRACER - RESTORE_GRAPH_REG_ARGS - call ftrace_graph_caller + addi a0, sp, PT_RA + REG_L a1, PT_EPC(sp) + addi a1, a1, -FENTRY_RA_OFFSET +#ifdef HAVE_FUNCTION_GRAPH_FP_TEST + mv a2, s0 +#endif +ftrace_graph_regs_call: + .global ftrace_graph_regs_call + call ftrace_stub #endif RESTORE_ALL diff --git a/arch/riscv/kernel/patch.c b/arch/riscv/kernel/patch.c index 3fe7a5296aa5897c4867a019e66315c1dfbe5888..0b552873a5778b397a3b25ca8e64124c57d4a5c3 100644 --- a/arch/riscv/kernel/patch.c +++ b/arch/riscv/kernel/patch.c @@ -20,7 +20,12 @@ struct patch_insn { }; #ifdef CONFIG_MMU -static void *patch_map(void *addr, int fixmap) +/* + * The fix_to_virt(, idx) needs a const value (not a dynamic variable of + * reg-a0) or BUILD_BUG_ON failed with "idx >= __end_of_fixed_addresses". + * So use '__always_inline' and 'const unsigned int fixmap' here. + */ +static __always_inline void *patch_map(void *addr, const unsigned int fixmap) { uintptr_t uintaddr = (uintptr_t) addr; struct page *page; @@ -37,7 +42,6 @@ static void *patch_map(void *addr, int fixmap) return (void *)set_fixmap_offset(fixmap, page_to_phys(page) + (uintaddr & ~PAGE_MASK)); } -NOKPROBE_SYMBOL(patch_map); static void patch_unmap(int fixmap) { diff --git a/arch/riscv/kernel/probes/Makefile b/arch/riscv/kernel/probes/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..7f0840dcc31bc78eeecd6c3ff45a5a645f2b2283 --- /dev/null +++ b/arch/riscv/kernel/probes/Makefile @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_KPROBES) += kprobes.o decode-insn.o simulate-insn.o +obj-$(CONFIG_KPROBES) += kprobes_trampoline.o +obj-$(CONFIG_KPROBES_ON_FTRACE) += ftrace.o +obj-$(CONFIG_UPROBES) += uprobes.o decode-insn.o simulate-insn.o +CFLAGS_REMOVE_simulate-insn.o = $(CC_FLAGS_FTRACE) diff --git a/arch/riscv/kernel/probes/decode-insn.c b/arch/riscv/kernel/probes/decode-insn.c new file mode 100644 index 0000000000000000000000000000000000000000..0ed043acc882cd40eab4a38a1424db3524b7574e --- /dev/null +++ b/arch/riscv/kernel/probes/decode-insn.c @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: GPL-2.0+ + +#include +#include +#include +#include +#include + +#include "decode-insn.h" +#include "simulate-insn.h" + +/* Return: + * INSN_REJECTED If instruction is one not allowed to kprobe, + * INSN_GOOD_NO_SLOT If instruction is supported but doesn't use its slot. + */ +enum probe_insn __kprobes +riscv_probe_decode_insn(probe_opcode_t *addr, struct arch_probe_insn *api) +{ + probe_opcode_t insn = *addr; + + /* + * Reject instructions list: + */ + RISCV_INSN_REJECTED(system, insn); + RISCV_INSN_REJECTED(fence, insn); + + /* + * Simulate instructions list: + * TODO: the REJECTED ones below need to be implemented + */ +#ifdef CONFIG_RISCV_ISA_C + RISCV_INSN_REJECTED(c_j, insn); + RISCV_INSN_REJECTED(c_jr, insn); + RISCV_INSN_REJECTED(c_jal, insn); + RISCV_INSN_REJECTED(c_jalr, insn); + RISCV_INSN_REJECTED(c_beqz, insn); + RISCV_INSN_REJECTED(c_bnez, insn); + RISCV_INSN_REJECTED(c_ebreak, insn); +#endif + + RISCV_INSN_REJECTED(auipc, insn); + RISCV_INSN_REJECTED(branch, insn); + + RISCV_INSN_SET_SIMULATE(jal, insn); + RISCV_INSN_SET_SIMULATE(jalr, insn); + + return INSN_GOOD; +} diff --git a/arch/riscv/kernel/probes/decode-insn.h b/arch/riscv/kernel/probes/decode-insn.h new file mode 100644 index 0000000000000000000000000000000000000000..42269a7d676d41c022ec3aad7f05b79e66687fdd --- /dev/null +++ b/arch/riscv/kernel/probes/decode-insn.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ + +#ifndef _RISCV_KERNEL_KPROBES_DECODE_INSN_H +#define _RISCV_KERNEL_KPROBES_DECODE_INSN_H + +#include +#include + +enum probe_insn { + INSN_REJECTED, + INSN_GOOD_NO_SLOT, + INSN_GOOD, +}; + +enum probe_insn __kprobes +riscv_probe_decode_insn(probe_opcode_t *addr, struct arch_probe_insn *asi); + +#endif /* _RISCV_KERNEL_KPROBES_DECODE_INSN_H */ diff --git a/arch/riscv/kernel/probes/ftrace.c b/arch/riscv/kernel/probes/ftrace.c new file mode 100644 index 0000000000000000000000000000000000000000..e6372490aa0baf276bf1f46da6933f7dafc45dfc --- /dev/null +++ b/arch/riscv/kernel/probes/ftrace.c @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +/* Ftrace callback handler for kprobes -- called under preepmt disabed */ +void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip, + struct ftrace_ops *ops, struct ftrace_regs *regs) +{ + struct kprobe *p; + struct kprobe_ctlblk *kcb; + + p = get_kprobe((kprobe_opcode_t *)ip); + if (unlikely(!p) || kprobe_disabled(p)) + return; + + kcb = get_kprobe_ctlblk(); + if (kprobe_running()) { + kprobes_inc_nmissed_count(p); + } else { + unsigned long orig_ip = instruction_pointer(&(regs->regs)); + + instruction_pointer_set(&(regs->regs), ip); + + __this_cpu_write(current_kprobe, p); + kcb->kprobe_status = KPROBE_HIT_ACTIVE; + if (!p->pre_handler || !p->pre_handler(p, &(regs->regs))) { + /* + * Emulate singlestep (and also recover regs->pc) + * as if there is a nop + */ + instruction_pointer_set(&(regs->regs), + (unsigned long)p->addr + MCOUNT_INSN_SIZE); + if (unlikely(p->post_handler)) { + kcb->kprobe_status = KPROBE_HIT_SSDONE; + p->post_handler(p, &(regs->regs), 0); + } + instruction_pointer_set(&(regs->regs), orig_ip); + } + + /* + * If pre_handler returns !0, it changes regs->pc. We have to + * skip emulating post_handler. + */ + __this_cpu_write(current_kprobe, NULL); + } +} +NOKPROBE_SYMBOL(kprobe_ftrace_handler); + +int arch_prepare_kprobe_ftrace(struct kprobe *p) +{ + p->ainsn.api.insn = NULL; + return 0; +} diff --git a/arch/riscv/kernel/probes/kprobes.c b/arch/riscv/kernel/probes/kprobes.c new file mode 100644 index 0000000000000000000000000000000000000000..a2ec18662fee09a7a7bbc41c8263751763b0af6d --- /dev/null +++ b/arch/riscv/kernel/probes/kprobes.c @@ -0,0 +1,398 @@ +// SPDX-License-Identifier: GPL-2.0+ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "decode-insn.h" + +DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL; +DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk); + +static void __kprobes +post_kprobe_handler(struct kprobe_ctlblk *, struct pt_regs *); + +static void __kprobes arch_prepare_ss_slot(struct kprobe *p) +{ + unsigned long offset = GET_INSN_LENGTH(p->opcode); + + p->ainsn.api.restore = (unsigned long)p->addr + offset; + + patch_text(p->ainsn.api.insn, p->opcode); + patch_text((void *)((unsigned long)(p->ainsn.api.insn) + offset), + __BUG_INSN_32); +} + +static void __kprobes arch_prepare_simulate(struct kprobe *p) +{ + p->ainsn.api.restore = 0; +} + +static void __kprobes arch_simulate_insn(struct kprobe *p, struct pt_regs *regs) +{ + struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); + + if (p->ainsn.api.handler) + p->ainsn.api.handler((u32)p->opcode, + (unsigned long)p->addr, regs); + + post_kprobe_handler(kcb, regs); +} + +int __kprobes arch_prepare_kprobe(struct kprobe *p) +{ + unsigned long probe_addr = (unsigned long)p->addr; + + if (probe_addr & 0x1) { + pr_warn("Address not aligned.\n"); + + return -EINVAL; + } + + /* copy instruction */ + p->opcode = *p->addr; + + /* decode instruction */ + switch (riscv_probe_decode_insn(p->addr, &p->ainsn.api)) { + case INSN_REJECTED: /* insn not supported */ + return -EINVAL; + + case INSN_GOOD_NO_SLOT: /* insn need simulation */ + p->ainsn.api.insn = NULL; + break; + + case INSN_GOOD: /* instruction uses slot */ + p->ainsn.api.insn = get_insn_slot(); + if (!p->ainsn.api.insn) + return -ENOMEM; + break; + } + + /* prepare the instruction */ + if (p->ainsn.api.insn) + arch_prepare_ss_slot(p); + else + arch_prepare_simulate(p); + + return 0; +} + +/* install breakpoint in text */ +void __kprobes arch_arm_kprobe(struct kprobe *p) +{ + if ((p->opcode & __INSN_LENGTH_MASK) == __INSN_LENGTH_32) + patch_text(p->addr, __BUG_INSN_32); + else + patch_text(p->addr, __BUG_INSN_16); +} + +/* remove breakpoint from text */ +void __kprobes arch_disarm_kprobe(struct kprobe *p) +{ + patch_text(p->addr, p->opcode); +} + +void __kprobes arch_remove_kprobe(struct kprobe *p) +{ +} + +static void __kprobes save_previous_kprobe(struct kprobe_ctlblk *kcb) +{ + kcb->prev_kprobe.kp = kprobe_running(); + kcb->prev_kprobe.status = kcb->kprobe_status; +} + +static void __kprobes restore_previous_kprobe(struct kprobe_ctlblk *kcb) +{ + __this_cpu_write(current_kprobe, kcb->prev_kprobe.kp); + kcb->kprobe_status = kcb->prev_kprobe.status; +} + +static void __kprobes set_current_kprobe(struct kprobe *p) +{ + __this_cpu_write(current_kprobe, p); +} + +/* + * Interrupts need to be disabled before single-step mode is set, and not + * reenabled until after single-step mode ends. + * Without disabling interrupt on local CPU, there is a chance of + * interrupt occurrence in the period of exception return and start of + * out-of-line single-step, that result in wrongly single stepping + * into the interrupt handler. + */ +static void __kprobes kprobes_save_local_irqflag(struct kprobe_ctlblk *kcb, + struct pt_regs *regs) +{ + kcb->saved_status = regs->status; + regs->status &= ~SR_SPIE; +} + +static void __kprobes kprobes_restore_local_irqflag(struct kprobe_ctlblk *kcb, + struct pt_regs *regs) +{ + regs->status = kcb->saved_status; +} + +static void __kprobes +set_ss_context(struct kprobe_ctlblk *kcb, unsigned long addr, struct kprobe *p) +{ + unsigned long offset = GET_INSN_LENGTH(p->opcode); + + kcb->ss_ctx.ss_pending = true; + kcb->ss_ctx.match_addr = addr + offset; +} + +static void __kprobes clear_ss_context(struct kprobe_ctlblk *kcb) +{ + kcb->ss_ctx.ss_pending = false; + kcb->ss_ctx.match_addr = 0; +} + +static void __kprobes setup_singlestep(struct kprobe *p, + struct pt_regs *regs, + struct kprobe_ctlblk *kcb, int reenter) +{ + unsigned long slot; + + if (reenter) { + save_previous_kprobe(kcb); + set_current_kprobe(p); + kcb->kprobe_status = KPROBE_REENTER; + } else { + kcb->kprobe_status = KPROBE_HIT_SS; + } + + if (p->ainsn.api.insn) { + /* prepare for single stepping */ + slot = (unsigned long)p->ainsn.api.insn; + + set_ss_context(kcb, slot, p); /* mark pending ss */ + + /* IRQs and single stepping do not mix well. */ + kprobes_save_local_irqflag(kcb, regs); + + instruction_pointer_set(regs, slot); + } else { + /* insn simulation */ + arch_simulate_insn(p, regs); + } +} + +static int __kprobes reenter_kprobe(struct kprobe *p, + struct pt_regs *regs, + struct kprobe_ctlblk *kcb) +{ + switch (kcb->kprobe_status) { + case KPROBE_HIT_SSDONE: + case KPROBE_HIT_ACTIVE: + kprobes_inc_nmissed_count(p); + setup_singlestep(p, regs, kcb, 1); + break; + case KPROBE_HIT_SS: + case KPROBE_REENTER: + pr_warn("Unrecoverable kprobe detected.\n"); + dump_kprobe(p); + BUG(); + break; + default: + WARN_ON(1); + return 0; + } + + return 1; +} + +static void __kprobes +post_kprobe_handler(struct kprobe_ctlblk *kcb, struct pt_regs *regs) +{ + struct kprobe *cur = kprobe_running(); + + if (!cur) + return; + + /* return addr restore if non-branching insn */ + if (cur->ainsn.api.restore != 0) + regs->epc = cur->ainsn.api.restore; + + /* restore back original saved kprobe variables and continue */ + if (kcb->kprobe_status == KPROBE_REENTER) { + restore_previous_kprobe(kcb); + return; + } + + /* call post handler */ + kcb->kprobe_status = KPROBE_HIT_SSDONE; + if (cur->post_handler) { + /* post_handler can hit breakpoint and single step + * again, so we enable D-flag for recursive exception. + */ + cur->post_handler(cur, regs, 0); + } + + reset_current_kprobe(); +} + +int __kprobes kprobe_fault_handler(struct pt_regs *regs, unsigned int trapnr) +{ + struct kprobe *cur = kprobe_running(); + struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); + + switch (kcb->kprobe_status) { + case KPROBE_HIT_SS: + case KPROBE_REENTER: + /* + * We are here because the instruction being single + * stepped caused a page fault. We reset the current + * kprobe and the ip points back to the probe address + * and allow the page fault handler to continue as a + * normal page fault. + */ + regs->epc = (unsigned long) cur->addr; + if (!instruction_pointer(regs)) + BUG(); + + if (kcb->kprobe_status == KPROBE_REENTER) + restore_previous_kprobe(kcb); + else + reset_current_kprobe(); + + break; + case KPROBE_HIT_ACTIVE: + case KPROBE_HIT_SSDONE: + /* + * We increment the nmissed count for accounting, + * we can also use npre/npostfault count for accounting + * these specific fault cases. + */ + kprobes_inc_nmissed_count(cur); + + /* + * We come here because instructions in the pre/post + * handler caused the page_fault, this could happen + * if handler tries to access user space by + * copy_from_user(), get_user() etc. Let the + * user-specified handler try to fix it first. + */ + if (cur->fault_handler && cur->fault_handler(cur, regs, trapnr)) + return 1; + + /* + * In case the user-specified fault handler returned + * zero, try to fix up. + */ + if (fixup_exception(regs)) + return 1; + } + return 0; +} + +bool __kprobes +kprobe_breakpoint_handler(struct pt_regs *regs) +{ + struct kprobe *p, *cur_kprobe; + struct kprobe_ctlblk *kcb; + unsigned long addr = instruction_pointer(regs); + + kcb = get_kprobe_ctlblk(); + cur_kprobe = kprobe_running(); + + p = get_kprobe((kprobe_opcode_t *) addr); + + if (p) { + if (cur_kprobe) { + if (reenter_kprobe(p, regs, kcb)) + return true; + } else { + /* Probe hit */ + set_current_kprobe(p); + kcb->kprobe_status = KPROBE_HIT_ACTIVE; + + /* + * If we have no pre-handler or it returned 0, we + * continue with normal processing. If we have a + * pre-handler and it returned non-zero, it will + * modify the execution path and no need to single + * stepping. Let's just reset current kprobe and exit. + * + * pre_handler can hit a breakpoint and can step thru + * before return. + */ + if (!p->pre_handler || !p->pre_handler(p, regs)) + setup_singlestep(p, regs, kcb, 0); + else + reset_current_kprobe(); + } + return true; + } + + /* + * The breakpoint instruction was removed right + * after we hit it. Another cpu has removed + * either a probepoint or a debugger breakpoint + * at this address. In either case, no further + * handling of this interrupt is appropriate. + * Return back to original instruction, and continue. + */ + return false; +} + +bool __kprobes +kprobe_single_step_handler(struct pt_regs *regs) +{ + struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); + + if ((kcb->ss_ctx.ss_pending) + && (kcb->ss_ctx.match_addr == instruction_pointer(regs))) { + clear_ss_context(kcb); /* clear pending ss */ + + kprobes_restore_local_irqflag(kcb, regs); + + post_kprobe_handler(kcb, regs); + return true; + } + return false; +} + +/* + * Provide a blacklist of symbols identifying ranges which cannot be kprobed. + * This blacklist is exposed to userspace via debugfs (kprobes/blacklist). + */ +int __init arch_populate_kprobe_blacklist(void) +{ + int ret; + + ret = kprobe_add_area_blacklist((unsigned long)__irqentry_text_start, + (unsigned long)__irqentry_text_end); + return ret; +} + +void __kprobes __used *trampoline_probe_handler(struct pt_regs *regs) +{ + return (void *)kretprobe_trampoline_handler(regs, &kretprobe_trampoline, NULL); +} + +void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri, + struct pt_regs *regs) +{ + ri->ret_addr = (kprobe_opcode_t *)regs->ra; + ri->fp = NULL; + regs->ra = (unsigned long) &kretprobe_trampoline; +} + +int __kprobes arch_trampoline_kprobe(struct kprobe *p) +{ + return 0; +} + +int __init arch_init_kprobes(void) +{ + return 0; +} diff --git a/arch/riscv/kernel/probes/kprobes_trampoline.S b/arch/riscv/kernel/probes/kprobes_trampoline.S new file mode 100644 index 0000000000000000000000000000000000000000..6e85d021e2a217929813d0d99905315c6f3d9cfb --- /dev/null +++ b/arch/riscv/kernel/probes/kprobes_trampoline.S @@ -0,0 +1,93 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* + * Author: Patrick Stählin + */ +#include + +#include +#include + + .text + .altmacro + + .macro save_all_base_regs + REG_S x1, PT_RA(sp) + REG_S x3, PT_GP(sp) + REG_S x4, PT_TP(sp) + REG_S x5, PT_T0(sp) + REG_S x6, PT_T1(sp) + REG_S x7, PT_T2(sp) + REG_S x8, PT_S0(sp) + REG_S x9, PT_S1(sp) + REG_S x10, PT_A0(sp) + REG_S x11, PT_A1(sp) + REG_S x12, PT_A2(sp) + REG_S x13, PT_A3(sp) + REG_S x14, PT_A4(sp) + REG_S x15, PT_A5(sp) + REG_S x16, PT_A6(sp) + REG_S x17, PT_A7(sp) + REG_S x18, PT_S2(sp) + REG_S x19, PT_S3(sp) + REG_S x20, PT_S4(sp) + REG_S x21, PT_S5(sp) + REG_S x22, PT_S6(sp) + REG_S x23, PT_S7(sp) + REG_S x24, PT_S8(sp) + REG_S x25, PT_S9(sp) + REG_S x26, PT_S10(sp) + REG_S x27, PT_S11(sp) + REG_S x28, PT_T3(sp) + REG_S x29, PT_T4(sp) + REG_S x30, PT_T5(sp) + REG_S x31, PT_T6(sp) + .endm + + .macro restore_all_base_regs + REG_L x3, PT_GP(sp) + REG_L x4, PT_TP(sp) + REG_L x5, PT_T0(sp) + REG_L x6, PT_T1(sp) + REG_L x7, PT_T2(sp) + REG_L x8, PT_S0(sp) + REG_L x9, PT_S1(sp) + REG_L x10, PT_A0(sp) + REG_L x11, PT_A1(sp) + REG_L x12, PT_A2(sp) + REG_L x13, PT_A3(sp) + REG_L x14, PT_A4(sp) + REG_L x15, PT_A5(sp) + REG_L x16, PT_A6(sp) + REG_L x17, PT_A7(sp) + REG_L x18, PT_S2(sp) + REG_L x19, PT_S3(sp) + REG_L x20, PT_S4(sp) + REG_L x21, PT_S5(sp) + REG_L x22, PT_S6(sp) + REG_L x23, PT_S7(sp) + REG_L x24, PT_S8(sp) + REG_L x25, PT_S9(sp) + REG_L x26, PT_S10(sp) + REG_L x27, PT_S11(sp) + REG_L x28, PT_T3(sp) + REG_L x29, PT_T4(sp) + REG_L x30, PT_T5(sp) + REG_L x31, PT_T6(sp) + .endm + +ENTRY(kretprobe_trampoline) + addi sp, sp, -(PT_SIZE_ON_STACK) + save_all_base_regs + + move a0, sp /* pt_regs */ + + call trampoline_probe_handler + + /* use the result as the return-address */ + move ra, a0 + + restore_all_base_regs + addi sp, sp, PT_SIZE_ON_STACK + + ret +ENDPROC(kretprobe_trampoline) diff --git a/arch/riscv/kernel/probes/simulate-insn.c b/arch/riscv/kernel/probes/simulate-insn.c new file mode 100644 index 0000000000000000000000000000000000000000..2519ce26377d15e9bd7c9fbd93e5da3fbcaeee99 --- /dev/null +++ b/arch/riscv/kernel/probes/simulate-insn.c @@ -0,0 +1,85 @@ +// SPDX-License-Identifier: GPL-2.0+ + +#include +#include +#include + +#include "decode-insn.h" +#include "simulate-insn.h" + +static inline bool rv_insn_reg_get_val(struct pt_regs *regs, u32 index, + unsigned long *ptr) +{ + if (index == 0) + *ptr = 0; + else if (index <= 31) + *ptr = *((unsigned long *)regs + index); + else + return false; + + return true; +} + +static inline bool rv_insn_reg_set_val(struct pt_regs *regs, u32 index, + unsigned long val) +{ + if (index == 0) + return false; + else if (index <= 31) + *((unsigned long *)regs + index) = val; + else + return false; + + return true; +} + +bool __kprobes simulate_jal(u32 opcode, unsigned long addr, struct pt_regs *regs) +{ + /* + * 31 30 21 20 19 12 11 7 6 0 + * imm [20] | imm[10:1] | imm[11] | imm[19:12] | rd | opcode + * 1 10 1 8 5 JAL/J + */ + bool ret; + u32 imm; + u32 index = (opcode >> 7) & 0x1f; + + ret = rv_insn_reg_set_val(regs, index, addr + 4); + if (!ret) + return ret; + + imm = ((opcode >> 21) & 0x3ff) << 1; + imm |= ((opcode >> 20) & 0x1) << 11; + imm |= ((opcode >> 12) & 0xff) << 12; + imm |= ((opcode >> 31) & 0x1) << 20; + + instruction_pointer_set(regs, addr + sign_extend32((imm), 20)); + + return ret; +} + +bool __kprobes simulate_jalr(u32 opcode, unsigned long addr, struct pt_regs *regs) +{ + /* + * 31 20 19 15 14 12 11 7 6 0 + * offset[11:0] | rs1 | 010 | rd | opcode + * 12 5 3 5 JALR/JR + */ + bool ret; + unsigned long base_addr; + u32 imm = (opcode >> 20) & 0xfff; + u32 rd_index = (opcode >> 7) & 0x1f; + u32 rs1_index = (opcode >> 15) & 0x1f; + + ret = rv_insn_reg_set_val(regs, rd_index, addr + 4); + if (!ret) + return ret; + + ret = rv_insn_reg_get_val(regs, rs1_index, &base_addr); + if (!ret) + return ret; + + instruction_pointer_set(regs, (base_addr + sign_extend32((imm), 11))&~1); + + return ret; +} diff --git a/arch/riscv/kernel/probes/simulate-insn.h b/arch/riscv/kernel/probes/simulate-insn.h new file mode 100644 index 0000000000000000000000000000000000000000..cb6ff7dccb92e18bde5f9b517e42ec851a98b986 --- /dev/null +++ b/arch/riscv/kernel/probes/simulate-insn.h @@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ + +#ifndef _RISCV_KERNEL_PROBES_SIMULATE_INSN_H +#define _RISCV_KERNEL_PROBES_SIMULATE_INSN_H + +#define __RISCV_INSN_FUNCS(name, mask, val) \ +static __always_inline bool riscv_insn_is_##name(probe_opcode_t code) \ +{ \ + BUILD_BUG_ON(~(mask) & (val)); \ + return (code & (mask)) == (val); \ +} \ +bool simulate_##name(u32 opcode, unsigned long addr, \ + struct pt_regs *regs) + +#define RISCV_INSN_REJECTED(name, code) \ + do { \ + if (riscv_insn_is_##name(code)) { \ + return INSN_REJECTED; \ + } \ + } while (0) + +__RISCV_INSN_FUNCS(system, 0x7f, 0x73); +__RISCV_INSN_FUNCS(fence, 0x7f, 0x0f); + +#define RISCV_INSN_SET_SIMULATE(name, code) \ + do { \ + if (riscv_insn_is_##name(code)) { \ + api->handler = simulate_##name; \ + return INSN_GOOD_NO_SLOT; \ + } \ + } while (0) + +__RISCV_INSN_FUNCS(c_j, 0xe003, 0xa001); +__RISCV_INSN_FUNCS(c_jr, 0xf007, 0x8002); +__RISCV_INSN_FUNCS(c_jal, 0xe003, 0x2001); +__RISCV_INSN_FUNCS(c_jalr, 0xf007, 0x9002); +__RISCV_INSN_FUNCS(c_beqz, 0xe003, 0xc001); +__RISCV_INSN_FUNCS(c_bnez, 0xe003, 0xe001); +__RISCV_INSN_FUNCS(c_ebreak, 0xffff, 0x9002); + +__RISCV_INSN_FUNCS(auipc, 0x7f, 0x17); +__RISCV_INSN_FUNCS(branch, 0x7f, 0x63); + +__RISCV_INSN_FUNCS(jal, 0x7f, 0x6f); +__RISCV_INSN_FUNCS(jalr, 0x707f, 0x67); + +#endif /* _RISCV_KERNEL_PROBES_SIMULATE_INSN_H */ diff --git a/arch/riscv/kernel/probes/uprobes.c b/arch/riscv/kernel/probes/uprobes.c new file mode 100644 index 0000000000000000000000000000000000000000..7a057b5f0adc73f1306e7ccc61f9cec28c50b66a --- /dev/null +++ b/arch/riscv/kernel/probes/uprobes.c @@ -0,0 +1,186 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include +#include +#include + +#include "decode-insn.h" + +#define UPROBE_TRAP_NR UINT_MAX + +bool is_swbp_insn(uprobe_opcode_t *insn) +{ +#ifdef CONFIG_RISCV_ISA_C + return (*insn & 0xffff) == UPROBE_SWBP_INSN; +#else + return *insn == UPROBE_SWBP_INSN; +#endif +} + +unsigned long uprobe_get_swbp_addr(struct pt_regs *regs) +{ + return instruction_pointer(regs); +} + +int arch_uprobe_analyze_insn(struct arch_uprobe *auprobe, struct mm_struct *mm, + unsigned long addr) +{ + probe_opcode_t opcode; + + opcode = *(probe_opcode_t *)(&auprobe->insn[0]); + + auprobe->insn_size = GET_INSN_LENGTH(opcode); + + switch (riscv_probe_decode_insn(&opcode, &auprobe->api)) { + case INSN_REJECTED: + return -EINVAL; + + case INSN_GOOD_NO_SLOT: + auprobe->simulate = true; + break; + + case INSN_GOOD: + auprobe->simulate = false; + break; + + default: + return -EINVAL; + } + + return 0; +} + +int arch_uprobe_pre_xol(struct arch_uprobe *auprobe, struct pt_regs *regs) +{ + struct uprobe_task *utask = current->utask; + + utask->autask.saved_cause = current->thread.bad_cause; + current->thread.bad_cause = UPROBE_TRAP_NR; + + instruction_pointer_set(regs, utask->xol_vaddr); + + regs->status &= ~SR_SPIE; + + return 0; +} + +int arch_uprobe_post_xol(struct arch_uprobe *auprobe, struct pt_regs *regs) +{ + struct uprobe_task *utask = current->utask; + + WARN_ON_ONCE(current->thread.bad_cause != UPROBE_TRAP_NR); + + instruction_pointer_set(regs, utask->vaddr + auprobe->insn_size); + + regs->status |= SR_SPIE; + + return 0; +} + +bool arch_uprobe_xol_was_trapped(struct task_struct *t) +{ + if (t->thread.bad_cause != UPROBE_TRAP_NR) + return true; + + return false; +} + +bool arch_uprobe_skip_sstep(struct arch_uprobe *auprobe, struct pt_regs *regs) +{ + probe_opcode_t insn; + unsigned long addr; + + if (!auprobe->simulate) + return false; + + insn = *(probe_opcode_t *)(&auprobe->insn[0]); + addr = instruction_pointer(regs); + + if (auprobe->api.handler) + auprobe->api.handler(insn, addr, regs); + + return true; +} + +void arch_uprobe_abort_xol(struct arch_uprobe *auprobe, struct pt_regs *regs) +{ + struct uprobe_task *utask = current->utask; + + /* + * Task has received a fatal signal, so reset back to probbed + * address. + */ + instruction_pointer_set(regs, utask->vaddr); + + regs->status &= ~SR_SPIE; +} + +bool arch_uretprobe_is_alive(struct return_instance *ret, enum rp_check ctx, + struct pt_regs *regs) +{ + if (ctx == RP_CHECK_CHAIN_CALL) + return regs->sp <= ret->stack; + else + return regs->sp < ret->stack; +} + +unsigned long +arch_uretprobe_hijack_return_addr(unsigned long trampoline_vaddr, + struct pt_regs *regs) +{ + unsigned long ra; + + ra = regs->ra; + + regs->ra = trampoline_vaddr; + + return ra; +} + +int arch_uprobe_exception_notify(struct notifier_block *self, + unsigned long val, void *data) +{ + return NOTIFY_DONE; +} + +bool uprobe_breakpoint_handler(struct pt_regs *regs) +{ + if (uprobe_pre_sstep_notifier(regs)) + return true; + + return false; +} + +bool uprobe_single_step_handler(struct pt_regs *regs) +{ + if (uprobe_post_sstep_notifier(regs)) + return true; + + return false; +} + +void arch_uprobe_copy_ixol(struct page *page, unsigned long vaddr, + void *src, unsigned long len) +{ + /* Initialize the slot */ + void *kaddr = kmap_atomic(page); + void *dst = kaddr + (vaddr & ~PAGE_MASK); + + memcpy(dst, src, len); + + /* Add ebreak behind opcode to simulate singlestep */ + if (vaddr) { + dst += GET_INSN_LENGTH(*(probe_opcode_t *)src); + *(uprobe_opcode_t *)dst = __BUG_INSN_32; + } + + kunmap_atomic(kaddr); + + /* + * We probably need flush_icache_user_page() but it needs vma. + * This should work on most of architectures by default. If + * architecture needs to do something different it can define + * its own version of the function. + */ + flush_dcache_page(page); +} diff --git a/arch/riscv/kernel/process.c b/arch/riscv/kernel/process.c index dd5f985b1f40e0418f4bdac872819275a0766cd0..6f728e731bedf56020516cbdc0076e53ce518b85 100644 --- a/arch/riscv/kernel/process.c +++ b/arch/riscv/kernel/process.c @@ -18,13 +18,14 @@ #include #include #include +#include #include #include #include register unsigned long gp_in_global __asm__("gp"); -#ifdef CONFIG_STACKPROTECTOR +#if defined(CONFIG_STACKPROTECTOR) && !defined(CONFIG_STACKPROTECTOR_PER_TASK) #include unsigned long __stack_chk_guard __read_mostly; EXPORT_SYMBOL(__stack_chk_guard); @@ -39,11 +40,16 @@ void arch_cpu_idle(void) raw_local_irq_enable(); } -void show_regs(struct pt_regs *regs) +void __show_regs(struct pt_regs *regs) { show_regs_print_info(KERN_DEFAULT); - pr_cont("epc: " REG_FMT " ra : " REG_FMT " sp : " REG_FMT "\n", + if (!user_mode(regs)) { + pr_cont("epc : %pS\n", (void *)regs->epc); + pr_cont(" ra : %pS\n", (void *)regs->ra); + } + + pr_cont("epc : " REG_FMT " ra : " REG_FMT " sp : " REG_FMT "\n", regs->epc, regs->ra, regs->sp); pr_cont(" gp : " REG_FMT " tp : " REG_FMT " t0 : " REG_FMT "\n", regs->gp, regs->tp, regs->t0); @@ -69,6 +75,12 @@ void show_regs(struct pt_regs *regs) pr_cont("status: " REG_FMT " badaddr: " REG_FMT " cause: " REG_FMT "\n", regs->status, regs->badaddr, regs->cause); } +void show_regs(struct pt_regs *regs) +{ + __show_regs(regs); + if (!user_mode(regs)) + dump_backtrace(regs, NULL, KERN_DEFAULT); +} void start_thread(struct pt_regs *regs, unsigned long pc, unsigned long sp) @@ -112,7 +124,7 @@ int copy_thread(unsigned long clone_flags, unsigned long usp, unsigned long arg, struct pt_regs *childregs = task_pt_regs(p); /* p->thread holds context to be restored by __switch_to() */ - if (unlikely(p->flags & PF_KTHREAD)) { + if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { /* Kernel thread */ memset(childregs, 0, sizeof(struct pt_regs)); childregs->gp = gp_in_global; diff --git a/arch/riscv/kernel/ptrace.c b/arch/riscv/kernel/ptrace.c index 2d6395f5ad54f0c93cc162671fd97e6acfe72771..1a85305720e848033fcbd0fb7e4c7b45883e5d52 100644 --- a/arch/riscv/kernel/ptrace.c +++ b/arch/riscv/kernel/ptrace.c @@ -114,6 +114,105 @@ const struct user_regset_view *task_user_regset_view(struct task_struct *task) return &riscv_user_native_view; } +struct pt_regs_offset { + const char *name; + int offset; +}; + +#define REG_OFFSET_NAME(r) {.name = #r, .offset = offsetof(struct pt_regs, r)} +#define REG_OFFSET_END {.name = NULL, .offset = 0} + +static const struct pt_regs_offset regoffset_table[] = { + REG_OFFSET_NAME(epc), + REG_OFFSET_NAME(ra), + REG_OFFSET_NAME(sp), + REG_OFFSET_NAME(gp), + REG_OFFSET_NAME(tp), + REG_OFFSET_NAME(t0), + REG_OFFSET_NAME(t1), + REG_OFFSET_NAME(t2), + REG_OFFSET_NAME(s0), + REG_OFFSET_NAME(s1), + REG_OFFSET_NAME(a0), + REG_OFFSET_NAME(a1), + REG_OFFSET_NAME(a2), + REG_OFFSET_NAME(a3), + REG_OFFSET_NAME(a4), + REG_OFFSET_NAME(a5), + REG_OFFSET_NAME(a6), + REG_OFFSET_NAME(a7), + REG_OFFSET_NAME(s2), + REG_OFFSET_NAME(s3), + REG_OFFSET_NAME(s4), + REG_OFFSET_NAME(s5), + REG_OFFSET_NAME(s6), + REG_OFFSET_NAME(s7), + REG_OFFSET_NAME(s8), + REG_OFFSET_NAME(s9), + REG_OFFSET_NAME(s10), + REG_OFFSET_NAME(s11), + REG_OFFSET_NAME(t3), + REG_OFFSET_NAME(t4), + REG_OFFSET_NAME(t5), + REG_OFFSET_NAME(t6), + REG_OFFSET_NAME(status), + REG_OFFSET_NAME(badaddr), + REG_OFFSET_NAME(cause), + REG_OFFSET_NAME(orig_a0), + REG_OFFSET_END, +}; + +/** + * regs_query_register_offset() - query register offset from its name + * @name: the name of a register + * + * regs_query_register_offset() returns the offset of a register in struct + * pt_regs from its name. If the name is invalid, this returns -EINVAL; + */ +int regs_query_register_offset(const char *name) +{ + const struct pt_regs_offset *roff; + + for (roff = regoffset_table; roff->name != NULL; roff++) + if (!strcmp(roff->name, name)) + return roff->offset; + return -EINVAL; +} + +/** + * regs_within_kernel_stack() - check the address in the stack + * @regs: pt_regs which contains kernel stack pointer. + * @addr: address which is checked. + * + * regs_within_kernel_stack() checks @addr is within the kernel stack page(s). + * If @addr is within the kernel stack, it returns true. If not, returns false. + */ +static bool regs_within_kernel_stack(struct pt_regs *regs, unsigned long addr) +{ + return (addr & ~(THREAD_SIZE - 1)) == + (kernel_stack_pointer(regs) & ~(THREAD_SIZE - 1)); +} + +/** + * regs_get_kernel_stack_nth() - get Nth entry of the stack + * @regs: pt_regs which contains kernel stack pointer. + * @n: stack entry number. + * + * regs_get_kernel_stack_nth() returns @n th entry of the kernel stack which + * is specified by @regs. If the @n th entry is NOT in the kernel stack, + * this returns 0. + */ +unsigned long regs_get_kernel_stack_nth(struct pt_regs *regs, unsigned int n) +{ + unsigned long *addr = (unsigned long *)kernel_stack_pointer(regs); + + addr += n; + if (regs_within_kernel_stack(regs, (unsigned long)addr)) + return *addr; + else + return 0; +} + void ptrace_disable(struct task_struct *child) { clear_tsk_thread_flag(child, TIF_SYSCALL_TRACE); diff --git a/arch/riscv/kernel/sbi.c b/arch/riscv/kernel/sbi.c index 226ccce0f9e0734a08ff3da652c898ed21e378ce..f4a7db3d309e69f4928b423532da1b96b762275d 100644 --- a/arch/riscv/kernel/sbi.c +++ b/arch/riscv/kernel/sbi.c @@ -351,7 +351,7 @@ static int __sbi_rfence_v02(int fid, const unsigned long *hart_mask, * sbi_set_timer() - Program the timer for next timer event. * @stime_value: The value after which next timer event should fire. * - * Return: None + * Return: None. */ void sbi_set_timer(uint64_t stime_value) { @@ -362,11 +362,11 @@ void sbi_set_timer(uint64_t stime_value) * sbi_send_ipi() - Send an IPI to any hart. * @hart_mask: A cpu mask containing all the target harts. * - * Return: None + * Return: 0 on success, appropriate linux error code otherwise. */ -void sbi_send_ipi(const unsigned long *hart_mask) +int sbi_send_ipi(const unsigned long *hart_mask) { - __sbi_send_ipi(hart_mask); + return __sbi_send_ipi(hart_mask); } EXPORT_SYMBOL(sbi_send_ipi); @@ -374,12 +374,12 @@ EXPORT_SYMBOL(sbi_send_ipi); * sbi_remote_fence_i() - Execute FENCE.I instruction on given remote harts. * @hart_mask: A cpu mask containing all the target harts. * - * Return: None + * Return: 0 on success, appropriate linux error code otherwise. */ -void sbi_remote_fence_i(const unsigned long *hart_mask) +int sbi_remote_fence_i(const unsigned long *hart_mask) { - __sbi_rfence(SBI_EXT_RFENCE_REMOTE_FENCE_I, - hart_mask, 0, 0, 0, 0); + return __sbi_rfence(SBI_EXT_RFENCE_REMOTE_FENCE_I, + hart_mask, 0, 0, 0, 0); } EXPORT_SYMBOL(sbi_remote_fence_i); @@ -390,14 +390,14 @@ EXPORT_SYMBOL(sbi_remote_fence_i); * @start: Start of the virtual address * @size: Total size of the virtual address range. * - * Return: None + * Return: 0 on success, appropriate linux error code otherwise. */ -void sbi_remote_sfence_vma(const unsigned long *hart_mask, +int sbi_remote_sfence_vma(const unsigned long *hart_mask, unsigned long start, unsigned long size) { - __sbi_rfence(SBI_EXT_RFENCE_REMOTE_SFENCE_VMA, - hart_mask, start, size, 0, 0); + return __sbi_rfence(SBI_EXT_RFENCE_REMOTE_SFENCE_VMA, + hart_mask, start, size, 0, 0); } EXPORT_SYMBOL(sbi_remote_sfence_vma); @@ -410,15 +410,15 @@ EXPORT_SYMBOL(sbi_remote_sfence_vma); * @size: Total size of the virtual address range. * @asid: The value of address space identifier (ASID). * - * Return: None + * Return: 0 on success, appropriate linux error code otherwise. */ -void sbi_remote_sfence_vma_asid(const unsigned long *hart_mask, +int sbi_remote_sfence_vma_asid(const unsigned long *hart_mask, unsigned long start, unsigned long size, unsigned long asid) { - __sbi_rfence(SBI_EXT_RFENCE_REMOTE_SFENCE_VMA_ASID, - hart_mask, start, size, asid, 0); + return __sbi_rfence(SBI_EXT_RFENCE_REMOTE_SFENCE_VMA_ASID, + hart_mask, start, size, asid, 0); } EXPORT_SYMBOL(sbi_remote_sfence_vma_asid); @@ -560,7 +560,7 @@ static struct riscv_ipi_ops sbi_ipi_ops = { .ipi_inject = sbi_send_cpumask_ipi }; -int __init sbi_init(void) +void __init sbi_init(void) { int ret; @@ -600,6 +600,4 @@ int __init sbi_init(void) } riscv_set_ipi_ops(&sbi_ipi_ops); - - return 0; } diff --git a/arch/riscv/kernel/setup.c b/arch/riscv/kernel/setup.c index c7c0655dd45b0f617a34d89b315e5829638c43af..e85bacff1b5075ee3b704f7a65e40d102d0a9de3 100644 --- a/arch/riscv/kernel/setup.c +++ b/arch/riscv/kernel/setup.c @@ -216,8 +216,15 @@ static void __init init_resources(void) static void __init parse_dtb(void) { /* Early scan of device tree from init memory */ - if (early_init_dt_scan(dtb_early_va)) + if (early_init_dt_scan(dtb_early_va)) { + const char *name = of_flat_dt_get_machine_name(); + + if (name) { + pr_info("Machine model: %s\n", name); + dump_stack_set_arch_desc("%s (DT)", name); + } return; + } pr_err("No DTB passed to the kernel\n"); #ifdef CONFIG_CMDLINE_FORCE @@ -252,9 +259,9 @@ void __init setup_arch(char **cmdline_p) else pr_err("No DTB found in kernel mappings\n"); #endif + misc_mem_init(); - if (IS_ENABLED(CONFIG_RISCV_SBI)) - sbi_init(); + sbi_init(); if (IS_ENABLED(CONFIG_STRICT_KERNEL_RWX)) protect_kernel_text_data(); @@ -275,13 +282,19 @@ void __init setup_arch(char **cmdline_p) static int __init topology_init(void) { - int i; + int i, ret; + + for_each_online_node(i) + register_one_node(i); for_each_possible_cpu(i) { struct cpu *cpu = &per_cpu(cpu_devices, i); cpu->hotpluggable = cpu_has_hotplug(i); - register_cpu(cpu, i); + ret = register_cpu(cpu, i); + if (unlikely(ret)) + pr_warn("Warning: %s: register_cpu %d failed (%d)\n", + __func__, i, ret); } return 0; diff --git a/arch/riscv/kernel/signal.c b/arch/riscv/kernel/signal.c index 469aef8ed922e862edd172e218b74590888e858c..65942b3748b41bb2eed79d2e284e06e5c20f8dee 100644 --- a/arch/riscv/kernel/signal.c +++ b/arch/riscv/kernel/signal.c @@ -309,6 +309,9 @@ static void do_signal(struct pt_regs *regs) asmlinkage __visible void do_notify_resume(struct pt_regs *regs, unsigned long thread_info_flags) { + if (thread_info_flags & _TIF_UPROBE) + uprobe_notify_resume(regs); + /* Handle pending signal delivery */ if (thread_info_flags & (_TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL)) do_signal(regs); diff --git a/arch/riscv/kernel/smpboot.c b/arch/riscv/kernel/smpboot.c index 96167d55ed9845a23debad56fcbeb009c600c7d1..5e276c25646fc0b583d2783f4508918af1bb83b6 100644 --- a/arch/riscv/kernel/smpboot.c +++ b/arch/riscv/kernel/smpboot.c @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -45,13 +46,18 @@ void __init smp_prepare_cpus(unsigned int max_cpus) { int cpuid; int ret; + unsigned int curr_cpuid; + + curr_cpuid = smp_processor_id(); + numa_store_cpu_info(curr_cpuid); + numa_add_cpu(curr_cpuid); /* This covers non-smp usecase mandated by "nosmp" option */ if (max_cpus == 0) return; for_each_possible_cpu(cpuid) { - if (cpuid == smp_processor_id()) + if (cpuid == curr_cpuid) continue; if (cpu_ops[cpuid]->cpu_prepare) { ret = cpu_ops[cpuid]->cpu_prepare(cpuid); @@ -59,6 +65,7 @@ void __init smp_prepare_cpus(unsigned int max_cpus) continue; } set_cpu_present(cpuid, true); + numa_store_cpu_info(cpuid); } } @@ -79,6 +86,7 @@ void __init setup_smp(void) if (hart == cpuid_to_hartid_map(0)) { BUG_ON(found_boot_cpu); found_boot_cpu = 1; + early_map_cpu_to_node(0, of_node_to_nid(dn)); continue; } if (cpuid >= NR_CPUS) { @@ -88,6 +96,7 @@ void __init setup_smp(void) } cpuid_to_hartid_map(cpuid) = hart; + early_map_cpu_to_node(cpuid, of_node_to_nid(dn)); cpuid++; } @@ -153,6 +162,7 @@ asmlinkage __visible void smp_callin(void) current->active_mm = mm; notify_cpu_starting(curr_cpuid); + numa_add_cpu(curr_cpuid); update_siblings_masks(curr_cpuid); set_cpu_online(curr_cpuid, 1); diff --git a/arch/riscv/kernel/soc.c b/arch/riscv/kernel/soc.c index c7b0a73e382e920bbfb1a1d6b6488cf906bf0cfa..a0516172a33c84f0f0ce3b58afd6f984aaa5e38d 100644 --- a/arch/riscv/kernel/soc.c +++ b/arch/riscv/kernel/soc.c @@ -26,30 +26,3 @@ void __init soc_early_init(void) } } } - -static bool soc_builtin_dtb_match(unsigned long vendor_id, - unsigned long arch_id, unsigned long imp_id, - const struct soc_builtin_dtb *entry) -{ - return entry->vendor_id == vendor_id && - entry->arch_id == arch_id && - entry->imp_id == imp_id; -} - -void * __init soc_lookup_builtin_dtb(void) -{ - unsigned long vendor_id, arch_id, imp_id; - const struct soc_builtin_dtb *s; - - __asm__ ("csrr %0, mvendorid" : "=r"(vendor_id)); - __asm__ ("csrr %0, marchid" : "=r"(arch_id)); - __asm__ ("csrr %0, mimpid" : "=r"(imp_id)); - - for (s = (void *)&__soc_builtin_dtb_table_start; - (void *)s < (void *)&__soc_builtin_dtb_table_end; s++) { - if (soc_builtin_dtb_match(vendor_id, arch_id, imp_id, s)) - return s->dtb_func(); - } - - return NULL; -} diff --git a/arch/riscv/kernel/stacktrace.c b/arch/riscv/kernel/stacktrace.c index df5d2da7c40be3fa9ec41e464911277b98321db3..3f893c9d9d85a37ab08b0e2a3e447902156dc6bc 100644 --- a/arch/riscv/kernel/stacktrace.c +++ b/arch/riscv/kernel/stacktrace.c @@ -53,9 +53,15 @@ void notrace walk_stackframe(struct task_struct *task, struct pt_regs *regs, /* Unwind stack frame */ frame = (struct stackframe *)fp - 1; sp = fp; - fp = frame->fp; - pc = ftrace_graph_ret_addr(current, NULL, frame->ra, - (unsigned long *)(fp - 8)); + if (regs && (regs->epc == pc) && (frame->fp & 0x7)) { + fp = frame->ra; + pc = regs->ra; + } else { + fp = frame->fp; + pc = ftrace_graph_ret_addr(current, NULL, frame->ra, + (unsigned long *)(fp - 8)); + } + } } @@ -100,10 +106,16 @@ static bool print_trace_address(void *arg, unsigned long pc) return true; } +void dump_backtrace(struct pt_regs *regs, struct task_struct *task, + const char *loglvl) +{ + pr_cont("%sCall Trace:\n", loglvl); + walk_stackframe(task, regs, print_trace_address, (void *)loglvl); +} + void show_stack(struct task_struct *task, unsigned long *sp, const char *loglvl) { - pr_cont("Call Trace:\n"); - walk_stackframe(task, NULL, print_trace_address, (void *)loglvl); + dump_backtrace(NULL, task, loglvl); } static bool save_wchan(void *arg, unsigned long pc) diff --git a/arch/riscv/kernel/traps.c b/arch/riscv/kernel/traps.c index ad14f4466d92421c9fff2a5c6cb973d7d98016cf..3ed2c23601a02959fe15230c192e70bb3d686a5d 100644 --- a/arch/riscv/kernel/traps.c +++ b/arch/riscv/kernel/traps.c @@ -12,10 +12,12 @@ #include #include #include +#include #include #include #include +#include #include #include #include @@ -66,7 +68,7 @@ void do_trap(struct pt_regs *regs, int signo, int code, unsigned long addr) tsk->comm, task_pid_nr(tsk), signo, code, addr); print_vma_addr(KERN_CONT " in ", instruction_pointer(regs)); pr_cont("\n"); - show_regs(regs); + __show_regs(regs); } force_sig_fault(signo, code, (void __user *)addr); @@ -75,6 +77,8 @@ void do_trap(struct pt_regs *regs, int signo, int code, unsigned long addr) static void do_trap_error(struct pt_regs *regs, int signo, int code, unsigned long addr, const char *str) { + current->thread.bad_cause = regs->cause; + if (user_mode(regs)) { do_trap(regs, signo, code, addr); } else { @@ -145,6 +149,22 @@ static inline unsigned long get_break_insn_length(unsigned long pc) asmlinkage __visible void do_trap_break(struct pt_regs *regs) { +#ifdef CONFIG_KPROBES + if (kprobe_single_step_handler(regs)) + return; + + if (kprobe_breakpoint_handler(regs)) + return; +#endif +#ifdef CONFIG_UPROBES + if (uprobe_single_step_handler(regs)) + return; + + if (uprobe_breakpoint_handler(regs)) + return; +#endif + current->thread.bad_cause = regs->cause; + if (user_mode(regs)) force_sig_fault(SIGTRAP, TRAP_BRKPT, (void __user *)regs->epc); #ifdef CONFIG_KGDB diff --git a/arch/riscv/kernel/vdso/Makefile b/arch/riscv/kernel/vdso/Makefile index 0cfd6da784f84187487a369b479a6dabd41dbbc8..71a315e73cbe7444031db9898ef08445a742a303 100644 --- a/arch/riscv/kernel/vdso/Makefile +++ b/arch/riscv/kernel/vdso/Makefile @@ -32,9 +32,10 @@ CPPFLAGS_vdso.lds += -P -C -U$(ARCH) # Disable -pg to prevent insert call site CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE) -Os -# Disable gcov profiling for VDSO code +# Disable profiling and instrumentation for VDSO code GCOV_PROFILE := n KCOV_INSTRUMENT := n +KASAN_SANITIZE := n # Force dependency $(obj)/vdso.o: $(obj)/vdso.so diff --git a/arch/riscv/lib/Makefile b/arch/riscv/lib/Makefile index ac6171e9c19e308aeaac2b69f961363bd8a045a5..25d5c9664e57e4f20e7c0c36ef47c1e1d096ac1c 100644 --- a/arch/riscv/lib/Makefile +++ b/arch/riscv/lib/Makefile @@ -5,3 +5,5 @@ lib-y += memset.o lib-y += memmove.o lib-$(CONFIG_MMU) += uaccess.o lib-$(CONFIG_64BIT) += tishift.o + +obj-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o diff --git a/arch/riscv/lib/error-inject.c b/arch/riscv/lib/error-inject.c new file mode 100644 index 0000000000000000000000000000000000000000..d667ade2bc41cdac5e695451e81ab27df4edd8c8 --- /dev/null +++ b/arch/riscv/lib/error-inject.c @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include + +void override_function_with_return(struct pt_regs *regs) +{ + instruction_pointer_set(regs, regs->ra); +} +NOKPROBE_SYMBOL(override_function_with_return); diff --git a/arch/riscv/mm/Makefile b/arch/riscv/mm/Makefile index c0185e556ca51b944e276267352119809ac1290e..7ebaef10ea1b69e1557c9d08fec4d288ac31e842 100644 --- a/arch/riscv/mm/Makefile +++ b/arch/riscv/mm/Makefile @@ -2,7 +2,8 @@ CFLAGS_init.o := -mcmodel=medany ifdef CONFIG_FTRACE -CFLAGS_REMOVE_init.o = -pg +CFLAGS_REMOVE_init.o = $(CC_FLAGS_FTRACE) +CFLAGS_REMOVE_cacheflush.o = $(CC_FLAGS_FTRACE) endif KCOV_INSTRUMENT_init.o := n diff --git a/arch/riscv/mm/context.c b/arch/riscv/mm/context.c index 613ec81a8979adc837390f3d676a56f78309563f..68aa312fc352458193b849a876781a6cf9c194e3 100644 --- a/arch/riscv/mm/context.c +++ b/arch/riscv/mm/context.c @@ -2,13 +2,273 @@ /* * Copyright (C) 2012 Regents of the University of California * Copyright (C) 2017 SiFive + * Copyright (C) 2021 Western Digital Corporation or its affiliates. */ +#include +#include #include +#include +#include +#include +#include #include #include #include +#ifdef CONFIG_MMU + +static DEFINE_STATIC_KEY_FALSE(use_asid_allocator); + +static unsigned long asid_bits; +static unsigned long num_asids; +static unsigned long asid_mask; + +static atomic_long_t current_version; + +static DEFINE_RAW_SPINLOCK(context_lock); +static cpumask_t context_tlb_flush_pending; +static unsigned long *context_asid_map; + +static DEFINE_PER_CPU(atomic_long_t, active_context); +static DEFINE_PER_CPU(unsigned long, reserved_context); + +static bool check_update_reserved_context(unsigned long cntx, + unsigned long newcntx) +{ + int cpu; + bool hit = false; + + /* + * Iterate over the set of reserved CONTEXT looking for a match. + * If we find one, then we can update our mm to use new CONTEXT + * (i.e. the same CONTEXT in the current_version) but we can't + * exit the loop early, since we need to ensure that all copies + * of the old CONTEXT are updated to reflect the mm. Failure to do + * so could result in us missing the reserved CONTEXT in a future + * version. + */ + for_each_possible_cpu(cpu) { + if (per_cpu(reserved_context, cpu) == cntx) { + hit = true; + per_cpu(reserved_context, cpu) = newcntx; + } + } + + return hit; +} + +static void __flush_context(void) +{ + int i; + unsigned long cntx; + + /* Must be called with context_lock held */ + lockdep_assert_held(&context_lock); + + /* Update the list of reserved ASIDs and the ASID bitmap. */ + bitmap_clear(context_asid_map, 0, num_asids); + + /* Mark already active ASIDs as used */ + for_each_possible_cpu(i) { + cntx = atomic_long_xchg_relaxed(&per_cpu(active_context, i), 0); + /* + * If this CPU has already been through a rollover, but + * hasn't run another task in the meantime, we must preserve + * its reserved CONTEXT, as this is the only trace we have of + * the process it is still running. + */ + if (cntx == 0) + cntx = per_cpu(reserved_context, i); + + __set_bit(cntx & asid_mask, context_asid_map); + per_cpu(reserved_context, i) = cntx; + } + + /* Mark ASID #0 as used because it is used at boot-time */ + __set_bit(0, context_asid_map); + + /* Queue a TLB invalidation for each CPU on next context-switch */ + cpumask_setall(&context_tlb_flush_pending); +} + +static unsigned long __new_context(struct mm_struct *mm) +{ + static u32 cur_idx = 1; + unsigned long cntx = atomic_long_read(&mm->context.id); + unsigned long asid, ver = atomic_long_read(¤t_version); + + /* Must be called with context_lock held */ + lockdep_assert_held(&context_lock); + + if (cntx != 0) { + unsigned long newcntx = ver | (cntx & asid_mask); + + /* + * If our current CONTEXT was active during a rollover, we + * can continue to use it and this was just a false alarm. + */ + if (check_update_reserved_context(cntx, newcntx)) + return newcntx; + + /* + * We had a valid CONTEXT in a previous life, so try to + * re-use it if possible. + */ + if (!__test_and_set_bit(cntx & asid_mask, context_asid_map)) + return newcntx; + } + + /* + * Allocate a free ASID. If we can't find one then increment + * current_version and flush all ASIDs. + */ + asid = find_next_zero_bit(context_asid_map, num_asids, cur_idx); + if (asid != num_asids) + goto set_asid; + + /* We're out of ASIDs, so increment current_version */ + ver = atomic_long_add_return_relaxed(num_asids, ¤t_version); + + /* Flush everything */ + __flush_context(); + + /* We have more ASIDs than CPUs, so this will always succeed */ + asid = find_next_zero_bit(context_asid_map, num_asids, 1); + +set_asid: + __set_bit(asid, context_asid_map); + cur_idx = asid; + return asid | ver; +} + +static void set_mm_asid(struct mm_struct *mm, unsigned int cpu) +{ + unsigned long flags; + bool need_flush_tlb = false; + unsigned long cntx, old_active_cntx; + + cntx = atomic_long_read(&mm->context.id); + + /* + * If our active_context is non-zero and the context matches the + * current_version, then we update the active_context entry with a + * relaxed cmpxchg. + * + * Following is how we handle racing with a concurrent rollover: + * + * - We get a zero back from the cmpxchg and end up waiting on the + * lock. Taking the lock synchronises with the rollover and so + * we are forced to see the updated verion. + * + * - We get a valid context back from the cmpxchg then we continue + * using old ASID because __flush_context() would have marked ASID + * of active_context as used and next context switch we will + * allocate new context. + */ + old_active_cntx = atomic_long_read(&per_cpu(active_context, cpu)); + if (old_active_cntx && + ((cntx & ~asid_mask) == atomic_long_read(¤t_version)) && + atomic_long_cmpxchg_relaxed(&per_cpu(active_context, cpu), + old_active_cntx, cntx)) + goto switch_mm_fast; + + raw_spin_lock_irqsave(&context_lock, flags); + + /* Check that our ASID belongs to the current_version. */ + cntx = atomic_long_read(&mm->context.id); + if ((cntx & ~asid_mask) != atomic_long_read(¤t_version)) { + cntx = __new_context(mm); + atomic_long_set(&mm->context.id, cntx); + } + + if (cpumask_test_and_clear_cpu(cpu, &context_tlb_flush_pending)) + need_flush_tlb = true; + + atomic_long_set(&per_cpu(active_context, cpu), cntx); + + raw_spin_unlock_irqrestore(&context_lock, flags); + +switch_mm_fast: + csr_write(CSR_SATP, virt_to_pfn(mm->pgd) | + ((cntx & asid_mask) << SATP_ASID_SHIFT) | + SATP_MODE); + + if (need_flush_tlb) + local_flush_tlb_all(); +} + +static void set_mm_noasid(struct mm_struct *mm) +{ + /* Switch the page table and blindly nuke entire local TLB */ + csr_write(CSR_SATP, virt_to_pfn(mm->pgd) | SATP_MODE); + local_flush_tlb_all(); +} + +static inline void set_mm(struct mm_struct *mm, unsigned int cpu) +{ + if (static_branch_unlikely(&use_asid_allocator)) + set_mm_asid(mm, cpu); + else + set_mm_noasid(mm); +} + +static int asids_init(void) +{ + unsigned long old; + + /* Figure-out number of ASID bits in HW */ + old = csr_read(CSR_SATP); + asid_bits = old | (SATP_ASID_MASK << SATP_ASID_SHIFT); + csr_write(CSR_SATP, asid_bits); + asid_bits = (csr_read(CSR_SATP) >> SATP_ASID_SHIFT) & SATP_ASID_MASK; + asid_bits = fls_long(asid_bits); + csr_write(CSR_SATP, old); + + /* + * In the process of determining number of ASID bits (above) + * we polluted the TLB of current HART so let's do TLB flushed + * to remove unwanted TLB enteries. + */ + local_flush_tlb_all(); + + /* Pre-compute ASID details */ + num_asids = 1 << asid_bits; + asid_mask = num_asids - 1; + + /* + * Use ASID allocator only if number of HW ASIDs are + * at-least twice more than CPUs + */ + if (num_asids > (2 * num_possible_cpus())) { + atomic_long_set(¤t_version, num_asids); + + context_asid_map = kcalloc(BITS_TO_LONGS(num_asids), + sizeof(*context_asid_map), GFP_KERNEL); + if (!context_asid_map) + panic("Failed to allocate bitmap for %lu ASIDs\n", + num_asids); + + __set_bit(0, context_asid_map); + + static_branch_enable(&use_asid_allocator); + + pr_info("ASID allocator using %lu bits (%lu entries)\n", + asid_bits, num_asids); + } else { + pr_info("ASID allocator disabled\n"); + } + + return 0; +} +early_initcall(asids_init); +#else +static inline void set_mm(struct mm_struct *mm, unsigned int cpu) +{ + /* Nothing to do here when there is no MMU */ +} +#endif + /* * When necessary, performs a deferred icache flush for the given MM context, * on the local CPU. RISC-V has no direct mechanism for instruction cache @@ -58,10 +318,7 @@ void switch_mm(struct mm_struct *prev, struct mm_struct *next, cpumask_clear_cpu(cpu, mm_cpumask(prev)); cpumask_set_cpu(cpu, mm_cpumask(next)); -#ifdef CONFIG_MMU - csr_write(CSR_SATP, virt_to_pfn(next->pgd) | SATP_MODE); - local_flush_tlb_all(); -#endif + set_mm(next, cpu); flush_icache_deferred(next); } diff --git a/arch/riscv/mm/fault.c b/arch/riscv/mm/fault.c index 3c8b9e433c673754e29444ffe2e75883c8ce0e99..8f17519208c756b7a08bf089ec53004c8a5b0c89 100644 --- a/arch/riscv/mm/fault.c +++ b/arch/riscv/mm/fault.c @@ -13,14 +13,30 @@ #include #include #include +#include #include #include #include "../kernel/head.h" +static void die_kernel_fault(const char *msg, unsigned long addr, + struct pt_regs *regs) +{ + bust_spinlocks(1); + + pr_alert("Unable to handle kernel %s at virtual address " REG_FMT "\n", msg, + addr); + + bust_spinlocks(0); + die(regs, "Oops"); + do_exit(SIGKILL); +} + static inline void no_context(struct pt_regs *regs, unsigned long addr) { + const char *msg; + /* Are we prepared to handle this kernel fault? */ if (fixup_exception(regs)) return; @@ -29,12 +45,8 @@ static inline void no_context(struct pt_regs *regs, unsigned long addr) * Oops. The kernel tried to access some bad page. We'll have to * terminate things with extreme prejudice. */ - bust_spinlocks(1); - pr_alert("Unable to handle kernel %s at virtual address " REG_FMT "\n", - (addr < PAGE_SIZE) ? "NULL pointer dereference" : - "paging request", addr); - die(regs, "Oops"); - do_exit(SIGKILL); + msg = (addr < PAGE_SIZE) ? "NULL pointer dereference" : "paging request"; + die_kernel_fault(msg, addr, regs); } static inline void mm_fault_error(struct pt_regs *regs, unsigned long addr, vm_fault_t fault) @@ -202,6 +214,9 @@ asmlinkage void do_page_fault(struct pt_regs *regs) tsk = current; mm = tsk->mm; + if (kprobe_page_fault(regs, cause)) + return; + /* * Fault-in kernel-space virtual memory on-demand. * The 'reference' page table is init_mm.pgd. @@ -225,6 +240,7 @@ asmlinkage void do_page_fault(struct pt_regs *regs) * in an atomic region, then we must not take the fault. */ if (unlikely(faulthandler_disabled() || !mm)) { + tsk->thread.bad_cause = cause; no_context(regs, addr); return; } @@ -232,6 +248,11 @@ asmlinkage void do_page_fault(struct pt_regs *regs) if (user_mode(regs)) flags |= FAULT_FLAG_USER; + if (!user_mode(regs) && addr < TASK_SIZE && + unlikely(!(regs->status & SR_SUM))) + die_kernel_fault("access to user memory without uaccess routines", + addr, regs); + perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, addr); if (cause == EXC_STORE_PAGE_FAULT) @@ -242,16 +263,19 @@ asmlinkage void do_page_fault(struct pt_regs *regs) mmap_read_lock(mm); vma = find_vma(mm, addr); if (unlikely(!vma)) { + tsk->thread.bad_cause = cause; bad_area(regs, mm, code, addr); return; } if (likely(vma->vm_start <= addr)) goto good_area; if (unlikely(!(vma->vm_flags & VM_GROWSDOWN))) { + tsk->thread.bad_cause = cause; bad_area(regs, mm, code, addr); return; } if (unlikely(expand_stack(vma, addr))) { + tsk->thread.bad_cause = cause; bad_area(regs, mm, code, addr); return; } @@ -264,6 +288,7 @@ asmlinkage void do_page_fault(struct pt_regs *regs) code = SEGV_ACCERR; if (unlikely(access_error(cause, vma))) { + tsk->thread.bad_cause = cause; bad_area(regs, mm, code, addr); return; } @@ -297,6 +322,7 @@ asmlinkage void do_page_fault(struct pt_regs *regs) mmap_read_unlock(mm); if (unlikely(fault & VM_FAULT_ERROR)) { + tsk->thread.bad_cause = cause; mm_fault_error(regs, addr, fault); return; } diff --git a/arch/riscv/mm/init.c b/arch/riscv/mm/init.c index f9f9568d689ef53e215c44de8b4468ae3033cfa9..067583ab1bd7f2e4b8cbe0fbab9fdd47ee0d8e85 100644 --- a/arch/riscv/mm/init.c +++ b/arch/riscv/mm/init.c @@ -21,6 +21,7 @@ #include #include #include +#include #include "../kernel/head.h" @@ -105,85 +106,19 @@ void __init mem_init(void) print_vm_layout(); } -#ifdef CONFIG_BLK_DEV_INITRD -static void __init setup_initrd(void) -{ - phys_addr_t start; - unsigned long size; - - /* Ignore the virtul address computed during device tree parsing */ - initrd_start = initrd_end = 0; - - if (!phys_initrd_size) - return; - /* - * Round the memory region to page boundaries as per free_initrd_mem() - * This allows us to detect whether the pages overlapping the initrd - * are in use, but more importantly, reserves the entire set of pages - * as we don't want these pages allocated for other purposes. - */ - start = round_down(phys_initrd_start, PAGE_SIZE); - size = phys_initrd_size + (phys_initrd_start - start); - size = round_up(size, PAGE_SIZE); - - if (!memblock_is_region_memory(start, size)) { - pr_err("INITRD: 0x%08llx+0x%08lx is not a memory region", - (u64)start, size); - goto disable; - } - - if (memblock_is_region_reserved(start, size)) { - pr_err("INITRD: 0x%08llx+0x%08lx overlaps in-use memory region\n", - (u64)start, size); - goto disable; - } - - memblock_reserve(start, size); - /* Now convert initrd to virtual addresses */ - initrd_start = (unsigned long)__va(phys_initrd_start); - initrd_end = initrd_start + phys_initrd_size; - initrd_below_start_ok = 1; - - pr_info("Initial ramdisk at: 0x%p (%lu bytes)\n", - (void *)(initrd_start), size); - return; -disable: - pr_cont(" - disabling initrd\n"); - initrd_start = 0; - initrd_end = 0; -} -#endif /* CONFIG_BLK_DEV_INITRD */ - void __init setup_bootmem(void) { - phys_addr_t mem_start = 0; - phys_addr_t start, dram_end, end = 0; phys_addr_t vmlinux_end = __pa_symbol(&_end); phys_addr_t vmlinux_start = __pa_symbol(&_start); + phys_addr_t dram_end = memblock_end_of_DRAM(); phys_addr_t max_mapped_addr = __pa(~(ulong)0); - u64 i; - /* Find the memory region containing the kernel */ - for_each_mem_range(i, &start, &end) { - phys_addr_t size = end - start; - if (!mem_start) - mem_start = start; - if (start <= vmlinux_start && vmlinux_end <= end) - BUG_ON(size == 0); - } - - /* - * The maximal physical memory size is -PAGE_OFFSET. - * Make sure that any memory beyond mem_start + (-PAGE_OFFSET) is removed - * as it is unusable by kernel. - */ + /* The maximal physical memory size is -PAGE_OFFSET. */ memblock_enforce_memory_limit(-PAGE_OFFSET); /* Reserve from the start of the kernel to the end of the kernel */ memblock_reserve(vmlinux_start, vmlinux_end - vmlinux_start); - dram_end = memblock_end_of_DRAM(); - /* * memblock allocator is not aware of the fact that last 4K bytes of * the addressable memory can not be mapped because of IS_ERR_VALUE @@ -198,20 +133,19 @@ void __init setup_bootmem(void) dma32_phys_limit = min(4UL * SZ_1G, (unsigned long)PFN_PHYS(max_low_pfn)); set_max_mapnr(max_low_pfn - ARCH_PFN_OFFSET); -#ifdef CONFIG_BLK_DEV_INITRD - setup_initrd(); -#endif /* CONFIG_BLK_DEV_INITRD */ - + reserve_initrd_mem(); /* - * Avoid using early_init_fdt_reserve_self() since __pa() does + * If DTB is built in, no need to reserve its memblock. + * Otherwise, do reserve it but avoid using + * early_init_fdt_reserve_self() since __pa() does * not work for DTB pointers that are fixmap addresses */ - memblock_reserve(dtb_early_pa, fdt_totalsize(dtb_early_va)); + if (!IS_ENABLED(CONFIG_BUILTIN_DTB)) + memblock_reserve(dtb_early_pa, fdt_totalsize(dtb_early_va)); early_init_fdt_scan_reserved_mem(); dma_contiguous_reserve(dma32_phys_limit); memblock_allow_resize(); - memblock_dump_all(); } #ifdef CONFIG_MMU @@ -226,8 +160,6 @@ pgd_t swapper_pg_dir[PTRS_PER_PGD] __page_aligned_bss; pgd_t trampoline_pg_dir[PTRS_PER_PGD] __page_aligned_bss; pte_t fixmap_pte[PTRS_PER_PTE] __page_aligned_bss; -#define MAX_EARLY_MAPPING_SIZE SZ_128M - pgd_t early_pg_dir[PTRS_PER_PGD] __initdata __aligned(PAGE_SIZE); void __set_fixmap(enum fixed_addresses idx, phys_addr_t phys, pgprot_t prot) @@ -302,13 +234,7 @@ static void __init create_pte_mapping(pte_t *ptep, pmd_t trampoline_pmd[PTRS_PER_PMD] __page_aligned_bss; pmd_t fixmap_pmd[PTRS_PER_PMD] __page_aligned_bss; - -#if MAX_EARLY_MAPPING_SIZE < PGDIR_SIZE -#define NUM_EARLY_PMDS 1UL -#else -#define NUM_EARLY_PMDS (1UL + MAX_EARLY_MAPPING_SIZE / PGDIR_SIZE) -#endif -pmd_t early_pmd[PTRS_PER_PMD * NUM_EARLY_PMDS] __initdata __aligned(PAGE_SIZE); +pmd_t early_pmd[PTRS_PER_PMD] __initdata __aligned(PAGE_SIZE); pmd_t early_dtb_pmd[PTRS_PER_PMD] __initdata __aligned(PAGE_SIZE); static pmd_t *__init get_pmd_virt_early(phys_addr_t pa) @@ -330,11 +256,9 @@ static pmd_t *get_pmd_virt_late(phys_addr_t pa) static phys_addr_t __init alloc_pmd_early(uintptr_t va) { - uintptr_t pmd_num; + BUG_ON((va - PAGE_OFFSET) >> PGDIR_SHIFT); - pmd_num = (va - PAGE_OFFSET) >> PGDIR_SHIFT; - BUG_ON(pmd_num >= NUM_EARLY_PMDS); - return (uintptr_t)&early_pmd[pmd_num * PTRS_PER_PMD]; + return (uintptr_t)early_pmd; } static phys_addr_t __init alloc_pmd_fixmap(uintptr_t va) @@ -452,7 +376,7 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa) uintptr_t va, pa, end_va; uintptr_t load_pa = (uintptr_t)(&_start); uintptr_t load_sz = (uintptr_t)(&_end) - load_pa; - uintptr_t map_size = best_map_size(load_pa, MAX_EARLY_MAPPING_SIZE); + uintptr_t map_size; #ifndef __PAGETABLE_PMD_FOLDED pmd_t fix_bmap_spmd, fix_bmap_epmd; #endif @@ -464,12 +388,11 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa) * Enforce boot alignment requirements of RV32 and * RV64 by only allowing PMD or PGD mappings. */ - BUG_ON(map_size == PAGE_SIZE); + map_size = PMD_SIZE; /* Sanity check alignment and size */ BUG_ON((PAGE_OFFSET % PGDIR_SIZE) != 0); BUG_ON((load_pa % map_size) != 0); - BUG_ON(load_sz > MAX_EARLY_MAPPING_SIZE); pt_ops.alloc_pte = alloc_pte_early; pt_ops.get_pte_virt = get_pte_virt_early; @@ -511,6 +434,7 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa) /* Setup early PMD for DTB */ create_pgd_mapping(early_pg_dir, DTB_EARLY_BASE_VA, (uintptr_t)early_dtb_pmd, PGDIR_SIZE, PAGE_TABLE); +#ifndef CONFIG_BUILTIN_DTB /* Create two consecutive PMD mappings for FDT early scan */ pa = dtb_pa & ~(PMD_SIZE - 1); create_pmd_mapping(early_dtb_pmd, DTB_EARLY_BASE_VA, @@ -518,7 +442,11 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa) create_pmd_mapping(early_dtb_pmd, DTB_EARLY_BASE_VA + PMD_SIZE, pa + PMD_SIZE, PMD_SIZE, PAGE_KERNEL); dtb_early_va = (void *)DTB_EARLY_BASE_VA + (dtb_pa & (PMD_SIZE - 1)); +#else /* CONFIG_BUILTIN_DTB */ + dtb_early_va = __va(dtb_pa); +#endif /* CONFIG_BUILTIN_DTB */ #else +#ifndef CONFIG_BUILTIN_DTB /* Create two consecutive PGD mappings for FDT early scan */ pa = dtb_pa & ~(PGDIR_SIZE - 1); create_pgd_mapping(early_pg_dir, DTB_EARLY_BASE_VA, @@ -526,6 +454,9 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa) create_pgd_mapping(early_pg_dir, DTB_EARLY_BASE_VA + PGDIR_SIZE, pa + PGDIR_SIZE, PGDIR_SIZE, PAGE_KERNEL); dtb_early_va = (void *)DTB_EARLY_BASE_VA + (dtb_pa & (PGDIR_SIZE - 1)); +#else /* CONFIG_BUILTIN_DTB */ + dtb_early_va = __va(dtb_pa); +#endif /* CONFIG_BUILTIN_DTB */ #endif dtb_early_pa = dtb_pa; @@ -616,15 +547,7 @@ static void __init setup_vm_final(void) #else asmlinkage void __init setup_vm(uintptr_t dtb_pa) { -#ifdef CONFIG_BUILTIN_DTB - dtb_early_va = soc_lookup_builtin_dtb(); - if (!dtb_early_va) { - /* Fallback to first available DTS */ - dtb_early_va = (void *) __dtb_start; - } -#else dtb_early_va = (void *)dtb_pa; -#endif dtb_early_pa = dtb_pa; } @@ -665,9 +588,15 @@ void mark_rodata_ro(void) void __init paging_init(void) { setup_vm_final(); - sparse_init(); setup_zero_page(); +} + +void __init misc_mem_init(void) +{ + arch_numa_init(); + sparse_init(); zone_sizes_init(); + memblock_dump_all(); } #ifdef CONFIG_SPARSEMEM_VMEMMAP diff --git a/arch/riscv/mm/kasan_init.c b/arch/riscv/mm/kasan_init.c index a8a2ffd9114aaa22c60661c7cd2d80023747cb2e..3fc18f469efbc6f1e3b05f8fbba71bc9345315fa 100644 --- a/arch/riscv/mm/kasan_init.c +++ b/arch/riscv/mm/kasan_init.c @@ -9,6 +9,19 @@ #include #include #include +#include + +static __init void *early_alloc(size_t size, int node) +{ + void *ptr = memblock_alloc_try_nid(size, size, + __pa(MAX_DMA_ADDRESS), MEMBLOCK_ALLOC_ACCESSIBLE, node); + + if (!ptr) + panic("%pS: Failed to allocate %zu bytes align=%zx nid=%d from=%llx\n", + __func__, size, size, node, (u64)__pa(MAX_DMA_ADDRESS)); + + return ptr; +} extern pgd_t early_pg_dir[PTRS_PER_PGD]; asmlinkage void __init kasan_early_init(void) @@ -47,40 +60,133 @@ asmlinkage void __init kasan_early_init(void) local_flush_tlb_all(); } -static void __init populate(void *start, void *end) +static void kasan_populate_pte(pmd_t *pmd, unsigned long vaddr, unsigned long end) +{ + phys_addr_t phys_addr; + pte_t *ptep, *base_pte; + + if (pmd_none(*pmd)) + base_pte = memblock_alloc(PTRS_PER_PTE * sizeof(pte_t), PAGE_SIZE); + else + base_pte = (pte_t *)pmd_page_vaddr(*pmd); + + ptep = base_pte + pte_index(vaddr); + + do { + if (pte_none(*ptep)) { + phys_addr = memblock_phys_alloc(PAGE_SIZE, PAGE_SIZE); + set_pte(ptep, pfn_pte(PFN_DOWN(phys_addr), PAGE_KERNEL)); + } + } while (ptep++, vaddr += PAGE_SIZE, vaddr != end); + + set_pmd(pmd, pfn_pmd(PFN_DOWN(__pa(base_pte)), PAGE_TABLE)); +} + +static void kasan_populate_pmd(pgd_t *pgd, unsigned long vaddr, unsigned long end) +{ + phys_addr_t phys_addr; + pmd_t *pmdp, *base_pmd; + unsigned long next; + + base_pmd = (pmd_t *)pgd_page_vaddr(*pgd); + if (base_pmd == lm_alias(kasan_early_shadow_pmd)) + base_pmd = memblock_alloc(PTRS_PER_PMD * sizeof(pmd_t), PAGE_SIZE); + + pmdp = base_pmd + pmd_index(vaddr); + + do { + next = pmd_addr_end(vaddr, end); + + if (pmd_none(*pmdp) && IS_ALIGNED(vaddr, PMD_SIZE) && (next - vaddr) >= PMD_SIZE) { + phys_addr = memblock_phys_alloc(PMD_SIZE, PMD_SIZE); + if (phys_addr) { + set_pmd(pmdp, pfn_pmd(PFN_DOWN(phys_addr), PAGE_KERNEL)); + continue; + } + } + + kasan_populate_pte(pmdp, vaddr, next); + } while (pmdp++, vaddr = next, vaddr != end); + + /* + * Wait for the whole PGD to be populated before setting the PGD in + * the page table, otherwise, if we did set the PGD before populating + * it entirely, memblock could allocate a page at a physical address + * where KASAN is not populated yet and then we'd get a page fault. + */ + set_pgd(pgd, pfn_pgd(PFN_DOWN(__pa(base_pmd)), PAGE_TABLE)); +} + +static void kasan_populate_pgd(unsigned long vaddr, unsigned long end) +{ + phys_addr_t phys_addr; + pgd_t *pgdp = pgd_offset_k(vaddr); + unsigned long next; + + do { + next = pgd_addr_end(vaddr, end); + + /* + * pgdp can't be none since kasan_early_init initialized all KASAN + * shadow region with kasan_early_shadow_pmd: if this is stillthe case, + * that means we can try to allocate a hugepage as a replacement. + */ + if (pgd_page_vaddr(*pgdp) == (unsigned long)lm_alias(kasan_early_shadow_pmd) && + IS_ALIGNED(vaddr, PGDIR_SIZE) && (next - vaddr) >= PGDIR_SIZE) { + phys_addr = memblock_phys_alloc(PGDIR_SIZE, PGDIR_SIZE); + if (phys_addr) { + set_pgd(pgdp, pfn_pgd(PFN_DOWN(phys_addr), PAGE_KERNEL)); + continue; + } + } + + kasan_populate_pmd(pgdp, vaddr, next); + } while (pgdp++, vaddr = next, vaddr != end); +} + +static void __init kasan_populate(void *start, void *end) { - unsigned long i, offset; unsigned long vaddr = (unsigned long)start & PAGE_MASK; unsigned long vend = PAGE_ALIGN((unsigned long)end); - unsigned long n_pages = (vend - vaddr) / PAGE_SIZE; - unsigned long n_ptes = - ((n_pages + PTRS_PER_PTE) & -PTRS_PER_PTE) / PTRS_PER_PTE; - unsigned long n_pmds = - ((n_ptes + PTRS_PER_PMD) & -PTRS_PER_PMD) / PTRS_PER_PMD; - - pte_t *pte = - memblock_alloc(n_ptes * PTRS_PER_PTE * sizeof(pte_t), PAGE_SIZE); - pmd_t *pmd = - memblock_alloc(n_pmds * PTRS_PER_PMD * sizeof(pmd_t), PAGE_SIZE); - pgd_t *pgd = pgd_offset_k(vaddr); - - for (i = 0; i < n_pages; i++) { - phys_addr_t phys = memblock_phys_alloc(PAGE_SIZE, PAGE_SIZE); - set_pte(&pte[i], pfn_pte(PHYS_PFN(phys), PAGE_KERNEL)); - } - - for (i = 0, offset = 0; i < n_ptes; i++, offset += PTRS_PER_PTE) - set_pmd(&pmd[i], - pfn_pmd(PFN_DOWN(__pa(&pte[offset])), - __pgprot(_PAGE_TABLE))); - for (i = 0, offset = 0; i < n_pmds; i++, offset += PTRS_PER_PMD) - set_pgd(&pgd[i], - pfn_pgd(PFN_DOWN(__pa(&pmd[offset])), - __pgprot(_PAGE_TABLE))); + kasan_populate_pgd(vaddr, vend); local_flush_tlb_all(); - memset(start, 0, end - start); + memset(start, KASAN_SHADOW_INIT, end - start); +} + +void __init kasan_shallow_populate(void *start, void *end) +{ + unsigned long vaddr = (unsigned long)start & PAGE_MASK; + unsigned long vend = PAGE_ALIGN((unsigned long)end); + unsigned long pfn; + int index; + void *p; + pud_t *pud_dir, *pud_k; + pgd_t *pgd_dir, *pgd_k; + p4d_t *p4d_dir, *p4d_k; + + while (vaddr < vend) { + index = pgd_index(vaddr); + pfn = csr_read(CSR_SATP) & SATP_PPN; + pgd_dir = (pgd_t *)pfn_to_virt(pfn) + index; + pgd_k = init_mm.pgd + index; + pgd_dir = pgd_offset_k(vaddr); + set_pgd(pgd_dir, *pgd_k); + + p4d_dir = p4d_offset(pgd_dir, vaddr); + p4d_k = p4d_offset(pgd_k, vaddr); + + vaddr = (vaddr + PUD_SIZE) & PUD_MASK; + pud_dir = pud_offset(p4d_dir, vaddr); + pud_k = pud_offset(p4d_k, vaddr); + + if (pud_present(*pud_dir)) { + p = early_alloc(PAGE_SIZE, NUMA_NO_NODE); + pud_populate(&init_mm, pud_dir, p); + } + vaddr += PAGE_SIZE; + } } void __init kasan_init(void) @@ -90,7 +196,15 @@ void __init kasan_init(void) kasan_populate_early_shadow((void *)KASAN_SHADOW_START, (void *)kasan_mem_to_shadow((void *) - VMALLOC_END)); + VMEMMAP_END)); + if (IS_ENABLED(CONFIG_KASAN_VMALLOC)) + kasan_shallow_populate( + (void *)kasan_mem_to_shadow((void *)VMALLOC_START), + (void *)kasan_mem_to_shadow((void *)VMALLOC_END)); + else + kasan_populate_early_shadow( + (void *)kasan_mem_to_shadow((void *)VMALLOC_START), + (void *)kasan_mem_to_shadow((void *)VMALLOC_END)); for_each_mem_range(i, &_start, &_end) { void *start = (void *)__va(_start); @@ -99,7 +213,7 @@ void __init kasan_init(void) if (start >= end) break; - populate(kasan_mem_to_shadow(start), kasan_mem_to_shadow(end)); + kasan_populate(kasan_mem_to_shadow(start), kasan_mem_to_shadow(end)); }; for (i = 0; i < PTRS_PER_PTE; i++) @@ -108,6 +222,6 @@ void __init kasan_init(void) __pgprot(_PAGE_PRESENT | _PAGE_READ | _PAGE_ACCESSED))); - memset(kasan_early_shadow_page, 0, PAGE_SIZE); + memset(kasan_early_shadow_page, KASAN_SHADOW_INIT, PAGE_SIZE); init_task.kasan_depth = 0; } diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index e8f7216f6c63d598fd0d1a07664366e9ee888861..c1ff874e6c2e63e278dd9a894b70f0f3df1b93ed 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -184,6 +184,7 @@ config S390 select HAVE_REGS_AND_STACK_ACCESS_API select HAVE_RELIABLE_STACKTRACE select HAVE_RSEQ + select HAVE_SOFTIRQ_ON_OWN_STACK select HAVE_SYSCALL_TRACEPOINTS select HAVE_VIRT_CPU_ACCOUNTING select HAVE_VIRT_CPU_ACCOUNTING_IDLE diff --git a/arch/s390/configs/debug_defconfig b/arch/s390/configs/debug_defconfig index 6896b4213c49cf589a431ce8b65adb58e433e5e3..02056b024091eb95ade5612922fa1616e9077b3d 100644 --- a/arch/s390/configs/debug_defconfig +++ b/arch/s390/configs/debug_defconfig @@ -71,7 +71,6 @@ CONFIG_MODULE_FORCE_UNLOAD=y CONFIG_MODVERSIONS=y CONFIG_MODULE_SRCVERSION_ALL=y CONFIG_MODULE_SIG_SHA256=y -CONFIG_UNUSED_SYMBOLS=y CONFIG_BLK_DEV_INTEGRITY=y CONFIG_BLK_DEV_THROTTLING=y CONFIG_BLK_WBT=y diff --git a/arch/s390/configs/defconfig b/arch/s390/configs/defconfig index ef4df9d60d6af4d52ef544ac48d2902933eab855..bac721a501da16a47313922369f4a61fff427a18 100644 --- a/arch/s390/configs/defconfig +++ b/arch/s390/configs/defconfig @@ -66,7 +66,6 @@ CONFIG_MODULE_FORCE_UNLOAD=y CONFIG_MODVERSIONS=y CONFIG_MODULE_SRCVERSION_ALL=y CONFIG_MODULE_SIG_SHA256=y -CONFIG_UNUSED_SYMBOLS=y CONFIG_BLK_DEV_THROTTLING=y CONFIG_BLK_WBT=y CONFIG_BLK_CGROUP_IOLATENCY=y diff --git a/arch/s390/crypto/aes_s390.c b/arch/s390/crypto/aes_s390.c index 73044634d3427f63e62506ed8bebc0a8f840a492..54c7536f2482d5cea7de657157d093985e1c1154 100644 --- a/arch/s390/crypto/aes_s390.c +++ b/arch/s390/crypto/aes_s390.c @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -1055,3 +1056,4 @@ MODULE_ALIAS_CRYPTO("aes-all"); MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm"); MODULE_LICENSE("GPL"); +MODULE_IMPORT_NS(CRYPTO_INTERNAL); diff --git a/arch/s390/include/asm/facility.h b/arch/s390/include/asm/facility.h index 68c476b20b57e25691cf990766c89c1c113490e1..91b5d714d28f64afc1d2cf32cbbfb1b657038510 100644 --- a/arch/s390/include/asm/facility.h +++ b/arch/s390/include/asm/facility.h @@ -44,7 +44,7 @@ static inline int __test_facility(unsigned long nr, void *facilities) } /* - * The test_facility function uses the bit odering where the MSB is bit 0. + * The test_facility function uses the bit ordering where the MSB is bit 0. * That makes it easier to query facility bits with the bit number as * documented in the Principles of Operation. */ diff --git a/arch/s390/include/asm/hardirq.h b/arch/s390/include/asm/hardirq.h index dfbc3c6c06746610c6eb04c980b8dcb185535db1..58668ffb54882fc31fd2d9fc9938a1ec2c4b7b82 100644 --- a/arch/s390/include/asm/hardirq.h +++ b/arch/s390/include/asm/hardirq.h @@ -18,7 +18,6 @@ #define or_softirq_pending(x) (S390_lowcore.softirq_pending |= (x)) #define __ARCH_IRQ_STAT -#define __ARCH_HAS_DO_SOFTIRQ #define __ARCH_IRQ_EXIT_IRQS_DISABLED static inline void ack_bad_irq(unsigned int irq) diff --git a/arch/s390/include/asm/irq_work.h b/arch/s390/include/asm/irq_work.h new file mode 100644 index 0000000000000000000000000000000000000000..603783766d0abb9a6e1b56ea8b588b51a806c351 --- /dev/null +++ b/arch/s390/include/asm/irq_work.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_S390_IRQ_WORK_H +#define _ASM_S390_IRQ_WORK_H + +static inline bool arch_irq_work_has_interrupt(void) +{ + return true; +} + +void arch_irq_work_raise(void); + +#endif /* _ASM_S390_IRQ_WORK_H */ diff --git a/arch/s390/include/asm/pgalloc.h b/arch/s390/include/asm/pgalloc.h index d1297d6bbdcf114a7c2783bdddce7cdf7bbe663f..6b187cd7225116b9a18859742e75f534bf5a51a7 100644 --- a/arch/s390/include/asm/pgalloc.h +++ b/arch/s390/include/asm/pgalloc.h @@ -135,7 +135,7 @@ static inline void pmd_populate(struct mm_struct *mm, #define pmd_populate_kernel(mm, pmd, pte) pmd_populate(mm, pmd, pte) #define pmd_pgtable(pmd) \ - (pgtable_t)(pmd_val(pmd) & -sizeof(pte_t)*PTRS_PER_PTE) + ((pgtable_t)__va(pmd_val(pmd) & -sizeof(pte_t)*PTRS_PER_PTE)) /* * page table entry allocation/free routines. diff --git a/arch/s390/include/asm/pgtable.h b/arch/s390/include/asm/pgtable.h index 794746a32806a391e4e81d971f6c0489140f1aa4..29c7ecd5ad1d5b2c96090e450932bf0603b044e0 100644 --- a/arch/s390/include/asm/pgtable.h +++ b/arch/s390/include/asm/pgtable.h @@ -1219,8 +1219,8 @@ static inline pte_t mk_pte(struct page *page, pgprot_t pgprot) #define pud_index(address) (((address) >> PUD_SHIFT) & (PTRS_PER_PUD-1)) #define pmd_index(address) (((address) >> PMD_SHIFT) & (PTRS_PER_PMD-1)) -#define p4d_deref(pud) (p4d_val(pud) & _REGION_ENTRY_ORIGIN) -#define pgd_deref(pgd) (pgd_val(pgd) & _REGION_ENTRY_ORIGIN) +#define p4d_deref(pud) ((unsigned long)__va(p4d_val(pud) & _REGION_ENTRY_ORIGIN)) +#define pgd_deref(pgd) ((unsigned long)__va(pgd_val(pgd) & _REGION_ENTRY_ORIGIN)) static inline unsigned long pmd_deref(pmd_t pmd) { @@ -1229,12 +1229,12 @@ static inline unsigned long pmd_deref(pmd_t pmd) origin_mask = _SEGMENT_ENTRY_ORIGIN; if (pmd_large(pmd)) origin_mask = _SEGMENT_ENTRY_ORIGIN_LARGE; - return pmd_val(pmd) & origin_mask; + return (unsigned long)__va(pmd_val(pmd) & origin_mask); } static inline unsigned long pmd_pfn(pmd_t pmd) { - return pmd_deref(pmd) >> PAGE_SHIFT; + return __pa(pmd_deref(pmd)) >> PAGE_SHIFT; } static inline unsigned long pud_deref(pud_t pud) @@ -1244,12 +1244,12 @@ static inline unsigned long pud_deref(pud_t pud) origin_mask = _REGION_ENTRY_ORIGIN; if (pud_large(pud)) origin_mask = _REGION3_ENTRY_ORIGIN_LARGE; - return pud_val(pud) & origin_mask; + return (unsigned long)__va(pud_val(pud) & origin_mask); } static inline unsigned long pud_pfn(pud_t pud) { - return pud_deref(pud) >> PAGE_SHIFT; + return __pa(pud_deref(pud)) >> PAGE_SHIFT; } /* @@ -1329,7 +1329,7 @@ static inline bool gup_fast_permitted(unsigned long start, unsigned long end) } #define gup_fast_permitted gup_fast_permitted -#define pfn_pte(pfn,pgprot) mk_pte_phys(__pa((pfn) << PAGE_SHIFT),(pgprot)) +#define pfn_pte(pfn, pgprot) mk_pte_phys(((pfn) << PAGE_SHIFT), (pgprot)) #define pte_pfn(x) (pte_val(x) >> PAGE_SHIFT) #define pte_page(x) pfn_to_page(pte_pfn(x)) @@ -1636,7 +1636,7 @@ static inline pmd_t pmdp_collapse_flush(struct vm_area_struct *vma, } #define pmdp_collapse_flush pmdp_collapse_flush -#define pfn_pmd(pfn, pgprot) mk_pmd_phys(__pa((pfn) << PAGE_SHIFT), (pgprot)) +#define pfn_pmd(pfn, pgprot) mk_pmd_phys(((pfn) << PAGE_SHIFT), (pgprot)) #define mk_pmd(page, pgprot) pfn_pmd(page_to_pfn(page), (pgprot)) static inline int pmd_trans_huge(pmd_t pmd) diff --git a/arch/s390/include/uapi/asm/perf_cpum_cf_diag.h b/arch/s390/include/uapi/asm/perf_cpum_cf_diag.h new file mode 100644 index 0000000000000000000000000000000000000000..3d8284b95f870e06e908f4329028871846539c1c --- /dev/null +++ b/arch/s390/include/uapi/asm/perf_cpum_cf_diag.h @@ -0,0 +1,51 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +/* + * Copyright IBM Corp. 2021 + * Interface implementation for communication with the CPU Measurement + * counter facility device driver. + * + * Author(s): Thomas Richter + * + * Define for ioctl() commands to communicate with the CPU Measurement + * counter facility device driver. + */ + +#ifndef _PERF_CPUM_CF_DIAG_H +#define _PERF_CPUM_CF_DIAG_H + +#include +#include + +#define S390_HWCTR_DEVICE "hwctr" +#define S390_HWCTR_START_VERSION 1 + +struct s390_ctrset_start { /* Set CPUs to operate on */ + __u64 version; /* Version of interface */ + __u64 data_bytes; /* # of bytes required */ + __u64 cpumask_len; /* Length of CPU mask in bytes */ + __u64 *cpumask; /* Pointer to CPU mask */ + __u64 counter_sets; /* Bit mask of counter sets to get */ +}; + +struct s390_ctrset_setdata { /* Counter set data */ + __u32 set; /* Counter set number */ + __u32 no_cnts; /* # of counters stored in cv[] */ + __u64 cv[0]; /* Counter values (variable length) */ +}; + +struct s390_ctrset_cpudata { /* Counter set data per CPU */ + __u32 cpu_nr; /* CPU number */ + __u32 no_sets; /* # of counters sets in data[] */ + struct s390_ctrset_setdata data[0]; +}; + +struct s390_ctrset_read { /* Structure to get all ctr sets */ + __u64 no_cpus; /* Total # of CPUs data taken from */ + struct s390_ctrset_cpudata data[0]; +}; + +#define S390_HWCTR_MAGIC 'C' /* Random magic # for ioctls */ +#define S390_HWCTR_START _IOWR(S390_HWCTR_MAGIC, 1, struct s390_ctrset_start) +#define S390_HWCTR_STOP _IO(S390_HWCTR_MAGIC, 2) +#define S390_HWCTR_READ _IOWR(S390_HWCTR_MAGIC, 3, struct s390_ctrset_read) +#endif diff --git a/arch/s390/kernel/irq.c b/arch/s390/kernel/irq.c index c6d40bcf4a680fae11fe0c93c5cb2ac4920f021f..601c2179133847d76db62b4f283522b1d76124e6 100644 --- a/arch/s390/kernel/irq.c +++ b/arch/s390/kernel/irq.c @@ -28,6 +28,7 @@ #include #include #include +#include #include "entry.h" DEFINE_PER_CPU_SHARED_ALIGNED(struct irq_stat, irq_stat); diff --git a/arch/s390/kernel/perf_cpum_cf_diag.c b/arch/s390/kernel/perf_cpum_cf_diag.c index b5c86fb70d633c07c4565e8ce4c182feacb9e924..db4877bbb9aa5626702bf6068de92accdaa9e81a 100644 --- a/arch/s390/kernel/perf_cpum_cf_diag.c +++ b/arch/s390/kernel/perf_cpum_cf_diag.c @@ -2,7 +2,7 @@ /* * Performance event support for s390x - CPU-measurement Counter Sets * - * Copyright IBM Corp. 2019 + * Copyright IBM Corp. 2019, 2021 * Author(s): Hendrik Brueckner * Thomas Richer */ @@ -17,6 +17,8 @@ #include #include #include +#include +#include #include #include @@ -24,15 +26,20 @@ #include #include -#define CF_DIAG_CTRSET_DEF 0xfeef /* Counter set header mark */ +#include +#define CF_DIAG_CTRSET_DEF 0xfeef /* Counter set header mark */ +#define CF_DIAG_MIN_INTERVAL 60 /* Minimum counter set read */ + /* interval in seconds */ +static unsigned long cf_diag_interval = CF_DIAG_MIN_INTERVAL; static unsigned int cf_diag_cpu_speed; static debug_info_t *cf_diag_dbg; -struct cf_diag_csd { /* Counter set data per CPU */ +struct cf_diag_csd { /* Counter set data per CPU */ size_t used; /* Bytes used in data/start */ unsigned char start[PAGE_SIZE]; /* Counter set at event start */ unsigned char data[PAGE_SIZE]; /* Counter set at event delete */ + unsigned int sets; /* # Counter set saved in data */ }; static DEFINE_PER_CPU(struct cf_diag_csd, cf_diag_csd); @@ -178,18 +185,35 @@ static void cf_diag_disable(struct pmu *pmu) /* Number of perf events counting hardware events */ static atomic_t cf_diag_events = ATOMIC_INIT(0); +/* Used to avoid races in calling reserve/release_cpumf_hardware */ +static DEFINE_MUTEX(cf_diag_reserve_mutex); /* Release the PMU if event is the last perf event */ static void cf_diag_perf_event_destroy(struct perf_event *event) { debug_sprintf_event(cf_diag_dbg, 5, "%s event %p cpu %d cf_diag_events %d\n", - __func__, event, event->cpu, + __func__, event, smp_processor_id(), atomic_read(&cf_diag_events)); if (atomic_dec_return(&cf_diag_events) == 0) __kernel_cpumcf_end(); } +static int get_authctrsets(void) +{ + struct cpu_cf_events *cpuhw; + unsigned long auth = 0; + enum cpumf_ctr_set i; + + cpuhw = &get_cpu_var(cpu_cf_events); + for (i = CPUMF_CTR_SET_BASIC; i < CPUMF_CTR_SET_MAX; ++i) { + if (cpuhw->info.auth_ctl & cpumf_ctr_ctl[i]) + auth |= cpumf_ctr_ctl[i]; + } + put_cpu_var(cpu_cf_events); + return auth; +} + /* Setup the event. Test for authorized counter sets and only include counter * sets which are authorized at the time of the setup. Including unauthorized * counter sets result in specification exception (and panic). @@ -197,15 +221,12 @@ static void cf_diag_perf_event_destroy(struct perf_event *event) static int __hw_perf_event_init(struct perf_event *event) { struct perf_event_attr *attr = &event->attr; - struct cpu_cf_events *cpuhw; - enum cpumf_ctr_set i; int err = 0; debug_sprintf_event(cf_diag_dbg, 5, "%s event %p cpu %d\n", __func__, event, event->cpu); event->hw.config = attr->config; - event->hw.config_base = 0; /* Add all authorized counter sets to config_base. The * the hardware init function is either called per-cpu or just once @@ -215,11 +236,7 @@ static int __hw_perf_event_init(struct perf_event *event) * Checking the authorization on any CPU is fine as the hardware * applies the same authorization settings to all CPUs. */ - cpuhw = &get_cpu_var(cpu_cf_events); - for (i = CPUMF_CTR_SET_BASIC; i < CPUMF_CTR_SET_MAX; ++i) - if (cpuhw->info.auth_ctl & cpumf_ctr_ctl[i]) - event->hw.config_base |= cpumf_ctr_ctl[i]; - put_cpu_var(cpu_cf_events); + event->hw.config_base = get_authctrsets(); /* No authorized counter sets, nothing to count/sample */ if (!event->hw.config_base) { @@ -237,6 +254,25 @@ static int __hw_perf_event_init(struct perf_event *event) return err; } +/* Return 0 if the CPU-measurement counter facility is currently free + * and an error otherwise. + */ +static int cf_diag_perf_event_inuse(void) +{ + int err = 0; + + if (!atomic_inc_not_zero(&cf_diag_events)) { + mutex_lock(&cf_diag_reserve_mutex); + if (atomic_read(&cf_diag_events) == 0 && + __kernel_cpumcf_begin()) + err = -EBUSY; + else + err = atomic_inc_return(&cf_diag_events); + mutex_unlock(&cf_diag_reserve_mutex); + } + return err; +} + static int cf_diag_event_init(struct perf_event *event) { struct perf_event_attr *attr = &event->attr; @@ -264,13 +300,9 @@ static int cf_diag_event_init(struct perf_event *event) } /* Initialize for using the CPU-measurement counter facility */ - if (atomic_inc_return(&cf_diag_events) == 1) { - if (__kernel_cpumcf_begin()) { - atomic_dec(&cf_diag_events); - err = -EBUSY; - goto out; - } - } + err = cf_diag_perf_event_inuse(); + if (err < 0) + goto out; event->destroy = cf_diag_perf_event_destroy; err = __hw_perf_event_init(event); @@ -599,6 +631,8 @@ static void cf_diag_del(struct perf_event *event, int flags) cpuhw->flags &= ~PMU_F_IN_USE; } +/* Default counter set events and format attribute groups */ + CPUMF_EVENT_ATTR(CF_DIAG, CF_DIAG, PERF_EVENT_CPUM_CF_DIAG); static struct attribute *cf_diag_events_attr[] = { @@ -663,6 +697,452 @@ static void cf_diag_get_cpu_speed(void) } } +/* Code to create device and file I/O operations */ +static atomic_t ctrset_opencnt = ATOMIC_INIT(0); /* Excl. access */ + +static int cf_diag_open(struct inode *inode, struct file *file) +{ + int err = 0; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + if (atomic_xchg(&ctrset_opencnt, 1)) + return -EBUSY; + + /* Avoid concurrent access with perf_event_open() system call */ + mutex_lock(&cf_diag_reserve_mutex); + if (atomic_read(&cf_diag_events) || __kernel_cpumcf_begin()) + err = -EBUSY; + mutex_unlock(&cf_diag_reserve_mutex); + if (err) { + atomic_set(&ctrset_opencnt, 0); + return err; + } + file->private_data = NULL; + debug_sprintf_event(cf_diag_dbg, 2, "%s\n", __func__); + /* nonseekable_open() never fails */ + return nonseekable_open(inode, file); +} + +/* Variables for ioctl() interface support */ +static DEFINE_MUTEX(cf_diag_ctrset_mutex); +static struct cf_diag_ctrset { + unsigned long ctrset; /* Bit mask of counter set to read */ + cpumask_t mask; /* CPU mask to read from */ + time64_t lastread; /* Epoch counter set last read */ +} cf_diag_ctrset; + +static void cf_diag_ctrset_clear(void) +{ + cpumask_clear(&cf_diag_ctrset.mask); + cf_diag_ctrset.ctrset = 0; +} + +static void cf_diag_release_cpu(void *p) +{ + struct cpu_cf_events *cpuhw = this_cpu_ptr(&cpu_cf_events); + + debug_sprintf_event(cf_diag_dbg, 3, "%s cpu %d\n", __func__, + smp_processor_id()); + lcctl(0); /* Reset counter sets */ + cpuhw->state = 0; /* Save state in CPU hardware state */ +} + +/* Release function is also called when application gets terminated without + * doing a proper ioctl(..., S390_HWCTR_STOP, ...) command. + * Since only one application is allowed to open the device, simple stop all + * CPU counter sets. + */ +static int cf_diag_release(struct inode *inode, struct file *file) +{ + on_each_cpu(cf_diag_release_cpu, NULL, 1); + cf_diag_ctrset_clear(); + atomic_set(&ctrset_opencnt, 0); + __kernel_cpumcf_end(); + debug_sprintf_event(cf_diag_dbg, 2, "%s\n", __func__); + return 0; +} + +struct cf_diag_call_on_cpu_parm { /* Parm struct for smp_call_on_cpu */ + unsigned int sets; /* Counter set bit mask */ + atomic_t cpus_ack; /* # CPUs successfully executed func */ +}; + +static int cf_diag_all_copy(unsigned long arg, cpumask_t *mask) +{ + struct s390_ctrset_read __user *ctrset_read; + unsigned int cpu, cpus, rc; + void __user *uptr; + + ctrset_read = (struct s390_ctrset_read __user *)arg; + uptr = ctrset_read->data; + for_each_cpu(cpu, mask) { + struct cf_diag_csd *csd = per_cpu_ptr(&cf_diag_csd, cpu); + struct s390_ctrset_cpudata __user *ctrset_cpudata; + + ctrset_cpudata = uptr; + debug_sprintf_event(cf_diag_dbg, 5, "%s cpu %d used %zd\n", + __func__, cpu, csd->used); + rc = put_user(cpu, &ctrset_cpudata->cpu_nr); + rc |= put_user(csd->sets, &ctrset_cpudata->no_sets); + rc |= copy_to_user(ctrset_cpudata->data, csd->data, csd->used); + if (rc) + return -EFAULT; + uptr += sizeof(struct s390_ctrset_cpudata) + csd->used; + cond_resched(); + } + cpus = cpumask_weight(mask); + if (put_user(cpus, &ctrset_read->no_cpus)) + return -EFAULT; + debug_sprintf_event(cf_diag_dbg, 5, "%s copied %ld\n", + __func__, uptr - (void __user *)ctrset_read->data); + return 0; +} + +static size_t cf_diag_cpuset_read(struct s390_ctrset_setdata *p, int ctrset, + int ctrset_size, size_t room) +{ + size_t need = 0; + int rc = -1; + + need = sizeof(*p) + sizeof(u64) * ctrset_size; + debug_sprintf_event(cf_diag_dbg, 5, + "%s room %zd need %zd set %#x set_size %d\n", + __func__, room, need, ctrset, ctrset_size); + if (need <= room) { + p->set = cpumf_ctr_ctl[ctrset]; + p->no_cnts = ctrset_size; + rc = ctr_stcctm(ctrset, ctrset_size, (u64 *)p->cv); + if (rc == 3) /* Nothing stored */ + need = 0; + } + debug_sprintf_event(cf_diag_dbg, 5, "%s need %zd rc %d\n", __func__, + need, rc); + return need; +} + +/* Read all counter sets. Since the perf_event_open() system call with + * event cpum_cf_diag/.../ is blocked when this interface is active, reuse + * the perf_event_open() data buffer to store the counter sets. + */ +static void cf_diag_cpu_read(void *parm) +{ + struct cpu_cf_events *cpuhw = this_cpu_ptr(&cpu_cf_events); + struct cf_diag_csd *csd = this_cpu_ptr(&cf_diag_csd); + struct cf_diag_call_on_cpu_parm *p = parm; + int set, set_size; + size_t space; + + debug_sprintf_event(cf_diag_dbg, 5, + "%s new %#x flags %#x state %#llx\n", + __func__, p->sets, cpuhw->flags, + cpuhw->state); + /* No data saved yet */ + csd->used = 0; + csd->sets = 0; + memset(csd->data, 0, sizeof(csd->data)); + + /* Scan the counter sets */ + for (set = CPUMF_CTR_SET_BASIC; set < CPUMF_CTR_SET_MAX; ++set) { + struct s390_ctrset_setdata *sp = (void *)csd->data + csd->used; + + if (!(p->sets & cpumf_ctr_ctl[set])) + continue; /* Counter set not in list */ + set_size = cf_diag_ctrset_size(set, &cpuhw->info); + space = sizeof(csd->data) - csd->used; + space = cf_diag_cpuset_read(sp, set, set_size, space); + if (space) { + csd->used += space; + csd->sets += 1; + } + debug_sprintf_event(cf_diag_dbg, 5, "%s sp %px space %zd\n", + __func__, sp, space); + } + debug_sprintf_event(cf_diag_dbg, 5, "%s sets %d used %zd\n", __func__, + csd->sets, csd->used); +} + +static int cf_diag_all_read(unsigned long arg) +{ + struct cf_diag_call_on_cpu_parm p; + cpumask_var_t mask; + time64_t now; + int rc = 0; + + debug_sprintf_event(cf_diag_dbg, 5, "%s\n", __func__); + if (!alloc_cpumask_var(&mask, GFP_KERNEL)) + return -ENOMEM; + now = ktime_get_seconds(); + if (cf_diag_ctrset.lastread + cf_diag_interval > now) { + debug_sprintf_event(cf_diag_dbg, 5, "%s now %lld " + " lastread %lld\n", __func__, now, + cf_diag_ctrset.lastread); + rc = -EAGAIN; + goto out; + } else { + cf_diag_ctrset.lastread = now; + } + p.sets = cf_diag_ctrset.ctrset; + cpumask_and(mask, &cf_diag_ctrset.mask, cpu_online_mask); + on_each_cpu_mask(mask, cf_diag_cpu_read, &p, 1); + rc = cf_diag_all_copy(arg, mask); +out: + free_cpumask_var(mask); + debug_sprintf_event(cf_diag_dbg, 5, "%s rc %d\n", __func__, rc); + return rc; +} + +/* Stop all counter sets via ioctl interface */ +static void cf_diag_ioctl_off(void *parm) +{ + struct cpu_cf_events *cpuhw = this_cpu_ptr(&cpu_cf_events); + struct cf_diag_call_on_cpu_parm *p = parm; + int rc; + + debug_sprintf_event(cf_diag_dbg, 5, + "%s new %#x flags %#x state %#llx\n", + __func__, p->sets, cpuhw->flags, + cpuhw->state); + + ctr_set_multiple_disable(&cpuhw->state, p->sets); + ctr_set_multiple_stop(&cpuhw->state, p->sets); + rc = lcctl(cpuhw->state); /* Stop counter sets */ + if (!cpuhw->state) + cpuhw->flags &= ~PMU_F_IN_USE; + debug_sprintf_event(cf_diag_dbg, 5, + "%s rc %d flags %#x state %#llx\n", __func__, + rc, cpuhw->flags, cpuhw->state); +} + +/* Start counter sets on particular CPU */ +static void cf_diag_ioctl_on(void *parm) +{ + struct cpu_cf_events *cpuhw = this_cpu_ptr(&cpu_cf_events); + struct cf_diag_call_on_cpu_parm *p = parm; + int rc; + + debug_sprintf_event(cf_diag_dbg, 5, + "%s new %#x flags %#x state %#llx\n", + __func__, p->sets, cpuhw->flags, + cpuhw->state); + + if (!(cpuhw->flags & PMU_F_IN_USE)) + cpuhw->state = 0; + cpuhw->flags |= PMU_F_IN_USE; + rc = lcctl(cpuhw->state); /* Reset unused counter sets */ + ctr_set_multiple_enable(&cpuhw->state, p->sets); + ctr_set_multiple_start(&cpuhw->state, p->sets); + rc |= lcctl(cpuhw->state); /* Start counter sets */ + if (!rc) + atomic_inc(&p->cpus_ack); + debug_sprintf_event(cf_diag_dbg, 5, "%s rc %d state %#llx\n", + __func__, rc, cpuhw->state); +} + +static int cf_diag_all_stop(void) +{ + struct cf_diag_call_on_cpu_parm p = { + .sets = cf_diag_ctrset.ctrset, + }; + cpumask_var_t mask; + + if (!alloc_cpumask_var(&mask, GFP_KERNEL)) + return -ENOMEM; + cpumask_and(mask, &cf_diag_ctrset.mask, cpu_online_mask); + on_each_cpu_mask(mask, cf_diag_ioctl_off, &p, 1); + free_cpumask_var(mask); + return 0; +} + +static int cf_diag_all_start(void) +{ + struct cf_diag_call_on_cpu_parm p = { + .sets = cf_diag_ctrset.ctrset, + .cpus_ack = ATOMIC_INIT(0), + }; + cpumask_var_t mask; + int rc = 0; + + if (!alloc_cpumask_var(&mask, GFP_KERNEL)) + return -ENOMEM; + cpumask_and(mask, &cf_diag_ctrset.mask, cpu_online_mask); + on_each_cpu_mask(mask, cf_diag_ioctl_on, &p, 1); + if (atomic_read(&p.cpus_ack) != cpumask_weight(mask)) { + on_each_cpu_mask(mask, cf_diag_ioctl_off, &p, 1); + rc = -EIO; + } + free_cpumask_var(mask); + return rc; +} + +/* Return the maximum required space for all possible CPUs in case one + * CPU will be onlined during the START, READ, STOP cycles. + * To find out the size of the counter sets, any one CPU will do. They + * all have the same counter sets. + */ +static size_t cf_diag_needspace(unsigned int sets) +{ + struct cpu_cf_events *cpuhw = this_cpu_ptr(&cpu_cf_events); + size_t bytes = 0; + int i; + + for (i = CPUMF_CTR_SET_BASIC; i < CPUMF_CTR_SET_MAX; ++i) { + if (!(sets & cpumf_ctr_ctl[i])) + continue; + bytes += cf_diag_ctrset_size(i, &cpuhw->info) * sizeof(u64) + + sizeof(((struct s390_ctrset_setdata *)0)->set) + + sizeof(((struct s390_ctrset_setdata *)0)->no_cnts); + } + bytes = sizeof(((struct s390_ctrset_read *)0)->no_cpus) + nr_cpu_ids * + (bytes + sizeof(((struct s390_ctrset_cpudata *)0)->cpu_nr) + + sizeof(((struct s390_ctrset_cpudata *)0)->no_sets)); + debug_sprintf_event(cf_diag_dbg, 5, "%s bytes %ld\n", __func__, + bytes); + return bytes; +} + +static long cf_diag_ioctl_read(unsigned long arg) +{ + struct s390_ctrset_read read; + int ret = 0; + + debug_sprintf_event(cf_diag_dbg, 5, "%s\n", __func__); + if (copy_from_user(&read, (char __user *)arg, sizeof(read))) + return -EFAULT; + ret = cf_diag_all_read(arg); + debug_sprintf_event(cf_diag_dbg, 5, "%s ret %d\n", __func__, ret); + return ret; +} + +static long cf_diag_ioctl_stop(void) +{ + int ret; + + debug_sprintf_event(cf_diag_dbg, 5, "%s\n", __func__); + ret = cf_diag_all_stop(); + cf_diag_ctrset_clear(); + debug_sprintf_event(cf_diag_dbg, 5, "%s ret %d\n", __func__, ret); + return ret; +} + +static long cf_diag_ioctl_start(unsigned long arg) +{ + struct s390_ctrset_start __user *ustart; + struct s390_ctrset_start start; + void __user *umask; + unsigned int len; + int ret = 0; + size_t need; + + if (cf_diag_ctrset.ctrset) + return -EBUSY; + ustart = (struct s390_ctrset_start __user *)arg; + if (copy_from_user(&start, ustart, sizeof(start))) + return -EFAULT; + if (start.version != S390_HWCTR_START_VERSION) + return -EINVAL; + if (start.counter_sets & ~(cpumf_ctr_ctl[CPUMF_CTR_SET_BASIC] | + cpumf_ctr_ctl[CPUMF_CTR_SET_USER] | + cpumf_ctr_ctl[CPUMF_CTR_SET_CRYPTO] | + cpumf_ctr_ctl[CPUMF_CTR_SET_EXT] | + cpumf_ctr_ctl[CPUMF_CTR_SET_MT_DIAG])) + return -EINVAL; /* Invalid counter set */ + if (!start.counter_sets) + return -EINVAL; /* No counter set at all? */ + cpumask_clear(&cf_diag_ctrset.mask); + len = min_t(u64, start.cpumask_len, cpumask_size()); + umask = (void __user *)start.cpumask; + if (copy_from_user(&cf_diag_ctrset.mask, umask, len)) + return -EFAULT; + if (cpumask_empty(&cf_diag_ctrset.mask)) + return -EINVAL; + need = cf_diag_needspace(start.counter_sets); + if (put_user(need, &ustart->data_bytes)) + ret = -EFAULT; + if (ret) + goto out; + cf_diag_ctrset.ctrset = start.counter_sets; + ret = cf_diag_all_start(); +out: + if (ret) + cf_diag_ctrset_clear(); + debug_sprintf_event(cf_diag_dbg, 2, "%s sets %#lx need %ld ret %d\n", + __func__, cf_diag_ctrset.ctrset, need, ret); + return ret; +} + +static long cf_diag_ioctl(struct file *file, unsigned int cmd, + unsigned long arg) +{ + int ret; + + debug_sprintf_event(cf_diag_dbg, 2, "%s cmd %#x arg %lx\n", __func__, + cmd, arg); + get_online_cpus(); + mutex_lock(&cf_diag_ctrset_mutex); + switch (cmd) { + case S390_HWCTR_START: + ret = cf_diag_ioctl_start(arg); + break; + case S390_HWCTR_STOP: + ret = cf_diag_ioctl_stop(); + break; + case S390_HWCTR_READ: + ret = cf_diag_ioctl_read(arg); + break; + default: + ret = -ENOTTY; + break; + } + mutex_unlock(&cf_diag_ctrset_mutex); + put_online_cpus(); + debug_sprintf_event(cf_diag_dbg, 2, "%s ret %d\n", __func__, ret); + return ret; +} + +static const struct file_operations cf_diag_fops = { + .owner = THIS_MODULE, + .open = cf_diag_open, + .release = cf_diag_release, + .unlocked_ioctl = cf_diag_ioctl, + .compat_ioctl = cf_diag_ioctl, + .llseek = no_llseek +}; + +static struct miscdevice cf_diag_dev = { + .name = S390_HWCTR_DEVICE, + .minor = MISC_DYNAMIC_MINOR, + .fops = &cf_diag_fops, +}; + +static int cf_diag_online_cpu(unsigned int cpu) +{ + struct cf_diag_call_on_cpu_parm p; + + mutex_lock(&cf_diag_ctrset_mutex); + if (!cf_diag_ctrset.ctrset) + goto out; + p.sets = cf_diag_ctrset.ctrset; + cf_diag_ioctl_on(&p); +out: + mutex_unlock(&cf_diag_ctrset_mutex); + return 0; +} + +static int cf_diag_offline_cpu(unsigned int cpu) +{ + struct cf_diag_call_on_cpu_parm p; + + mutex_lock(&cf_diag_ctrset_mutex); + if (!cf_diag_ctrset.ctrset) + goto out; + p.sets = cf_diag_ctrset.ctrset; + cf_diag_ioctl_off(&p); +out: + mutex_unlock(&cf_diag_ctrset_mutex); + return 0; +} + /* Initialize the counter set PMU to generate complete counter set data as * event raw data. This relies on the CPU Measurement Counter Facility device * already being loaded and initialized. @@ -685,21 +1165,43 @@ static int __init cf_diag_init(void) return -ENOMEM; } + rc = misc_register(&cf_diag_dev); + if (rc) { + pr_err("Registration of /dev/" S390_HWCTR_DEVICE + "failed rc=%d\n", rc); + goto out; + } + /* Setup s390dbf facility */ cf_diag_dbg = debug_register(KMSG_COMPONENT, 2, 1, 128); if (!cf_diag_dbg) { pr_err("Registration of s390dbf(cpum_cf_diag) failed\n"); - return -ENOMEM; + rc = -ENOMEM; + goto out_dbf; } debug_register_view(cf_diag_dbg, &debug_sprintf_view); rc = perf_pmu_register(&cf_diag, "cpum_cf_diag", -1); if (rc) { - debug_unregister_view(cf_diag_dbg, &debug_sprintf_view); - debug_unregister(cf_diag_dbg); pr_err("Registration of PMU(cpum_cf_diag) failed with rc=%i\n", rc); + goto out_perf; } + rc = cpuhp_setup_state_nocalls(CPUHP_AP_PERF_S390_CFD_ONLINE, + "perf/s390/cfd:online", + cf_diag_online_cpu, cf_diag_offline_cpu); + if (!rc) + goto out; + + pr_err("Registration of CPUHP_AP_PERF_S390_CFD_ONLINE failed rc=%i\n", + rc); + perf_pmu_unregister(&cf_diag); +out_perf: + debug_unregister_view(cf_diag_dbg, &debug_sprintf_view); + debug_unregister(cf_diag_dbg); +out_dbf: + misc_deregister(&cf_diag_dev); +out: return rc; } -arch_initcall(cf_diag_init); +device_initcall(cf_diag_init); diff --git a/arch/s390/kernel/process.c b/arch/s390/kernel/process.c index 367bd000f6d15dbb0aced89b3608bcd3f8ff1d94..e20bed1ed34a2d8691cd8bd000832875fcf85ebd 100644 --- a/arch/s390/kernel/process.c +++ b/arch/s390/kernel/process.c @@ -130,7 +130,7 @@ int copy_thread(unsigned long clone_flags, unsigned long new_stackp, frame->sf.gprs[9] = (unsigned long)frame; /* Store access registers to kernel stack of new process. */ - if (unlikely(p->flags & PF_KTHREAD)) { + if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { /* kernel thread */ memset(&frame->childregs, 0, sizeof(struct pt_regs)); frame->childregs.psw.mask = PSW_KERNEL_BITS | PSW_MASK_DAT | diff --git a/arch/s390/kernel/smp.c b/arch/s390/kernel/smp.c index e299892440b6dc031e11d7579abd22ce1402a445..58c8afa3da65d4e42a34ed1f82de5bb62882619a 100644 --- a/arch/s390/kernel/smp.c +++ b/arch/s390/kernel/smp.c @@ -30,6 +30,7 @@ #include #include #include +#include #include #include #include @@ -62,6 +63,7 @@ enum { ec_call_function_single, ec_stop_cpu, ec_mcck_pending, + ec_irq_work, }; enum { @@ -434,10 +436,12 @@ void notrace smp_yield_cpu(int cpu) */ void notrace smp_emergency_stop(void) { - cpumask_t cpumask; + static arch_spinlock_t lock = __ARCH_SPIN_LOCK_UNLOCKED; + static cpumask_t cpumask; u64 end; int cpu; + arch_spin_lock(&lock); cpumask_copy(&cpumask, cpu_online_mask); cpumask_clear_cpu(smp_processor_id(), &cpumask); @@ -458,6 +462,7 @@ void notrace smp_emergency_stop(void) break; cpu_relax(); } + arch_spin_unlock(&lock); } NOKPROBE_SYMBOL(smp_emergency_stop); @@ -505,6 +510,8 @@ static void smp_handle_ext_call(void) generic_smp_call_function_single_interrupt(); if (test_bit(ec_mcck_pending, &bits)) __s390_handle_mcck(); + if (test_bit(ec_irq_work, &bits)) + irq_work_run(); } static void do_ext_call_interrupt(struct ext_code ext_code, @@ -537,6 +544,13 @@ void smp_send_reschedule(int cpu) pcpu_ec_call(pcpu_devices + cpu, ec_schedule); } +#ifdef CONFIG_IRQ_WORK +void arch_irq_work_raise(void) +{ + pcpu_ec_call(pcpu_devices + smp_processor_id(), ec_irq_work); +} +#endif + /* * parameter area for the set/clear control bit callbacks */ @@ -775,11 +789,13 @@ static int smp_add_core(struct sclp_core_entry *core, cpumask_t *avail, static int __smp_rescan_cpus(struct sclp_core_info *info, bool early) { struct sclp_core_entry *core; - cpumask_t avail; + static cpumask_t avail; bool configured; u16 core_id; int nr, i; + get_online_cpus(); + mutex_lock(&smp_cpu_state_mutex); nr = 0; cpumask_xor(&avail, cpu_possible_mask, cpu_present_mask); /* @@ -800,6 +816,8 @@ static int __smp_rescan_cpus(struct sclp_core_info *info, bool early) configured = i < info->configured; nr += smp_add_core(&info->core[i], &avail, configured, early); } + mutex_unlock(&smp_cpu_state_mutex); + put_online_cpus(); return nr; } @@ -847,9 +865,7 @@ void __init smp_detect_cpus(void) pr_info("%d configured CPUs, %d standby CPUs\n", c_cpus, s_cpus); /* Add CPUs present at boot */ - get_online_cpus(); __smp_rescan_cpus(info, true); - put_online_cpus(); memblock_free_early((unsigned long)info, sizeof(*info)); } @@ -1178,11 +1194,7 @@ int __ref smp_rescan_cpus(void) if (!info) return -ENOMEM; smp_get_core_info(info, 0); - get_online_cpus(); - mutex_lock(&smp_cpu_state_mutex); nr = __smp_rescan_cpus(info, false); - mutex_unlock(&smp_cpu_state_mutex); - put_online_cpus(); kfree(info); if (nr) topology_schedule_update(); diff --git a/arch/s390/kernel/syscalls/syscall.tbl b/arch/s390/kernel/syscalls/syscall.tbl index d443423495e565e49a8e57527f00dfdd4ffbea17..3abef2144dac79b69b25852ce28ffb2e5b284afe 100644 --- a/arch/s390/kernel/syscalls/syscall.tbl +++ b/arch/s390/kernel/syscalls/syscall.tbl @@ -444,3 +444,4 @@ 439 common faccessat2 sys_faccessat2 sys_faccessat2 440 common process_madvise sys_process_madvise sys_process_madvise 441 common epoll_pwait2 sys_epoll_pwait2 compat_sys_epoll_pwait2 +442 common mount_setattr sys_mount_setattr sys_mount_setattr diff --git a/arch/s390/kernel/topology.c b/arch/s390/kernel/topology.c index ca47141a5be9028658eccfd7a9d4e514033e0f0f..e7ce447651b905c20dce60dfc41555ee359d5d55 100644 --- a/arch/s390/kernel/topology.c +++ b/arch/s390/kernel/topology.c @@ -62,16 +62,16 @@ static struct mask_info drawer_info; struct cpu_topology_s390 cpu_topology[NR_CPUS]; EXPORT_SYMBOL_GPL(cpu_topology); -static cpumask_t cpu_group_map(struct mask_info *info, unsigned int cpu) +static void cpu_group_map(cpumask_t *dst, struct mask_info *info, unsigned int cpu) { - cpumask_t mask; + static cpumask_t mask; cpumask_copy(&mask, cpumask_of(cpu)); switch (topology_mode) { case TOPOLOGY_MODE_HW: while (info) { if (cpumask_test_cpu(cpu, &info->mask)) { - mask = info->mask; + cpumask_copy(&mask, &info->mask); break; } info = info->next; @@ -89,23 +89,24 @@ static cpumask_t cpu_group_map(struct mask_info *info, unsigned int cpu) break; } cpumask_and(&mask, &mask, cpu_online_mask); - return mask; + cpumask_copy(dst, &mask); } -static cpumask_t cpu_thread_map(unsigned int cpu) +static void cpu_thread_map(cpumask_t *dst, unsigned int cpu) { - cpumask_t mask; + static cpumask_t mask; int i; cpumask_copy(&mask, cpumask_of(cpu)); if (topology_mode != TOPOLOGY_MODE_HW) - return mask; + goto out; cpu -= cpu % (smp_cpu_mtid + 1); for (i = 0; i <= smp_cpu_mtid; i++) if (cpu_present(cpu + i)) cpumask_set_cpu(cpu + i, &mask); cpumask_and(&mask, &mask, cpu_online_mask); - return mask; +out: + cpumask_copy(dst, &mask); } #define TOPOLOGY_CORE_BITS 64 @@ -250,10 +251,10 @@ void update_cpu_masks(void) for_each_possible_cpu(cpu) { topo = &cpu_topology[cpu]; - topo->thread_mask = cpu_thread_map(cpu); - topo->core_mask = cpu_group_map(&socket_info, cpu); - topo->book_mask = cpu_group_map(&book_info, cpu); - topo->drawer_mask = cpu_group_map(&drawer_info, cpu); + cpu_thread_map(&topo->thread_mask, cpu); + cpu_group_map(&topo->core_mask, &socket_info, cpu); + cpu_group_map(&topo->book_mask, &book_info, cpu); + cpu_group_map(&topo->drawer_mask, &drawer_info, cpu); topo->booted_cores = 0; if (topology_mode != TOPOLOGY_MODE_HW) { id = topology_mode == TOPOLOGY_MODE_PACKAGE ? 0 : cpu; diff --git a/arch/s390/mm/init.c b/arch/s390/mm/init.c index 73a163065b95f6d7d3cb960cac4b649c708b25e1..0e76b2127dc6b81b54d74613a07aefa086ea637b 100644 --- a/arch/s390/mm/init.c +++ b/arch/s390/mm/init.c @@ -297,6 +297,7 @@ int arch_add_memory(int nid, u64 start, u64 size, if (WARN_ON_ONCE(params->pgprot.pgprot != PAGE_KERNEL.pgprot)) return -EINVAL; + VM_BUG_ON(!mhp_range_allowed(start, size, true)); rc = vmem_add_mapping(start, size); if (rc) return rc; diff --git a/arch/s390/mm/pgalloc.c b/arch/s390/mm/pgalloc.c index 4e87c819ddea7e00ce290cf5835acd34cfca0ab0..781965f7210ebf1bbae7a3c546220cd30fd1f161 100644 --- a/arch/s390/mm/pgalloc.c +++ b/arch/s390/mm/pgalloc.c @@ -58,7 +58,7 @@ unsigned long *crst_table_alloc(struct mm_struct *mm) if (!page) return NULL; arch_set_page_dat(page, 2); - return (unsigned long *) page_to_phys(page); + return (unsigned long *) page_to_virt(page); } void crst_table_free(struct mm_struct *mm, unsigned long *table) @@ -161,7 +161,7 @@ struct page *page_table_alloc_pgste(struct mm_struct *mm) page = alloc_page(GFP_KERNEL); if (page) { - table = (u64 *)page_to_phys(page); + table = (u64 *)page_to_virt(page); memset64(table, _PAGE_INVALID, PTRS_PER_PTE); memset64(table + PTRS_PER_PTE, 0, PTRS_PER_PTE); } @@ -194,7 +194,7 @@ unsigned long *page_table_alloc(struct mm_struct *mm) mask = atomic_read(&page->_refcount) >> 24; mask = (mask | (mask >> 4)) & 3; if (mask != 3) { - table = (unsigned long *) page_to_phys(page); + table = (unsigned long *) page_to_virt(page); bit = mask & 1; /* =1 -> second 2K */ if (bit) table += PTRS_PER_PTE; @@ -217,7 +217,7 @@ unsigned long *page_table_alloc(struct mm_struct *mm) } arch_set_page_dat(page, 0); /* Initialize page table */ - table = (unsigned long *) page_to_phys(page); + table = (unsigned long *) page_to_virt(page); if (mm_alloc_pgste(mm)) { /* Return 4K page table with PGSTEs */ atomic_xor_bits(&page->_refcount, 3 << 24); @@ -239,10 +239,10 @@ void page_table_free(struct mm_struct *mm, unsigned long *table) struct page *page; unsigned int bit, mask; - page = pfn_to_page(__pa(table) >> PAGE_SHIFT); + page = virt_to_page(table); if (!mm_alloc_pgste(mm)) { /* Free 2K page table fragment of a 4K page */ - bit = (__pa(table) & ~PAGE_MASK)/(PTRS_PER_PTE*sizeof(pte_t)); + bit = ((unsigned long) table & ~PAGE_MASK)/(PTRS_PER_PTE*sizeof(pte_t)); spin_lock_bh(&mm->context.lock); mask = atomic_xor_bits(&page->_refcount, 1U << (bit + 24)); mask >>= 24; @@ -269,14 +269,14 @@ void page_table_free_rcu(struct mmu_gather *tlb, unsigned long *table, unsigned int bit, mask; mm = tlb->mm; - page = pfn_to_page(__pa(table) >> PAGE_SHIFT); + page = virt_to_page(table); if (mm_alloc_pgste(mm)) { gmap_unlink(mm, table, vmaddr); - table = (unsigned long *) (__pa(table) | 3); + table = (unsigned long *) ((unsigned long)table | 3); tlb_remove_table(tlb, table); return; } - bit = (__pa(table) & ~PAGE_MASK) / (PTRS_PER_PTE*sizeof(pte_t)); + bit = ((unsigned long) table & ~PAGE_MASK) / (PTRS_PER_PTE*sizeof(pte_t)); spin_lock_bh(&mm->context.lock); mask = atomic_xor_bits(&page->_refcount, 0x11U << (bit + 24)); mask >>= 24; @@ -285,7 +285,7 @@ void page_table_free_rcu(struct mmu_gather *tlb, unsigned long *table, else list_del(&page->lru); spin_unlock_bh(&mm->context.lock); - table = (unsigned long *) (__pa(table) | (1U << bit)); + table = (unsigned long *) ((unsigned long) table | (1U << bit)); tlb_remove_table(tlb, table); } @@ -293,7 +293,7 @@ void __tlb_remove_table(void *_table) { unsigned int mask = (unsigned long) _table & 3; void *table = (void *)((unsigned long) _table ^ mask); - struct page *page = pfn_to_page(__pa(table) >> PAGE_SHIFT); + struct page *page = virt_to_page(table); switch (mask) { case 0: /* pmd, pud, or p4d */ diff --git a/arch/s390/mm/vmem.c b/arch/s390/mm/vmem.c index 01f3a5f58e6420b2d704dcf7955417e9a2e45787..96897fab89dc899baddb63e7953354823726d2c1 100644 --- a/arch/s390/mm/vmem.c +++ b/arch/s390/mm/vmem.c @@ -4,6 +4,7 @@ * Author(s): Heiko Carstens */ +#include #include #include #include @@ -26,14 +27,14 @@ static void __ref *vmem_alloc_pages(unsigned int order) if (slab_is_available()) return (void *)__get_free_pages(GFP_KERNEL, order); - return (void *) memblock_phys_alloc(size, size); + return memblock_alloc(size, size); } static void vmem_free_pages(unsigned long addr, int order) { /* We don't expect boot memory to be removed ever. */ if (!slab_is_available() || - WARN_ON_ONCE(PageReserved(phys_to_page(addr)))) + WARN_ON_ONCE(PageReserved(virt_to_page(addr)))) return; free_pages(addr, order); } @@ -56,7 +57,7 @@ pte_t __ref *vmem_pte_alloc(void) if (slab_is_available()) pte = (pte_t *) page_table_alloc(&init_mm); else - pte = (pte_t *) memblock_phys_alloc(size, size); + pte = (pte_t *) memblock_alloc(size, size); if (!pte) return NULL; memset64((u64 *)pte, _PAGE_INVALID, PTRS_PER_PTE); @@ -84,7 +85,7 @@ static void vmemmap_flush_unused_sub_pmd(void) { if (!unused_sub_pmd_start) return; - memset(__va(unused_sub_pmd_start), PAGE_UNUSED, + memset((void *)unused_sub_pmd_start, PAGE_UNUSED, ALIGN(unused_sub_pmd_start, PMD_SIZE) - unused_sub_pmd_start); unused_sub_pmd_start = 0; } @@ -97,7 +98,7 @@ static void vmemmap_mark_sub_pmd_used(unsigned long start, unsigned long end) * getting removed (just in case the memmap never gets initialized, * e.g., because the memory block never gets onlined). */ - memset(__va(start), 0, sizeof(struct page)); + memset((void *)start, 0, sizeof(struct page)); } static void vmemmap_use_sub_pmd(unsigned long start, unsigned long end) @@ -118,7 +119,7 @@ static void vmemmap_use_sub_pmd(unsigned long start, unsigned long end) static void vmemmap_use_new_sub_pmd(unsigned long start, unsigned long end) { - void *page = __va(ALIGN_DOWN(start, PMD_SIZE)); + unsigned long page = ALIGN_DOWN(start, PMD_SIZE); vmemmap_flush_unused_sub_pmd(); @@ -127,7 +128,7 @@ static void vmemmap_use_new_sub_pmd(unsigned long start, unsigned long end) /* Mark the unused parts of the new memmap page PAGE_UNUSED. */ if (!IS_ALIGNED(start, PMD_SIZE)) - memset(page, PAGE_UNUSED, start - __pa(page)); + memset((void *)page, PAGE_UNUSED, start - page); /* * We want to avoid memset(PAGE_UNUSED) when populating the vmemmap of * consecutive sections. Remember for the last added PMD the last @@ -140,11 +141,11 @@ static void vmemmap_use_new_sub_pmd(unsigned long start, unsigned long end) /* Returns true if the PMD is completely unused and can be freed. */ static bool vmemmap_unuse_sub_pmd(unsigned long start, unsigned long end) { - void *page = __va(ALIGN_DOWN(start, PMD_SIZE)); + unsigned long page = ALIGN_DOWN(start, PMD_SIZE); vmemmap_flush_unused_sub_pmd(); - memset(__va(start), PAGE_UNUSED, end - start); - return !memchr_inv(page, PAGE_UNUSED, PMD_SIZE); + memset((void *)start, PAGE_UNUSED, end - start); + return !memchr_inv((void *)page, PAGE_UNUSED, PMD_SIZE); } /* __ref: we'll only call vmemmap_alloc_block() via vmemmap_populate() */ @@ -165,7 +166,7 @@ static int __ref modify_pte_table(pmd_t *pmd, unsigned long addr, if (pte_none(*pte)) continue; if (!direct) - vmem_free_pages(pfn_to_phys(pte_pfn(*pte)), 0); + vmem_free_pages((unsigned long) pfn_to_virt(pte_pfn(*pte)), 0); pte_clear(&init_mm, addr, pte); } else if (pte_none(*pte)) { if (!direct) { @@ -175,7 +176,7 @@ static int __ref modify_pte_table(pmd_t *pmd, unsigned long addr, goto out; pte_val(*pte) = __pa(new_page) | prot; } else { - pte_val(*pte) = addr | prot; + pte_val(*pte) = __pa(addr) | prot; } } else { continue; @@ -200,7 +201,7 @@ static void try_free_pte_table(pmd_t *pmd, unsigned long start) if (!pte_none(*pte)) return; } - vmem_pte_free(__va(pmd_deref(*pmd))); + vmem_pte_free((unsigned long *) pmd_deref(*pmd)); pmd_clear(pmd); } @@ -241,7 +242,7 @@ static int __ref modify_pmd_table(pud_t *pud, unsigned long addr, IS_ALIGNED(next, PMD_SIZE) && MACHINE_HAS_EDAT1 && addr && direct && !debug_pagealloc_enabled()) { - pmd_val(*pmd) = addr | prot; + pmd_val(*pmd) = __pa(addr) | prot; pages++; continue; } else if (!direct && MACHINE_HAS_EDAT1) { @@ -337,7 +338,7 @@ static int modify_pud_table(p4d_t *p4d, unsigned long addr, unsigned long end, IS_ALIGNED(next, PUD_SIZE) && MACHINE_HAS_EDAT2 && addr && direct && !debug_pagealloc_enabled()) { - pud_val(*pud) = addr | prot; + pud_val(*pud) = __pa(addr) | prot; pages++; continue; } @@ -532,11 +533,22 @@ void vmem_remove_mapping(unsigned long start, unsigned long size) mutex_unlock(&vmem_mutex); } +struct range arch_get_mappable_range(void) +{ + struct range mhp_range; + + mhp_range.start = 0; + mhp_range.end = VMEM_MAX_PHYS - 1; + return mhp_range; +} + int vmem_add_mapping(unsigned long start, unsigned long size) { + struct range range = arch_get_mappable_range(); int ret; - if (start + size > VMEM_MAX_PHYS || + if (start < range.start || + start + size > range.end + 1 || start + size < start) return -ERANGE; diff --git a/arch/s390/tools/opcodes.txt b/arch/s390/tools/opcodes.txt index 46d8ed96cf0644b506964bd935beec43ca7e1500..0e207c46e8da1021971937a5e21bc3824738058f 100644 --- a/arch/s390/tools/opcodes.txt +++ b/arch/s390/tools/opcodes.txt @@ -597,7 +597,7 @@ b9b3 cu42 RRE_RR b9bd trtre RRF_U0RR b9be srstu RRE_RR b9bf trte RRF_U0RR -b9c0 selhhhr RRF_RURR +b9c0 selfhr RRF_RURR b9c8 ahhhr RRF_R0RR2 b9c9 shhhr RRF_R0RR2 b9ca alhhhr RRF_R0RR2 diff --git a/arch/sh/Kconfig b/arch/sh/Kconfig index 7ac847ca6356e89d16c3c218cebeec0d6867da62..e798e55915c2337f875a525fbeb8db58400c31ad 100644 --- a/arch/sh/Kconfig +++ b/arch/sh/Kconfig @@ -54,6 +54,7 @@ config SUPERH select HAVE_PERF_EVENTS select HAVE_REGS_AND_STACK_ACCESS_API select HAVE_UID16 + select HAVE_SOFTIRQ_ON_OWN_STACK if IRQSTACKS select HAVE_STACKPROTECTOR select HAVE_SYSCALL_TRACEPOINTS select IRQ_FORCED_THREADING diff --git a/arch/sh/boards/mach-landisk/gio.c b/arch/sh/boards/mach-landisk/gio.c index 1c0da99dfc6059d882e6491738d452d8ed8f0017..ff2200fec29a76391801bceb5e38a498a81271bb 100644 --- a/arch/sh/boards/mach-landisk/gio.c +++ b/arch/sh/boards/mach-landisk/gio.c @@ -27,11 +27,10 @@ static int openCnt; static int gio_open(struct inode *inode, struct file *filp) { - int minor; + int minor = iminor(inode); int ret = -ENOENT; preempt_disable(); - minor = MINOR(inode->i_rdev); if (minor < DEVCOUNT) { if (openCnt > 0) { ret = -EALREADY; @@ -46,9 +45,8 @@ static int gio_open(struct inode *inode, struct file *filp) static int gio_close(struct inode *inode, struct file *filp) { - int minor; + int minor = iminor(inode); - minor = MINOR(inode->i_rdev); if (minor < DEVCOUNT) { openCnt--; } diff --git a/arch/sh/configs/edosk7760_defconfig b/arch/sh/configs/edosk7760_defconfig index 02ba622985769d146153a97aabaa9c03b1cda9d3..d77f54e906fd04fece65ad96cbd6a3fa402db36f 100644 --- a/arch/sh/configs/edosk7760_defconfig +++ b/arch/sh/configs/edosk7760_defconfig @@ -102,7 +102,6 @@ CONFIG_NLS_UTF8=y CONFIG_PRINTK_TIME=y # CONFIG_ENABLE_MUST_CHECK is not set CONFIG_MAGIC_SYSRQ=y -CONFIG_UNUSED_SYMBOLS=y CONFIG_DEBUG_KERNEL=y CONFIG_DEBUG_SHIRQ=y CONFIG_DETECT_HUNG_TASK=y diff --git a/arch/sh/configs/sdk7780_defconfig b/arch/sh/configs/sdk7780_defconfig index d00376eb044f8ab45c1bfaf44e1be3922011fa7b..6c719ab4332ac9eba787eaff34310db8f8bc27bc 100644 --- a/arch/sh/configs/sdk7780_defconfig +++ b/arch/sh/configs/sdk7780_defconfig @@ -128,7 +128,6 @@ CONFIG_NLS_ISO8859_15=y CONFIG_NLS_UTF8=y # CONFIG_ENABLE_MUST_CHECK is not set CONFIG_MAGIC_SYSRQ=y -CONFIG_UNUSED_SYMBOLS=y CONFIG_DEBUG_KERNEL=y CONFIG_DETECT_HUNG_TASK=y # CONFIG_SCHED_DEBUG is not set diff --git a/arch/sh/include/asm/irq.h b/arch/sh/include/asm/irq.h index 6d44c32ef0475beef268773dca2f2fdea8a2d795..839551ce398c5bfb4c2c4d16f5624c0013ff5014 100644 --- a/arch/sh/include/asm/irq.h +++ b/arch/sh/include/asm/irq.h @@ -51,7 +51,6 @@ asmlinkage int do_IRQ(unsigned int irq, struct pt_regs *regs); #ifdef CONFIG_IRQSTACKS extern void irq_ctx_init(int cpu); extern void irq_ctx_exit(int cpu); -# define __ARCH_HAS_DO_SOFTIRQ #else # define irq_ctx_init(cpu) do { } while (0) # define irq_ctx_exit(cpu) do { } while (0) diff --git a/arch/sh/kernel/irq.c b/arch/sh/kernel/irq.c index ab5f790b0cd2778465530f38959721678a25b632..ef0f0827cf5756beb32e38d887520efe7edae6ff 100644 --- a/arch/sh/kernel/irq.c +++ b/arch/sh/kernel/irq.c @@ -20,6 +20,7 @@ #include #include #include +#include atomic_t irq_err_count; diff --git a/arch/sh/kernel/process_32.c b/arch/sh/kernel/process_32.c index 80a5d1c66a5175ffa4af49fe8ddd086d41eac978..1aa508eb0823ae3d0ffd70437d22227a585b21b9 100644 --- a/arch/sh/kernel/process_32.c +++ b/arch/sh/kernel/process_32.c @@ -114,7 +114,7 @@ int copy_thread(unsigned long clone_flags, unsigned long usp, unsigned long arg, childregs = task_pt_regs(p); p->thread.sp = (unsigned long) childregs; - if (unlikely(p->flags & PF_KTHREAD)) { + if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { memset(childregs, 0, sizeof(struct pt_regs)); p->thread.pc = (unsigned long) ret_from_kernel_thread; childregs->regs[4] = arg; diff --git a/arch/sh/kernel/syscalls/Makefile b/arch/sh/kernel/syscalls/Makefile index 659faefdcb1da5f0ffbafc40fd598dd98d9a5fde..285aaba832d93936c3f2b7edb2938dbfe13fc21f 100644 --- a/arch/sh/kernel/syscalls/Makefile +++ b/arch/sh/kernel/syscalls/Makefile @@ -5,7 +5,7 @@ uapi := arch/$(SRCARCH)/include/generated/uapi/asm _dummy := $(shell [ -d '$(uapi)' ] || mkdir -p '$(uapi)') \ $(shell [ -d '$(kapi)' ] || mkdir -p '$(kapi)') -syscall := $(srctree)/$(src)/syscall.tbl +syscall := $(src)/syscall.tbl syshdr := $(srctree)/$(src)/syscallhdr.sh systbl := $(srctree)/$(src)/syscalltbl.sh @@ -21,18 +21,19 @@ quiet_cmd_systbl = SYSTBL $@ '$(systbl_abi_$(basetarget))' \ '$(systbl_offset_$(basetarget))' -$(uapi)/unistd_32.h: $(syscall) $(syshdr) +$(uapi)/unistd_32.h: $(syscall) $(syshdr) FORCE $(call if_changed,syshdr) -$(kapi)/syscall_table.h: $(syscall) $(systbl) +$(kapi)/syscall_table.h: $(syscall) $(systbl) FORCE $(call if_changed,systbl) uapisyshdr-y += unistd_32.h kapisyshdr-y += syscall_table.h -targets += $(uapisyshdr-y) $(kapisyshdr-y) +uapisyshdr-y := $(addprefix $(uapi)/, $(uapisyshdr-y)) +kapisyshdr-y := $(addprefix $(kapi)/, $(kapisyshdr-y)) +targets += $(addprefix ../../../../, $(uapisyshdr-y) $(kapisyshdr-y)) PHONY += all -all: $(addprefix $(uapi)/,$(uapisyshdr-y)) -all: $(addprefix $(kapi)/,$(kapisyshdr-y)) +all: $(uapisyshdr-y) $(kapisyshdr-y) @: diff --git a/arch/sh/kernel/syscalls/syscall.tbl b/arch/sh/kernel/syscalls/syscall.tbl index 9df40ac0ebc05f584b43dc23777f027cddeff6b3..d08eebad6b7f394cf3bd10a469a179edd14368e3 100644 --- a/arch/sh/kernel/syscalls/syscall.tbl +++ b/arch/sh/kernel/syscalls/syscall.tbl @@ -444,3 +444,4 @@ 439 common faccessat2 sys_faccessat2 440 common process_madvise sys_process_madvise 441 common epoll_pwait2 sys_epoll_pwait2 +442 common mount_setattr sys_mount_setattr diff --git a/arch/sparc/Kconfig b/arch/sparc/Kconfig index 2c1cee9eed730ed84e8191108faac56dde2ee249..164a5254c91c0d5224b8ce93dc29e496a599fe3a 100644 --- a/arch/sparc/Kconfig +++ b/arch/sparc/Kconfig @@ -96,6 +96,7 @@ config SPARC64 select ARCH_HAS_PTE_SPECIAL select PCI_DOMAINS if PCI select ARCH_HAS_GIGANTIC_PAGE + select HAVE_SOFTIRQ_ON_OWN_STACK config ARCH_PROC_KCORE_TEXT def_bool y @@ -175,7 +176,7 @@ config SMP Management" code will be disabled if you say Y here. See also and the SMP-HOWTO - available at . + available at . If you don't know what to do here, say N. diff --git a/arch/sparc/boot/piggyback.c b/arch/sparc/boot/piggyback.c index a7a38fb4ece0f2308ea29125bfcdfc3aeea61ac2..6d74064add0a8828a3817ca54ffa71f605d2e5b1 100644 --- a/arch/sparc/boot/piggyback.c +++ b/arch/sparc/boot/piggyback.c @@ -154,6 +154,10 @@ static off_t get_hdrs_offset(int kernelfd, const char *filename) offset -= LOOKBACK; /* skip a.out header */ offset += AOUT_TEXT_OFFSET; + if (offset < 0) { + errno = -EINVAL; + die("Calculated a negative offset, probably elftoaout generated an invalid image. Did you use a recent elftoaout ?"); + } if (lseek(kernelfd, offset, SEEK_SET) < 0) die("lseek"); if (read(kernelfd, buffer, BUFSIZE) != BUFSIZE) diff --git a/arch/sparc/configs/sparc64_defconfig b/arch/sparc/configs/sparc64_defconfig index d91eb6a76dd1fa703dbdbae2b126e950976ab907..148f44b33890e220039f1f345b8ccfaf4a9f0006 100644 --- a/arch/sparc/configs/sparc64_defconfig +++ b/arch/sparc/configs/sparc64_defconfig @@ -65,9 +65,8 @@ CONFIG_CDROM_PKTCDVD=m CONFIG_CDROM_PKTCDVD_WCACHE=y CONFIG_ATA_OVER_ETH=m CONFIG_SUNVDC=m -CONFIG_IDE=y -CONFIG_BLK_DEV_IDECD=y -CONFIG_BLK_DEV_ALI15X3=y +CONFIG_ATA=y +CONFIG_PATA_ALI=y CONFIG_RAID_ATTRS=m CONFIG_SCSI=y CONFIG_BLK_DEV_SD=y @@ -235,3 +234,9 @@ CONFIG_CRYPTO_TWOFISH=m CONFIG_CRC16=m CONFIG_LIBCRC32C=m CONFIG_VCC=m +CONFIG_ATA=y +CONFIG_PATA_CMD64X=y +CONFIG_HAPPYMEAL=y +CONFIG_IP_PNP=y +CONFIG_IP_PNP_DHCP=y +CONFIG_DEVTMPFS=y diff --git a/arch/sparc/include/asm/backoff.h b/arch/sparc/include/asm/backoff.h index 8625946d8d00651d01c7275c99560dfc19e13195..597a22953bc5b54ef0eca2e7a74050af0dc5e8bd 100644 --- a/arch/sparc/include/asm/backoff.h +++ b/arch/sparc/include/asm/backoff.h @@ -18,7 +18,7 @@ * * When we spin, we try to use an operation that will cause the * current cpu strand to block, and therefore make the core fully - * available to any other other runnable strands. There are two + * available to any other runnable strands. There are two * options, based upon cpu capabilities. * * On all cpus prior to SPARC-T4 we do three dummy reads of the diff --git a/arch/sparc/include/asm/cmpxchg_32.h b/arch/sparc/include/asm/cmpxchg_32.h index c73b5a3ab7b91f971d7ca9f3f09e7814fd113856..a53d744d4212507520c7b15958b8014f874caa4c 100644 --- a/arch/sparc/include/asm/cmpxchg_32.h +++ b/arch/sparc/include/asm/cmpxchg_32.h @@ -25,7 +25,7 @@ static inline unsigned long __xchg(unsigned long x, __volatile__ void * ptr, int return x; } -#define xchg(ptr,x) ((__typeof__(*(ptr)))__xchg((unsigned long)(x),(ptr),sizeof(*(ptr)))) +#define xchg(ptr,x) ({(__typeof__(*(ptr)))__xchg((unsigned long)(x),(ptr),sizeof(*(ptr)));}) /* Emulate cmpxchg() the same way we emulate atomics, * by hashing the object address and indexing into an array diff --git a/arch/sparc/include/asm/irq_64.h b/arch/sparc/include/asm/irq_64.h index 4d748e93b97426e7f4148d2548b5cebcad876d77..154df2cf19f42d8fdf1ea2bc49a56a31fdf09398 100644 --- a/arch/sparc/include/asm/irq_64.h +++ b/arch/sparc/include/asm/irq_64.h @@ -93,7 +93,6 @@ void arch_trigger_cpumask_backtrace(const struct cpumask *mask, extern void *hardirq_stack[NR_CPUS]; extern void *softirq_stack[NR_CPUS]; -#define __ARCH_HAS_DO_SOFTIRQ #define NO_IRQ 0xffffffff diff --git a/arch/sparc/include/asm/mman.h b/arch/sparc/include/asm/mman.h index f94532f25db14014b06ef2b69055e6daf96f828e..274217e7ed70206dfd32591ac3fb30735bc83ccb 100644 --- a/arch/sparc/include/asm/mman.h +++ b/arch/sparc/include/asm/mman.h @@ -57,35 +57,39 @@ static inline int sparc_validate_prot(unsigned long prot, unsigned long addr) { if (prot & ~(PROT_READ | PROT_WRITE | PROT_EXEC | PROT_SEM | PROT_ADI)) return 0; - if (prot & PROT_ADI) { - if (!adi_capable()) - return 0; + return 1; +} - if (addr) { - struct vm_area_struct *vma; +#define arch_validate_flags(vm_flags) arch_validate_flags(vm_flags) +/* arch_validate_flags() - Ensure combination of flags is valid for a + * VMA. + */ +static inline bool arch_validate_flags(unsigned long vm_flags) +{ + /* If ADI is being enabled on this VMA, check for ADI + * capability on the platform and ensure VMA is suitable + * for ADI + */ + if (vm_flags & VM_SPARC_ADI) { + if (!adi_capable()) + return false; - vma = find_vma(current->mm, addr); - if (vma) { - /* ADI can not be enabled on PFN - * mapped pages - */ - if (vma->vm_flags & (VM_PFNMAP | VM_MIXEDMAP)) - return 0; + /* ADI can not be enabled on PFN mapped pages */ + if (vm_flags & (VM_PFNMAP | VM_MIXEDMAP)) + return false; - /* Mergeable pages can become unmergeable - * if ADI is enabled on them even if they - * have identical data on them. This can be - * because ADI enabled pages with identical - * data may still not have identical ADI - * tags on them. Disallow ADI on mergeable - * pages. - */ - if (vma->vm_flags & VM_MERGEABLE) - return 0; - } - } + /* Mergeable pages can become unmergeable + * if ADI is enabled on them even if they + * have identical data on them. This can be + * because ADI enabled pages with identical + * data may still not have identical ADI + * tags on them. Disallow ADI on mergeable + * pages. + */ + if (vm_flags & VM_MERGEABLE) + return false; } - return 1; + return true; } #endif /* CONFIG_SPARC64 */ diff --git a/arch/sparc/include/asm/pgtsrmmu.h b/arch/sparc/include/asm/pgtsrmmu.h index 7708d015712b63e95ec7690f77e8937f25bae4a5..6067925972d9ddaf99dd83cd0845960cd64c908f 100644 --- a/arch/sparc/include/asm/pgtsrmmu.h +++ b/arch/sparc/include/asm/pgtsrmmu.h @@ -113,7 +113,7 @@ extern unsigned long last_valid_pfn; extern void *srmmu_nocache_pool; #define __nocache_pa(VADDR) (((unsigned long)VADDR) - SRMMU_NOCACHE_VADDR + __pa((unsigned long)srmmu_nocache_pool)) #define __nocache_va(PADDR) (__va((unsigned long)PADDR) - (unsigned long)srmmu_nocache_pool + SRMMU_NOCACHE_VADDR) -#define __nocache_fix(VADDR) __va(__nocache_pa(VADDR)) +#define __nocache_fix(VADDR) ((__typeof__(VADDR))__va(__nocache_pa(VADDR))) /* Accessing the MMU control register. */ unsigned int srmmu_get_mmureg(void); diff --git a/arch/sparc/include/asm/signal.h b/arch/sparc/include/asm/signal.h index 827b73a97f8a7e48418408ff4011571da2c49573..28f81081e37daf2ad5f986a0bc9223ced3a208be 100644 --- a/arch/sparc/include/asm/signal.h +++ b/arch/sparc/include/asm/signal.h @@ -9,18 +9,6 @@ #include #ifndef __ASSEMBLY__ -/* - * DJHR - * SA_STATIC_ALLOC is used for the sparc32 system to indicate that this - * interrupt handler's irq structure should be statically allocated - * by the request_irq routine. - * The alternative is that arch/sparc/kernel/irq.c has carnal knowledge - * of interrupt usage and that sucks. Also without a flag like this - * it may be possible for the free_irq routine to attempt to free - * statically allocated data.. which is NOT GOOD. - * - */ -#define SA_STATIC_ALLOC 0x8000 #define __ARCH_HAS_KA_RESTORER #define __ARCH_HAS_SA_RESTORER diff --git a/arch/sparc/kernel/entry.S b/arch/sparc/kernel/entry.S index d58940280f8d690c1b22f99b195e7601c5605b3d..a269ad2fe6dfeae29115a8757f07ad5396ea8ada 100644 --- a/arch/sparc/kernel/entry.S +++ b/arch/sparc/kernel/entry.S @@ -994,7 +994,7 @@ do_syscall: andcc %l5, _TIF_SYSCALL_TRACE, %g0 mov %i4, %o4 bne linux_syscall_trace - mov %i0, %l5 + mov %i0, %l6 2: call %l7 mov %i5, %o5 @@ -1003,16 +1003,15 @@ do_syscall: st %o0, [%sp + STACKFRAME_SZ + PT_I0] ret_sys_call: - ld [%curptr + TI_FLAGS], %l6 + ld [%curptr + TI_FLAGS], %l5 cmp %o0, -ERESTART_RESTARTBLOCK ld [%sp + STACKFRAME_SZ + PT_PSR], %g3 set PSR_C, %g2 bgeu 1f - andcc %l6, _TIF_SYSCALL_TRACE, %g0 + andcc %l5, _TIF_SYSCALL_TRACE, %g0 /* System call success, clear Carry condition code. */ andn %g3, %g2, %g3 - clr %l6 st %g3, [%sp + STACKFRAME_SZ + PT_PSR] bne linux_syscall_trace2 ld [%sp + STACKFRAME_SZ + PT_NPC], %l1 /* pc = npc */ @@ -1027,7 +1026,6 @@ ret_sys_call: sub %g0, %o0, %o0 or %g3, %g2, %g3 st %o0, [%sp + STACKFRAME_SZ + PT_I0] - mov 1, %l6 st %g3, [%sp + STACKFRAME_SZ + PT_PSR] bne linux_syscall_trace2 ld [%sp + STACKFRAME_SZ + PT_NPC], %l1 /* pc = npc */ diff --git a/arch/sparc/kernel/irq_64.c b/arch/sparc/kernel/irq_64.c index 3ec9f1402aad357bacb23e73e65801e3ee58253d..c8848bb681a11537497d63e1f029c12fdeac2cee 100644 --- a/arch/sparc/kernel/irq_64.c +++ b/arch/sparc/kernel/irq_64.c @@ -42,6 +42,7 @@ #include #include #include +#include #include "entry.h" #include "cpumap.h" diff --git a/arch/sparc/kernel/led.c b/arch/sparc/kernel/led.c index bd48575172c323e2f21696b16de3b078c7ddb76a..3a66e62eb2a0e6949d0d3af828ca8e20da7cb32a 100644 --- a/arch/sparc/kernel/led.c +++ b/arch/sparc/kernel/led.c @@ -50,6 +50,7 @@ static void led_blink(struct timer_list *unused) add_timer(&led_blink_timer); } +#ifdef CONFIG_PROC_FS static int led_proc_show(struct seq_file *m, void *v) { if (get_auxio() & AUXIO_LED) @@ -111,6 +112,7 @@ static const struct proc_ops led_proc_ops = { .proc_release = single_release, .proc_write = led_proc_write, }; +#endif static struct proc_dir_entry *led; diff --git a/arch/sparc/kernel/pci.c b/arch/sparc/kernel/pci.c index 5d45b6d766d6a2a350c30957769b1d097dbe2888..9c2b720bfd20d784907171668c4db7ce7d8e9fa1 100644 --- a/arch/sparc/kernel/pci.c +++ b/arch/sparc/kernel/pci.c @@ -552,9 +552,8 @@ static void pci_of_scan_bus(struct pci_pbm_info *pbm, pci_info(bus, "scan_bus[%pOF] bus no %d\n", node, bus->number); - child = NULL; prev_devfn = -1; - while ((child = of_get_next_child(node, child)) != NULL) { + for_each_child_of_node(node, child) { if (ofpci_verbose) pci_info(bus, " * %pOF\n", child); reg = of_get_property(child, "reg", ®len); diff --git a/arch/sparc/kernel/process_32.c b/arch/sparc/kernel/process_32.c index a023637359154e9bbd874a574bd9f95ba50c5b10..b91e88058e0ce39675eb095c9c4876e4ea44ad97 100644 --- a/arch/sparc/kernel/process_32.c +++ b/arch/sparc/kernel/process_32.c @@ -183,7 +183,7 @@ void exit_thread(struct task_struct *tsk) #ifndef CONFIG_SMP if (last_task_used_math == tsk) { #else - if (test_ti_thread_flag(task_thread_info(tsk), TIF_USEDFPU)) { + if (test_tsk_thread_flag(tsk, TIF_USEDFPU)) { #endif /* Keep process from leaving FPU in a bogon state. */ put_psr(get_psr() | PSR_EF); @@ -309,7 +309,7 @@ int copy_thread(unsigned long clone_flags, unsigned long sp, unsigned long arg, ti->ksp = (unsigned long) new_stack; p->thread.kregs = childregs; - if (unlikely(p->flags & PF_KTHREAD)) { + if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { extern int nwindows; unsigned long psr; memset(new_stack, 0, STACKFRAME_SZ + TRACEREG_SZ); diff --git a/arch/sparc/kernel/process_64.c b/arch/sparc/kernel/process_64.c index 6f8c7822fc065d394c0e573aa5eae10851a34542..7afd0a859a78c8aa0389de0e6409df7defa7adc7 100644 --- a/arch/sparc/kernel/process_64.c +++ b/arch/sparc/kernel/process_64.c @@ -597,7 +597,7 @@ int copy_thread(unsigned long clone_flags, unsigned long sp, unsigned long arg, sizeof(struct sparc_stackf)); t->fpsaved[0] = 0; - if (unlikely(p->flags & PF_KTHREAD)) { + if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { memset(child_trap_frame, 0, child_stack_sz); __thread_flag_byte_ptr(t)[TI_FLAG_BYTE_CWP] = (current_pt_regs()->tstate + 1) & TSTATE_CWP; diff --git a/arch/sparc/kernel/rtrap_32.S b/arch/sparc/kernel/rtrap_32.S index dca8ed81004660458e6961c5270c738340013936..8931fe266346a3275cfffa7e6c3e6de6ec459b0a 100644 --- a/arch/sparc/kernel/rtrap_32.S +++ b/arch/sparc/kernel/rtrap_32.S @@ -75,7 +75,7 @@ signal_p: ld [%sp + STACKFRAME_SZ + PT_PSR], %t_psr mov %g2, %o2 - mov %l5, %o1 + mov %l6, %o1 call do_notify_resume add %sp, STACKFRAME_SZ, %o0 ! pt_regs ptr diff --git a/arch/sparc/kernel/signal_32.c b/arch/sparc/kernel/signal_32.c index 11cf2281b5817a274ab08314522f1057e574fd52..02f3ad55dfe3195c39bdfe5941d63d19f4e2a017 100644 --- a/arch/sparc/kernel/signal_32.c +++ b/arch/sparc/kernel/signal_32.c @@ -400,8 +400,8 @@ static int setup_rt_frame(struct ksignal *ksig, struct pt_regs *regs, else { regs->u_regs[UREG_I7] = (unsigned long)(&(sf->insns[0]) - 2); - /* mov __NR_sigreturn, %g1 */ - err |= __put_user(0x821020d8, &sf->insns[0]); + /* mov __NR_rt_sigreturn, %g1 */ + err |= __put_user(0x82102065, &sf->insns[0]); /* t 0x10 */ err |= __put_user(0x91d02010, &sf->insns[1]); diff --git a/arch/sparc/kernel/syscalls/Makefile b/arch/sparc/kernel/syscalls/Makefile index c22a21c39f308429ccaaa7d3b1d29dc23010a1fb..283f64407b07a40944e842c44a15ed2629b85db9 100644 --- a/arch/sparc/kernel/syscalls/Makefile +++ b/arch/sparc/kernel/syscalls/Makefile @@ -5,7 +5,7 @@ uapi := arch/$(SRCARCH)/include/generated/uapi/asm _dummy := $(shell [ -d '$(uapi)' ] || mkdir -p '$(uapi)') \ $(shell [ -d '$(kapi)' ] || mkdir -p '$(kapi)') -syscall := $(srctree)/$(src)/syscall.tbl +syscall := $(src)/syscall.tbl syshdr := $(srctree)/$(src)/syscallhdr.sh systbl := $(srctree)/$(src)/syscalltbl.sh @@ -22,24 +22,24 @@ quiet_cmd_systbl = SYSTBL $@ '$(systbl_offset_$(basetarget))' syshdr_abis_unistd_32 := common,32 -$(uapi)/unistd_32.h: $(syscall) $(syshdr) +$(uapi)/unistd_32.h: $(syscall) $(syshdr) FORCE $(call if_changed,syshdr) syshdr_abis_unistd_64 := common,64 -$(uapi)/unistd_64.h: $(syscall) $(syshdr) +$(uapi)/unistd_64.h: $(syscall) $(syshdr) FORCE $(call if_changed,syshdr) systbl_abis_syscall_table_32 := common,32 -$(kapi)/syscall_table_32.h: $(syscall) $(systbl) +$(kapi)/syscall_table_32.h: $(syscall) $(systbl) FORCE $(call if_changed,systbl) systbl_abis_syscall_table_64 := common,64 -$(kapi)/syscall_table_64.h: $(syscall) $(systbl) +$(kapi)/syscall_table_64.h: $(syscall) $(systbl) FORCE $(call if_changed,systbl) systbl_abis_syscall_table_c32 := common,32 systbl_abi_syscall_table_c32 := c32 -$(kapi)/syscall_table_c32.h: $(syscall) $(systbl) +$(kapi)/syscall_table_c32.h: $(syscall) $(systbl) FORCE $(call if_changed,systbl) uapisyshdr-y += unistd_32.h unistd_64.h @@ -47,9 +47,10 @@ kapisyshdr-y += syscall_table_32.h \ syscall_table_64.h \ syscall_table_c32.h -targets += $(uapisyshdr-y) $(kapisyshdr-y) +uapisyshdr-y := $(addprefix $(uapi)/, $(uapisyshdr-y)) +kapisyshdr-y := $(addprefix $(kapi)/, $(kapisyshdr-y)) +targets += $(addprefix ../../../../, $(uapisyshdr-y) $(kapisyshdr-y)) PHONY += all -all: $(addprefix $(uapi)/,$(uapisyshdr-y)) -all: $(addprefix $(kapi)/,$(kapisyshdr-y)) +all: $(uapisyshdr-y) $(kapisyshdr-y) @: diff --git a/arch/sparc/kernel/syscalls/syscall.tbl b/arch/sparc/kernel/syscalls/syscall.tbl index 40d8c7cd82984b44ddf9f95074835961114547eb..84403a99039c819c1c43b3600105b5b7cd87a49f 100644 --- a/arch/sparc/kernel/syscalls/syscall.tbl +++ b/arch/sparc/kernel/syscalls/syscall.tbl @@ -487,3 +487,4 @@ 439 common faccessat2 sys_faccessat2 440 common process_madvise sys_process_madvise 441 common epoll_pwait2 sys_epoll_pwait2 compat_sys_epoll_pwait2 +442 common mount_setattr sys_mount_setattr diff --git a/arch/sparc/kernel/viohs.c b/arch/sparc/kernel/viohs.c index 7db5aabe9708576109bd028c241532150339ff59..e27afd233bf52628f89e471638673f81842eaa4b 100644 --- a/arch/sparc/kernel/viohs.c +++ b/arch/sparc/kernel/viohs.c @@ -428,7 +428,7 @@ static int process_dreg_info(struct vio_driver_state *vio, struct vio_dring_register *pkt) { struct vio_dring_state *dr; - int i, len; + int i; viodbg(HS, "GOT DRING_REG INFO ident[%llx] " "ndesc[%u] dsz[%u] opt[0x%x] ncookies[%u]\n", @@ -482,9 +482,7 @@ static int process_dreg_info(struct vio_driver_state *vio, pkt->num_descr, pkt->descr_size, pkt->options, pkt->num_cookies); - len = (sizeof(*pkt) + - (dr->ncookies * sizeof(struct ldc_trans_cookie))); - if (send_ctrl(vio, &pkt->tag, len) < 0) + if (send_ctrl(vio, &pkt->tag, struct_size(pkt, cookies, dr->ncookies)) < 0) goto send_nack; vio->dr_state |= VIO_DR_STATE_RXREG; diff --git a/arch/sparc/lib/memset.S b/arch/sparc/lib/memset.S index b89d42b29e344a4ad9bfff0335c84a20c0e25bd3..f427f34b8b79bd6798fb90f756035d8f989d3909 100644 --- a/arch/sparc/lib/memset.S +++ b/arch/sparc/lib/memset.S @@ -142,6 +142,7 @@ __bzero: ZERO_LAST_BLOCKS(%o0, 0x48, %g2) ZERO_LAST_BLOCKS(%o0, 0x08, %g2) 13: + EXT(12b, 13b, 21f) be 8f andcc %o1, 4, %g0 diff --git a/arch/sparc/mm/init_32.c b/arch/sparc/mm/init_32.c index eb2946b1df8a43a792a08eb083d28127d066049a..6139c5700ccc9cf2ac477af4e795b875b88b6f9f 100644 --- a/arch/sparc/mm/init_32.c +++ b/arch/sparc/mm/init_32.c @@ -197,6 +197,9 @@ unsigned long __init bootmem_init(unsigned long *pages_avail) size = memblock_phys_mem_size() - memblock_reserved_size(); *pages_avail = (size >> PAGE_SHIFT) - high_pages; + /* Only allow low memory to be allocated via memblock allocation */ + memblock_set_current_limit(max_low_pfn << PAGE_SHIFT); + return max_pfn; } diff --git a/arch/sparc/mm/srmmu.c b/arch/sparc/mm/srmmu.c index a03caa5f6628df8432d7a39323e52abba8d67cf8..a9aa6a92c7fee280b5ae408e6b4f65c9e96d36dc 100644 --- a/arch/sparc/mm/srmmu.c +++ b/arch/sparc/mm/srmmu.c @@ -351,7 +351,7 @@ pgtable_t pte_alloc_one(struct mm_struct *mm) pte_t *ptep; struct page *page; - if ((ptep = pte_alloc_one_kernel(mm)) == 0) + if (!(ptep = pte_alloc_one_kernel(mm))) return NULL; page = pfn_to_page(__nocache_pa((unsigned long)ptep) >> PAGE_SHIFT); spin_lock(&mm->page_table_lock); @@ -689,7 +689,7 @@ static void __init srmmu_early_allocate_ptable_skeleton(unsigned long start, pgdp = pgd_offset_k(start); p4dp = p4d_offset(pgdp, start); pudp = pud_offset(p4dp, start); - if (pud_none(*(pud_t *)__nocache_fix(pudp))) { + if (pud_none(*__nocache_fix(pudp))) { pmdp = __srmmu_get_nocache( SRMMU_PMD_TABLE_SIZE, SRMMU_PMD_TABLE_SIZE); if (pmdp == NULL) @@ -698,7 +698,7 @@ static void __init srmmu_early_allocate_ptable_skeleton(unsigned long start, pud_set(__nocache_fix(pudp), pmdp); } pmdp = pmd_offset(__nocache_fix(pudp), start); - if (srmmu_pmd_none(*(pmd_t *)__nocache_fix(pmdp))) { + if (srmmu_pmd_none(*__nocache_fix(pmdp))) { ptep = __srmmu_get_nocache(PTE_SIZE, PTE_SIZE); if (ptep == NULL) early_pgtable_allocfail("pte"); @@ -810,11 +810,11 @@ static void __init srmmu_inherit_prom_mappings(unsigned long start, p4dp = p4d_offset(pgdp, start); pudp = pud_offset(p4dp, start); if (what == 2) { - *(pgd_t *)__nocache_fix(pgdp) = __pgd(probed); + *__nocache_fix(pgdp) = __pgd(probed); start += PGDIR_SIZE; continue; } - if (pud_none(*(pud_t *)__nocache_fix(pudp))) { + if (pud_none(*__nocache_fix(pudp))) { pmdp = __srmmu_get_nocache(SRMMU_PMD_TABLE_SIZE, SRMMU_PMD_TABLE_SIZE); if (pmdp == NULL) @@ -822,13 +822,13 @@ static void __init srmmu_inherit_prom_mappings(unsigned long start, memset(__nocache_fix(pmdp), 0, SRMMU_PMD_TABLE_SIZE); pud_set(__nocache_fix(pudp), pmdp); } - pmdp = pmd_offset(__nocache_fix(pgdp), start); + pmdp = pmd_offset(__nocache_fix(pudp), start); if (what == 1) { *(pmd_t *)__nocache_fix(pmdp) = __pmd(probed); start += PMD_SIZE; continue; } - if (srmmu_pmd_none(*(pmd_t *)__nocache_fix(pmdp))) { + if (srmmu_pmd_none(*__nocache_fix(pmdp))) { ptep = __srmmu_get_nocache(PTE_SIZE, PTE_SIZE); if (ptep == NULL) early_pgtable_allocfail("pte"); @@ -836,7 +836,7 @@ static void __init srmmu_inherit_prom_mappings(unsigned long start, pmd_set(__nocache_fix(pmdp), ptep); } ptep = pte_offset_kernel(__nocache_fix(pmdp), start); - *(pte_t *)__nocache_fix(ptep) = __pte(probed); + *__nocache_fix(ptep) = __pte(probed); start += PAGE_SIZE; } } @@ -850,7 +850,7 @@ static void __init do_large_mapping(unsigned long vaddr, unsigned long phys_base unsigned long big_pte; big_pte = KERNEL_PTE(phys_base >> 4); - *(pgd_t *)__nocache_fix(pgdp) = __pgd(big_pte); + *__nocache_fix(pgdp) = __pgd(big_pte); } /* Map sp_bank entry SP_ENTRY, starting at virtual address VBASE. */ @@ -940,7 +940,7 @@ void __init srmmu_paging_init(void) srmmu_ctx_table_phys = (ctxd_t *)__nocache_pa(srmmu_context_table); for (i = 0; i < num_contexts; i++) - srmmu_ctxd_set((ctxd_t *)__nocache_fix(&srmmu_context_table[i]), srmmu_swapper_pg_dir); + srmmu_ctxd_set(__nocache_fix(&srmmu_context_table[i]), srmmu_swapper_pg_dir); flush_cache_all(); srmmu_set_ctable_ptr((unsigned long)srmmu_ctx_table_phys); diff --git a/arch/um/include/asm/Kbuild b/arch/um/include/asm/Kbuild index a58811dc054cdecc36dbe3c60289161fa65b360b..d7492e5a1bbba093d09cce79bcb83396a2e0625a 100644 --- a/arch/um/include/asm/Kbuild +++ b/arch/um/include/asm/Kbuild @@ -20,6 +20,7 @@ generic-y += param.h generic-y += pci.h generic-y += percpu.h generic-y += preempt.h +generic-y += softirq_stack.h generic-y += switch_to.h generic-y += topology.h generic-y += trace_clock.h diff --git a/arch/um/kernel/process.c b/arch/um/kernel/process.c index 81d508daf67cbc4299e28899a8811dfb143180d5..c5011064b5dd4207c564cd7e8a10e45e0be63249 100644 --- a/arch/um/kernel/process.c +++ b/arch/um/kernel/process.c @@ -157,7 +157,7 @@ int copy_thread(unsigned long clone_flags, unsigned long sp, unsigned long arg, struct task_struct * p, unsigned long tls) { void (*handler)(void); - int kthread = current->flags & PF_KTHREAD; + int kthread = current->flags & (PF_KTHREAD | PF_IO_WORKER); int ret = 0; p->thread = (struct thread_struct) INIT_THREAD; diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 595193bc2d31e32f39d19dd767310ef51087f1dc..2792879d398ee41fa0d4858815b33e18664297ae 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -97,6 +97,8 @@ config X86 select ARCH_SUPPORTS_DEBUG_PAGEALLOC select ARCH_SUPPORTS_NUMA_BALANCING if X86_64 select ARCH_SUPPORTS_KMAP_LOCAL_FORCE_MAP if NR_CPUS <= 4096 + select ARCH_SUPPORTS_LTO_CLANG if X86_64 + select ARCH_SUPPORTS_LTO_CLANG_THIN if X86_64 select ARCH_USE_BUILTIN_BSWAP select ARCH_USE_QUEUED_RWLOCKS select ARCH_USE_QUEUED_SPINLOCKS @@ -149,6 +151,7 @@ config X86 select HAVE_ARCH_JUMP_LABEL_RELATIVE select HAVE_ARCH_KASAN if X86_64 select HAVE_ARCH_KASAN_VMALLOC if X86_64 + select HAVE_ARCH_KFENCE select HAVE_ARCH_KGDB select HAVE_ARCH_MMAP_RND_BITS if MMU select HAVE_ARCH_MMAP_RND_COMPAT_BITS if MMU && COMPAT @@ -169,6 +172,7 @@ config X86 select HAVE_CONTEXT_TRACKING if X86_64 select HAVE_CONTEXT_TRACKING_OFFSTACK if HAVE_CONTEXT_TRACKING select HAVE_C_RECORDMCOUNT + select HAVE_OBJTOOL_MCOUNT if STACK_VALIDATION select HAVE_DEBUG_KMEMLEAK select HAVE_DMA_CONTIGUOUS select HAVE_DYNAMIC_FTRACE @@ -188,6 +192,7 @@ config X86 select HAVE_HW_BREAKPOINT select HAVE_IDE select HAVE_IOREMAP_PROT + select HAVE_IRQ_EXIT_ON_IRQ_STACK if X86_64 select HAVE_IRQ_TIME_ACCOUNTING select HAVE_KERNEL_BZIP2 select HAVE_KERNEL_GZIP @@ -220,6 +225,7 @@ config X86 select HAVE_REGS_AND_STACK_ACCESS_API select HAVE_RELIABLE_STACKTRACE if X86_64 && (UNWINDER_FRAME_POINTER || UNWINDER_ORC) && STACK_VALIDATION select HAVE_FUNCTION_ARG_ACCESS_API + select HAVE_SOFTIRQ_ON_OWN_STACK select HAVE_STACKPROTECTOR if CC_HAS_SANE_STACKPROTECTOR select HAVE_STACK_VALIDATION if X86_64 select HAVE_STATIC_CALL @@ -445,7 +451,7 @@ config X86_X2APIC If you don't know what to do here, say N. config X86_MPPARSE - bool "Enable MPS table" if ACPI || SFI + bool "Enable MPS table" if ACPI default y depends on X86_LOCAL_APIC help @@ -604,7 +610,6 @@ config X86_INTEL_MID depends on PCI depends on X86_64 || (PCI_GOANY && X86_32) depends on X86_IO_APIC - select SFI select I2C select DW_APB_TIMER select APB_TIMER @@ -893,18 +898,6 @@ config HPET_EMULATE_RTC def_bool y depends on HPET_TIMER && (RTC_DRV_CMOS=m || RTC_DRV_CMOS=y) -config APB_TIMER - def_bool y if X86_INTEL_MID - prompt "Intel MID APB Timer Support" if X86_INTEL_MID - select DW_APB_TIMER - depends on X86_INTEL_MID && SFI - help - APB timer is the replacement for 8254, HPET on X86 MID platforms. - The APBT provides a stable time base on SMP - systems, unlike the TSC, but it is more expensive to access, - as it is off-chip. APB timers are always running regardless of CPU - C states, they are used as per CPU clockevent device when possible. - # Mark as expert because too many people got it wrong. # The code disables itself when not needed. config DMI @@ -2466,8 +2459,6 @@ source "kernel/power/Kconfig" source "drivers/acpi/Kconfig" -source "drivers/sfi/Kconfig" - config X86_APM_BOOT def_bool y depends on APM @@ -2654,7 +2645,7 @@ config PCI_DIRECT config PCI_MMCONFIG bool "Support mmconfig PCI config space access" if X86_64 default y - depends on PCI && (ACPI || SFI || JAILHOUSE_GUEST) + depends on PCI && (ACPI || JAILHOUSE_GUEST) depends on X86_64 || (PCI_GOANY || PCI_GOMMCONFIG) config PCI_OLPC diff --git a/arch/x86/Makefile b/arch/x86/Makefile index b797f1561943912a4a8fc28cfc44bc664ac1627b..2d6d5a28c3bf791b586d50b8bcb18039c0b62ffc 100644 --- a/arch/x86/Makefile +++ b/arch/x86/Makefile @@ -169,6 +169,11 @@ ifeq ($(ACCUMULATE_OUTGOING_ARGS), 1) KBUILD_CFLAGS += $(call cc-option,-maccumulate-outgoing-args,) endif +ifdef CONFIG_LTO_CLANG +KBUILD_LDFLAGS += -plugin-opt=-code-model=kernel \ + -plugin-opt=-stack-alignment=$(if $(CONFIG_X86_32),4,8) +endif + # Workaround for a gcc prelease that unfortunately was shipped in a suse release KBUILD_CFLAGS += -Wno-sign-compare # diff --git a/arch/x86/configs/i386_defconfig b/arch/x86/configs/i386_defconfig index 78210793d357cf027b8a44419ec3b9447bdad98a..9c9c4a888b1dbf65dfe29b3b91a6666d2dff14cd 100644 --- a/arch/x86/configs/i386_defconfig +++ b/arch/x86/configs/i386_defconfig @@ -50,7 +50,6 @@ CONFIG_JUMP_LABEL=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y CONFIG_MODULE_FORCE_UNLOAD=y -# CONFIG_UNUSED_SYMBOLS is not set CONFIG_BINFMT_MISC=y CONFIG_NET=y CONFIG_PACKET=y diff --git a/arch/x86/configs/x86_64_defconfig b/arch/x86/configs/x86_64_defconfig index 9936528e19393afad0f87a89196e43dd2c52e171..b60bd2d8603499a88dcf0bedb164154cb818c982 100644 --- a/arch/x86/configs/x86_64_defconfig +++ b/arch/x86/configs/x86_64_defconfig @@ -48,7 +48,6 @@ CONFIG_JUMP_LABEL=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y CONFIG_MODULE_FORCE_UNLOAD=y -# CONFIG_UNUSED_SYMBOLS is not set CONFIG_BINFMT_MISC=y CONFIG_NET=y CONFIG_PACKET=y diff --git a/arch/x86/crypto/Makefile b/arch/x86/crypto/Makefile index a31de0c6ccde2111874b25da5594d9454d3e2be0..b28e36b7c96bef7baa7f8924a1e705b964b123f8 100644 --- a/arch/x86/crypto/Makefile +++ b/arch/x86/crypto/Makefile @@ -4,8 +4,6 @@ OBJECT_FILES_NON_STANDARD := y -obj-$(CONFIG_CRYPTO_GLUE_HELPER_X86) += glue_helper.o - obj-$(CONFIG_CRYPTO_TWOFISH_586) += twofish-i586.o twofish-i586-y := twofish-i586-asm_32.o twofish_glue.o obj-$(CONFIG_CRYPTO_TWOFISH_X86_64) += twofish-x86_64.o diff --git a/arch/x86/crypto/aesni-intel_asm.S b/arch/x86/crypto/aesni-intel_asm.S index d1436c37008b4ed6379c5b94a3d80d18a21b94ea..4e3972570916ed973c9abdeb2609069275715bc6 100644 --- a/arch/x86/crypto/aesni-intel_asm.S +++ b/arch/x86/crypto/aesni-intel_asm.S @@ -43,10 +43,6 @@ #ifdef __x86_64__ # constants in mergeable sections, linker can reorder and merge -.section .rodata.cst16.gf128mul_x_ble_mask, "aM", @progbits, 16 -.align 16 -.Lgf128mul_x_ble_mask: - .octa 0x00000000000000010000000000000087 .section .rodata.cst16.POLY, "aM", @progbits, 16 .align 16 POLY: .octa 0xC2000000000000000000000000000001 @@ -146,7 +142,7 @@ ALL_F: .octa 0xffffffffffffffffffffffffffffffff #define CTR %xmm11 #define INC %xmm12 -#define GF128MUL_MASK %xmm10 +#define GF128MUL_MASK %xmm7 #ifdef __x86_64__ #define AREG %rax @@ -2577,13 +2573,140 @@ SYM_FUNC_START(aesni_cbc_dec) ret SYM_FUNC_END(aesni_cbc_dec) -#ifdef __x86_64__ +/* + * void aesni_cts_cbc_enc(struct crypto_aes_ctx *ctx, const u8 *dst, u8 *src, + * size_t len, u8 *iv) + */ +SYM_FUNC_START(aesni_cts_cbc_enc) + FRAME_BEGIN +#ifndef __x86_64__ + pushl IVP + pushl LEN + pushl KEYP + pushl KLEN + movl (FRAME_OFFSET+20)(%esp), KEYP # ctx + movl (FRAME_OFFSET+24)(%esp), OUTP # dst + movl (FRAME_OFFSET+28)(%esp), INP # src + movl (FRAME_OFFSET+32)(%esp), LEN # len + movl (FRAME_OFFSET+36)(%esp), IVP # iv + lea .Lcts_permute_table, T1 +#else + lea .Lcts_permute_table(%rip), T1 +#endif + mov 480(KEYP), KLEN + movups (IVP), STATE + sub $16, LEN + mov T1, IVP + add $32, IVP + add LEN, T1 + sub LEN, IVP + movups (T1), %xmm4 + movups (IVP), %xmm5 + + movups (INP), IN1 + add LEN, INP + movups (INP), IN2 + + pxor IN1, STATE + call _aesni_enc1 + + pshufb %xmm5, IN2 + pxor STATE, IN2 + pshufb %xmm4, STATE + add OUTP, LEN + movups STATE, (LEN) + + movaps IN2, STATE + call _aesni_enc1 + movups STATE, (OUTP) + +#ifndef __x86_64__ + popl KLEN + popl KEYP + popl LEN + popl IVP +#endif + FRAME_END + ret +SYM_FUNC_END(aesni_cts_cbc_enc) + +/* + * void aesni_cts_cbc_dec(struct crypto_aes_ctx *ctx, const u8 *dst, u8 *src, + * size_t len, u8 *iv) + */ +SYM_FUNC_START(aesni_cts_cbc_dec) + FRAME_BEGIN +#ifndef __x86_64__ + pushl IVP + pushl LEN + pushl KEYP + pushl KLEN + movl (FRAME_OFFSET+20)(%esp), KEYP # ctx + movl (FRAME_OFFSET+24)(%esp), OUTP # dst + movl (FRAME_OFFSET+28)(%esp), INP # src + movl (FRAME_OFFSET+32)(%esp), LEN # len + movl (FRAME_OFFSET+36)(%esp), IVP # iv + lea .Lcts_permute_table, T1 +#else + lea .Lcts_permute_table(%rip), T1 +#endif + mov 480(KEYP), KLEN + add $240, KEYP + movups (IVP), IV + sub $16, LEN + mov T1, IVP + add $32, IVP + add LEN, T1 + sub LEN, IVP + movups (T1), %xmm4 + + movups (INP), STATE + add LEN, INP + movups (INP), IN1 + + call _aesni_dec1 + movaps STATE, IN2 + pshufb %xmm4, STATE + pxor IN1, STATE + + add OUTP, LEN + movups STATE, (LEN) + + movups (IVP), %xmm0 + pshufb %xmm0, IN1 + pblendvb IN2, IN1 + movaps IN1, STATE + call _aesni_dec1 + + pxor IV, STATE + movups STATE, (OUTP) + +#ifndef __x86_64__ + popl KLEN + popl KEYP + popl LEN + popl IVP +#endif + FRAME_END + ret +SYM_FUNC_END(aesni_cts_cbc_dec) + .pushsection .rodata .align 16 +.Lcts_permute_table: + .byte 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80 + .byte 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80 + .byte 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07 + .byte 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f + .byte 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80 + .byte 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80 +#ifdef __x86_64__ .Lbswap_mask: .byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 +#endif .popsection +#ifdef __x86_64__ /* * _aesni_inc_init: internal ABI * setup registers used by _aesni_inc @@ -2696,6 +2819,14 @@ SYM_FUNC_START(aesni_ctr_enc) ret SYM_FUNC_END(aesni_ctr_enc) +#endif + +.section .rodata.cst16.gf128mul_x_ble_mask, "aM", @progbits, 16 +.align 16 +.Lgf128mul_x_ble_mask: + .octa 0x00000000000000010000000000000087 +.previous + /* * _aesni_gf128mul_x_ble: internal ABI * Multiply in GF(2^128) for XTS IVs @@ -2708,120 +2839,325 @@ SYM_FUNC_END(aesni_ctr_enc) * CTR: == temporary value */ #define _aesni_gf128mul_x_ble() \ - pshufd $0x13, IV, CTR; \ + pshufd $0x13, IV, KEY; \ paddq IV, IV; \ - psrad $31, CTR; \ - pand GF128MUL_MASK, CTR; \ - pxor CTR, IV; + psrad $31, KEY; \ + pand GF128MUL_MASK, KEY; \ + pxor KEY, IV; /* - * void aesni_xts_crypt8(const struct crypto_aes_ctx *ctx, u8 *dst, - * const u8 *src, bool enc, le128 *iv) + * void aesni_xts_encrypt(const struct crypto_aes_ctx *ctx, u8 *dst, + * const u8 *src, unsigned int len, le128 *iv) */ -SYM_FUNC_START(aesni_xts_crypt8) +SYM_FUNC_START(aesni_xts_encrypt) FRAME_BEGIN - testb %cl, %cl - movl $0, %ecx - movl $240, %r10d - leaq _aesni_enc4, %r11 - leaq _aesni_dec4, %rax - cmovel %r10d, %ecx - cmoveq %rax, %r11 - +#ifndef __x86_64__ + pushl IVP + pushl LEN + pushl KEYP + pushl KLEN + movl (FRAME_OFFSET+20)(%esp), KEYP # ctx + movl (FRAME_OFFSET+24)(%esp), OUTP # dst + movl (FRAME_OFFSET+28)(%esp), INP # src + movl (FRAME_OFFSET+32)(%esp), LEN # len + movl (FRAME_OFFSET+36)(%esp), IVP # iv movdqa .Lgf128mul_x_ble_mask, GF128MUL_MASK +#else + movdqa .Lgf128mul_x_ble_mask(%rip), GF128MUL_MASK +#endif movups (IVP), IV mov 480(KEYP), KLEN - addq %rcx, KEYP + +.Lxts_enc_loop4: + sub $64, LEN + jl .Lxts_enc_1x movdqa IV, STATE1 - movdqu 0x00(INP), INC - pxor INC, STATE1 + movdqu 0x00(INP), IN + pxor IN, STATE1 movdqu IV, 0x00(OUTP) _aesni_gf128mul_x_ble() movdqa IV, STATE2 - movdqu 0x10(INP), INC - pxor INC, STATE2 + movdqu 0x10(INP), IN + pxor IN, STATE2 movdqu IV, 0x10(OUTP) _aesni_gf128mul_x_ble() movdqa IV, STATE3 - movdqu 0x20(INP), INC - pxor INC, STATE3 + movdqu 0x20(INP), IN + pxor IN, STATE3 movdqu IV, 0x20(OUTP) _aesni_gf128mul_x_ble() movdqa IV, STATE4 - movdqu 0x30(INP), INC - pxor INC, STATE4 + movdqu 0x30(INP), IN + pxor IN, STATE4 movdqu IV, 0x30(OUTP) - CALL_NOSPEC r11 + call _aesni_enc4 - movdqu 0x00(OUTP), INC - pxor INC, STATE1 + movdqu 0x00(OUTP), IN + pxor IN, STATE1 movdqu STATE1, 0x00(OUTP) + movdqu 0x10(OUTP), IN + pxor IN, STATE2 + movdqu STATE2, 0x10(OUTP) + + movdqu 0x20(OUTP), IN + pxor IN, STATE3 + movdqu STATE3, 0x20(OUTP) + + movdqu 0x30(OUTP), IN + pxor IN, STATE4 + movdqu STATE4, 0x30(OUTP) + _aesni_gf128mul_x_ble() - movdqa IV, STATE1 - movdqu 0x40(INP), INC - pxor INC, STATE1 - movdqu IV, 0x40(OUTP) - movdqu 0x10(OUTP), INC - pxor INC, STATE2 - movdqu STATE2, 0x10(OUTP) + add $64, INP + add $64, OUTP + test LEN, LEN + jnz .Lxts_enc_loop4 +.Lxts_enc_ret_iv: + movups IV, (IVP) + +.Lxts_enc_ret: +#ifndef __x86_64__ + popl KLEN + popl KEYP + popl LEN + popl IVP +#endif + FRAME_END + ret + +.Lxts_enc_1x: + add $64, LEN + jz .Lxts_enc_ret_iv + sub $16, LEN + jl .Lxts_enc_cts4 + +.Lxts_enc_loop1: + movdqu (INP), STATE + pxor IV, STATE + call _aesni_enc1 + pxor IV, STATE _aesni_gf128mul_x_ble() - movdqa IV, STATE2 - movdqu 0x50(INP), INC - pxor INC, STATE2 - movdqu IV, 0x50(OUTP) - movdqu 0x20(OUTP), INC - pxor INC, STATE3 - movdqu STATE3, 0x20(OUTP) + test LEN, LEN + jz .Lxts_enc_out + + add $16, INP + sub $16, LEN + jl .Lxts_enc_cts1 + + movdqu STATE, (OUTP) + add $16, OUTP + jmp .Lxts_enc_loop1 + +.Lxts_enc_out: + movdqu STATE, (OUTP) + jmp .Lxts_enc_ret_iv + +.Lxts_enc_cts4: + movdqa STATE4, STATE + sub $16, OUTP + +.Lxts_enc_cts1: +#ifndef __x86_64__ + lea .Lcts_permute_table, T1 +#else + lea .Lcts_permute_table(%rip), T1 +#endif + add LEN, INP /* rewind input pointer */ + add $16, LEN /* # bytes in final block */ + movups (INP), IN1 + + mov T1, IVP + add $32, IVP + add LEN, T1 + sub LEN, IVP + add OUTP, LEN + + movups (T1), %xmm4 + movaps STATE, IN2 + pshufb %xmm4, STATE + movups STATE, (LEN) + + movups (IVP), %xmm0 + pshufb %xmm0, IN1 + pblendvb IN2, IN1 + movaps IN1, STATE + + pxor IV, STATE + call _aesni_enc1 + pxor IV, STATE + + movups STATE, (OUTP) + jmp .Lxts_enc_ret +SYM_FUNC_END(aesni_xts_encrypt) + +/* + * void aesni_xts_decrypt(const struct crypto_aes_ctx *ctx, u8 *dst, + * const u8 *src, unsigned int len, le128 *iv) + */ +SYM_FUNC_START(aesni_xts_decrypt) + FRAME_BEGIN +#ifndef __x86_64__ + pushl IVP + pushl LEN + pushl KEYP + pushl KLEN + movl (FRAME_OFFSET+20)(%esp), KEYP # ctx + movl (FRAME_OFFSET+24)(%esp), OUTP # dst + movl (FRAME_OFFSET+28)(%esp), INP # src + movl (FRAME_OFFSET+32)(%esp), LEN # len + movl (FRAME_OFFSET+36)(%esp), IVP # iv + movdqa .Lgf128mul_x_ble_mask, GF128MUL_MASK +#else + movdqa .Lgf128mul_x_ble_mask(%rip), GF128MUL_MASK +#endif + movups (IVP), IV + + mov 480(KEYP), KLEN + add $240, KEYP + + test $15, LEN + jz .Lxts_dec_loop4 + sub $16, LEN + +.Lxts_dec_loop4: + sub $64, LEN + jl .Lxts_dec_1x + + movdqa IV, STATE1 + movdqu 0x00(INP), IN + pxor IN, STATE1 + movdqu IV, 0x00(OUTP) _aesni_gf128mul_x_ble() - movdqa IV, STATE3 - movdqu 0x60(INP), INC - pxor INC, STATE3 - movdqu IV, 0x60(OUTP) + movdqa IV, STATE2 + movdqu 0x10(INP), IN + pxor IN, STATE2 + movdqu IV, 0x10(OUTP) - movdqu 0x30(OUTP), INC - pxor INC, STATE4 - movdqu STATE4, 0x30(OUTP) + _aesni_gf128mul_x_ble() + movdqa IV, STATE3 + movdqu 0x20(INP), IN + pxor IN, STATE3 + movdqu IV, 0x20(OUTP) _aesni_gf128mul_x_ble() movdqa IV, STATE4 - movdqu 0x70(INP), INC - pxor INC, STATE4 - movdqu IV, 0x70(OUTP) + movdqu 0x30(INP), IN + pxor IN, STATE4 + movdqu IV, 0x30(OUTP) - _aesni_gf128mul_x_ble() - movups IV, (IVP) + call _aesni_dec4 + + movdqu 0x00(OUTP), IN + pxor IN, STATE1 + movdqu STATE1, 0x00(OUTP) + + movdqu 0x10(OUTP), IN + pxor IN, STATE2 + movdqu STATE2, 0x10(OUTP) - CALL_NOSPEC r11 + movdqu 0x20(OUTP), IN + pxor IN, STATE3 + movdqu STATE3, 0x20(OUTP) - movdqu 0x40(OUTP), INC - pxor INC, STATE1 - movdqu STATE1, 0x40(OUTP) + movdqu 0x30(OUTP), IN + pxor IN, STATE4 + movdqu STATE4, 0x30(OUTP) - movdqu 0x50(OUTP), INC - pxor INC, STATE2 - movdqu STATE2, 0x50(OUTP) + _aesni_gf128mul_x_ble() - movdqu 0x60(OUTP), INC - pxor INC, STATE3 - movdqu STATE3, 0x60(OUTP) + add $64, INP + add $64, OUTP + test LEN, LEN + jnz .Lxts_dec_loop4 - movdqu 0x70(OUTP), INC - pxor INC, STATE4 - movdqu STATE4, 0x70(OUTP) +.Lxts_dec_ret_iv: + movups IV, (IVP) +.Lxts_dec_ret: +#ifndef __x86_64__ + popl KLEN + popl KEYP + popl LEN + popl IVP +#endif FRAME_END ret -SYM_FUNC_END(aesni_xts_crypt8) +.Lxts_dec_1x: + add $64, LEN + jz .Lxts_dec_ret_iv + +.Lxts_dec_loop1: + movdqu (INP), STATE + + add $16, INP + sub $16, LEN + jl .Lxts_dec_cts1 + + pxor IV, STATE + call _aesni_dec1 + pxor IV, STATE + _aesni_gf128mul_x_ble() + + test LEN, LEN + jz .Lxts_dec_out + + movdqu STATE, (OUTP) + add $16, OUTP + jmp .Lxts_dec_loop1 + +.Lxts_dec_out: + movdqu STATE, (OUTP) + jmp .Lxts_dec_ret_iv + +.Lxts_dec_cts1: + movdqa IV, STATE4 + _aesni_gf128mul_x_ble() + + pxor IV, STATE + call _aesni_dec1 + pxor IV, STATE + +#ifndef __x86_64__ + lea .Lcts_permute_table, T1 +#else + lea .Lcts_permute_table(%rip), T1 #endif + add LEN, INP /* rewind input pointer */ + add $16, LEN /* # bytes in final block */ + movups (INP), IN1 + + mov T1, IVP + add $32, IVP + add LEN, T1 + sub LEN, IVP + add OUTP, LEN + + movups (T1), %xmm4 + movaps STATE, IN2 + pshufb %xmm4, STATE + movups STATE, (LEN) + + movups (IVP), %xmm0 + pshufb %xmm0, IN1 + pblendvb IN2, IN1 + movaps IN1, STATE + + pxor STATE4, STATE + call _aesni_dec1 + pxor STATE4, STATE + + movups STATE, (OUTP) + jmp .Lxts_dec_ret +SYM_FUNC_END(aesni_xts_decrypt) diff --git a/arch/x86/crypto/aesni-intel_glue.c b/arch/x86/crypto/aesni-intel_glue.c index ad8a7188a2bf757a0e455ae0e812b375a12db8e4..2144e54a6c89212ded6eddb65e2e3bb3027030ea 100644 --- a/arch/x86/crypto/aesni-intel_glue.c +++ b/arch/x86/crypto/aesni-intel_glue.c @@ -31,11 +31,10 @@ #include #include #include +#include #include #include -#ifdef CONFIG_X86_64 -#include -#endif +#include #define AESNI_ALIGN 16 @@ -93,62 +92,25 @@ asmlinkage void aesni_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out, const u8 *in, unsigned int len, u8 *iv); asmlinkage void aesni_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out, const u8 *in, unsigned int len, u8 *iv); +asmlinkage void aesni_cts_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out, + const u8 *in, unsigned int len, u8 *iv); +asmlinkage void aesni_cts_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out, + const u8 *in, unsigned int len, u8 *iv); #define AVX_GEN2_OPTSIZE 640 #define AVX_GEN4_OPTSIZE 4096 +asmlinkage void aesni_xts_encrypt(const struct crypto_aes_ctx *ctx, u8 *out, + const u8 *in, unsigned int len, u8 *iv); + +asmlinkage void aesni_xts_decrypt(const struct crypto_aes_ctx *ctx, u8 *out, + const u8 *in, unsigned int len, u8 *iv); + #ifdef CONFIG_X86_64 -static void (*aesni_ctr_enc_tfm)(struct crypto_aes_ctx *ctx, u8 *out, - const u8 *in, unsigned int len, u8 *iv); asmlinkage void aesni_ctr_enc(struct crypto_aes_ctx *ctx, u8 *out, const u8 *in, unsigned int len, u8 *iv); - -asmlinkage void aesni_xts_crypt8(const struct crypto_aes_ctx *ctx, u8 *out, - const u8 *in, bool enc, le128 *iv); - -/* asmlinkage void aesni_gcm_enc() - * void *ctx, AES Key schedule. Starts on a 16 byte boundary. - * struct gcm_context_data. May be uninitialized. - * u8 *out, Ciphertext output. Encrypt in-place is allowed. - * const u8 *in, Plaintext input - * unsigned long plaintext_len, Length of data in bytes for encryption. - * u8 *iv, Pre-counter block j0: 12 byte IV concatenated with 0x00000001. - * 16-byte aligned pointer. - * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary. - * const u8 *aad, Additional Authentication Data (AAD) - * unsigned long aad_len, Length of AAD in bytes. - * u8 *auth_tag, Authenticated Tag output. - * unsigned long auth_tag_len), Authenticated Tag Length in bytes. - * Valid values are 16 (most likely), 12 or 8. - */ -asmlinkage void aesni_gcm_enc(void *ctx, - struct gcm_context_data *gdata, u8 *out, - const u8 *in, unsigned long plaintext_len, u8 *iv, - u8 *hash_subkey, const u8 *aad, unsigned long aad_len, - u8 *auth_tag, unsigned long auth_tag_len); - -/* asmlinkage void aesni_gcm_dec() - * void *ctx, AES Key schedule. Starts on a 16 byte boundary. - * struct gcm_context_data. May be uninitialized. - * u8 *out, Plaintext output. Decrypt in-place is allowed. - * const u8 *in, Ciphertext input - * unsigned long ciphertext_len, Length of data in bytes for decryption. - * u8 *iv, Pre-counter block j0: 12 byte IV concatenated with 0x00000001. - * 16-byte aligned pointer. - * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary. - * const u8 *aad, Additional Authentication Data (AAD) - * unsigned long aad_len, Length of AAD in bytes. With RFC4106 this is going - * to be 8 or 12 bytes - * u8 *auth_tag, Authenticated Tag output. - * unsigned long auth_tag_len) Authenticated Tag Length in bytes. - * Valid values are 16 (most likely), 12 or 8. - */ -asmlinkage void aesni_gcm_dec(void *ctx, - struct gcm_context_data *gdata, u8 *out, - const u8 *in, unsigned long ciphertext_len, u8 *iv, - u8 *hash_subkey, const u8 *aad, unsigned long aad_len, - u8 *auth_tag, unsigned long auth_tag_len); +DEFINE_STATIC_CALL(aesni_ctr_enc_tfm, aesni_ctr_enc); /* Scatter / Gather routines, with args similar to above */ asmlinkage void aesni_gcm_init(void *ctx, @@ -167,24 +129,6 @@ asmlinkage void aesni_gcm_finalize(void *ctx, struct gcm_context_data *gdata, u8 *auth_tag, unsigned long auth_tag_len); -static const struct aesni_gcm_tfm_s { - void (*init)(void *ctx, struct gcm_context_data *gdata, u8 *iv, - u8 *hash_subkey, const u8 *aad, unsigned long aad_len); - void (*enc_update)(void *ctx, struct gcm_context_data *gdata, u8 *out, - const u8 *in, unsigned long plaintext_len); - void (*dec_update)(void *ctx, struct gcm_context_data *gdata, u8 *out, - const u8 *in, unsigned long ciphertext_len); - void (*finalize)(void *ctx, struct gcm_context_data *gdata, - u8 *auth_tag, unsigned long auth_tag_len); -} *aesni_gcm_tfm; - -static const struct aesni_gcm_tfm_s aesni_gcm_tfm_sse = { - .init = &aesni_gcm_init, - .enc_update = &aesni_gcm_enc_update, - .dec_update = &aesni_gcm_dec_update, - .finalize = &aesni_gcm_finalize, -}; - asmlinkage void aes_ctr_enc_128_avx_by8(const u8 *in, u8 *iv, void *keys, u8 *out, unsigned int num_bytes); asmlinkage void aes_ctr_enc_192_avx_by8(const u8 *in, u8 *iv, @@ -214,25 +158,6 @@ asmlinkage void aesni_gcm_finalize_avx_gen2(void *ctx, struct gcm_context_data *gdata, u8 *auth_tag, unsigned long auth_tag_len); -asmlinkage void aesni_gcm_enc_avx_gen2(void *ctx, - struct gcm_context_data *gdata, u8 *out, - const u8 *in, unsigned long plaintext_len, u8 *iv, - const u8 *aad, unsigned long aad_len, - u8 *auth_tag, unsigned long auth_tag_len); - -asmlinkage void aesni_gcm_dec_avx_gen2(void *ctx, - struct gcm_context_data *gdata, u8 *out, - const u8 *in, unsigned long ciphertext_len, u8 *iv, - const u8 *aad, unsigned long aad_len, - u8 *auth_tag, unsigned long auth_tag_len); - -static const struct aesni_gcm_tfm_s aesni_gcm_tfm_avx_gen2 = { - .init = &aesni_gcm_init_avx_gen2, - .enc_update = &aesni_gcm_enc_update_avx_gen2, - .dec_update = &aesni_gcm_dec_update_avx_gen2, - .finalize = &aesni_gcm_finalize_avx_gen2, -}; - /* * asmlinkage void aesni_gcm_init_avx_gen4() * gcm_data *my_ctx_data, context data @@ -256,24 +181,8 @@ asmlinkage void aesni_gcm_finalize_avx_gen4(void *ctx, struct gcm_context_data *gdata, u8 *auth_tag, unsigned long auth_tag_len); -asmlinkage void aesni_gcm_enc_avx_gen4(void *ctx, - struct gcm_context_data *gdata, u8 *out, - const u8 *in, unsigned long plaintext_len, u8 *iv, - const u8 *aad, unsigned long aad_len, - u8 *auth_tag, unsigned long auth_tag_len); - -asmlinkage void aesni_gcm_dec_avx_gen4(void *ctx, - struct gcm_context_data *gdata, u8 *out, - const u8 *in, unsigned long ciphertext_len, u8 *iv, - const u8 *aad, unsigned long aad_len, - u8 *auth_tag, unsigned long auth_tag_len); - -static const struct aesni_gcm_tfm_s aesni_gcm_tfm_avx_gen4 = { - .init = &aesni_gcm_init_avx_gen4, - .enc_update = &aesni_gcm_enc_update_avx_gen4, - .dec_update = &aesni_gcm_dec_update_avx_gen4, - .finalize = &aesni_gcm_finalize_avx_gen4, -}; +static __ro_after_init DEFINE_STATIC_KEY_FALSE(gcm_use_avx); +static __ro_after_init DEFINE_STATIC_KEY_FALSE(gcm_use_avx2); static inline struct aesni_rfc4106_gcm_ctx *aesni_rfc4106_gcm_ctx_get(struct crypto_aead *tfm) @@ -374,16 +283,16 @@ static int ecb_encrypt(struct skcipher_request *req) unsigned int nbytes; int err; - err = skcipher_walk_virt(&walk, req, true); + err = skcipher_walk_virt(&walk, req, false); - kernel_fpu_begin(); while ((nbytes = walk.nbytes)) { + kernel_fpu_begin(); aesni_ecb_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr, nbytes & AES_BLOCK_MASK); + kernel_fpu_end(); nbytes &= AES_BLOCK_SIZE - 1; err = skcipher_walk_done(&walk, nbytes); } - kernel_fpu_end(); return err; } @@ -396,16 +305,16 @@ static int ecb_decrypt(struct skcipher_request *req) unsigned int nbytes; int err; - err = skcipher_walk_virt(&walk, req, true); + err = skcipher_walk_virt(&walk, req, false); - kernel_fpu_begin(); while ((nbytes = walk.nbytes)) { + kernel_fpu_begin(); aesni_ecb_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr, nbytes & AES_BLOCK_MASK); + kernel_fpu_end(); nbytes &= AES_BLOCK_SIZE - 1; err = skcipher_walk_done(&walk, nbytes); } - kernel_fpu_end(); return err; } @@ -418,16 +327,16 @@ static int cbc_encrypt(struct skcipher_request *req) unsigned int nbytes; int err; - err = skcipher_walk_virt(&walk, req, true); + err = skcipher_walk_virt(&walk, req, false); - kernel_fpu_begin(); while ((nbytes = walk.nbytes)) { + kernel_fpu_begin(); aesni_cbc_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr, nbytes & AES_BLOCK_MASK, walk.iv); + kernel_fpu_end(); nbytes &= AES_BLOCK_SIZE - 1; err = skcipher_walk_done(&walk, nbytes); } - kernel_fpu_end(); return err; } @@ -440,36 +349,133 @@ static int cbc_decrypt(struct skcipher_request *req) unsigned int nbytes; int err; - err = skcipher_walk_virt(&walk, req, true); + err = skcipher_walk_virt(&walk, req, false); - kernel_fpu_begin(); while ((nbytes = walk.nbytes)) { + kernel_fpu_begin(); aesni_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr, nbytes & AES_BLOCK_MASK, walk.iv); + kernel_fpu_end(); nbytes &= AES_BLOCK_SIZE - 1; err = skcipher_walk_done(&walk, nbytes); } - kernel_fpu_end(); return err; } -#ifdef CONFIG_X86_64 -static void ctr_crypt_final(struct crypto_aes_ctx *ctx, - struct skcipher_walk *walk) +static int cts_cbc_encrypt(struct skcipher_request *req) { - u8 *ctrblk = walk->iv; - u8 keystream[AES_BLOCK_SIZE]; - u8 *src = walk->src.virt.addr; - u8 *dst = walk->dst.virt.addr; - unsigned int nbytes = walk->nbytes; + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm)); + int cbc_blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2; + struct scatterlist *src = req->src, *dst = req->dst; + struct scatterlist sg_src[2], sg_dst[2]; + struct skcipher_request subreq; + struct skcipher_walk walk; + int err; + + skcipher_request_set_tfm(&subreq, tfm); + skcipher_request_set_callback(&subreq, skcipher_request_flags(req), + NULL, NULL); + + if (req->cryptlen <= AES_BLOCK_SIZE) { + if (req->cryptlen < AES_BLOCK_SIZE) + return -EINVAL; + cbc_blocks = 1; + } + + if (cbc_blocks > 0) { + skcipher_request_set_crypt(&subreq, req->src, req->dst, + cbc_blocks * AES_BLOCK_SIZE, + req->iv); + + err = cbc_encrypt(&subreq); + if (err) + return err; + + if (req->cryptlen == AES_BLOCK_SIZE) + return 0; + + dst = src = scatterwalk_ffwd(sg_src, req->src, subreq.cryptlen); + if (req->dst != req->src) + dst = scatterwalk_ffwd(sg_dst, req->dst, + subreq.cryptlen); + } + + /* handle ciphertext stealing */ + skcipher_request_set_crypt(&subreq, src, dst, + req->cryptlen - cbc_blocks * AES_BLOCK_SIZE, + req->iv); + + err = skcipher_walk_virt(&walk, &subreq, false); + if (err) + return err; + + kernel_fpu_begin(); + aesni_cts_cbc_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr, + walk.nbytes, walk.iv); + kernel_fpu_end(); + + return skcipher_walk_done(&walk, 0); +} + +static int cts_cbc_decrypt(struct skcipher_request *req) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm)); + int cbc_blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2; + struct scatterlist *src = req->src, *dst = req->dst; + struct scatterlist sg_src[2], sg_dst[2]; + struct skcipher_request subreq; + struct skcipher_walk walk; + int err; + + skcipher_request_set_tfm(&subreq, tfm); + skcipher_request_set_callback(&subreq, skcipher_request_flags(req), + NULL, NULL); + + if (req->cryptlen <= AES_BLOCK_SIZE) { + if (req->cryptlen < AES_BLOCK_SIZE) + return -EINVAL; + cbc_blocks = 1; + } + + if (cbc_blocks > 0) { + skcipher_request_set_crypt(&subreq, req->src, req->dst, + cbc_blocks * AES_BLOCK_SIZE, + req->iv); + + err = cbc_decrypt(&subreq); + if (err) + return err; + + if (req->cryptlen == AES_BLOCK_SIZE) + return 0; + + dst = src = scatterwalk_ffwd(sg_src, req->src, subreq.cryptlen); + if (req->dst != req->src) + dst = scatterwalk_ffwd(sg_dst, req->dst, + subreq.cryptlen); + } + + /* handle ciphertext stealing */ + skcipher_request_set_crypt(&subreq, src, dst, + req->cryptlen - cbc_blocks * AES_BLOCK_SIZE, + req->iv); - aesni_enc(ctx, keystream, ctrblk); - crypto_xor_cpy(dst, keystream, src, nbytes); + err = skcipher_walk_virt(&walk, &subreq, false); + if (err) + return err; + + kernel_fpu_begin(); + aesni_cts_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr, + walk.nbytes, walk.iv); + kernel_fpu_end(); - crypto_inc(ctrblk, AES_BLOCK_SIZE); + return skcipher_walk_done(&walk, 0); } +#ifdef CONFIG_X86_64 static void aesni_ctr_enc_avx_tfm(struct crypto_aes_ctx *ctx, u8 *out, const u8 *in, unsigned int len, u8 *iv) { @@ -491,120 +497,36 @@ static int ctr_crypt(struct skcipher_request *req) { struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); struct crypto_aes_ctx *ctx = aes_ctx(crypto_skcipher_ctx(tfm)); + u8 keystream[AES_BLOCK_SIZE]; struct skcipher_walk walk; unsigned int nbytes; int err; - err = skcipher_walk_virt(&walk, req, true); + err = skcipher_walk_virt(&walk, req, false); - kernel_fpu_begin(); - while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) { - aesni_ctr_enc_tfm(ctx, walk.dst.virt.addr, walk.src.virt.addr, - nbytes & AES_BLOCK_MASK, walk.iv); - nbytes &= AES_BLOCK_SIZE - 1; + while ((nbytes = walk.nbytes) > 0) { + kernel_fpu_begin(); + if (nbytes & AES_BLOCK_MASK) + static_call(aesni_ctr_enc_tfm)(ctx, walk.dst.virt.addr, + walk.src.virt.addr, + nbytes & AES_BLOCK_MASK, + walk.iv); + nbytes &= ~AES_BLOCK_MASK; + + if (walk.nbytes == walk.total && nbytes > 0) { + aesni_enc(ctx, keystream, walk.iv); + crypto_xor_cpy(walk.dst.virt.addr + walk.nbytes - nbytes, + walk.src.virt.addr + walk.nbytes - nbytes, + keystream, nbytes); + crypto_inc(walk.iv, AES_BLOCK_SIZE); + nbytes = 0; + } + kernel_fpu_end(); err = skcipher_walk_done(&walk, nbytes); } - if (walk.nbytes) { - ctr_crypt_final(ctx, &walk); - err = skcipher_walk_done(&walk, 0); - } - kernel_fpu_end(); - return err; } -static int xts_aesni_setkey(struct crypto_skcipher *tfm, const u8 *key, - unsigned int keylen) -{ - struct aesni_xts_ctx *ctx = crypto_skcipher_ctx(tfm); - int err; - - err = xts_verify_key(tfm, key, keylen); - if (err) - return err; - - keylen /= 2; - - /* first half of xts-key is for crypt */ - err = aes_set_key_common(crypto_skcipher_tfm(tfm), ctx->raw_crypt_ctx, - key, keylen); - if (err) - return err; - - /* second half of xts-key is for tweak */ - return aes_set_key_common(crypto_skcipher_tfm(tfm), ctx->raw_tweak_ctx, - key + keylen, keylen); -} - - -static void aesni_xts_enc(const void *ctx, u8 *dst, const u8 *src, le128 *iv) -{ - glue_xts_crypt_128bit_one(ctx, dst, src, iv, aesni_enc); -} - -static void aesni_xts_dec(const void *ctx, u8 *dst, const u8 *src, le128 *iv) -{ - glue_xts_crypt_128bit_one(ctx, dst, src, iv, aesni_dec); -} - -static void aesni_xts_enc8(const void *ctx, u8 *dst, const u8 *src, le128 *iv) -{ - aesni_xts_crypt8(ctx, dst, src, true, iv); -} - -static void aesni_xts_dec8(const void *ctx, u8 *dst, const u8 *src, le128 *iv) -{ - aesni_xts_crypt8(ctx, dst, src, false, iv); -} - -static const struct common_glue_ctx aesni_enc_xts = { - .num_funcs = 2, - .fpu_blocks_limit = 1, - - .funcs = { { - .num_blocks = 8, - .fn_u = { .xts = aesni_xts_enc8 } - }, { - .num_blocks = 1, - .fn_u = { .xts = aesni_xts_enc } - } } -}; - -static const struct common_glue_ctx aesni_dec_xts = { - .num_funcs = 2, - .fpu_blocks_limit = 1, - - .funcs = { { - .num_blocks = 8, - .fn_u = { .xts = aesni_xts_dec8 } - }, { - .num_blocks = 1, - .fn_u = { .xts = aesni_xts_dec } - } } -}; - -static int xts_encrypt(struct skcipher_request *req) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct aesni_xts_ctx *ctx = crypto_skcipher_ctx(tfm); - - return glue_xts_req_128bit(&aesni_enc_xts, req, aesni_enc, - aes_ctx(ctx->raw_tweak_ctx), - aes_ctx(ctx->raw_crypt_ctx), - false); -} - -static int xts_decrypt(struct skcipher_request *req) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct aesni_xts_ctx *ctx = crypto_skcipher_ctx(tfm); - - return glue_xts_req_128bit(&aesni_dec_xts, req, aesni_enc, - aes_ctx(ctx->raw_tweak_ctx), - aes_ctx(ctx->raw_crypt_ctx), - true); -} - static int rfc4106_set_hash_subkey(u8 *hash_subkey, const u8 *key, unsigned int key_len) { @@ -681,42 +603,35 @@ static int generic_gcmaes_set_authsize(struct crypto_aead *tfm, static int gcmaes_crypt_by_sg(bool enc, struct aead_request *req, unsigned int assoclen, u8 *hash_subkey, - u8 *iv, void *aes_ctx) + u8 *iv, void *aes_ctx, u8 *auth_tag, + unsigned long auth_tag_len) { - struct crypto_aead *tfm = crypto_aead_reqtfm(req); - unsigned long auth_tag_len = crypto_aead_authsize(tfm); - const struct aesni_gcm_tfm_s *gcm_tfm = aesni_gcm_tfm; - struct gcm_context_data data AESNI_ALIGN_ATTR; - struct scatter_walk dst_sg_walk = {}; + u8 databuf[sizeof(struct gcm_context_data) + (AESNI_ALIGN - 8)] __aligned(8); + struct gcm_context_data *data = PTR_ALIGN((void *)databuf, AESNI_ALIGN); unsigned long left = req->cryptlen; - unsigned long len, srclen, dstlen; struct scatter_walk assoc_sg_walk; - struct scatter_walk src_sg_walk; - struct scatterlist src_start[2]; - struct scatterlist dst_start[2]; - struct scatterlist *src_sg; - struct scatterlist *dst_sg; - u8 *src, *dst, *assoc; + struct skcipher_walk walk; + bool do_avx, do_avx2; u8 *assocmem = NULL; - u8 authTag[16]; + u8 *assoc; + int err; if (!enc) left -= auth_tag_len; - if (left < AVX_GEN4_OPTSIZE && gcm_tfm == &aesni_gcm_tfm_avx_gen4) - gcm_tfm = &aesni_gcm_tfm_avx_gen2; - if (left < AVX_GEN2_OPTSIZE && gcm_tfm == &aesni_gcm_tfm_avx_gen2) - gcm_tfm = &aesni_gcm_tfm_sse; + do_avx = (left >= AVX_GEN2_OPTSIZE); + do_avx2 = (left >= AVX_GEN4_OPTSIZE); /* Linearize assoc, if not already linear */ - if (req->src->length >= assoclen && req->src->length && - (!PageHighMem(sg_page(req->src)) || - req->src->offset + req->src->length <= PAGE_SIZE)) { + if (req->src->length >= assoclen && req->src->length) { scatterwalk_start(&assoc_sg_walk, req->src); assoc = scatterwalk_map(&assoc_sg_walk); } else { + gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC; + /* assoc can be any length, so must be on heap */ - assocmem = kmalloc(assoclen, GFP_ATOMIC); + assocmem = kmalloc(assoclen, flags); if (unlikely(!assocmem)) return -ENOMEM; assoc = assocmem; @@ -724,62 +639,15 @@ static int gcmaes_crypt_by_sg(bool enc, struct aead_request *req, scatterwalk_map_and_copy(assoc, req->src, 0, assoclen, 0); } - if (left) { - src_sg = scatterwalk_ffwd(src_start, req->src, req->assoclen); - scatterwalk_start(&src_sg_walk, src_sg); - if (req->src != req->dst) { - dst_sg = scatterwalk_ffwd(dst_start, req->dst, - req->assoclen); - scatterwalk_start(&dst_sg_walk, dst_sg); - } - } - kernel_fpu_begin(); - gcm_tfm->init(aes_ctx, &data, iv, - hash_subkey, assoc, assoclen); - if (req->src != req->dst) { - while (left) { - src = scatterwalk_map(&src_sg_walk); - dst = scatterwalk_map(&dst_sg_walk); - srclen = scatterwalk_clamp(&src_sg_walk, left); - dstlen = scatterwalk_clamp(&dst_sg_walk, left); - len = min(srclen, dstlen); - if (len) { - if (enc) - gcm_tfm->enc_update(aes_ctx, &data, - dst, src, len); - else - gcm_tfm->dec_update(aes_ctx, &data, - dst, src, len); - } - left -= len; - - scatterwalk_unmap(src); - scatterwalk_unmap(dst); - scatterwalk_advance(&src_sg_walk, len); - scatterwalk_advance(&dst_sg_walk, len); - scatterwalk_done(&src_sg_walk, 0, left); - scatterwalk_done(&dst_sg_walk, 1, left); - } - } else { - while (left) { - dst = src = scatterwalk_map(&src_sg_walk); - len = scatterwalk_clamp(&src_sg_walk, left); - if (len) { - if (enc) - gcm_tfm->enc_update(aes_ctx, &data, - src, src, len); - else - gcm_tfm->dec_update(aes_ctx, &data, - src, src, len); - } - left -= len; - scatterwalk_unmap(src); - scatterwalk_advance(&src_sg_walk, len); - scatterwalk_done(&src_sg_walk, 1, left); - } - } - gcm_tfm->finalize(aes_ctx, &data, authTag, auth_tag_len); + if (static_branch_likely(&gcm_use_avx2) && do_avx2) + aesni_gcm_init_avx_gen4(aes_ctx, data, iv, hash_subkey, assoc, + assoclen); + else if (static_branch_likely(&gcm_use_avx) && do_avx) + aesni_gcm_init_avx_gen2(aes_ctx, data, iv, hash_subkey, assoc, + assoclen); + else + aesni_gcm_init(aes_ctx, data, iv, hash_subkey, assoc, assoclen); kernel_fpu_end(); if (!assocmem) @@ -787,24 +655,58 @@ static int gcmaes_crypt_by_sg(bool enc, struct aead_request *req, else kfree(assocmem); - if (!enc) { - u8 authTagMsg[16]; + err = enc ? skcipher_walk_aead_encrypt(&walk, req, false) + : skcipher_walk_aead_decrypt(&walk, req, false); - /* Copy out original authTag */ - scatterwalk_map_and_copy(authTagMsg, req->src, - req->assoclen + req->cryptlen - - auth_tag_len, - auth_tag_len, 0); + while (walk.nbytes > 0) { + kernel_fpu_begin(); + if (static_branch_likely(&gcm_use_avx2) && do_avx2) { + if (enc) + aesni_gcm_enc_update_avx_gen4(aes_ctx, data, + walk.dst.virt.addr, + walk.src.virt.addr, + walk.nbytes); + else + aesni_gcm_dec_update_avx_gen4(aes_ctx, data, + walk.dst.virt.addr, + walk.src.virt.addr, + walk.nbytes); + } else if (static_branch_likely(&gcm_use_avx) && do_avx) { + if (enc) + aesni_gcm_enc_update_avx_gen2(aes_ctx, data, + walk.dst.virt.addr, + walk.src.virt.addr, + walk.nbytes); + else + aesni_gcm_dec_update_avx_gen2(aes_ctx, data, + walk.dst.virt.addr, + walk.src.virt.addr, + walk.nbytes); + } else if (enc) { + aesni_gcm_enc_update(aes_ctx, data, walk.dst.virt.addr, + walk.src.virt.addr, walk.nbytes); + } else { + aesni_gcm_dec_update(aes_ctx, data, walk.dst.virt.addr, + walk.src.virt.addr, walk.nbytes); + } + kernel_fpu_end(); - /* Compare generated tag with passed in tag. */ - return crypto_memneq(authTagMsg, authTag, auth_tag_len) ? - -EBADMSG : 0; + err = skcipher_walk_done(&walk, 0); } - /* Copy in the authTag */ - scatterwalk_map_and_copy(authTag, req->dst, - req->assoclen + req->cryptlen, - auth_tag_len, 1); + if (err) + return err; + + kernel_fpu_begin(); + if (static_branch_likely(&gcm_use_avx2) && do_avx2) + aesni_gcm_finalize_avx_gen4(aes_ctx, data, auth_tag, + auth_tag_len); + else if (static_branch_likely(&gcm_use_avx) && do_avx) + aesni_gcm_finalize_avx_gen2(aes_ctx, data, auth_tag, + auth_tag_len); + else + aesni_gcm_finalize(aes_ctx, data, auth_tag, auth_tag_len); + kernel_fpu_end(); return 0; } @@ -812,15 +714,47 @@ static int gcmaes_crypt_by_sg(bool enc, struct aead_request *req, static int gcmaes_encrypt(struct aead_request *req, unsigned int assoclen, u8 *hash_subkey, u8 *iv, void *aes_ctx) { - return gcmaes_crypt_by_sg(true, req, assoclen, hash_subkey, iv, - aes_ctx); + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + unsigned long auth_tag_len = crypto_aead_authsize(tfm); + u8 auth_tag[16]; + int err; + + err = gcmaes_crypt_by_sg(true, req, assoclen, hash_subkey, iv, aes_ctx, + auth_tag, auth_tag_len); + if (err) + return err; + + scatterwalk_map_and_copy(auth_tag, req->dst, + req->assoclen + req->cryptlen, + auth_tag_len, 1); + return 0; } static int gcmaes_decrypt(struct aead_request *req, unsigned int assoclen, u8 *hash_subkey, u8 *iv, void *aes_ctx) { - return gcmaes_crypt_by_sg(false, req, assoclen, hash_subkey, iv, - aes_ctx); + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + unsigned long auth_tag_len = crypto_aead_authsize(tfm); + u8 auth_tag_msg[16]; + u8 auth_tag[16]; + int err; + + err = gcmaes_crypt_by_sg(false, req, assoclen, hash_subkey, iv, aes_ctx, + auth_tag, auth_tag_len); + if (err) + return err; + + /* Copy out original auth_tag */ + scatterwalk_map_and_copy(auth_tag_msg, req->src, + req->assoclen + req->cryptlen - auth_tag_len, + auth_tag_len, 0); + + /* Compare generated tag with passed in tag. */ + if (crypto_memneq(auth_tag_msg, auth_tag, auth_tag_len)) { + memzero_explicit(auth_tag, sizeof(auth_tag)); + return -EBADMSG; + } + return 0; } static int helper_rfc4106_encrypt(struct aead_request *req) @@ -828,7 +762,8 @@ static int helper_rfc4106_encrypt(struct aead_request *req) struct crypto_aead *tfm = crypto_aead_reqtfm(req); struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm); void *aes_ctx = &(ctx->aes_key_expanded); - u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN))); + u8 ivbuf[16 + (AESNI_ALIGN - 8)] __aligned(8); + u8 *iv = PTR_ALIGN(&ivbuf[0], AESNI_ALIGN); unsigned int i; __be32 counter = cpu_to_be32(1); @@ -855,7 +790,8 @@ static int helper_rfc4106_decrypt(struct aead_request *req) struct crypto_aead *tfm = crypto_aead_reqtfm(req); struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm); void *aes_ctx = &(ctx->aes_key_expanded); - u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN))); + u8 ivbuf[16 + (AESNI_ALIGN - 8)] __aligned(8); + u8 *iv = PTR_ALIGN(&ivbuf[0], AESNI_ALIGN); unsigned int i; if (unlikely(req->assoclen != 16 && req->assoclen != 20)) @@ -877,6 +813,128 @@ static int helper_rfc4106_decrypt(struct aead_request *req) } #endif +static int xts_aesni_setkey(struct crypto_skcipher *tfm, const u8 *key, + unsigned int keylen) +{ + struct aesni_xts_ctx *ctx = crypto_skcipher_ctx(tfm); + int err; + + err = xts_verify_key(tfm, key, keylen); + if (err) + return err; + + keylen /= 2; + + /* first half of xts-key is for crypt */ + err = aes_set_key_common(crypto_skcipher_tfm(tfm), ctx->raw_crypt_ctx, + key, keylen); + if (err) + return err; + + /* second half of xts-key is for tweak */ + return aes_set_key_common(crypto_skcipher_tfm(tfm), ctx->raw_tweak_ctx, + key + keylen, keylen); +} + +static int xts_crypt(struct skcipher_request *req, bool encrypt) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct aesni_xts_ctx *ctx = crypto_skcipher_ctx(tfm); + int tail = req->cryptlen % AES_BLOCK_SIZE; + struct skcipher_request subreq; + struct skcipher_walk walk; + int err; + + if (req->cryptlen < AES_BLOCK_SIZE) + return -EINVAL; + + err = skcipher_walk_virt(&walk, req, false); + + if (unlikely(tail > 0 && walk.nbytes < walk.total)) { + int blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2; + + skcipher_walk_abort(&walk); + + skcipher_request_set_tfm(&subreq, tfm); + skcipher_request_set_callback(&subreq, + skcipher_request_flags(req), + NULL, NULL); + skcipher_request_set_crypt(&subreq, req->src, req->dst, + blocks * AES_BLOCK_SIZE, req->iv); + req = &subreq; + err = skcipher_walk_virt(&walk, req, false); + } else { + tail = 0; + } + + kernel_fpu_begin(); + + /* calculate first value of T */ + aesni_enc(aes_ctx(ctx->raw_tweak_ctx), walk.iv, walk.iv); + + while (walk.nbytes > 0) { + int nbytes = walk.nbytes; + + if (nbytes < walk.total) + nbytes &= ~(AES_BLOCK_SIZE - 1); + + if (encrypt) + aesni_xts_encrypt(aes_ctx(ctx->raw_crypt_ctx), + walk.dst.virt.addr, walk.src.virt.addr, + nbytes, walk.iv); + else + aesni_xts_decrypt(aes_ctx(ctx->raw_crypt_ctx), + walk.dst.virt.addr, walk.src.virt.addr, + nbytes, walk.iv); + kernel_fpu_end(); + + err = skcipher_walk_done(&walk, walk.nbytes - nbytes); + + if (walk.nbytes > 0) + kernel_fpu_begin(); + } + + if (unlikely(tail > 0 && !err)) { + struct scatterlist sg_src[2], sg_dst[2]; + struct scatterlist *src, *dst; + + dst = src = scatterwalk_ffwd(sg_src, req->src, req->cryptlen); + if (req->dst != req->src) + dst = scatterwalk_ffwd(sg_dst, req->dst, req->cryptlen); + + skcipher_request_set_crypt(req, src, dst, AES_BLOCK_SIZE + tail, + req->iv); + + err = skcipher_walk_virt(&walk, &subreq, false); + if (err) + return err; + + kernel_fpu_begin(); + if (encrypt) + aesni_xts_encrypt(aes_ctx(ctx->raw_crypt_ctx), + walk.dst.virt.addr, walk.src.virt.addr, + walk.nbytes, walk.iv); + else + aesni_xts_decrypt(aes_ctx(ctx->raw_crypt_ctx), + walk.dst.virt.addr, walk.src.virt.addr, + walk.nbytes, walk.iv); + kernel_fpu_end(); + + err = skcipher_walk_done(&walk, 0); + } + return err; +} + +static int xts_encrypt(struct skcipher_request *req) +{ + return xts_crypt(req, true); +} + +static int xts_decrypt(struct skcipher_request *req) +{ + return xts_crypt(req, false); +} + static struct crypto_alg aesni_cipher_alg = { .cra_name = "aes", .cra_driver_name = "aes-aesni", @@ -928,6 +986,23 @@ static struct skcipher_alg aesni_skciphers[] = { .setkey = aesni_skcipher_setkey, .encrypt = cbc_encrypt, .decrypt = cbc_decrypt, + }, { + .base = { + .cra_name = "__cts(cbc(aes))", + .cra_driver_name = "__cts-cbc-aes-aesni", + .cra_priority = 400, + .cra_flags = CRYPTO_ALG_INTERNAL, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = CRYPTO_AES_CTX_SIZE, + .cra_module = THIS_MODULE, + }, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .walksize = 2 * AES_BLOCK_SIZE, + .setkey = aesni_skcipher_setkey, + .encrypt = cts_cbc_encrypt, + .decrypt = cts_cbc_decrypt, #ifdef CONFIG_X86_64 }, { .base = { @@ -946,6 +1021,7 @@ static struct skcipher_alg aesni_skciphers[] = { .setkey = aesni_skcipher_setkey, .encrypt = ctr_crypt, .decrypt = ctr_crypt, +#endif }, { .base = { .cra_name = "__xts(aes)", @@ -959,10 +1035,10 @@ static struct skcipher_alg aesni_skciphers[] = { .min_keysize = 2 * AES_MIN_KEY_SIZE, .max_keysize = 2 * AES_MAX_KEY_SIZE, .ivsize = AES_BLOCK_SIZE, + .walksize = 2 * AES_BLOCK_SIZE, .setkey = xts_aesni_setkey, .encrypt = xts_encrypt, .decrypt = xts_decrypt, -#endif } }; @@ -985,7 +1061,8 @@ static int generic_gcmaes_encrypt(struct aead_request *req) struct crypto_aead *tfm = crypto_aead_reqtfm(req); struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(tfm); void *aes_ctx = &(ctx->aes_key_expanded); - u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN))); + u8 ivbuf[16 + (AESNI_ALIGN - 8)] __aligned(8); + u8 *iv = PTR_ALIGN(&ivbuf[0], AESNI_ALIGN); __be32 counter = cpu_to_be32(1); memcpy(iv, req->iv, 12); @@ -1001,7 +1078,8 @@ static int generic_gcmaes_decrypt(struct aead_request *req) struct crypto_aead *tfm = crypto_aead_reqtfm(req); struct generic_gcmaes_ctx *ctx = generic_gcmaes_ctx_get(tfm); void *aes_ctx = &(ctx->aes_key_expanded); - u8 iv[16] __attribute__ ((__aligned__(AESNI_ALIGN))); + u8 ivbuf[16 + (AESNI_ALIGN - 8)] __aligned(8); + u8 *iv = PTR_ALIGN(&ivbuf[0], AESNI_ALIGN); memcpy(iv, req->iv, 12); *((__be32 *)(iv+12)) = counter; @@ -1066,19 +1144,18 @@ static int __init aesni_init(void) #ifdef CONFIG_X86_64 if (boot_cpu_has(X86_FEATURE_AVX2)) { pr_info("AVX2 version of gcm_enc/dec engaged.\n"); - aesni_gcm_tfm = &aesni_gcm_tfm_avx_gen4; + static_branch_enable(&gcm_use_avx); + static_branch_enable(&gcm_use_avx2); } else if (boot_cpu_has(X86_FEATURE_AVX)) { pr_info("AVX version of gcm_enc/dec engaged.\n"); - aesni_gcm_tfm = &aesni_gcm_tfm_avx_gen2; + static_branch_enable(&gcm_use_avx); } else { pr_info("SSE version of gcm_enc/dec engaged.\n"); - aesni_gcm_tfm = &aesni_gcm_tfm_sse; } - aesni_ctr_enc_tfm = aesni_ctr_enc; if (boot_cpu_has(X86_FEATURE_AVX)) { /* optimize performance of ctr mode encryption transform */ - aesni_ctr_enc_tfm = aesni_ctr_enc_avx_tfm; + static_call_update(aesni_ctr_enc_tfm, aesni_ctr_enc_avx_tfm); pr_info("AES CTR mode by8 optimization enabled\n"); } #endif diff --git a/arch/x86/crypto/blake2s-glue.c b/arch/x86/crypto/blake2s-glue.c index c025a01cf70841bc69d8a9797cb18d86c8c5ac02..a40365ab301eef7ad3bb86083c02700943f0956a 100644 --- a/arch/x86/crypto/blake2s-glue.c +++ b/arch/x86/crypto/blake2s-glue.c @@ -58,138 +58,40 @@ void blake2s_compress_arch(struct blake2s_state *state, } EXPORT_SYMBOL(blake2s_compress_arch); -static int crypto_blake2s_setkey(struct crypto_shash *tfm, const u8 *key, - unsigned int keylen) +static int crypto_blake2s_update_x86(struct shash_desc *desc, + const u8 *in, unsigned int inlen) { - struct blake2s_tfm_ctx *tctx = crypto_shash_ctx(tfm); - - if (keylen == 0 || keylen > BLAKE2S_KEY_SIZE) - return -EINVAL; - - memcpy(tctx->key, key, keylen); - tctx->keylen = keylen; - - return 0; + return crypto_blake2s_update(desc, in, inlen, blake2s_compress_arch); } -static int crypto_blake2s_init(struct shash_desc *desc) +static int crypto_blake2s_final_x86(struct shash_desc *desc, u8 *out) { - struct blake2s_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); - struct blake2s_state *state = shash_desc_ctx(desc); - const int outlen = crypto_shash_digestsize(desc->tfm); - - if (tctx->keylen) - blake2s_init_key(state, outlen, tctx->key, tctx->keylen); - else - blake2s_init(state, outlen); - - return 0; + return crypto_blake2s_final(desc, out, blake2s_compress_arch); } -static int crypto_blake2s_update(struct shash_desc *desc, const u8 *in, - unsigned int inlen) -{ - struct blake2s_state *state = shash_desc_ctx(desc); - const size_t fill = BLAKE2S_BLOCK_SIZE - state->buflen; - - if (unlikely(!inlen)) - return 0; - if (inlen > fill) { - memcpy(state->buf + state->buflen, in, fill); - blake2s_compress_arch(state, state->buf, 1, BLAKE2S_BLOCK_SIZE); - state->buflen = 0; - in += fill; - inlen -= fill; +#define BLAKE2S_ALG(name, driver_name, digest_size) \ + { \ + .base.cra_name = name, \ + .base.cra_driver_name = driver_name, \ + .base.cra_priority = 200, \ + .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY, \ + .base.cra_blocksize = BLAKE2S_BLOCK_SIZE, \ + .base.cra_ctxsize = sizeof(struct blake2s_tfm_ctx), \ + .base.cra_module = THIS_MODULE, \ + .digestsize = digest_size, \ + .setkey = crypto_blake2s_setkey, \ + .init = crypto_blake2s_init, \ + .update = crypto_blake2s_update_x86, \ + .final = crypto_blake2s_final_x86, \ + .descsize = sizeof(struct blake2s_state), \ } - if (inlen > BLAKE2S_BLOCK_SIZE) { - const size_t nblocks = DIV_ROUND_UP(inlen, BLAKE2S_BLOCK_SIZE); - /* Hash one less (full) block than strictly possible */ - blake2s_compress_arch(state, in, nblocks - 1, BLAKE2S_BLOCK_SIZE); - in += BLAKE2S_BLOCK_SIZE * (nblocks - 1); - inlen -= BLAKE2S_BLOCK_SIZE * (nblocks - 1); - } - memcpy(state->buf + state->buflen, in, inlen); - state->buflen += inlen; - - return 0; -} - -static int crypto_blake2s_final(struct shash_desc *desc, u8 *out) -{ - struct blake2s_state *state = shash_desc_ctx(desc); - - blake2s_set_lastblock(state); - memset(state->buf + state->buflen, 0, - BLAKE2S_BLOCK_SIZE - state->buflen); /* Padding */ - blake2s_compress_arch(state, state->buf, 1, state->buflen); - cpu_to_le32_array(state->h, ARRAY_SIZE(state->h)); - memcpy(out, state->h, state->outlen); - memzero_explicit(state, sizeof(*state)); - - return 0; -} -static struct shash_alg blake2s_algs[] = {{ - .base.cra_name = "blake2s-128", - .base.cra_driver_name = "blake2s-128-x86", - .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY, - .base.cra_ctxsize = sizeof(struct blake2s_tfm_ctx), - .base.cra_priority = 200, - .base.cra_blocksize = BLAKE2S_BLOCK_SIZE, - .base.cra_module = THIS_MODULE, - - .digestsize = BLAKE2S_128_HASH_SIZE, - .setkey = crypto_blake2s_setkey, - .init = crypto_blake2s_init, - .update = crypto_blake2s_update, - .final = crypto_blake2s_final, - .descsize = sizeof(struct blake2s_state), -}, { - .base.cra_name = "blake2s-160", - .base.cra_driver_name = "blake2s-160-x86", - .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY, - .base.cra_ctxsize = sizeof(struct blake2s_tfm_ctx), - .base.cra_priority = 200, - .base.cra_blocksize = BLAKE2S_BLOCK_SIZE, - .base.cra_module = THIS_MODULE, - - .digestsize = BLAKE2S_160_HASH_SIZE, - .setkey = crypto_blake2s_setkey, - .init = crypto_blake2s_init, - .update = crypto_blake2s_update, - .final = crypto_blake2s_final, - .descsize = sizeof(struct blake2s_state), -}, { - .base.cra_name = "blake2s-224", - .base.cra_driver_name = "blake2s-224-x86", - .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY, - .base.cra_ctxsize = sizeof(struct blake2s_tfm_ctx), - .base.cra_priority = 200, - .base.cra_blocksize = BLAKE2S_BLOCK_SIZE, - .base.cra_module = THIS_MODULE, - - .digestsize = BLAKE2S_224_HASH_SIZE, - .setkey = crypto_blake2s_setkey, - .init = crypto_blake2s_init, - .update = crypto_blake2s_update, - .final = crypto_blake2s_final, - .descsize = sizeof(struct blake2s_state), -}, { - .base.cra_name = "blake2s-256", - .base.cra_driver_name = "blake2s-256-x86", - .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY, - .base.cra_ctxsize = sizeof(struct blake2s_tfm_ctx), - .base.cra_priority = 200, - .base.cra_blocksize = BLAKE2S_BLOCK_SIZE, - .base.cra_module = THIS_MODULE, - - .digestsize = BLAKE2S_256_HASH_SIZE, - .setkey = crypto_blake2s_setkey, - .init = crypto_blake2s_init, - .update = crypto_blake2s_update, - .final = crypto_blake2s_final, - .descsize = sizeof(struct blake2s_state), -}}; +static struct shash_alg blake2s_algs[] = { + BLAKE2S_ALG("blake2s-128", "blake2s-128-x86", BLAKE2S_128_HASH_SIZE), + BLAKE2S_ALG("blake2s-160", "blake2s-160-x86", BLAKE2S_160_HASH_SIZE), + BLAKE2S_ALG("blake2s-224", "blake2s-224-x86", BLAKE2S_224_HASH_SIZE), + BLAKE2S_ALG("blake2s-256", "blake2s-256-x86", BLAKE2S_256_HASH_SIZE), +}; static int __init blake2s_mod_init(void) { diff --git a/arch/x86/crypto/blowfish_glue.c b/arch/x86/crypto/blowfish_glue.c index cedfdba69ce3203a5dddb78634e2e12730464d96..a880e0b1c2555fc6d4d8e4055af221f2e617c587 100644 --- a/arch/x86/crypto/blowfish_glue.c +++ b/arch/x86/crypto/blowfish_glue.c @@ -6,8 +6,6 @@ * * CBC & ECB parts based on code (crypto/cbc.c,ecb.c) by: * Copyright (c) 2006 Herbert Xu - * CTR part based on code (crypto/ctr.c) by: - * (C) Copyright IBM Corp. 2007 - Joy Latten */ #include @@ -247,97 +245,6 @@ static int cbc_decrypt(struct skcipher_request *req) return err; } -static void ctr_crypt_final(struct bf_ctx *ctx, struct skcipher_walk *walk) -{ - u8 *ctrblk = walk->iv; - u8 keystream[BF_BLOCK_SIZE]; - u8 *src = walk->src.virt.addr; - u8 *dst = walk->dst.virt.addr; - unsigned int nbytes = walk->nbytes; - - blowfish_enc_blk(ctx, keystream, ctrblk); - crypto_xor_cpy(dst, keystream, src, nbytes); - - crypto_inc(ctrblk, BF_BLOCK_SIZE); -} - -static unsigned int __ctr_crypt(struct bf_ctx *ctx, struct skcipher_walk *walk) -{ - unsigned int bsize = BF_BLOCK_SIZE; - unsigned int nbytes = walk->nbytes; - u64 *src = (u64 *)walk->src.virt.addr; - u64 *dst = (u64 *)walk->dst.virt.addr; - u64 ctrblk = be64_to_cpu(*(__be64 *)walk->iv); - __be64 ctrblocks[4]; - - /* Process four block batch */ - if (nbytes >= bsize * 4) { - do { - if (dst != src) { - dst[0] = src[0]; - dst[1] = src[1]; - dst[2] = src[2]; - dst[3] = src[3]; - } - - /* create ctrblks for parallel encrypt */ - ctrblocks[0] = cpu_to_be64(ctrblk++); - ctrblocks[1] = cpu_to_be64(ctrblk++); - ctrblocks[2] = cpu_to_be64(ctrblk++); - ctrblocks[3] = cpu_to_be64(ctrblk++); - - blowfish_enc_blk_xor_4way(ctx, (u8 *)dst, - (u8 *)ctrblocks); - - src += 4; - dst += 4; - } while ((nbytes -= bsize * 4) >= bsize * 4); - - if (nbytes < bsize) - goto done; - } - - /* Handle leftovers */ - do { - if (dst != src) - *dst = *src; - - ctrblocks[0] = cpu_to_be64(ctrblk++); - - blowfish_enc_blk_xor(ctx, (u8 *)dst, (u8 *)ctrblocks); - - src += 1; - dst += 1; - } while ((nbytes -= bsize) >= bsize); - -done: - *(__be64 *)walk->iv = cpu_to_be64(ctrblk); - return nbytes; -} - -static int ctr_crypt(struct skcipher_request *req) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct bf_ctx *ctx = crypto_skcipher_ctx(tfm); - struct skcipher_walk walk; - unsigned int nbytes; - int err; - - err = skcipher_walk_virt(&walk, req, false); - - while ((nbytes = walk.nbytes) >= BF_BLOCK_SIZE) { - nbytes = __ctr_crypt(ctx, &walk); - err = skcipher_walk_done(&walk, nbytes); - } - - if (nbytes) { - ctr_crypt_final(ctx, &walk); - err = skcipher_walk_done(&walk, 0); - } - - return err; -} - static struct crypto_alg bf_cipher_alg = { .cra_name = "blowfish", .cra_driver_name = "blowfish-asm", @@ -384,20 +291,6 @@ static struct skcipher_alg bf_skcipher_algs[] = { .setkey = blowfish_setkey_skcipher, .encrypt = cbc_encrypt, .decrypt = cbc_decrypt, - }, { - .base.cra_name = "ctr(blowfish)", - .base.cra_driver_name = "ctr-blowfish-asm", - .base.cra_priority = 300, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct bf_ctx), - .base.cra_module = THIS_MODULE, - .min_keysize = BF_MIN_KEY_SIZE, - .max_keysize = BF_MAX_KEY_SIZE, - .ivsize = BF_BLOCK_SIZE, - .chunksize = BF_BLOCK_SIZE, - .setkey = blowfish_setkey_skcipher, - .encrypt = ctr_crypt, - .decrypt = ctr_crypt, }, }; diff --git a/arch/x86/crypto/camellia-aesni-avx-asm_64.S b/arch/x86/crypto/camellia-aesni-avx-asm_64.S index ecc0a9a905c481998e934781a76069917cf0d120..e2a0e0f4bf9d8402f0d50041ac039fc603b31058 100644 --- a/arch/x86/crypto/camellia-aesni-avx-asm_64.S +++ b/arch/x86/crypto/camellia-aesni-avx-asm_64.S @@ -17,7 +17,6 @@ #include #include -#include #define CAMELLIA_TABLE_BYTE_LEN 272 @@ -589,14 +588,6 @@ SYM_FUNC_END(roundsm16_x4_x5_x6_x7_x0_x1_x2_x3_y4_y5_y6_y7_y0_y1_y2_y3_ab) .long 0x80808080 .long 0x80808080 -/* For CTR-mode IV byteswap */ -.Lbswap128_mask: - .byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 - -/* For XTS mode IV generation */ -.Lxts_gf128mul_and_shl1_mask: - .byte 0x87, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0 - /* * pre-SubByte transform * @@ -998,292 +989,3 @@ SYM_FUNC_START(camellia_cbc_dec_16way) FRAME_END ret; SYM_FUNC_END(camellia_cbc_dec_16way) - -#define inc_le128(x, minus_one, tmp) \ - vpcmpeqq minus_one, x, tmp; \ - vpsubq minus_one, x, x; \ - vpslldq $8, tmp, tmp; \ - vpsubq tmp, x, x; - -SYM_FUNC_START(camellia_ctr_16way) - /* input: - * %rdi: ctx, CTX - * %rsi: dst (16 blocks) - * %rdx: src (16 blocks) - * %rcx: iv (little endian, 128bit) - */ - FRAME_BEGIN - - subq $(16 * 16), %rsp; - movq %rsp, %rax; - - vmovdqa .Lbswap128_mask, %xmm14; - - /* load IV and byteswap */ - vmovdqu (%rcx), %xmm0; - vpshufb %xmm14, %xmm0, %xmm15; - vmovdqu %xmm15, 15 * 16(%rax); - - vpcmpeqd %xmm15, %xmm15, %xmm15; - vpsrldq $8, %xmm15, %xmm15; /* low: -1, high: 0 */ - - /* construct IVs */ - inc_le128(%xmm0, %xmm15, %xmm13); - vpshufb %xmm14, %xmm0, %xmm13; - vmovdqu %xmm13, 14 * 16(%rax); - inc_le128(%xmm0, %xmm15, %xmm13); - vpshufb %xmm14, %xmm0, %xmm13; - vmovdqu %xmm13, 13 * 16(%rax); - inc_le128(%xmm0, %xmm15, %xmm13); - vpshufb %xmm14, %xmm0, %xmm12; - inc_le128(%xmm0, %xmm15, %xmm13); - vpshufb %xmm14, %xmm0, %xmm11; - inc_le128(%xmm0, %xmm15, %xmm13); - vpshufb %xmm14, %xmm0, %xmm10; - inc_le128(%xmm0, %xmm15, %xmm13); - vpshufb %xmm14, %xmm0, %xmm9; - inc_le128(%xmm0, %xmm15, %xmm13); - vpshufb %xmm14, %xmm0, %xmm8; - inc_le128(%xmm0, %xmm15, %xmm13); - vpshufb %xmm14, %xmm0, %xmm7; - inc_le128(%xmm0, %xmm15, %xmm13); - vpshufb %xmm14, %xmm0, %xmm6; - inc_le128(%xmm0, %xmm15, %xmm13); - vpshufb %xmm14, %xmm0, %xmm5; - inc_le128(%xmm0, %xmm15, %xmm13); - vpshufb %xmm14, %xmm0, %xmm4; - inc_le128(%xmm0, %xmm15, %xmm13); - vpshufb %xmm14, %xmm0, %xmm3; - inc_le128(%xmm0, %xmm15, %xmm13); - vpshufb %xmm14, %xmm0, %xmm2; - inc_le128(%xmm0, %xmm15, %xmm13); - vpshufb %xmm14, %xmm0, %xmm1; - inc_le128(%xmm0, %xmm15, %xmm13); - vmovdqa %xmm0, %xmm13; - vpshufb %xmm14, %xmm0, %xmm0; - inc_le128(%xmm13, %xmm15, %xmm14); - vmovdqu %xmm13, (%rcx); - - /* inpack16_pre: */ - vmovq (key_table)(CTX), %xmm15; - vpshufb .Lpack_bswap, %xmm15, %xmm15; - vpxor %xmm0, %xmm15, %xmm0; - vpxor %xmm1, %xmm15, %xmm1; - vpxor %xmm2, %xmm15, %xmm2; - vpxor %xmm3, %xmm15, %xmm3; - vpxor %xmm4, %xmm15, %xmm4; - vpxor %xmm5, %xmm15, %xmm5; - vpxor %xmm6, %xmm15, %xmm6; - vpxor %xmm7, %xmm15, %xmm7; - vpxor %xmm8, %xmm15, %xmm8; - vpxor %xmm9, %xmm15, %xmm9; - vpxor %xmm10, %xmm15, %xmm10; - vpxor %xmm11, %xmm15, %xmm11; - vpxor %xmm12, %xmm15, %xmm12; - vpxor 13 * 16(%rax), %xmm15, %xmm13; - vpxor 14 * 16(%rax), %xmm15, %xmm14; - vpxor 15 * 16(%rax), %xmm15, %xmm15; - - call __camellia_enc_blk16; - - addq $(16 * 16), %rsp; - - vpxor 0 * 16(%rdx), %xmm7, %xmm7; - vpxor 1 * 16(%rdx), %xmm6, %xmm6; - vpxor 2 * 16(%rdx), %xmm5, %xmm5; - vpxor 3 * 16(%rdx), %xmm4, %xmm4; - vpxor 4 * 16(%rdx), %xmm3, %xmm3; - vpxor 5 * 16(%rdx), %xmm2, %xmm2; - vpxor 6 * 16(%rdx), %xmm1, %xmm1; - vpxor 7 * 16(%rdx), %xmm0, %xmm0; - vpxor 8 * 16(%rdx), %xmm15, %xmm15; - vpxor 9 * 16(%rdx), %xmm14, %xmm14; - vpxor 10 * 16(%rdx), %xmm13, %xmm13; - vpxor 11 * 16(%rdx), %xmm12, %xmm12; - vpxor 12 * 16(%rdx), %xmm11, %xmm11; - vpxor 13 * 16(%rdx), %xmm10, %xmm10; - vpxor 14 * 16(%rdx), %xmm9, %xmm9; - vpxor 15 * 16(%rdx), %xmm8, %xmm8; - write_output(%xmm7, %xmm6, %xmm5, %xmm4, %xmm3, %xmm2, %xmm1, %xmm0, - %xmm15, %xmm14, %xmm13, %xmm12, %xmm11, %xmm10, %xmm9, - %xmm8, %rsi); - - FRAME_END - ret; -SYM_FUNC_END(camellia_ctr_16way) - -#define gf128mul_x_ble(iv, mask, tmp) \ - vpsrad $31, iv, tmp; \ - vpaddq iv, iv, iv; \ - vpshufd $0x13, tmp, tmp; \ - vpand mask, tmp, tmp; \ - vpxor tmp, iv, iv; - -.align 8 -SYM_FUNC_START_LOCAL(camellia_xts_crypt_16way) - /* input: - * %rdi: ctx, CTX - * %rsi: dst (16 blocks) - * %rdx: src (16 blocks) - * %rcx: iv (t ⊕ α⿠∈ GF(2¹²â¸)) - * %r8: index for input whitening key - * %r9: pointer to __camellia_enc_blk16 or __camellia_dec_blk16 - */ - FRAME_BEGIN - - subq $(16 * 16), %rsp; - movq %rsp, %rax; - - vmovdqa .Lxts_gf128mul_and_shl1_mask, %xmm14; - - /* load IV */ - vmovdqu (%rcx), %xmm0; - vpxor 0 * 16(%rdx), %xmm0, %xmm15; - vmovdqu %xmm15, 15 * 16(%rax); - vmovdqu %xmm0, 0 * 16(%rsi); - - /* construct IVs */ - gf128mul_x_ble(%xmm0, %xmm14, %xmm15); - vpxor 1 * 16(%rdx), %xmm0, %xmm15; - vmovdqu %xmm15, 14 * 16(%rax); - vmovdqu %xmm0, 1 * 16(%rsi); - - gf128mul_x_ble(%xmm0, %xmm14, %xmm15); - vpxor 2 * 16(%rdx), %xmm0, %xmm13; - vmovdqu %xmm0, 2 * 16(%rsi); - - gf128mul_x_ble(%xmm0, %xmm14, %xmm15); - vpxor 3 * 16(%rdx), %xmm0, %xmm12; - vmovdqu %xmm0, 3 * 16(%rsi); - - gf128mul_x_ble(%xmm0, %xmm14, %xmm15); - vpxor 4 * 16(%rdx), %xmm0, %xmm11; - vmovdqu %xmm0, 4 * 16(%rsi); - - gf128mul_x_ble(%xmm0, %xmm14, %xmm15); - vpxor 5 * 16(%rdx), %xmm0, %xmm10; - vmovdqu %xmm0, 5 * 16(%rsi); - - gf128mul_x_ble(%xmm0, %xmm14, %xmm15); - vpxor 6 * 16(%rdx), %xmm0, %xmm9; - vmovdqu %xmm0, 6 * 16(%rsi); - - gf128mul_x_ble(%xmm0, %xmm14, %xmm15); - vpxor 7 * 16(%rdx), %xmm0, %xmm8; - vmovdqu %xmm0, 7 * 16(%rsi); - - gf128mul_x_ble(%xmm0, %xmm14, %xmm15); - vpxor 8 * 16(%rdx), %xmm0, %xmm7; - vmovdqu %xmm0, 8 * 16(%rsi); - - gf128mul_x_ble(%xmm0, %xmm14, %xmm15); - vpxor 9 * 16(%rdx), %xmm0, %xmm6; - vmovdqu %xmm0, 9 * 16(%rsi); - - gf128mul_x_ble(%xmm0, %xmm14, %xmm15); - vpxor 10 * 16(%rdx), %xmm0, %xmm5; - vmovdqu %xmm0, 10 * 16(%rsi); - - gf128mul_x_ble(%xmm0, %xmm14, %xmm15); - vpxor 11 * 16(%rdx), %xmm0, %xmm4; - vmovdqu %xmm0, 11 * 16(%rsi); - - gf128mul_x_ble(%xmm0, %xmm14, %xmm15); - vpxor 12 * 16(%rdx), %xmm0, %xmm3; - vmovdqu %xmm0, 12 * 16(%rsi); - - gf128mul_x_ble(%xmm0, %xmm14, %xmm15); - vpxor 13 * 16(%rdx), %xmm0, %xmm2; - vmovdqu %xmm0, 13 * 16(%rsi); - - gf128mul_x_ble(%xmm0, %xmm14, %xmm15); - vpxor 14 * 16(%rdx), %xmm0, %xmm1; - vmovdqu %xmm0, 14 * 16(%rsi); - - gf128mul_x_ble(%xmm0, %xmm14, %xmm15); - vpxor 15 * 16(%rdx), %xmm0, %xmm15; - vmovdqu %xmm15, 0 * 16(%rax); - vmovdqu %xmm0, 15 * 16(%rsi); - - gf128mul_x_ble(%xmm0, %xmm14, %xmm15); - vmovdqu %xmm0, (%rcx); - - /* inpack16_pre: */ - vmovq (key_table)(CTX, %r8, 8), %xmm15; - vpshufb .Lpack_bswap, %xmm15, %xmm15; - vpxor 0 * 16(%rax), %xmm15, %xmm0; - vpxor %xmm1, %xmm15, %xmm1; - vpxor %xmm2, %xmm15, %xmm2; - vpxor %xmm3, %xmm15, %xmm3; - vpxor %xmm4, %xmm15, %xmm4; - vpxor %xmm5, %xmm15, %xmm5; - vpxor %xmm6, %xmm15, %xmm6; - vpxor %xmm7, %xmm15, %xmm7; - vpxor %xmm8, %xmm15, %xmm8; - vpxor %xmm9, %xmm15, %xmm9; - vpxor %xmm10, %xmm15, %xmm10; - vpxor %xmm11, %xmm15, %xmm11; - vpxor %xmm12, %xmm15, %xmm12; - vpxor %xmm13, %xmm15, %xmm13; - vpxor 14 * 16(%rax), %xmm15, %xmm14; - vpxor 15 * 16(%rax), %xmm15, %xmm15; - - CALL_NOSPEC r9; - - addq $(16 * 16), %rsp; - - vpxor 0 * 16(%rsi), %xmm7, %xmm7; - vpxor 1 * 16(%rsi), %xmm6, %xmm6; - vpxor 2 * 16(%rsi), %xmm5, %xmm5; - vpxor 3 * 16(%rsi), %xmm4, %xmm4; - vpxor 4 * 16(%rsi), %xmm3, %xmm3; - vpxor 5 * 16(%rsi), %xmm2, %xmm2; - vpxor 6 * 16(%rsi), %xmm1, %xmm1; - vpxor 7 * 16(%rsi), %xmm0, %xmm0; - vpxor 8 * 16(%rsi), %xmm15, %xmm15; - vpxor 9 * 16(%rsi), %xmm14, %xmm14; - vpxor 10 * 16(%rsi), %xmm13, %xmm13; - vpxor 11 * 16(%rsi), %xmm12, %xmm12; - vpxor 12 * 16(%rsi), %xmm11, %xmm11; - vpxor 13 * 16(%rsi), %xmm10, %xmm10; - vpxor 14 * 16(%rsi), %xmm9, %xmm9; - vpxor 15 * 16(%rsi), %xmm8, %xmm8; - write_output(%xmm7, %xmm6, %xmm5, %xmm4, %xmm3, %xmm2, %xmm1, %xmm0, - %xmm15, %xmm14, %xmm13, %xmm12, %xmm11, %xmm10, %xmm9, - %xmm8, %rsi); - - FRAME_END - ret; -SYM_FUNC_END(camellia_xts_crypt_16way) - -SYM_FUNC_START(camellia_xts_enc_16way) - /* input: - * %rdi: ctx, CTX - * %rsi: dst (16 blocks) - * %rdx: src (16 blocks) - * %rcx: iv (t ⊕ α⿠∈ GF(2¹²â¸)) - */ - xorl %r8d, %r8d; /* input whitening key, 0 for enc */ - - leaq __camellia_enc_blk16, %r9; - - jmp camellia_xts_crypt_16way; -SYM_FUNC_END(camellia_xts_enc_16way) - -SYM_FUNC_START(camellia_xts_dec_16way) - /* input: - * %rdi: ctx, CTX - * %rsi: dst (16 blocks) - * %rdx: src (16 blocks) - * %rcx: iv (t ⊕ α⿠∈ GF(2¹²â¸)) - */ - - cmpl $16, key_length(CTX); - movl $32, %r8d; - movl $24, %eax; - cmovel %eax, %r8d; /* input whitening key, last for dec */ - - leaq __camellia_dec_blk16, %r9; - - jmp camellia_xts_crypt_16way; -SYM_FUNC_END(camellia_xts_dec_16way) diff --git a/arch/x86/crypto/camellia-aesni-avx2-asm_64.S b/arch/x86/crypto/camellia-aesni-avx2-asm_64.S index 0907243c501cdca38907d06601c696559e21358a..782e9712a1ec6f29ce7aff3b1c9d1f4acd68018c 100644 --- a/arch/x86/crypto/camellia-aesni-avx2-asm_64.S +++ b/arch/x86/crypto/camellia-aesni-avx2-asm_64.S @@ -7,7 +7,6 @@ #include #include -#include #define CAMELLIA_TABLE_BYTE_LEN 272 @@ -625,16 +624,6 @@ SYM_FUNC_END(roundsm32_x4_x5_x6_x7_x0_x1_x2_x3_y4_y5_y6_y7_y0_y1_y2_y3_ab) .section .rodata.cst16, "aM", @progbits, 16 .align 16 -/* For CTR-mode IV byteswap */ -.Lbswap128_mask: - .byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 - -/* For XTS mode */ -.Lxts_gf128mul_and_shl1_mask_0: - .byte 0x87, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0 -.Lxts_gf128mul_and_shl1_mask_1: - .byte 0x0e, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0 - /* * pre-SubByte transform * @@ -1061,343 +1050,3 @@ SYM_FUNC_START(camellia_cbc_dec_32way) FRAME_END ret; SYM_FUNC_END(camellia_cbc_dec_32way) - -#define inc_le128(x, minus_one, tmp) \ - vpcmpeqq minus_one, x, tmp; \ - vpsubq minus_one, x, x; \ - vpslldq $8, tmp, tmp; \ - vpsubq tmp, x, x; - -#define add2_le128(x, minus_one, minus_two, tmp1, tmp2) \ - vpcmpeqq minus_one, x, tmp1; \ - vpcmpeqq minus_two, x, tmp2; \ - vpsubq minus_two, x, x; \ - vpor tmp2, tmp1, tmp1; \ - vpslldq $8, tmp1, tmp1; \ - vpsubq tmp1, x, x; - -SYM_FUNC_START(camellia_ctr_32way) - /* input: - * %rdi: ctx, CTX - * %rsi: dst (32 blocks) - * %rdx: src (32 blocks) - * %rcx: iv (little endian, 128bit) - */ - FRAME_BEGIN - - vzeroupper; - - movq %rsp, %r10; - cmpq %rsi, %rdx; - je .Lctr_use_stack; - - /* dst can be used as temporary storage, src is not overwritten. */ - movq %rsi, %rax; - jmp .Lctr_continue; - -.Lctr_use_stack: - subq $(16 * 32), %rsp; - movq %rsp, %rax; - -.Lctr_continue: - vpcmpeqd %ymm15, %ymm15, %ymm15; - vpsrldq $8, %ymm15, %ymm15; /* ab: -1:0 ; cd: -1:0 */ - vpaddq %ymm15, %ymm15, %ymm12; /* ab: -2:0 ; cd: -2:0 */ - - /* load IV and byteswap */ - vmovdqu (%rcx), %xmm0; - vmovdqa %xmm0, %xmm1; - inc_le128(%xmm0, %xmm15, %xmm14); - vbroadcasti128 .Lbswap128_mask, %ymm14; - vinserti128 $1, %xmm0, %ymm1, %ymm0; - vpshufb %ymm14, %ymm0, %ymm13; - vmovdqu %ymm13, 15 * 32(%rax); - - /* construct IVs */ - add2_le128(%ymm0, %ymm15, %ymm12, %ymm11, %ymm13); /* ab:le2 ; cd:le3 */ - vpshufb %ymm14, %ymm0, %ymm13; - vmovdqu %ymm13, 14 * 32(%rax); - add2_le128(%ymm0, %ymm15, %ymm12, %ymm11, %ymm13); - vpshufb %ymm14, %ymm0, %ymm13; - vmovdqu %ymm13, 13 * 32(%rax); - add2_le128(%ymm0, %ymm15, %ymm12, %ymm11, %ymm13); - vpshufb %ymm14, %ymm0, %ymm13; - vmovdqu %ymm13, 12 * 32(%rax); - add2_le128(%ymm0, %ymm15, %ymm12, %ymm11, %ymm13); - vpshufb %ymm14, %ymm0, %ymm13; - vmovdqu %ymm13, 11 * 32(%rax); - add2_le128(%ymm0, %ymm15, %ymm12, %ymm11, %ymm13); - vpshufb %ymm14, %ymm0, %ymm10; - add2_le128(%ymm0, %ymm15, %ymm12, %ymm11, %ymm13); - vpshufb %ymm14, %ymm0, %ymm9; - add2_le128(%ymm0, %ymm15, %ymm12, %ymm11, %ymm13); - vpshufb %ymm14, %ymm0, %ymm8; - add2_le128(%ymm0, %ymm15, %ymm12, %ymm11, %ymm13); - vpshufb %ymm14, %ymm0, %ymm7; - add2_le128(%ymm0, %ymm15, %ymm12, %ymm11, %ymm13); - vpshufb %ymm14, %ymm0, %ymm6; - add2_le128(%ymm0, %ymm15, %ymm12, %ymm11, %ymm13); - vpshufb %ymm14, %ymm0, %ymm5; - add2_le128(%ymm0, %ymm15, %ymm12, %ymm11, %ymm13); - vpshufb %ymm14, %ymm0, %ymm4; - add2_le128(%ymm0, %ymm15, %ymm12, %ymm11, %ymm13); - vpshufb %ymm14, %ymm0, %ymm3; - add2_le128(%ymm0, %ymm15, %ymm12, %ymm11, %ymm13); - vpshufb %ymm14, %ymm0, %ymm2; - add2_le128(%ymm0, %ymm15, %ymm12, %ymm11, %ymm13); - vpshufb %ymm14, %ymm0, %ymm1; - add2_le128(%ymm0, %ymm15, %ymm12, %ymm11, %ymm13); - vextracti128 $1, %ymm0, %xmm13; - vpshufb %ymm14, %ymm0, %ymm0; - inc_le128(%xmm13, %xmm15, %xmm14); - vmovdqu %xmm13, (%rcx); - - /* inpack32_pre: */ - vpbroadcastq (key_table)(CTX), %ymm15; - vpshufb .Lpack_bswap, %ymm15, %ymm15; - vpxor %ymm0, %ymm15, %ymm0; - vpxor %ymm1, %ymm15, %ymm1; - vpxor %ymm2, %ymm15, %ymm2; - vpxor %ymm3, %ymm15, %ymm3; - vpxor %ymm4, %ymm15, %ymm4; - vpxor %ymm5, %ymm15, %ymm5; - vpxor %ymm6, %ymm15, %ymm6; - vpxor %ymm7, %ymm15, %ymm7; - vpxor %ymm8, %ymm15, %ymm8; - vpxor %ymm9, %ymm15, %ymm9; - vpxor %ymm10, %ymm15, %ymm10; - vpxor 11 * 32(%rax), %ymm15, %ymm11; - vpxor 12 * 32(%rax), %ymm15, %ymm12; - vpxor 13 * 32(%rax), %ymm15, %ymm13; - vpxor 14 * 32(%rax), %ymm15, %ymm14; - vpxor 15 * 32(%rax), %ymm15, %ymm15; - - call __camellia_enc_blk32; - - movq %r10, %rsp; - - vpxor 0 * 32(%rdx), %ymm7, %ymm7; - vpxor 1 * 32(%rdx), %ymm6, %ymm6; - vpxor 2 * 32(%rdx), %ymm5, %ymm5; - vpxor 3 * 32(%rdx), %ymm4, %ymm4; - vpxor 4 * 32(%rdx), %ymm3, %ymm3; - vpxor 5 * 32(%rdx), %ymm2, %ymm2; - vpxor 6 * 32(%rdx), %ymm1, %ymm1; - vpxor 7 * 32(%rdx), %ymm0, %ymm0; - vpxor 8 * 32(%rdx), %ymm15, %ymm15; - vpxor 9 * 32(%rdx), %ymm14, %ymm14; - vpxor 10 * 32(%rdx), %ymm13, %ymm13; - vpxor 11 * 32(%rdx), %ymm12, %ymm12; - vpxor 12 * 32(%rdx), %ymm11, %ymm11; - vpxor 13 * 32(%rdx), %ymm10, %ymm10; - vpxor 14 * 32(%rdx), %ymm9, %ymm9; - vpxor 15 * 32(%rdx), %ymm8, %ymm8; - write_output(%ymm7, %ymm6, %ymm5, %ymm4, %ymm3, %ymm2, %ymm1, %ymm0, - %ymm15, %ymm14, %ymm13, %ymm12, %ymm11, %ymm10, %ymm9, - %ymm8, %rsi); - - vzeroupper; - - FRAME_END - ret; -SYM_FUNC_END(camellia_ctr_32way) - -#define gf128mul_x_ble(iv, mask, tmp) \ - vpsrad $31, iv, tmp; \ - vpaddq iv, iv, iv; \ - vpshufd $0x13, tmp, tmp; \ - vpand mask, tmp, tmp; \ - vpxor tmp, iv, iv; - -#define gf128mul_x2_ble(iv, mask1, mask2, tmp0, tmp1) \ - vpsrad $31, iv, tmp0; \ - vpaddq iv, iv, tmp1; \ - vpsllq $2, iv, iv; \ - vpshufd $0x13, tmp0, tmp0; \ - vpsrad $31, tmp1, tmp1; \ - vpand mask2, tmp0, tmp0; \ - vpshufd $0x13, tmp1, tmp1; \ - vpxor tmp0, iv, iv; \ - vpand mask1, tmp1, tmp1; \ - vpxor tmp1, iv, iv; - -.align 8 -SYM_FUNC_START_LOCAL(camellia_xts_crypt_32way) - /* input: - * %rdi: ctx, CTX - * %rsi: dst (32 blocks) - * %rdx: src (32 blocks) - * %rcx: iv (t ⊕ α⿠∈ GF(2¹²â¸)) - * %r8: index for input whitening key - * %r9: pointer to __camellia_enc_blk32 or __camellia_dec_blk32 - */ - FRAME_BEGIN - - vzeroupper; - - subq $(16 * 32), %rsp; - movq %rsp, %rax; - - vbroadcasti128 .Lxts_gf128mul_and_shl1_mask_0, %ymm12; - - /* load IV and construct second IV */ - vmovdqu (%rcx), %xmm0; - vmovdqa %xmm0, %xmm15; - gf128mul_x_ble(%xmm0, %xmm12, %xmm13); - vbroadcasti128 .Lxts_gf128mul_and_shl1_mask_1, %ymm13; - vinserti128 $1, %xmm0, %ymm15, %ymm0; - vpxor 0 * 32(%rdx), %ymm0, %ymm15; - vmovdqu %ymm15, 15 * 32(%rax); - vmovdqu %ymm0, 0 * 32(%rsi); - - /* construct IVs */ - gf128mul_x2_ble(%ymm0, %ymm12, %ymm13, %ymm14, %ymm15); - vpxor 1 * 32(%rdx), %ymm0, %ymm15; - vmovdqu %ymm15, 14 * 32(%rax); - vmovdqu %ymm0, 1 * 32(%rsi); - - gf128mul_x2_ble(%ymm0, %ymm12, %ymm13, %ymm14, %ymm15); - vpxor 2 * 32(%rdx), %ymm0, %ymm15; - vmovdqu %ymm15, 13 * 32(%rax); - vmovdqu %ymm0, 2 * 32(%rsi); - - gf128mul_x2_ble(%ymm0, %ymm12, %ymm13, %ymm14, %ymm15); - vpxor 3 * 32(%rdx), %ymm0, %ymm15; - vmovdqu %ymm15, 12 * 32(%rax); - vmovdqu %ymm0, 3 * 32(%rsi); - - gf128mul_x2_ble(%ymm0, %ymm12, %ymm13, %ymm14, %ymm15); - vpxor 4 * 32(%rdx), %ymm0, %ymm11; - vmovdqu %ymm0, 4 * 32(%rsi); - - gf128mul_x2_ble(%ymm0, %ymm12, %ymm13, %ymm14, %ymm15); - vpxor 5 * 32(%rdx), %ymm0, %ymm10; - vmovdqu %ymm0, 5 * 32(%rsi); - - gf128mul_x2_ble(%ymm0, %ymm12, %ymm13, %ymm14, %ymm15); - vpxor 6 * 32(%rdx), %ymm0, %ymm9; - vmovdqu %ymm0, 6 * 32(%rsi); - - gf128mul_x2_ble(%ymm0, %ymm12, %ymm13, %ymm14, %ymm15); - vpxor 7 * 32(%rdx), %ymm0, %ymm8; - vmovdqu %ymm0, 7 * 32(%rsi); - - gf128mul_x2_ble(%ymm0, %ymm12, %ymm13, %ymm14, %ymm15); - vpxor 8 * 32(%rdx), %ymm0, %ymm7; - vmovdqu %ymm0, 8 * 32(%rsi); - - gf128mul_x2_ble(%ymm0, %ymm12, %ymm13, %ymm14, %ymm15); - vpxor 9 * 32(%rdx), %ymm0, %ymm6; - vmovdqu %ymm0, 9 * 32(%rsi); - - gf128mul_x2_ble(%ymm0, %ymm12, %ymm13, %ymm14, %ymm15); - vpxor 10 * 32(%rdx), %ymm0, %ymm5; - vmovdqu %ymm0, 10 * 32(%rsi); - - gf128mul_x2_ble(%ymm0, %ymm12, %ymm13, %ymm14, %ymm15); - vpxor 11 * 32(%rdx), %ymm0, %ymm4; - vmovdqu %ymm0, 11 * 32(%rsi); - - gf128mul_x2_ble(%ymm0, %ymm12, %ymm13, %ymm14, %ymm15); - vpxor 12 * 32(%rdx), %ymm0, %ymm3; - vmovdqu %ymm0, 12 * 32(%rsi); - - gf128mul_x2_ble(%ymm0, %ymm12, %ymm13, %ymm14, %ymm15); - vpxor 13 * 32(%rdx), %ymm0, %ymm2; - vmovdqu %ymm0, 13 * 32(%rsi); - - gf128mul_x2_ble(%ymm0, %ymm12, %ymm13, %ymm14, %ymm15); - vpxor 14 * 32(%rdx), %ymm0, %ymm1; - vmovdqu %ymm0, 14 * 32(%rsi); - - gf128mul_x2_ble(%ymm0, %ymm12, %ymm13, %ymm14, %ymm15); - vpxor 15 * 32(%rdx), %ymm0, %ymm15; - vmovdqu %ymm15, 0 * 32(%rax); - vmovdqu %ymm0, 15 * 32(%rsi); - - vextracti128 $1, %ymm0, %xmm0; - gf128mul_x_ble(%xmm0, %xmm12, %xmm15); - vmovdqu %xmm0, (%rcx); - - /* inpack32_pre: */ - vpbroadcastq (key_table)(CTX, %r8, 8), %ymm15; - vpshufb .Lpack_bswap, %ymm15, %ymm15; - vpxor 0 * 32(%rax), %ymm15, %ymm0; - vpxor %ymm1, %ymm15, %ymm1; - vpxor %ymm2, %ymm15, %ymm2; - vpxor %ymm3, %ymm15, %ymm3; - vpxor %ymm4, %ymm15, %ymm4; - vpxor %ymm5, %ymm15, %ymm5; - vpxor %ymm6, %ymm15, %ymm6; - vpxor %ymm7, %ymm15, %ymm7; - vpxor %ymm8, %ymm15, %ymm8; - vpxor %ymm9, %ymm15, %ymm9; - vpxor %ymm10, %ymm15, %ymm10; - vpxor %ymm11, %ymm15, %ymm11; - vpxor 12 * 32(%rax), %ymm15, %ymm12; - vpxor 13 * 32(%rax), %ymm15, %ymm13; - vpxor 14 * 32(%rax), %ymm15, %ymm14; - vpxor 15 * 32(%rax), %ymm15, %ymm15; - - CALL_NOSPEC r9; - - addq $(16 * 32), %rsp; - - vpxor 0 * 32(%rsi), %ymm7, %ymm7; - vpxor 1 * 32(%rsi), %ymm6, %ymm6; - vpxor 2 * 32(%rsi), %ymm5, %ymm5; - vpxor 3 * 32(%rsi), %ymm4, %ymm4; - vpxor 4 * 32(%rsi), %ymm3, %ymm3; - vpxor 5 * 32(%rsi), %ymm2, %ymm2; - vpxor 6 * 32(%rsi), %ymm1, %ymm1; - vpxor 7 * 32(%rsi), %ymm0, %ymm0; - vpxor 8 * 32(%rsi), %ymm15, %ymm15; - vpxor 9 * 32(%rsi), %ymm14, %ymm14; - vpxor 10 * 32(%rsi), %ymm13, %ymm13; - vpxor 11 * 32(%rsi), %ymm12, %ymm12; - vpxor 12 * 32(%rsi), %ymm11, %ymm11; - vpxor 13 * 32(%rsi), %ymm10, %ymm10; - vpxor 14 * 32(%rsi), %ymm9, %ymm9; - vpxor 15 * 32(%rsi), %ymm8, %ymm8; - write_output(%ymm7, %ymm6, %ymm5, %ymm4, %ymm3, %ymm2, %ymm1, %ymm0, - %ymm15, %ymm14, %ymm13, %ymm12, %ymm11, %ymm10, %ymm9, - %ymm8, %rsi); - - vzeroupper; - - FRAME_END - ret; -SYM_FUNC_END(camellia_xts_crypt_32way) - -SYM_FUNC_START(camellia_xts_enc_32way) - /* input: - * %rdi: ctx, CTX - * %rsi: dst (32 blocks) - * %rdx: src (32 blocks) - * %rcx: iv (t ⊕ α⿠∈ GF(2¹²â¸)) - */ - - xorl %r8d, %r8d; /* input whitening key, 0 for enc */ - - leaq __camellia_enc_blk32, %r9; - - jmp camellia_xts_crypt_32way; -SYM_FUNC_END(camellia_xts_enc_32way) - -SYM_FUNC_START(camellia_xts_dec_32way) - /* input: - * %rdi: ctx, CTX - * %rsi: dst (32 blocks) - * %rdx: src (32 blocks) - * %rcx: iv (t ⊕ α⿠∈ GF(2¹²â¸)) - */ - - cmpl $16, key_length(CTX); - movl $32, %r8d; - movl $24, %eax; - cmovel %eax, %r8d; /* input whitening key, last for dec */ - - leaq __camellia_dec_blk32, %r9; - - jmp camellia_xts_crypt_32way; -SYM_FUNC_END(camellia_xts_dec_32way) diff --git a/arch/x86/include/asm/crypto/camellia.h b/arch/x86/crypto/camellia.h similarity index 69% rename from arch/x86/include/asm/crypto/camellia.h rename to arch/x86/crypto/camellia.h index f6d91861cb14d4b4992846fd6c9405a08fb15025..1dcea79e8f8eb5a5f7f4c2987d59e8938a7e845d 100644 --- a/arch/x86/include/asm/crypto/camellia.h +++ b/arch/x86/crypto/camellia.h @@ -19,18 +19,10 @@ struct camellia_ctx { u32 key_length; }; -struct camellia_xts_ctx { - struct camellia_ctx tweak_ctx; - struct camellia_ctx crypt_ctx; -}; - extern int __camellia_setkey(struct camellia_ctx *cctx, const unsigned char *key, unsigned int key_len); -extern int xts_camellia_setkey(struct crypto_skcipher *tfm, const u8 *key, - unsigned int keylen); - /* regular block cipher functions */ asmlinkage void __camellia_enc_blk(const void *ctx, u8 *dst, const u8 *src, bool xor); @@ -46,13 +38,6 @@ asmlinkage void camellia_ecb_enc_16way(const void *ctx, u8 *dst, const u8 *src); asmlinkage void camellia_ecb_dec_16way(const void *ctx, u8 *dst, const u8 *src); asmlinkage void camellia_cbc_dec_16way(const void *ctx, u8 *dst, const u8 *src); -asmlinkage void camellia_ctr_16way(const void *ctx, u8 *dst, const u8 *src, - le128 *iv); - -asmlinkage void camellia_xts_enc_16way(const void *ctx, u8 *dst, const u8 *src, - le128 *iv); -asmlinkage void camellia_xts_dec_16way(const void *ctx, u8 *dst, const u8 *src, - le128 *iv); static inline void camellia_enc_blk(const void *ctx, u8 *dst, const u8 *src) { @@ -78,14 +63,5 @@ static inline void camellia_enc_blk_xor_2way(const void *ctx, u8 *dst, /* glue helpers */ extern void camellia_decrypt_cbc_2way(const void *ctx, u8 *dst, const u8 *src); -extern void camellia_crypt_ctr(const void *ctx, u8 *dst, const u8 *src, - le128 *iv); -extern void camellia_crypt_ctr_2way(const void *ctx, u8 *dst, const u8 *src, - le128 *iv); - -extern void camellia_xts_enc(const void *ctx, u8 *dst, const u8 *src, - le128 *iv); -extern void camellia_xts_dec(const void *ctx, u8 *dst, const u8 *src, - le128 *iv); #endif /* ASM_X86_CAMELLIA_H */ diff --git a/arch/x86/crypto/camellia_aesni_avx2_glue.c b/arch/x86/crypto/camellia_aesni_avx2_glue.c index ccda647422d696c0cd75937b6cf90550430f5ecb..e7e4d64e9577e9956d49df01b491aac580b1fcbf 100644 --- a/arch/x86/crypto/camellia_aesni_avx2_glue.c +++ b/arch/x86/crypto/camellia_aesni_avx2_glue.c @@ -5,16 +5,16 @@ * Copyright © 2013 Jussi Kivilinna */ -#include -#include #include #include -#include #include #include #include #include +#include "camellia.h" +#include "ecb_cbc_helpers.h" + #define CAMELLIA_AESNI_PARALLEL_BLOCKS 16 #define CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS 32 @@ -23,121 +23,6 @@ asmlinkage void camellia_ecb_enc_32way(const void *ctx, u8 *dst, const u8 *src); asmlinkage void camellia_ecb_dec_32way(const void *ctx, u8 *dst, const u8 *src); asmlinkage void camellia_cbc_dec_32way(const void *ctx, u8 *dst, const u8 *src); -asmlinkage void camellia_ctr_32way(const void *ctx, u8 *dst, const u8 *src, - le128 *iv); - -asmlinkage void camellia_xts_enc_32way(const void *ctx, u8 *dst, const u8 *src, - le128 *iv); -asmlinkage void camellia_xts_dec_32way(const void *ctx, u8 *dst, const u8 *src, - le128 *iv); - -static const struct common_glue_ctx camellia_enc = { - .num_funcs = 4, - .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS, - - .funcs = { { - .num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS, - .fn_u = { .ecb = camellia_ecb_enc_32way } - }, { - .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS, - .fn_u = { .ecb = camellia_ecb_enc_16way } - }, { - .num_blocks = 2, - .fn_u = { .ecb = camellia_enc_blk_2way } - }, { - .num_blocks = 1, - .fn_u = { .ecb = camellia_enc_blk } - } } -}; - -static const struct common_glue_ctx camellia_ctr = { - .num_funcs = 4, - .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS, - - .funcs = { { - .num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS, - .fn_u = { .ctr = camellia_ctr_32way } - }, { - .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS, - .fn_u = { .ctr = camellia_ctr_16way } - }, { - .num_blocks = 2, - .fn_u = { .ctr = camellia_crypt_ctr_2way } - }, { - .num_blocks = 1, - .fn_u = { .ctr = camellia_crypt_ctr } - } } -}; - -static const struct common_glue_ctx camellia_enc_xts = { - .num_funcs = 3, - .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS, - - .funcs = { { - .num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS, - .fn_u = { .xts = camellia_xts_enc_32way } - }, { - .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS, - .fn_u = { .xts = camellia_xts_enc_16way } - }, { - .num_blocks = 1, - .fn_u = { .xts = camellia_xts_enc } - } } -}; - -static const struct common_glue_ctx camellia_dec = { - .num_funcs = 4, - .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS, - - .funcs = { { - .num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS, - .fn_u = { .ecb = camellia_ecb_dec_32way } - }, { - .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS, - .fn_u = { .ecb = camellia_ecb_dec_16way } - }, { - .num_blocks = 2, - .fn_u = { .ecb = camellia_dec_blk_2way } - }, { - .num_blocks = 1, - .fn_u = { .ecb = camellia_dec_blk } - } } -}; - -static const struct common_glue_ctx camellia_dec_cbc = { - .num_funcs = 4, - .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS, - - .funcs = { { - .num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS, - .fn_u = { .cbc = camellia_cbc_dec_32way } - }, { - .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS, - .fn_u = { .cbc = camellia_cbc_dec_16way } - }, { - .num_blocks = 2, - .fn_u = { .cbc = camellia_decrypt_cbc_2way } - }, { - .num_blocks = 1, - .fn_u = { .cbc = camellia_dec_blk } - } } -}; - -static const struct common_glue_ctx camellia_dec_xts = { - .num_funcs = 3, - .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS, - - .funcs = { { - .num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS, - .fn_u = { .xts = camellia_xts_dec_32way } - }, { - .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS, - .fn_u = { .xts = camellia_xts_dec_16way } - }, { - .num_blocks = 1, - .fn_u = { .xts = camellia_xts_dec } - } } -}; static int camellia_setkey(struct crypto_skcipher *tfm, const u8 *key, unsigned int keylen) @@ -147,45 +32,39 @@ static int camellia_setkey(struct crypto_skcipher *tfm, const u8 *key, static int ecb_encrypt(struct skcipher_request *req) { - return glue_ecb_req_128bit(&camellia_enc, req); + ECB_WALK_START(req, CAMELLIA_BLOCK_SIZE, CAMELLIA_AESNI_PARALLEL_BLOCKS); + ECB_BLOCK(CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS, camellia_ecb_enc_32way); + ECB_BLOCK(CAMELLIA_AESNI_PARALLEL_BLOCKS, camellia_ecb_enc_16way); + ECB_BLOCK(2, camellia_enc_blk_2way); + ECB_BLOCK(1, camellia_enc_blk); + ECB_WALK_END(); } static int ecb_decrypt(struct skcipher_request *req) { - return glue_ecb_req_128bit(&camellia_dec, req); + ECB_WALK_START(req, CAMELLIA_BLOCK_SIZE, CAMELLIA_AESNI_PARALLEL_BLOCKS); + ECB_BLOCK(CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS, camellia_ecb_dec_32way); + ECB_BLOCK(CAMELLIA_AESNI_PARALLEL_BLOCKS, camellia_ecb_dec_16way); + ECB_BLOCK(2, camellia_dec_blk_2way); + ECB_BLOCK(1, camellia_dec_blk); + ECB_WALK_END(); } static int cbc_encrypt(struct skcipher_request *req) { - return glue_cbc_encrypt_req_128bit(camellia_enc_blk, req); + CBC_WALK_START(req, CAMELLIA_BLOCK_SIZE, -1); + CBC_ENC_BLOCK(camellia_enc_blk); + CBC_WALK_END(); } static int cbc_decrypt(struct skcipher_request *req) { - return glue_cbc_decrypt_req_128bit(&camellia_dec_cbc, req); -} - -static int ctr_crypt(struct skcipher_request *req) -{ - return glue_ctr_req_128bit(&camellia_ctr, req); -} - -static int xts_encrypt(struct skcipher_request *req) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct camellia_xts_ctx *ctx = crypto_skcipher_ctx(tfm); - - return glue_xts_req_128bit(&camellia_enc_xts, req, camellia_enc_blk, - &ctx->tweak_ctx, &ctx->crypt_ctx, false); -} - -static int xts_decrypt(struct skcipher_request *req) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct camellia_xts_ctx *ctx = crypto_skcipher_ctx(tfm); - - return glue_xts_req_128bit(&camellia_dec_xts, req, camellia_enc_blk, - &ctx->tweak_ctx, &ctx->crypt_ctx, true); + CBC_WALK_START(req, CAMELLIA_BLOCK_SIZE, CAMELLIA_AESNI_PARALLEL_BLOCKS); + CBC_DEC_BLOCK(CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS, camellia_cbc_dec_32way); + CBC_DEC_BLOCK(CAMELLIA_AESNI_PARALLEL_BLOCKS, camellia_cbc_dec_16way); + CBC_DEC_BLOCK(2, camellia_decrypt_cbc_2way); + CBC_DEC_BLOCK(1, camellia_dec_blk); + CBC_WALK_END(); } static struct skcipher_alg camellia_algs[] = { @@ -216,35 +95,6 @@ static struct skcipher_alg camellia_algs[] = { .setkey = camellia_setkey, .encrypt = cbc_encrypt, .decrypt = cbc_decrypt, - }, { - .base.cra_name = "__ctr(camellia)", - .base.cra_driver_name = "__ctr-camellia-aesni-avx2", - .base.cra_priority = 500, - .base.cra_flags = CRYPTO_ALG_INTERNAL, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct camellia_ctx), - .base.cra_module = THIS_MODULE, - .min_keysize = CAMELLIA_MIN_KEY_SIZE, - .max_keysize = CAMELLIA_MAX_KEY_SIZE, - .ivsize = CAMELLIA_BLOCK_SIZE, - .chunksize = CAMELLIA_BLOCK_SIZE, - .setkey = camellia_setkey, - .encrypt = ctr_crypt, - .decrypt = ctr_crypt, - }, { - .base.cra_name = "__xts(camellia)", - .base.cra_driver_name = "__xts-camellia-aesni-avx2", - .base.cra_priority = 500, - .base.cra_flags = CRYPTO_ALG_INTERNAL, - .base.cra_blocksize = CAMELLIA_BLOCK_SIZE, - .base.cra_ctxsize = sizeof(struct camellia_xts_ctx), - .base.cra_module = THIS_MODULE, - .min_keysize = 2 * CAMELLIA_MIN_KEY_SIZE, - .max_keysize = 2 * CAMELLIA_MAX_KEY_SIZE, - .ivsize = CAMELLIA_BLOCK_SIZE, - .setkey = xts_camellia_setkey, - .encrypt = xts_encrypt, - .decrypt = xts_decrypt, }, }; diff --git a/arch/x86/crypto/camellia_aesni_avx_glue.c b/arch/x86/crypto/camellia_aesni_avx_glue.c index 4e5de6ef206e1394cf20c0605e0d9b84164effd4..c7ccf63e741e16a9ec3386e63a206763bf3dee45 100644 --- a/arch/x86/crypto/camellia_aesni_avx_glue.c +++ b/arch/x86/crypto/camellia_aesni_avx_glue.c @@ -5,16 +5,16 @@ * Copyright © 2012-2013 Jussi Kivilinna */ -#include -#include #include #include -#include #include #include #include #include +#include "camellia.h" +#include "ecb_cbc_helpers.h" + #define CAMELLIA_AESNI_PARALLEL_BLOCKS 16 /* 16-way parallel cipher functions (avx/aes-ni) */ @@ -27,120 +27,6 @@ EXPORT_SYMBOL_GPL(camellia_ecb_dec_16way); asmlinkage void camellia_cbc_dec_16way(const void *ctx, u8 *dst, const u8 *src); EXPORT_SYMBOL_GPL(camellia_cbc_dec_16way); -asmlinkage void camellia_ctr_16way(const void *ctx, u8 *dst, const u8 *src, - le128 *iv); -EXPORT_SYMBOL_GPL(camellia_ctr_16way); - -asmlinkage void camellia_xts_enc_16way(const void *ctx, u8 *dst, const u8 *src, - le128 *iv); -EXPORT_SYMBOL_GPL(camellia_xts_enc_16way); - -asmlinkage void camellia_xts_dec_16way(const void *ctx, u8 *dst, const u8 *src, - le128 *iv); -EXPORT_SYMBOL_GPL(camellia_xts_dec_16way); - -void camellia_xts_enc(const void *ctx, u8 *dst, const u8 *src, le128 *iv) -{ - glue_xts_crypt_128bit_one(ctx, dst, src, iv, camellia_enc_blk); -} -EXPORT_SYMBOL_GPL(camellia_xts_enc); - -void camellia_xts_dec(const void *ctx, u8 *dst, const u8 *src, le128 *iv) -{ - glue_xts_crypt_128bit_one(ctx, dst, src, iv, camellia_dec_blk); -} -EXPORT_SYMBOL_GPL(camellia_xts_dec); - -static const struct common_glue_ctx camellia_enc = { - .num_funcs = 3, - .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS, - - .funcs = { { - .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS, - .fn_u = { .ecb = camellia_ecb_enc_16way } - }, { - .num_blocks = 2, - .fn_u = { .ecb = camellia_enc_blk_2way } - }, { - .num_blocks = 1, - .fn_u = { .ecb = camellia_enc_blk } - } } -}; - -static const struct common_glue_ctx camellia_ctr = { - .num_funcs = 3, - .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS, - - .funcs = { { - .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS, - .fn_u = { .ctr = camellia_ctr_16way } - }, { - .num_blocks = 2, - .fn_u = { .ctr = camellia_crypt_ctr_2way } - }, { - .num_blocks = 1, - .fn_u = { .ctr = camellia_crypt_ctr } - } } -}; - -static const struct common_glue_ctx camellia_enc_xts = { - .num_funcs = 2, - .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS, - - .funcs = { { - .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS, - .fn_u = { .xts = camellia_xts_enc_16way } - }, { - .num_blocks = 1, - .fn_u = { .xts = camellia_xts_enc } - } } -}; - -static const struct common_glue_ctx camellia_dec = { - .num_funcs = 3, - .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS, - - .funcs = { { - .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS, - .fn_u = { .ecb = camellia_ecb_dec_16way } - }, { - .num_blocks = 2, - .fn_u = { .ecb = camellia_dec_blk_2way } - }, { - .num_blocks = 1, - .fn_u = { .ecb = camellia_dec_blk } - } } -}; - -static const struct common_glue_ctx camellia_dec_cbc = { - .num_funcs = 3, - .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS, - - .funcs = { { - .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS, - .fn_u = { .cbc = camellia_cbc_dec_16way } - }, { - .num_blocks = 2, - .fn_u = { .cbc = camellia_decrypt_cbc_2way } - }, { - .num_blocks = 1, - .fn_u = { .cbc = camellia_dec_blk } - } } -}; - -static const struct common_glue_ctx camellia_dec_xts = { - .num_funcs = 2, - .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS, - - .funcs = { { - .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS, - .fn_u = { .xts = camellia_xts_dec_16way } - }, { - .num_blocks = 1, - .fn_u = { .xts = camellia_xts_dec } - } } -}; - static int camellia_setkey(struct crypto_skcipher *tfm, const u8 *key, unsigned int keylen) { @@ -149,65 +35,36 @@ static int camellia_setkey(struct crypto_skcipher *tfm, const u8 *key, static int ecb_encrypt(struct skcipher_request *req) { - return glue_ecb_req_128bit(&camellia_enc, req); + ECB_WALK_START(req, CAMELLIA_BLOCK_SIZE, CAMELLIA_AESNI_PARALLEL_BLOCKS); + ECB_BLOCK(CAMELLIA_AESNI_PARALLEL_BLOCKS, camellia_ecb_enc_16way); + ECB_BLOCK(2, camellia_enc_blk_2way); + ECB_BLOCK(1, camellia_enc_blk); + ECB_WALK_END(); } static int ecb_decrypt(struct skcipher_request *req) { - return glue_ecb_req_128bit(&camellia_dec, req); + ECB_WALK_START(req, CAMELLIA_BLOCK_SIZE, CAMELLIA_AESNI_PARALLEL_BLOCKS); + ECB_BLOCK(CAMELLIA_AESNI_PARALLEL_BLOCKS, camellia_ecb_dec_16way); + ECB_BLOCK(2, camellia_dec_blk_2way); + ECB_BLOCK(1, camellia_dec_blk); + ECB_WALK_END(); } static int cbc_encrypt(struct skcipher_request *req) { - return glue_cbc_encrypt_req_128bit(camellia_enc_blk, req); + CBC_WALK_START(req, CAMELLIA_BLOCK_SIZE, -1); + CBC_ENC_BLOCK(camellia_enc_blk); + CBC_WALK_END(); } static int cbc_decrypt(struct skcipher_request *req) { - return glue_cbc_decrypt_req_128bit(&camellia_dec_cbc, req); -} - -static int ctr_crypt(struct skcipher_request *req) -{ - return glue_ctr_req_128bit(&camellia_ctr, req); -} - -int xts_camellia_setkey(struct crypto_skcipher *tfm, const u8 *key, - unsigned int keylen) -{ - struct camellia_xts_ctx *ctx = crypto_skcipher_ctx(tfm); - int err; - - err = xts_verify_key(tfm, key, keylen); - if (err) - return err; - - /* first half of xts-key is for crypt */ - err = __camellia_setkey(&ctx->crypt_ctx, key, keylen / 2); - if (err) - return err; - - /* second half of xts-key is for tweak */ - return __camellia_setkey(&ctx->tweak_ctx, key + keylen / 2, keylen / 2); -} -EXPORT_SYMBOL_GPL(xts_camellia_setkey); - -static int xts_encrypt(struct skcipher_request *req) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct camellia_xts_ctx *ctx = crypto_skcipher_ctx(tfm); - - return glue_xts_req_128bit(&camellia_enc_xts, req, camellia_enc_blk, - &ctx->tweak_ctx, &ctx->crypt_ctx, false); -} - -static int xts_decrypt(struct skcipher_request *req) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct camellia_xts_ctx *ctx = crypto_skcipher_ctx(tfm); - - return glue_xts_req_128bit(&camellia_dec_xts, req, camellia_enc_blk, - &ctx->tweak_ctx, &ctx->crypt_ctx, true); + CBC_WALK_START(req, CAMELLIA_BLOCK_SIZE, CAMELLIA_AESNI_PARALLEL_BLOCKS); + CBC_DEC_BLOCK(CAMELLIA_AESNI_PARALLEL_BLOCKS, camellia_cbc_dec_16way); + CBC_DEC_BLOCK(2, camellia_decrypt_cbc_2way); + CBC_DEC_BLOCK(1, camellia_dec_blk); + CBC_WALK_END(); } static struct skcipher_alg camellia_algs[] = { @@ -238,36 +95,7 @@ static struct skcipher_alg camellia_algs[] = { .setkey = camellia_setkey, .encrypt = cbc_encrypt, .decrypt = cbc_decrypt, - }, { - .base.cra_name = "__ctr(camellia)", - .base.cra_driver_name = "__ctr-camellia-aesni", - .base.cra_priority = 400, - .base.cra_flags = CRYPTO_ALG_INTERNAL, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct camellia_ctx), - .base.cra_module = THIS_MODULE, - .min_keysize = CAMELLIA_MIN_KEY_SIZE, - .max_keysize = CAMELLIA_MAX_KEY_SIZE, - .ivsize = CAMELLIA_BLOCK_SIZE, - .chunksize = CAMELLIA_BLOCK_SIZE, - .setkey = camellia_setkey, - .encrypt = ctr_crypt, - .decrypt = ctr_crypt, - }, { - .base.cra_name = "__xts(camellia)", - .base.cra_driver_name = "__xts-camellia-aesni", - .base.cra_priority = 400, - .base.cra_flags = CRYPTO_ALG_INTERNAL, - .base.cra_blocksize = CAMELLIA_BLOCK_SIZE, - .base.cra_ctxsize = sizeof(struct camellia_xts_ctx), - .base.cra_module = THIS_MODULE, - .min_keysize = 2 * CAMELLIA_MIN_KEY_SIZE, - .max_keysize = 2 * CAMELLIA_MAX_KEY_SIZE, - .ivsize = CAMELLIA_BLOCK_SIZE, - .setkey = xts_camellia_setkey, - .encrypt = xts_encrypt, - .decrypt = xts_decrypt, - }, + } }; static struct simd_skcipher_alg *camellia_simd_algs[ARRAY_SIZE(camellia_algs)]; diff --git a/arch/x86/crypto/camellia_glue.c b/arch/x86/crypto/camellia_glue.c index 242c056e5fa85149eb266ea775d583d8350446db..66c435ba9d3da0a6ffde0b114bde995631ab1b33 100644 --- a/arch/x86/crypto/camellia_glue.c +++ b/arch/x86/crypto/camellia_glue.c @@ -14,8 +14,9 @@ #include #include #include -#include -#include + +#include "camellia.h" +#include "ecb_cbc_helpers.h" /* regular block cipher functions */ asmlinkage void __camellia_enc_blk(const void *ctx, u8 *dst, const u8 *src, @@ -1262,129 +1263,47 @@ static int camellia_setkey_skcipher(struct crypto_skcipher *tfm, const u8 *key, return camellia_setkey(&tfm->base, key, key_len); } -void camellia_decrypt_cbc_2way(const void *ctx, u8 *d, const u8 *s) +void camellia_decrypt_cbc_2way(const void *ctx, u8 *dst, const u8 *src) { - u128 *dst = (u128 *)d; - const u128 *src = (const u128 *)s; - u128 iv = *src; - - camellia_dec_blk_2way(ctx, (u8 *)dst, (u8 *)src); + u8 buf[CAMELLIA_BLOCK_SIZE]; + const u8 *iv = src; - u128_xor(&dst[1], &dst[1], &iv); + if (dst == src) + iv = memcpy(buf, iv, sizeof(buf)); + camellia_dec_blk_2way(ctx, dst, src); + crypto_xor(dst + CAMELLIA_BLOCK_SIZE, iv, CAMELLIA_BLOCK_SIZE); } EXPORT_SYMBOL_GPL(camellia_decrypt_cbc_2way); -void camellia_crypt_ctr(const void *ctx, u8 *d, const u8 *s, le128 *iv) -{ - be128 ctrblk; - u128 *dst = (u128 *)d; - const u128 *src = (const u128 *)s; - - if (dst != src) - *dst = *src; - - le128_to_be128(&ctrblk, iv); - le128_inc(iv); - - camellia_enc_blk_xor(ctx, (u8 *)dst, (u8 *)&ctrblk); -} -EXPORT_SYMBOL_GPL(camellia_crypt_ctr); - -void camellia_crypt_ctr_2way(const void *ctx, u8 *d, const u8 *s, le128 *iv) -{ - be128 ctrblks[2]; - u128 *dst = (u128 *)d; - const u128 *src = (const u128 *)s; - - if (dst != src) { - dst[0] = src[0]; - dst[1] = src[1]; - } - - le128_to_be128(&ctrblks[0], iv); - le128_inc(iv); - le128_to_be128(&ctrblks[1], iv); - le128_inc(iv); - - camellia_enc_blk_xor_2way(ctx, (u8 *)dst, (u8 *)ctrblks); -} -EXPORT_SYMBOL_GPL(camellia_crypt_ctr_2way); - -static const struct common_glue_ctx camellia_enc = { - .num_funcs = 2, - .fpu_blocks_limit = -1, - - .funcs = { { - .num_blocks = 2, - .fn_u = { .ecb = camellia_enc_blk_2way } - }, { - .num_blocks = 1, - .fn_u = { .ecb = camellia_enc_blk } - } } -}; - -static const struct common_glue_ctx camellia_ctr = { - .num_funcs = 2, - .fpu_blocks_limit = -1, - - .funcs = { { - .num_blocks = 2, - .fn_u = { .ctr = camellia_crypt_ctr_2way } - }, { - .num_blocks = 1, - .fn_u = { .ctr = camellia_crypt_ctr } - } } -}; - -static const struct common_glue_ctx camellia_dec = { - .num_funcs = 2, - .fpu_blocks_limit = -1, - - .funcs = { { - .num_blocks = 2, - .fn_u = { .ecb = camellia_dec_blk_2way } - }, { - .num_blocks = 1, - .fn_u = { .ecb = camellia_dec_blk } - } } -}; - -static const struct common_glue_ctx camellia_dec_cbc = { - .num_funcs = 2, - .fpu_blocks_limit = -1, - - .funcs = { { - .num_blocks = 2, - .fn_u = { .cbc = camellia_decrypt_cbc_2way } - }, { - .num_blocks = 1, - .fn_u = { .cbc = camellia_dec_blk } - } } -}; - static int ecb_encrypt(struct skcipher_request *req) { - return glue_ecb_req_128bit(&camellia_enc, req); + ECB_WALK_START(req, CAMELLIA_BLOCK_SIZE, -1); + ECB_BLOCK(2, camellia_enc_blk_2way); + ECB_BLOCK(1, camellia_enc_blk); + ECB_WALK_END(); } static int ecb_decrypt(struct skcipher_request *req) { - return glue_ecb_req_128bit(&camellia_dec, req); + ECB_WALK_START(req, CAMELLIA_BLOCK_SIZE, -1); + ECB_BLOCK(2, camellia_dec_blk_2way); + ECB_BLOCK(1, camellia_dec_blk); + ECB_WALK_END(); } static int cbc_encrypt(struct skcipher_request *req) { - return glue_cbc_encrypt_req_128bit(camellia_enc_blk, req); + CBC_WALK_START(req, CAMELLIA_BLOCK_SIZE, -1); + CBC_ENC_BLOCK(camellia_enc_blk); + CBC_WALK_END(); } static int cbc_decrypt(struct skcipher_request *req) { - return glue_cbc_decrypt_req_128bit(&camellia_dec_cbc, req); -} - -static int ctr_crypt(struct skcipher_request *req) -{ - return glue_ctr_req_128bit(&camellia_ctr, req); + CBC_WALK_START(req, CAMELLIA_BLOCK_SIZE, -1); + CBC_DEC_BLOCK(2, camellia_decrypt_cbc_2way); + CBC_DEC_BLOCK(1, camellia_dec_blk); + CBC_WALK_END(); } static struct crypto_alg camellia_cipher_alg = { @@ -1433,20 +1352,6 @@ static struct skcipher_alg camellia_skcipher_algs[] = { .setkey = camellia_setkey_skcipher, .encrypt = cbc_encrypt, .decrypt = cbc_decrypt, - }, { - .base.cra_name = "ctr(camellia)", - .base.cra_driver_name = "ctr-camellia-asm", - .base.cra_priority = 300, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct camellia_ctx), - .base.cra_module = THIS_MODULE, - .min_keysize = CAMELLIA_MIN_KEY_SIZE, - .max_keysize = CAMELLIA_MAX_KEY_SIZE, - .ivsize = CAMELLIA_BLOCK_SIZE, - .chunksize = CAMELLIA_BLOCK_SIZE, - .setkey = camellia_setkey_skcipher, - .encrypt = ctr_crypt, - .decrypt = ctr_crypt, } }; diff --git a/arch/x86/crypto/cast5_avx_glue.c b/arch/x86/crypto/cast5_avx_glue.c index 384ccb00f9e15ab274d41900aab2e24dd0162117..3976a87f92ad504665d7d75f9d8eb59a490b2674 100644 --- a/arch/x86/crypto/cast5_avx_glue.c +++ b/arch/x86/crypto/cast5_avx_glue.c @@ -6,7 +6,6 @@ * */ -#include #include #include #include @@ -15,6 +14,8 @@ #include #include +#include "ecb_cbc_helpers.h" + #define CAST5_PARALLEL_BLOCKS 16 asmlinkage void cast5_ecb_enc_16way(struct cast5_ctx *ctx, u8 *dst, @@ -23,8 +24,6 @@ asmlinkage void cast5_ecb_dec_16way(struct cast5_ctx *ctx, u8 *dst, const u8 *src); asmlinkage void cast5_cbc_dec_16way(struct cast5_ctx *ctx, u8 *dst, const u8 *src); -asmlinkage void cast5_ctr_16way(struct cast5_ctx *ctx, u8 *dst, const u8 *src, - __be64 *iv); static int cast5_setkey_skcipher(struct crypto_skcipher *tfm, const u8 *key, unsigned int keylen) @@ -32,272 +31,35 @@ static int cast5_setkey_skcipher(struct crypto_skcipher *tfm, const u8 *key, return cast5_setkey(&tfm->base, key, keylen); } -static inline bool cast5_fpu_begin(bool fpu_enabled, struct skcipher_walk *walk, - unsigned int nbytes) -{ - return glue_fpu_begin(CAST5_BLOCK_SIZE, CAST5_PARALLEL_BLOCKS, - walk, fpu_enabled, nbytes); -} - -static inline void cast5_fpu_end(bool fpu_enabled) -{ - return glue_fpu_end(fpu_enabled); -} - -static int ecb_crypt(struct skcipher_request *req, bool enc) -{ - bool fpu_enabled = false; - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct cast5_ctx *ctx = crypto_skcipher_ctx(tfm); - struct skcipher_walk walk; - const unsigned int bsize = CAST5_BLOCK_SIZE; - unsigned int nbytes; - void (*fn)(struct cast5_ctx *ctx, u8 *dst, const u8 *src); - int err; - - err = skcipher_walk_virt(&walk, req, false); - - while ((nbytes = walk.nbytes)) { - u8 *wsrc = walk.src.virt.addr; - u8 *wdst = walk.dst.virt.addr; - - fpu_enabled = cast5_fpu_begin(fpu_enabled, &walk, nbytes); - - /* Process multi-block batch */ - if (nbytes >= bsize * CAST5_PARALLEL_BLOCKS) { - fn = (enc) ? cast5_ecb_enc_16way : cast5_ecb_dec_16way; - do { - fn(ctx, wdst, wsrc); - - wsrc += bsize * CAST5_PARALLEL_BLOCKS; - wdst += bsize * CAST5_PARALLEL_BLOCKS; - nbytes -= bsize * CAST5_PARALLEL_BLOCKS; - } while (nbytes >= bsize * CAST5_PARALLEL_BLOCKS); - - if (nbytes < bsize) - goto done; - } - - fn = (enc) ? __cast5_encrypt : __cast5_decrypt; - - /* Handle leftovers */ - do { - fn(ctx, wdst, wsrc); - - wsrc += bsize; - wdst += bsize; - nbytes -= bsize; - } while (nbytes >= bsize); - -done: - err = skcipher_walk_done(&walk, nbytes); - } - - cast5_fpu_end(fpu_enabled); - return err; -} - static int ecb_encrypt(struct skcipher_request *req) { - return ecb_crypt(req, true); + ECB_WALK_START(req, CAST5_BLOCK_SIZE, CAST5_PARALLEL_BLOCKS); + ECB_BLOCK(CAST5_PARALLEL_BLOCKS, cast5_ecb_enc_16way); + ECB_BLOCK(1, __cast5_encrypt); + ECB_WALK_END(); } static int ecb_decrypt(struct skcipher_request *req) { - return ecb_crypt(req, false); + ECB_WALK_START(req, CAST5_BLOCK_SIZE, CAST5_PARALLEL_BLOCKS); + ECB_BLOCK(CAST5_PARALLEL_BLOCKS, cast5_ecb_dec_16way); + ECB_BLOCK(1, __cast5_decrypt); + ECB_WALK_END(); } static int cbc_encrypt(struct skcipher_request *req) { - const unsigned int bsize = CAST5_BLOCK_SIZE; - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct cast5_ctx *ctx = crypto_skcipher_ctx(tfm); - struct skcipher_walk walk; - unsigned int nbytes; - int err; - - err = skcipher_walk_virt(&walk, req, false); - - while ((nbytes = walk.nbytes)) { - u64 *src = (u64 *)walk.src.virt.addr; - u64 *dst = (u64 *)walk.dst.virt.addr; - u64 *iv = (u64 *)walk.iv; - - do { - *dst = *src ^ *iv; - __cast5_encrypt(ctx, (u8 *)dst, (u8 *)dst); - iv = dst; - src++; - dst++; - nbytes -= bsize; - } while (nbytes >= bsize); - - *(u64 *)walk.iv = *iv; - err = skcipher_walk_done(&walk, nbytes); - } - - return err; -} - -static unsigned int __cbc_decrypt(struct cast5_ctx *ctx, - struct skcipher_walk *walk) -{ - const unsigned int bsize = CAST5_BLOCK_SIZE; - unsigned int nbytes = walk->nbytes; - u64 *src = (u64 *)walk->src.virt.addr; - u64 *dst = (u64 *)walk->dst.virt.addr; - u64 last_iv; - - /* Start of the last block. */ - src += nbytes / bsize - 1; - dst += nbytes / bsize - 1; - - last_iv = *src; - - /* Process multi-block batch */ - if (nbytes >= bsize * CAST5_PARALLEL_BLOCKS) { - do { - nbytes -= bsize * (CAST5_PARALLEL_BLOCKS - 1); - src -= CAST5_PARALLEL_BLOCKS - 1; - dst -= CAST5_PARALLEL_BLOCKS - 1; - - cast5_cbc_dec_16way(ctx, (u8 *)dst, (u8 *)src); - - nbytes -= bsize; - if (nbytes < bsize) - goto done; - - *dst ^= *(src - 1); - src -= 1; - dst -= 1; - } while (nbytes >= bsize * CAST5_PARALLEL_BLOCKS); - } - - /* Handle leftovers */ - for (;;) { - __cast5_decrypt(ctx, (u8 *)dst, (u8 *)src); - - nbytes -= bsize; - if (nbytes < bsize) - break; - - *dst ^= *(src - 1); - src -= 1; - dst -= 1; - } - -done: - *dst ^= *(u64 *)walk->iv; - *(u64 *)walk->iv = last_iv; - - return nbytes; + CBC_WALK_START(req, CAST5_BLOCK_SIZE, -1); + CBC_ENC_BLOCK(__cast5_encrypt); + CBC_WALK_END(); } static int cbc_decrypt(struct skcipher_request *req) { - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct cast5_ctx *ctx = crypto_skcipher_ctx(tfm); - bool fpu_enabled = false; - struct skcipher_walk walk; - unsigned int nbytes; - int err; - - err = skcipher_walk_virt(&walk, req, false); - - while ((nbytes = walk.nbytes)) { - fpu_enabled = cast5_fpu_begin(fpu_enabled, &walk, nbytes); - nbytes = __cbc_decrypt(ctx, &walk); - err = skcipher_walk_done(&walk, nbytes); - } - - cast5_fpu_end(fpu_enabled); - return err; -} - -static void ctr_crypt_final(struct skcipher_walk *walk, struct cast5_ctx *ctx) -{ - u8 *ctrblk = walk->iv; - u8 keystream[CAST5_BLOCK_SIZE]; - u8 *src = walk->src.virt.addr; - u8 *dst = walk->dst.virt.addr; - unsigned int nbytes = walk->nbytes; - - __cast5_encrypt(ctx, keystream, ctrblk); - crypto_xor_cpy(dst, keystream, src, nbytes); - - crypto_inc(ctrblk, CAST5_BLOCK_SIZE); -} - -static unsigned int __ctr_crypt(struct skcipher_walk *walk, - struct cast5_ctx *ctx) -{ - const unsigned int bsize = CAST5_BLOCK_SIZE; - unsigned int nbytes = walk->nbytes; - u64 *src = (u64 *)walk->src.virt.addr; - u64 *dst = (u64 *)walk->dst.virt.addr; - - /* Process multi-block batch */ - if (nbytes >= bsize * CAST5_PARALLEL_BLOCKS) { - do { - cast5_ctr_16way(ctx, (u8 *)dst, (u8 *)src, - (__be64 *)walk->iv); - - src += CAST5_PARALLEL_BLOCKS; - dst += CAST5_PARALLEL_BLOCKS; - nbytes -= bsize * CAST5_PARALLEL_BLOCKS; - } while (nbytes >= bsize * CAST5_PARALLEL_BLOCKS); - - if (nbytes < bsize) - goto done; - } - - /* Handle leftovers */ - do { - u64 ctrblk; - - if (dst != src) - *dst = *src; - - ctrblk = *(u64 *)walk->iv; - be64_add_cpu((__be64 *)walk->iv, 1); - - __cast5_encrypt(ctx, (u8 *)&ctrblk, (u8 *)&ctrblk); - *dst ^= ctrblk; - - src += 1; - dst += 1; - nbytes -= bsize; - } while (nbytes >= bsize); - -done: - return nbytes; -} - -static int ctr_crypt(struct skcipher_request *req) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct cast5_ctx *ctx = crypto_skcipher_ctx(tfm); - bool fpu_enabled = false; - struct skcipher_walk walk; - unsigned int nbytes; - int err; - - err = skcipher_walk_virt(&walk, req, false); - - while ((nbytes = walk.nbytes) >= CAST5_BLOCK_SIZE) { - fpu_enabled = cast5_fpu_begin(fpu_enabled, &walk, nbytes); - nbytes = __ctr_crypt(&walk, ctx); - err = skcipher_walk_done(&walk, nbytes); - } - - cast5_fpu_end(fpu_enabled); - - if (walk.nbytes) { - ctr_crypt_final(&walk, ctx); - err = skcipher_walk_done(&walk, 0); - } - - return err; + CBC_WALK_START(req, CAST5_BLOCK_SIZE, CAST5_PARALLEL_BLOCKS); + CBC_DEC_BLOCK(CAST5_PARALLEL_BLOCKS, cast5_cbc_dec_16way); + CBC_DEC_BLOCK(1, __cast5_decrypt); + CBC_WALK_END(); } static struct skcipher_alg cast5_algs[] = { @@ -328,21 +90,6 @@ static struct skcipher_alg cast5_algs[] = { .setkey = cast5_setkey_skcipher, .encrypt = cbc_encrypt, .decrypt = cbc_decrypt, - }, { - .base.cra_name = "__ctr(cast5)", - .base.cra_driver_name = "__ctr-cast5-avx", - .base.cra_priority = 200, - .base.cra_flags = CRYPTO_ALG_INTERNAL, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct cast5_ctx), - .base.cra_module = THIS_MODULE, - .min_keysize = CAST5_MIN_KEY_SIZE, - .max_keysize = CAST5_MAX_KEY_SIZE, - .ivsize = CAST5_BLOCK_SIZE, - .chunksize = CAST5_BLOCK_SIZE, - .setkey = cast5_setkey_skcipher, - .encrypt = ctr_crypt, - .decrypt = ctr_crypt, } }; diff --git a/arch/x86/crypto/cast6-avx-x86_64-asm_64.S b/arch/x86/crypto/cast6-avx-x86_64-asm_64.S index 932a3ce32a88e2c78bd121ea18ae0b21734bc74a..fbddcecc3e3fc463e9e8af1deae1dcf270264e88 100644 --- a/arch/x86/crypto/cast6-avx-x86_64-asm_64.S +++ b/arch/x86/crypto/cast6-avx-x86_64-asm_64.S @@ -212,8 +212,6 @@ .section .rodata.cst16, "aM", @progbits, 16 .align 16 -.Lxts_gf128mul_and_shl1_mask: - .byte 0x87, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0 .Lbswap_mask: .byte 3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12 .Lbswap128_mask: @@ -412,85 +410,3 @@ SYM_FUNC_START(cast6_cbc_dec_8way) FRAME_END ret; SYM_FUNC_END(cast6_cbc_dec_8way) - -SYM_FUNC_START(cast6_ctr_8way) - /* input: - * %rdi: ctx, CTX - * %rsi: dst - * %rdx: src - * %rcx: iv (little endian, 128bit) - */ - FRAME_BEGIN - pushq %r12; - pushq %r15 - - movq %rdi, CTX; - movq %rsi, %r11; - movq %rdx, %r12; - - load_ctr_8way(%rcx, .Lbswap128_mask, RA1, RB1, RC1, RD1, RA2, RB2, RC2, - RD2, RX, RKR, RKM); - - call __cast6_enc_blk8; - - store_ctr_8way(%r12, %r11, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2); - - popq %r15; - popq %r12; - FRAME_END - ret; -SYM_FUNC_END(cast6_ctr_8way) - -SYM_FUNC_START(cast6_xts_enc_8way) - /* input: - * %rdi: ctx, CTX - * %rsi: dst - * %rdx: src - * %rcx: iv (t ⊕ α⿠∈ GF(2¹²â¸)) - */ - FRAME_BEGIN - pushq %r15; - - movq %rdi, CTX - movq %rsi, %r11; - - /* regs <= src, dst <= IVs, regs <= regs xor IVs */ - load_xts_8way(%rcx, %rdx, %rsi, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2, - RX, RKR, RKM, .Lxts_gf128mul_and_shl1_mask); - - call __cast6_enc_blk8; - - /* dst <= regs xor IVs(in dst) */ - store_xts_8way(%r11, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2); - - popq %r15; - FRAME_END - ret; -SYM_FUNC_END(cast6_xts_enc_8way) - -SYM_FUNC_START(cast6_xts_dec_8way) - /* input: - * %rdi: ctx, CTX - * %rsi: dst - * %rdx: src - * %rcx: iv (t ⊕ α⿠∈ GF(2¹²â¸)) - */ - FRAME_BEGIN - pushq %r15; - - movq %rdi, CTX - movq %rsi, %r11; - - /* regs <= src, dst <= IVs, regs <= regs xor IVs */ - load_xts_8way(%rcx, %rdx, %rsi, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2, - RX, RKR, RKM, .Lxts_gf128mul_and_shl1_mask); - - call __cast6_dec_blk8; - - /* dst <= regs xor IVs(in dst) */ - store_xts_8way(%r11, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2); - - popq %r15; - FRAME_END - ret; -SYM_FUNC_END(cast6_xts_dec_8way) diff --git a/arch/x86/crypto/cast6_avx_glue.c b/arch/x86/crypto/cast6_avx_glue.c index 48e0f37796fa0215dd2296255ff018b80f6f3273..7e2aea372349085bcb090a2117dd32019c67a81f 100644 --- a/arch/x86/crypto/cast6_avx_glue.c +++ b/arch/x86/crypto/cast6_avx_glue.c @@ -15,8 +15,8 @@ #include #include #include -#include -#include + +#include "ecb_cbc_helpers.h" #define CAST6_PARALLEL_BLOCKS 8 @@ -24,13 +24,6 @@ asmlinkage void cast6_ecb_enc_8way(const void *ctx, u8 *dst, const u8 *src); asmlinkage void cast6_ecb_dec_8way(const void *ctx, u8 *dst, const u8 *src); asmlinkage void cast6_cbc_dec_8way(const void *ctx, u8 *dst, const u8 *src); -asmlinkage void cast6_ctr_8way(const void *ctx, u8 *dst, const u8 *src, - le128 *iv); - -asmlinkage void cast6_xts_enc_8way(const void *ctx, u8 *dst, const u8 *src, - le128 *iv); -asmlinkage void cast6_xts_dec_8way(const void *ctx, u8 *dst, const u8 *src, - le128 *iv); static int cast6_setkey_skcipher(struct crypto_skcipher *tfm, const u8 *key, unsigned int keylen) @@ -38,172 +31,35 @@ static int cast6_setkey_skcipher(struct crypto_skcipher *tfm, return cast6_setkey(&tfm->base, key, keylen); } -static void cast6_xts_enc(const void *ctx, u8 *dst, const u8 *src, le128 *iv) -{ - glue_xts_crypt_128bit_one(ctx, dst, src, iv, __cast6_encrypt); -} - -static void cast6_xts_dec(const void *ctx, u8 *dst, const u8 *src, le128 *iv) -{ - glue_xts_crypt_128bit_one(ctx, dst, src, iv, __cast6_decrypt); -} - -static void cast6_crypt_ctr(const void *ctx, u8 *d, const u8 *s, le128 *iv) -{ - be128 ctrblk; - u128 *dst = (u128 *)d; - const u128 *src = (const u128 *)s; - - le128_to_be128(&ctrblk, iv); - le128_inc(iv); - - __cast6_encrypt(ctx, (u8 *)&ctrblk, (u8 *)&ctrblk); - u128_xor(dst, src, (u128 *)&ctrblk); -} - -static const struct common_glue_ctx cast6_enc = { - .num_funcs = 2, - .fpu_blocks_limit = CAST6_PARALLEL_BLOCKS, - - .funcs = { { - .num_blocks = CAST6_PARALLEL_BLOCKS, - .fn_u = { .ecb = cast6_ecb_enc_8way } - }, { - .num_blocks = 1, - .fn_u = { .ecb = __cast6_encrypt } - } } -}; - -static const struct common_glue_ctx cast6_ctr = { - .num_funcs = 2, - .fpu_blocks_limit = CAST6_PARALLEL_BLOCKS, - - .funcs = { { - .num_blocks = CAST6_PARALLEL_BLOCKS, - .fn_u = { .ctr = cast6_ctr_8way } - }, { - .num_blocks = 1, - .fn_u = { .ctr = cast6_crypt_ctr } - } } -}; - -static const struct common_glue_ctx cast6_enc_xts = { - .num_funcs = 2, - .fpu_blocks_limit = CAST6_PARALLEL_BLOCKS, - - .funcs = { { - .num_blocks = CAST6_PARALLEL_BLOCKS, - .fn_u = { .xts = cast6_xts_enc_8way } - }, { - .num_blocks = 1, - .fn_u = { .xts = cast6_xts_enc } - } } -}; - -static const struct common_glue_ctx cast6_dec = { - .num_funcs = 2, - .fpu_blocks_limit = CAST6_PARALLEL_BLOCKS, - - .funcs = { { - .num_blocks = CAST6_PARALLEL_BLOCKS, - .fn_u = { .ecb = cast6_ecb_dec_8way } - }, { - .num_blocks = 1, - .fn_u = { .ecb = __cast6_decrypt } - } } -}; - -static const struct common_glue_ctx cast6_dec_cbc = { - .num_funcs = 2, - .fpu_blocks_limit = CAST6_PARALLEL_BLOCKS, - - .funcs = { { - .num_blocks = CAST6_PARALLEL_BLOCKS, - .fn_u = { .cbc = cast6_cbc_dec_8way } - }, { - .num_blocks = 1, - .fn_u = { .cbc = __cast6_decrypt } - } } -}; - -static const struct common_glue_ctx cast6_dec_xts = { - .num_funcs = 2, - .fpu_blocks_limit = CAST6_PARALLEL_BLOCKS, - - .funcs = { { - .num_blocks = CAST6_PARALLEL_BLOCKS, - .fn_u = { .xts = cast6_xts_dec_8way } - }, { - .num_blocks = 1, - .fn_u = { .xts = cast6_xts_dec } - } } -}; - static int ecb_encrypt(struct skcipher_request *req) { - return glue_ecb_req_128bit(&cast6_enc, req); + ECB_WALK_START(req, CAST6_BLOCK_SIZE, CAST6_PARALLEL_BLOCKS); + ECB_BLOCK(CAST6_PARALLEL_BLOCKS, cast6_ecb_enc_8way); + ECB_BLOCK(1, __cast6_encrypt); + ECB_WALK_END(); } static int ecb_decrypt(struct skcipher_request *req) { - return glue_ecb_req_128bit(&cast6_dec, req); + ECB_WALK_START(req, CAST6_BLOCK_SIZE, CAST6_PARALLEL_BLOCKS); + ECB_BLOCK(CAST6_PARALLEL_BLOCKS, cast6_ecb_dec_8way); + ECB_BLOCK(1, __cast6_decrypt); + ECB_WALK_END(); } static int cbc_encrypt(struct skcipher_request *req) { - return glue_cbc_encrypt_req_128bit(__cast6_encrypt, req); + CBC_WALK_START(req, CAST6_BLOCK_SIZE, -1); + CBC_ENC_BLOCK(__cast6_encrypt); + CBC_WALK_END(); } static int cbc_decrypt(struct skcipher_request *req) { - return glue_cbc_decrypt_req_128bit(&cast6_dec_cbc, req); -} - -static int ctr_crypt(struct skcipher_request *req) -{ - return glue_ctr_req_128bit(&cast6_ctr, req); -} - -struct cast6_xts_ctx { - struct cast6_ctx tweak_ctx; - struct cast6_ctx crypt_ctx; -}; - -static int xts_cast6_setkey(struct crypto_skcipher *tfm, const u8 *key, - unsigned int keylen) -{ - struct cast6_xts_ctx *ctx = crypto_skcipher_ctx(tfm); - int err; - - err = xts_verify_key(tfm, key, keylen); - if (err) - return err; - - /* first half of xts-key is for crypt */ - err = __cast6_setkey(&ctx->crypt_ctx, key, keylen / 2); - if (err) - return err; - - /* second half of xts-key is for tweak */ - return __cast6_setkey(&ctx->tweak_ctx, key + keylen / 2, keylen / 2); -} - -static int xts_encrypt(struct skcipher_request *req) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct cast6_xts_ctx *ctx = crypto_skcipher_ctx(tfm); - - return glue_xts_req_128bit(&cast6_enc_xts, req, __cast6_encrypt, - &ctx->tweak_ctx, &ctx->crypt_ctx, false); -} - -static int xts_decrypt(struct skcipher_request *req) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct cast6_xts_ctx *ctx = crypto_skcipher_ctx(tfm); - - return glue_xts_req_128bit(&cast6_dec_xts, req, __cast6_encrypt, - &ctx->tweak_ctx, &ctx->crypt_ctx, true); + CBC_WALK_START(req, CAST6_BLOCK_SIZE, CAST6_PARALLEL_BLOCKS); + CBC_DEC_BLOCK(CAST6_PARALLEL_BLOCKS, cast6_cbc_dec_8way); + CBC_DEC_BLOCK(1, __cast6_decrypt); + CBC_WALK_END(); } static struct skcipher_alg cast6_algs[] = { @@ -234,35 +90,6 @@ static struct skcipher_alg cast6_algs[] = { .setkey = cast6_setkey_skcipher, .encrypt = cbc_encrypt, .decrypt = cbc_decrypt, - }, { - .base.cra_name = "__ctr(cast6)", - .base.cra_driver_name = "__ctr-cast6-avx", - .base.cra_priority = 200, - .base.cra_flags = CRYPTO_ALG_INTERNAL, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct cast6_ctx), - .base.cra_module = THIS_MODULE, - .min_keysize = CAST6_MIN_KEY_SIZE, - .max_keysize = CAST6_MAX_KEY_SIZE, - .ivsize = CAST6_BLOCK_SIZE, - .chunksize = CAST6_BLOCK_SIZE, - .setkey = cast6_setkey_skcipher, - .encrypt = ctr_crypt, - .decrypt = ctr_crypt, - }, { - .base.cra_name = "__xts(cast6)", - .base.cra_driver_name = "__xts-cast6-avx", - .base.cra_priority = 200, - .base.cra_flags = CRYPTO_ALG_INTERNAL, - .base.cra_blocksize = CAST6_BLOCK_SIZE, - .base.cra_ctxsize = sizeof(struct cast6_xts_ctx), - .base.cra_module = THIS_MODULE, - .min_keysize = 2 * CAST6_MIN_KEY_SIZE, - .max_keysize = 2 * CAST6_MAX_KEY_SIZE, - .ivsize = CAST6_BLOCK_SIZE, - .setkey = xts_cast6_setkey, - .encrypt = xts_encrypt, - .decrypt = xts_decrypt, }, }; diff --git a/arch/x86/crypto/des3_ede_glue.c b/arch/x86/crypto/des3_ede_glue.c index 89830e531350350cb8ea8c10ee213a4868652340..e7cb68a3db3be4738ec8cda0c25999d135db8e19 100644 --- a/arch/x86/crypto/des3_ede_glue.c +++ b/arch/x86/crypto/des3_ede_glue.c @@ -6,8 +6,6 @@ * * CBC & ECB parts based on code (crypto/cbc.c,ecb.c) by: * Copyright (c) 2006 Herbert Xu - * CTR part based on code (crypto/ctr.c) by: - * (C) Copyright IBM Corp. 2007 - Joy Latten */ #include @@ -253,94 +251,6 @@ static int cbc_decrypt(struct skcipher_request *req) return err; } -static void ctr_crypt_final(struct des3_ede_x86_ctx *ctx, - struct skcipher_walk *walk) -{ - u8 *ctrblk = walk->iv; - u8 keystream[DES3_EDE_BLOCK_SIZE]; - u8 *src = walk->src.virt.addr; - u8 *dst = walk->dst.virt.addr; - unsigned int nbytes = walk->nbytes; - - des3_ede_enc_blk(ctx, keystream, ctrblk); - crypto_xor_cpy(dst, keystream, src, nbytes); - - crypto_inc(ctrblk, DES3_EDE_BLOCK_SIZE); -} - -static unsigned int __ctr_crypt(struct des3_ede_x86_ctx *ctx, - struct skcipher_walk *walk) -{ - unsigned int bsize = DES3_EDE_BLOCK_SIZE; - unsigned int nbytes = walk->nbytes; - __be64 *src = (__be64 *)walk->src.virt.addr; - __be64 *dst = (__be64 *)walk->dst.virt.addr; - u64 ctrblk = be64_to_cpu(*(__be64 *)walk->iv); - __be64 ctrblocks[3]; - - /* Process four block batch */ - if (nbytes >= bsize * 3) { - do { - /* create ctrblks for parallel encrypt */ - ctrblocks[0] = cpu_to_be64(ctrblk++); - ctrblocks[1] = cpu_to_be64(ctrblk++); - ctrblocks[2] = cpu_to_be64(ctrblk++); - - des3_ede_enc_blk_3way(ctx, (u8 *)ctrblocks, - (u8 *)ctrblocks); - - dst[0] = src[0] ^ ctrblocks[0]; - dst[1] = src[1] ^ ctrblocks[1]; - dst[2] = src[2] ^ ctrblocks[2]; - - src += 3; - dst += 3; - } while ((nbytes -= bsize * 3) >= bsize * 3); - - if (nbytes < bsize) - goto done; - } - - /* Handle leftovers */ - do { - ctrblocks[0] = cpu_to_be64(ctrblk++); - - des3_ede_enc_blk(ctx, (u8 *)ctrblocks, (u8 *)ctrblocks); - - dst[0] = src[0] ^ ctrblocks[0]; - - src += 1; - dst += 1; - } while ((nbytes -= bsize) >= bsize); - -done: - *(__be64 *)walk->iv = cpu_to_be64(ctrblk); - return nbytes; -} - -static int ctr_crypt(struct skcipher_request *req) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct des3_ede_x86_ctx *ctx = crypto_skcipher_ctx(tfm); - struct skcipher_walk walk; - unsigned int nbytes; - int err; - - err = skcipher_walk_virt(&walk, req, false); - - while ((nbytes = walk.nbytes) >= DES3_EDE_BLOCK_SIZE) { - nbytes = __ctr_crypt(ctx, &walk); - err = skcipher_walk_done(&walk, nbytes); - } - - if (nbytes) { - ctr_crypt_final(ctx, &walk); - err = skcipher_walk_done(&walk, 0); - } - - return err; -} - static int des3_ede_x86_setkey(struct crypto_tfm *tfm, const u8 *key, unsigned int keylen) { @@ -428,20 +338,6 @@ static struct skcipher_alg des3_ede_skciphers[] = { .setkey = des3_ede_x86_setkey_skcipher, .encrypt = cbc_encrypt, .decrypt = cbc_decrypt, - }, { - .base.cra_name = "ctr(des3_ede)", - .base.cra_driver_name = "ctr-des3_ede-asm", - .base.cra_priority = 300, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct des3_ede_x86_ctx), - .base.cra_module = THIS_MODULE, - .min_keysize = DES3_EDE_KEY_SIZE, - .max_keysize = DES3_EDE_KEY_SIZE, - .ivsize = DES3_EDE_BLOCK_SIZE, - .chunksize = DES3_EDE_BLOCK_SIZE, - .setkey = des3_ede_x86_setkey_skcipher, - .encrypt = ctr_crypt, - .decrypt = ctr_crypt, } }; diff --git a/arch/x86/crypto/ecb_cbc_helpers.h b/arch/x86/crypto/ecb_cbc_helpers.h new file mode 100644 index 0000000000000000000000000000000000000000..eaa15c7b29d62962bc51151396d8593cda0caff1 --- /dev/null +++ b/arch/x86/crypto/ecb_cbc_helpers.h @@ -0,0 +1,76 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef _CRYPTO_ECB_CBC_HELPER_H +#define _CRYPTO_ECB_CBC_HELPER_H + +#include +#include + +/* + * Mode helpers to instantiate parameterized skcipher ECB/CBC modes without + * having to rely on indirect calls and retpolines. + */ + +#define ECB_WALK_START(req, bsize, fpu_blocks) do { \ + void *ctx = crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); \ + const int __bsize = (bsize); \ + struct skcipher_walk walk; \ + int err = skcipher_walk_virt(&walk, (req), false); \ + while (walk.nbytes > 0) { \ + unsigned int nbytes = walk.nbytes; \ + bool do_fpu = (fpu_blocks) != -1 && \ + nbytes >= (fpu_blocks) * __bsize; \ + const u8 *src = walk.src.virt.addr; \ + u8 *dst = walk.dst.virt.addr; \ + u8 __maybe_unused buf[(bsize)]; \ + if (do_fpu) kernel_fpu_begin() + +#define CBC_WALK_START(req, bsize, fpu_blocks) \ + ECB_WALK_START(req, bsize, fpu_blocks) + +#define ECB_WALK_ADVANCE(blocks) do { \ + dst += (blocks) * __bsize; \ + src += (blocks) * __bsize; \ + nbytes -= (blocks) * __bsize; \ +} while (0) + +#define ECB_BLOCK(blocks, func) do { \ + while (nbytes >= (blocks) * __bsize) { \ + (func)(ctx, dst, src); \ + ECB_WALK_ADVANCE(blocks); \ + } \ +} while (0) + +#define CBC_ENC_BLOCK(func) do { \ + const u8 *__iv = walk.iv; \ + while (nbytes >= __bsize) { \ + crypto_xor_cpy(dst, src, __iv, __bsize); \ + (func)(ctx, dst, dst); \ + __iv = dst; \ + ECB_WALK_ADVANCE(1); \ + } \ + memcpy(walk.iv, __iv, __bsize); \ +} while (0) + +#define CBC_DEC_BLOCK(blocks, func) do { \ + while (nbytes >= (blocks) * __bsize) { \ + const u8 *__iv = src + ((blocks) - 1) * __bsize; \ + if (dst == src) \ + __iv = memcpy(buf, __iv, __bsize); \ + (func)(ctx, dst, src); \ + crypto_xor(dst, walk.iv, __bsize); \ + memcpy(walk.iv, __iv, __bsize); \ + ECB_WALK_ADVANCE(blocks); \ + } \ +} while (0) + +#define ECB_WALK_END() \ + if (do_fpu) kernel_fpu_end(); \ + err = skcipher_walk_done(&walk, nbytes); \ + } \ + return err; \ +} while (0) + +#define CBC_WALK_END() ECB_WALK_END() + +#endif diff --git a/arch/x86/crypto/glue_helper-asm-avx.S b/arch/x86/crypto/glue_helper-asm-avx.S index d08fc575ef7f8e3364ac5ba62152289baa1ab132..3da38527122745a03ec2acc7a4f319191ef146d4 100644 --- a/arch/x86/crypto/glue_helper-asm-avx.S +++ b/arch/x86/crypto/glue_helper-asm-avx.S @@ -34,107 +34,3 @@ vpxor (5*16)(src), x6, x6; \ vpxor (6*16)(src), x7, x7; \ store_8way(dst, x0, x1, x2, x3, x4, x5, x6, x7); - -#define inc_le128(x, minus_one, tmp) \ - vpcmpeqq minus_one, x, tmp; \ - vpsubq minus_one, x, x; \ - vpslldq $8, tmp, tmp; \ - vpsubq tmp, x, x; - -#define load_ctr_8way(iv, bswap, x0, x1, x2, x3, x4, x5, x6, x7, t0, t1, t2) \ - vpcmpeqd t0, t0, t0; \ - vpsrldq $8, t0, t0; /* low: -1, high: 0 */ \ - vmovdqa bswap, t1; \ - \ - /* load IV and byteswap */ \ - vmovdqu (iv), x7; \ - vpshufb t1, x7, x0; \ - \ - /* construct IVs */ \ - inc_le128(x7, t0, t2); \ - vpshufb t1, x7, x1; \ - inc_le128(x7, t0, t2); \ - vpshufb t1, x7, x2; \ - inc_le128(x7, t0, t2); \ - vpshufb t1, x7, x3; \ - inc_le128(x7, t0, t2); \ - vpshufb t1, x7, x4; \ - inc_le128(x7, t0, t2); \ - vpshufb t1, x7, x5; \ - inc_le128(x7, t0, t2); \ - vpshufb t1, x7, x6; \ - inc_le128(x7, t0, t2); \ - vmovdqa x7, t2; \ - vpshufb t1, x7, x7; \ - inc_le128(t2, t0, t1); \ - vmovdqu t2, (iv); - -#define store_ctr_8way(src, dst, x0, x1, x2, x3, x4, x5, x6, x7) \ - vpxor (0*16)(src), x0, x0; \ - vpxor (1*16)(src), x1, x1; \ - vpxor (2*16)(src), x2, x2; \ - vpxor (3*16)(src), x3, x3; \ - vpxor (4*16)(src), x4, x4; \ - vpxor (5*16)(src), x5, x5; \ - vpxor (6*16)(src), x6, x6; \ - vpxor (7*16)(src), x7, x7; \ - store_8way(dst, x0, x1, x2, x3, x4, x5, x6, x7); - -#define gf128mul_x_ble(iv, mask, tmp) \ - vpsrad $31, iv, tmp; \ - vpaddq iv, iv, iv; \ - vpshufd $0x13, tmp, tmp; \ - vpand mask, tmp, tmp; \ - vpxor tmp, iv, iv; - -#define load_xts_8way(iv, src, dst, x0, x1, x2, x3, x4, x5, x6, x7, tiv, t0, \ - t1, xts_gf128mul_and_shl1_mask) \ - vmovdqa xts_gf128mul_and_shl1_mask, t0; \ - \ - /* load IV */ \ - vmovdqu (iv), tiv; \ - vpxor (0*16)(src), tiv, x0; \ - vmovdqu tiv, (0*16)(dst); \ - \ - /* construct and store IVs, also xor with source */ \ - gf128mul_x_ble(tiv, t0, t1); \ - vpxor (1*16)(src), tiv, x1; \ - vmovdqu tiv, (1*16)(dst); \ - \ - gf128mul_x_ble(tiv, t0, t1); \ - vpxor (2*16)(src), tiv, x2; \ - vmovdqu tiv, (2*16)(dst); \ - \ - gf128mul_x_ble(tiv, t0, t1); \ - vpxor (3*16)(src), tiv, x3; \ - vmovdqu tiv, (3*16)(dst); \ - \ - gf128mul_x_ble(tiv, t0, t1); \ - vpxor (4*16)(src), tiv, x4; \ - vmovdqu tiv, (4*16)(dst); \ - \ - gf128mul_x_ble(tiv, t0, t1); \ - vpxor (5*16)(src), tiv, x5; \ - vmovdqu tiv, (5*16)(dst); \ - \ - gf128mul_x_ble(tiv, t0, t1); \ - vpxor (6*16)(src), tiv, x6; \ - vmovdqu tiv, (6*16)(dst); \ - \ - gf128mul_x_ble(tiv, t0, t1); \ - vpxor (7*16)(src), tiv, x7; \ - vmovdqu tiv, (7*16)(dst); \ - \ - gf128mul_x_ble(tiv, t0, t1); \ - vmovdqu tiv, (iv); - -#define store_xts_8way(dst, x0, x1, x2, x3, x4, x5, x6, x7) \ - vpxor (0*16)(dst), x0, x0; \ - vpxor (1*16)(dst), x1, x1; \ - vpxor (2*16)(dst), x2, x2; \ - vpxor (3*16)(dst), x3, x3; \ - vpxor (4*16)(dst), x4, x4; \ - vpxor (5*16)(dst), x5, x5; \ - vpxor (6*16)(dst), x6, x6; \ - vpxor (7*16)(dst), x7, x7; \ - store_8way(dst, x0, x1, x2, x3, x4, x5, x6, x7); diff --git a/arch/x86/crypto/glue_helper-asm-avx2.S b/arch/x86/crypto/glue_helper-asm-avx2.S index d84508c85c136030c36d03be6b22b62fad664ea5..c77e9049431f399dd9305d70db0d02685f6fb9c2 100644 --- a/arch/x86/crypto/glue_helper-asm-avx2.S +++ b/arch/x86/crypto/glue_helper-asm-avx2.S @@ -37,139 +37,3 @@ vpxor (5*32+16)(src), x6, x6; \ vpxor (6*32+16)(src), x7, x7; \ store_16way(dst, x0, x1, x2, x3, x4, x5, x6, x7); - -#define inc_le128(x, minus_one, tmp) \ - vpcmpeqq minus_one, x, tmp; \ - vpsubq minus_one, x, x; \ - vpslldq $8, tmp, tmp; \ - vpsubq tmp, x, x; - -#define add2_le128(x, minus_one, minus_two, tmp1, tmp2) \ - vpcmpeqq minus_one, x, tmp1; \ - vpcmpeqq minus_two, x, tmp2; \ - vpsubq minus_two, x, x; \ - vpor tmp2, tmp1, tmp1; \ - vpslldq $8, tmp1, tmp1; \ - vpsubq tmp1, x, x; - -#define load_ctr_16way(iv, bswap, x0, x1, x2, x3, x4, x5, x6, x7, t0, t0x, t1, \ - t1x, t2, t2x, t3, t3x, t4, t5) \ - vpcmpeqd t0, t0, t0; \ - vpsrldq $8, t0, t0; /* ab: -1:0 ; cd: -1:0 */ \ - vpaddq t0, t0, t4; /* ab: -2:0 ; cd: -2:0 */\ - \ - /* load IV and byteswap */ \ - vmovdqu (iv), t2x; \ - vmovdqa t2x, t3x; \ - inc_le128(t2x, t0x, t1x); \ - vbroadcasti128 bswap, t1; \ - vinserti128 $1, t2x, t3, t2; /* ab: le0 ; cd: le1 */ \ - vpshufb t1, t2, x0; \ - \ - /* construct IVs */ \ - add2_le128(t2, t0, t4, t3, t5); /* ab: le2 ; cd: le3 */ \ - vpshufb t1, t2, x1; \ - add2_le128(t2, t0, t4, t3, t5); \ - vpshufb t1, t2, x2; \ - add2_le128(t2, t0, t4, t3, t5); \ - vpshufb t1, t2, x3; \ - add2_le128(t2, t0, t4, t3, t5); \ - vpshufb t1, t2, x4; \ - add2_le128(t2, t0, t4, t3, t5); \ - vpshufb t1, t2, x5; \ - add2_le128(t2, t0, t4, t3, t5); \ - vpshufb t1, t2, x6; \ - add2_le128(t2, t0, t4, t3, t5); \ - vpshufb t1, t2, x7; \ - vextracti128 $1, t2, t2x; \ - inc_le128(t2x, t0x, t3x); \ - vmovdqu t2x, (iv); - -#define store_ctr_16way(src, dst, x0, x1, x2, x3, x4, x5, x6, x7) \ - vpxor (0*32)(src), x0, x0; \ - vpxor (1*32)(src), x1, x1; \ - vpxor (2*32)(src), x2, x2; \ - vpxor (3*32)(src), x3, x3; \ - vpxor (4*32)(src), x4, x4; \ - vpxor (5*32)(src), x5, x5; \ - vpxor (6*32)(src), x6, x6; \ - vpxor (7*32)(src), x7, x7; \ - store_16way(dst, x0, x1, x2, x3, x4, x5, x6, x7); - -#define gf128mul_x_ble(iv, mask, tmp) \ - vpsrad $31, iv, tmp; \ - vpaddq iv, iv, iv; \ - vpshufd $0x13, tmp, tmp; \ - vpand mask, tmp, tmp; \ - vpxor tmp, iv, iv; - -#define gf128mul_x2_ble(iv, mask1, mask2, tmp0, tmp1) \ - vpsrad $31, iv, tmp0; \ - vpaddq iv, iv, tmp1; \ - vpsllq $2, iv, iv; \ - vpshufd $0x13, tmp0, tmp0; \ - vpsrad $31, tmp1, tmp1; \ - vpand mask2, tmp0, tmp0; \ - vpshufd $0x13, tmp1, tmp1; \ - vpxor tmp0, iv, iv; \ - vpand mask1, tmp1, tmp1; \ - vpxor tmp1, iv, iv; - -#define load_xts_16way(iv, src, dst, x0, x1, x2, x3, x4, x5, x6, x7, tiv, \ - tivx, t0, t0x, t1, t1x, t2, t2x, t3, \ - xts_gf128mul_and_shl1_mask_0, \ - xts_gf128mul_and_shl1_mask_1) \ - vbroadcasti128 xts_gf128mul_and_shl1_mask_0, t1; \ - \ - /* load IV and construct second IV */ \ - vmovdqu (iv), tivx; \ - vmovdqa tivx, t0x; \ - gf128mul_x_ble(tivx, t1x, t2x); \ - vbroadcasti128 xts_gf128mul_and_shl1_mask_1, t2; \ - vinserti128 $1, tivx, t0, tiv; \ - vpxor (0*32)(src), tiv, x0; \ - vmovdqu tiv, (0*32)(dst); \ - \ - /* construct and store IVs, also xor with source */ \ - gf128mul_x2_ble(tiv, t1, t2, t0, t3); \ - vpxor (1*32)(src), tiv, x1; \ - vmovdqu tiv, (1*32)(dst); \ - \ - gf128mul_x2_ble(tiv, t1, t2, t0, t3); \ - vpxor (2*32)(src), tiv, x2; \ - vmovdqu tiv, (2*32)(dst); \ - \ - gf128mul_x2_ble(tiv, t1, t2, t0, t3); \ - vpxor (3*32)(src), tiv, x3; \ - vmovdqu tiv, (3*32)(dst); \ - \ - gf128mul_x2_ble(tiv, t1, t2, t0, t3); \ - vpxor (4*32)(src), tiv, x4; \ - vmovdqu tiv, (4*32)(dst); \ - \ - gf128mul_x2_ble(tiv, t1, t2, t0, t3); \ - vpxor (5*32)(src), tiv, x5; \ - vmovdqu tiv, (5*32)(dst); \ - \ - gf128mul_x2_ble(tiv, t1, t2, t0, t3); \ - vpxor (6*32)(src), tiv, x6; \ - vmovdqu tiv, (6*32)(dst); \ - \ - gf128mul_x2_ble(tiv, t1, t2, t0, t3); \ - vpxor (7*32)(src), tiv, x7; \ - vmovdqu tiv, (7*32)(dst); \ - \ - vextracti128 $1, tiv, tivx; \ - gf128mul_x_ble(tivx, t1x, t2x); \ - vmovdqu tivx, (iv); - -#define store_xts_16way(dst, x0, x1, x2, x3, x4, x5, x6, x7) \ - vpxor (0*32)(dst), x0, x0; \ - vpxor (1*32)(dst), x1, x1; \ - vpxor (2*32)(dst), x2, x2; \ - vpxor (3*32)(dst), x3, x3; \ - vpxor (4*32)(dst), x4, x4; \ - vpxor (5*32)(dst), x5, x5; \ - vpxor (6*32)(dst), x6, x6; \ - vpxor (7*32)(dst), x7, x7; \ - store_16way(dst, x0, x1, x2, x3, x4, x5, x6, x7); diff --git a/arch/x86/crypto/glue_helper.c b/arch/x86/crypto/glue_helper.c deleted file mode 100644 index d3d91a0abf88f7f4507222e536b95a872fde856f..0000000000000000000000000000000000000000 --- a/arch/x86/crypto/glue_helper.c +++ /dev/null @@ -1,381 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Shared glue code for 128bit block ciphers - * - * Copyright © 2012-2013 Jussi Kivilinna - * - * CBC & ECB parts based on code (crypto/cbc.c,ecb.c) by: - * Copyright (c) 2006 Herbert Xu - * CTR part based on code (crypto/ctr.c) by: - * (C) Copyright IBM Corp. 2007 - Joy Latten - */ - -#include -#include -#include -#include -#include -#include -#include - -int glue_ecb_req_128bit(const struct common_glue_ctx *gctx, - struct skcipher_request *req) -{ - void *ctx = crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); - const unsigned int bsize = 128 / 8; - struct skcipher_walk walk; - bool fpu_enabled = false; - unsigned int nbytes; - int err; - - err = skcipher_walk_virt(&walk, req, false); - - while ((nbytes = walk.nbytes)) { - const u8 *src = walk.src.virt.addr; - u8 *dst = walk.dst.virt.addr; - unsigned int func_bytes; - unsigned int i; - - fpu_enabled = glue_fpu_begin(bsize, gctx->fpu_blocks_limit, - &walk, fpu_enabled, nbytes); - for (i = 0; i < gctx->num_funcs; i++) { - func_bytes = bsize * gctx->funcs[i].num_blocks; - - if (nbytes < func_bytes) - continue; - - /* Process multi-block batch */ - do { - gctx->funcs[i].fn_u.ecb(ctx, dst, src); - src += func_bytes; - dst += func_bytes; - nbytes -= func_bytes; - } while (nbytes >= func_bytes); - - if (nbytes < bsize) - break; - } - err = skcipher_walk_done(&walk, nbytes); - } - - glue_fpu_end(fpu_enabled); - return err; -} -EXPORT_SYMBOL_GPL(glue_ecb_req_128bit); - -int glue_cbc_encrypt_req_128bit(const common_glue_func_t fn, - struct skcipher_request *req) -{ - void *ctx = crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); - const unsigned int bsize = 128 / 8; - struct skcipher_walk walk; - unsigned int nbytes; - int err; - - err = skcipher_walk_virt(&walk, req, false); - - while ((nbytes = walk.nbytes)) { - const u128 *src = (u128 *)walk.src.virt.addr; - u128 *dst = (u128 *)walk.dst.virt.addr; - u128 *iv = (u128 *)walk.iv; - - do { - u128_xor(dst, src, iv); - fn(ctx, (u8 *)dst, (u8 *)dst); - iv = dst; - src++; - dst++; - nbytes -= bsize; - } while (nbytes >= bsize); - - *(u128 *)walk.iv = *iv; - err = skcipher_walk_done(&walk, nbytes); - } - return err; -} -EXPORT_SYMBOL_GPL(glue_cbc_encrypt_req_128bit); - -int glue_cbc_decrypt_req_128bit(const struct common_glue_ctx *gctx, - struct skcipher_request *req) -{ - void *ctx = crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); - const unsigned int bsize = 128 / 8; - struct skcipher_walk walk; - bool fpu_enabled = false; - unsigned int nbytes; - int err; - - err = skcipher_walk_virt(&walk, req, false); - - while ((nbytes = walk.nbytes)) { - const u128 *src = walk.src.virt.addr; - u128 *dst = walk.dst.virt.addr; - unsigned int func_bytes, num_blocks; - unsigned int i; - u128 last_iv; - - fpu_enabled = glue_fpu_begin(bsize, gctx->fpu_blocks_limit, - &walk, fpu_enabled, nbytes); - /* Start of the last block. */ - src += nbytes / bsize - 1; - dst += nbytes / bsize - 1; - - last_iv = *src; - - for (i = 0; i < gctx->num_funcs; i++) { - num_blocks = gctx->funcs[i].num_blocks; - func_bytes = bsize * num_blocks; - - if (nbytes < func_bytes) - continue; - - /* Process multi-block batch */ - do { - src -= num_blocks - 1; - dst -= num_blocks - 1; - - gctx->funcs[i].fn_u.cbc(ctx, (u8 *)dst, - (const u8 *)src); - - nbytes -= func_bytes; - if (nbytes < bsize) - goto done; - - u128_xor(dst, dst, --src); - dst--; - } while (nbytes >= func_bytes); - } -done: - u128_xor(dst, dst, (u128 *)walk.iv); - *(u128 *)walk.iv = last_iv; - err = skcipher_walk_done(&walk, nbytes); - } - - glue_fpu_end(fpu_enabled); - return err; -} -EXPORT_SYMBOL_GPL(glue_cbc_decrypt_req_128bit); - -int glue_ctr_req_128bit(const struct common_glue_ctx *gctx, - struct skcipher_request *req) -{ - void *ctx = crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); - const unsigned int bsize = 128 / 8; - struct skcipher_walk walk; - bool fpu_enabled = false; - unsigned int nbytes; - int err; - - err = skcipher_walk_virt(&walk, req, false); - - while ((nbytes = walk.nbytes) >= bsize) { - const u128 *src = walk.src.virt.addr; - u128 *dst = walk.dst.virt.addr; - unsigned int func_bytes, num_blocks; - unsigned int i; - le128 ctrblk; - - fpu_enabled = glue_fpu_begin(bsize, gctx->fpu_blocks_limit, - &walk, fpu_enabled, nbytes); - - be128_to_le128(&ctrblk, (be128 *)walk.iv); - - for (i = 0; i < gctx->num_funcs; i++) { - num_blocks = gctx->funcs[i].num_blocks; - func_bytes = bsize * num_blocks; - - if (nbytes < func_bytes) - continue; - - /* Process multi-block batch */ - do { - gctx->funcs[i].fn_u.ctr(ctx, (u8 *)dst, - (const u8 *)src, - &ctrblk); - src += num_blocks; - dst += num_blocks; - nbytes -= func_bytes; - } while (nbytes >= func_bytes); - - if (nbytes < bsize) - break; - } - - le128_to_be128((be128 *)walk.iv, &ctrblk); - err = skcipher_walk_done(&walk, nbytes); - } - - glue_fpu_end(fpu_enabled); - - if (nbytes) { - le128 ctrblk; - u128 tmp; - - be128_to_le128(&ctrblk, (be128 *)walk.iv); - memcpy(&tmp, walk.src.virt.addr, nbytes); - gctx->funcs[gctx->num_funcs - 1].fn_u.ctr(ctx, (u8 *)&tmp, - (const u8 *)&tmp, - &ctrblk); - memcpy(walk.dst.virt.addr, &tmp, nbytes); - le128_to_be128((be128 *)walk.iv, &ctrblk); - - err = skcipher_walk_done(&walk, 0); - } - - return err; -} -EXPORT_SYMBOL_GPL(glue_ctr_req_128bit); - -static unsigned int __glue_xts_req_128bit(const struct common_glue_ctx *gctx, - void *ctx, - struct skcipher_walk *walk) -{ - const unsigned int bsize = 128 / 8; - unsigned int nbytes = walk->nbytes; - u128 *src = walk->src.virt.addr; - u128 *dst = walk->dst.virt.addr; - unsigned int num_blocks, func_bytes; - unsigned int i; - - /* Process multi-block batch */ - for (i = 0; i < gctx->num_funcs; i++) { - num_blocks = gctx->funcs[i].num_blocks; - func_bytes = bsize * num_blocks; - - if (nbytes >= func_bytes) { - do { - gctx->funcs[i].fn_u.xts(ctx, (u8 *)dst, - (const u8 *)src, - walk->iv); - - src += num_blocks; - dst += num_blocks; - nbytes -= func_bytes; - } while (nbytes >= func_bytes); - - if (nbytes < bsize) - goto done; - } - } - -done: - return nbytes; -} - -int glue_xts_req_128bit(const struct common_glue_ctx *gctx, - struct skcipher_request *req, - common_glue_func_t tweak_fn, void *tweak_ctx, - void *crypt_ctx, bool decrypt) -{ - const bool cts = (req->cryptlen % XTS_BLOCK_SIZE); - const unsigned int bsize = 128 / 8; - struct skcipher_request subreq; - struct skcipher_walk walk; - bool fpu_enabled = false; - unsigned int nbytes, tail; - int err; - - if (req->cryptlen < XTS_BLOCK_SIZE) - return -EINVAL; - - if (unlikely(cts)) { - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - - tail = req->cryptlen % XTS_BLOCK_SIZE + XTS_BLOCK_SIZE; - - skcipher_request_set_tfm(&subreq, tfm); - skcipher_request_set_callback(&subreq, - crypto_skcipher_get_flags(tfm), - NULL, NULL); - skcipher_request_set_crypt(&subreq, req->src, req->dst, - req->cryptlen - tail, req->iv); - req = &subreq; - } - - err = skcipher_walk_virt(&walk, req, false); - nbytes = walk.nbytes; - if (err) - return err; - - /* set minimum length to bsize, for tweak_fn */ - fpu_enabled = glue_fpu_begin(bsize, gctx->fpu_blocks_limit, - &walk, fpu_enabled, - nbytes < bsize ? bsize : nbytes); - - /* calculate first value of T */ - tweak_fn(tweak_ctx, walk.iv, walk.iv); - - while (nbytes) { - nbytes = __glue_xts_req_128bit(gctx, crypt_ctx, &walk); - - err = skcipher_walk_done(&walk, nbytes); - nbytes = walk.nbytes; - } - - if (unlikely(cts)) { - u8 *next_tweak, *final_tweak = req->iv; - struct scatterlist *src, *dst; - struct scatterlist s[2], d[2]; - le128 b[2]; - - dst = src = scatterwalk_ffwd(s, req->src, req->cryptlen); - if (req->dst != req->src) - dst = scatterwalk_ffwd(d, req->dst, req->cryptlen); - - if (decrypt) { - next_tweak = memcpy(b, req->iv, XTS_BLOCK_SIZE); - gf128mul_x_ble(b, b); - } else { - next_tweak = req->iv; - } - - skcipher_request_set_crypt(&subreq, src, dst, XTS_BLOCK_SIZE, - next_tweak); - - err = skcipher_walk_virt(&walk, req, false) ?: - skcipher_walk_done(&walk, - __glue_xts_req_128bit(gctx, crypt_ctx, &walk)); - if (err) - goto out; - - scatterwalk_map_and_copy(b, dst, 0, XTS_BLOCK_SIZE, 0); - memcpy(b + 1, b, tail - XTS_BLOCK_SIZE); - scatterwalk_map_and_copy(b, src, XTS_BLOCK_SIZE, - tail - XTS_BLOCK_SIZE, 0); - scatterwalk_map_and_copy(b, dst, 0, tail, 1); - - skcipher_request_set_crypt(&subreq, dst, dst, XTS_BLOCK_SIZE, - final_tweak); - - err = skcipher_walk_virt(&walk, req, false) ?: - skcipher_walk_done(&walk, - __glue_xts_req_128bit(gctx, crypt_ctx, &walk)); - } - -out: - glue_fpu_end(fpu_enabled); - - return err; -} -EXPORT_SYMBOL_GPL(glue_xts_req_128bit); - -void glue_xts_crypt_128bit_one(const void *ctx, u8 *dst, const u8 *src, - le128 *iv, common_glue_func_t fn) -{ - le128 ivblk = *iv; - - /* generate next IV */ - gf128mul_x_ble(iv, &ivblk); - - /* CC <- T xor C */ - u128_xor((u128 *)dst, (const u128 *)src, (u128 *)&ivblk); - - /* PP <- D(Key2,CC) */ - fn(ctx, dst, dst); - - /* P <- T xor PP */ - u128_xor((u128 *)dst, (u128 *)dst, (u128 *)&ivblk); -} -EXPORT_SYMBOL_GPL(glue_xts_crypt_128bit_one); - -MODULE_LICENSE("GPL"); diff --git a/arch/x86/crypto/serpent-avx-x86_64-asm_64.S b/arch/x86/crypto/serpent-avx-x86_64-asm_64.S index ba9e4c1e7f5c75d38f2b4b05deb7da1a67a2b6b7..b7ee24df7fbae4db3243c3cb263fa4cd1c9a8406 100644 --- a/arch/x86/crypto/serpent-avx-x86_64-asm_64.S +++ b/arch/x86/crypto/serpent-avx-x86_64-asm_64.S @@ -18,10 +18,6 @@ .align 16 .Lbswap128_mask: .byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 -.section .rodata.cst16.xts_gf128mul_and_shl1_mask, "aM", @progbits, 16 -.align 16 -.Lxts_gf128mul_and_shl1_mask: - .byte 0x87, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0 .text @@ -715,67 +711,3 @@ SYM_FUNC_START(serpent_cbc_dec_8way_avx) FRAME_END ret; SYM_FUNC_END(serpent_cbc_dec_8way_avx) - -SYM_FUNC_START(serpent_ctr_8way_avx) - /* input: - * %rdi: ctx, CTX - * %rsi: dst - * %rdx: src - * %rcx: iv (little endian, 128bit) - */ - FRAME_BEGIN - - load_ctr_8way(%rcx, .Lbswap128_mask, RA1, RB1, RC1, RD1, RA2, RB2, RC2, - RD2, RK0, RK1, RK2); - - call __serpent_enc_blk8_avx; - - store_ctr_8way(%rdx, %rsi, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2); - - FRAME_END - ret; -SYM_FUNC_END(serpent_ctr_8way_avx) - -SYM_FUNC_START(serpent_xts_enc_8way_avx) - /* input: - * %rdi: ctx, CTX - * %rsi: dst - * %rdx: src - * %rcx: iv (t ⊕ α⿠∈ GF(2¹²â¸)) - */ - FRAME_BEGIN - - /* regs <= src, dst <= IVs, regs <= regs xor IVs */ - load_xts_8way(%rcx, %rdx, %rsi, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2, - RK0, RK1, RK2, .Lxts_gf128mul_and_shl1_mask); - - call __serpent_enc_blk8_avx; - - /* dst <= regs xor IVs(in dst) */ - store_xts_8way(%rsi, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2); - - FRAME_END - ret; -SYM_FUNC_END(serpent_xts_enc_8way_avx) - -SYM_FUNC_START(serpent_xts_dec_8way_avx) - /* input: - * %rdi: ctx, CTX - * %rsi: dst - * %rdx: src - * %rcx: iv (t ⊕ α⿠∈ GF(2¹²â¸)) - */ - FRAME_BEGIN - - /* regs <= src, dst <= IVs, regs <= regs xor IVs */ - load_xts_8way(%rcx, %rdx, %rsi, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2, - RK0, RK1, RK2, .Lxts_gf128mul_and_shl1_mask); - - call __serpent_dec_blk8_avx; - - /* dst <= regs xor IVs(in dst) */ - store_xts_8way(%rsi, RC1, RD1, RB1, RE1, RC2, RD2, RB2, RE2); - - FRAME_END - ret; -SYM_FUNC_END(serpent_xts_dec_8way_avx) diff --git a/arch/x86/crypto/serpent-avx.h b/arch/x86/crypto/serpent-avx.h new file mode 100644 index 0000000000000000000000000000000000000000..23f3361a0e72d712dcd09062315820379025dd2a --- /dev/null +++ b/arch/x86/crypto/serpent-avx.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef ASM_X86_SERPENT_AVX_H +#define ASM_X86_SERPENT_AVX_H + +#include +#include +#include + +struct crypto_skcipher; + +#define SERPENT_PARALLEL_BLOCKS 8 + +asmlinkage void serpent_ecb_enc_8way_avx(const void *ctx, u8 *dst, + const u8 *src); +asmlinkage void serpent_ecb_dec_8way_avx(const void *ctx, u8 *dst, + const u8 *src); + +asmlinkage void serpent_cbc_dec_8way_avx(const void *ctx, u8 *dst, + const u8 *src); + +#endif diff --git a/arch/x86/crypto/serpent-avx2-asm_64.S b/arch/x86/crypto/serpent-avx2-asm_64.S index c9648aeae7053af241759431615766d58318c9b6..9161b6e441f31e5fea7014be27802cf5b6b94011 100644 --- a/arch/x86/crypto/serpent-avx2-asm_64.S +++ b/arch/x86/crypto/serpent-avx2-asm_64.S @@ -20,16 +20,6 @@ .Lbswap128_mask: .byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 -.section .rodata.cst16.xts_gf128mul_and_shl1_mask_0, "aM", @progbits, 16 -.align 16 -.Lxts_gf128mul_and_shl1_mask_0: - .byte 0x87, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0 - -.section .rodata.cst16.xts_gf128mul_and_shl1_mask_1, "aM", @progbits, 16 -.align 16 -.Lxts_gf128mul_and_shl1_mask_1: - .byte 0x0e, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0 - .text #define CTX %rdi @@ -734,80 +724,3 @@ SYM_FUNC_START(serpent_cbc_dec_16way) FRAME_END ret; SYM_FUNC_END(serpent_cbc_dec_16way) - -SYM_FUNC_START(serpent_ctr_16way) - /* input: - * %rdi: ctx, CTX - * %rsi: dst (16 blocks) - * %rdx: src (16 blocks) - * %rcx: iv (little endian, 128bit) - */ - FRAME_BEGIN - - vzeroupper; - - load_ctr_16way(%rcx, .Lbswap128_mask, RA1, RB1, RC1, RD1, RA2, RB2, RC2, - RD2, RK0, RK0x, RK1, RK1x, RK2, RK2x, RK3, RK3x, RNOT, - tp); - - call __serpent_enc_blk16; - - store_ctr_16way(%rdx, %rsi, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2); - - vzeroupper; - - FRAME_END - ret; -SYM_FUNC_END(serpent_ctr_16way) - -SYM_FUNC_START(serpent_xts_enc_16way) - /* input: - * %rdi: ctx, CTX - * %rsi: dst (16 blocks) - * %rdx: src (16 blocks) - * %rcx: iv (t ⊕ α⿠∈ GF(2¹²â¸)) - */ - FRAME_BEGIN - - vzeroupper; - - load_xts_16way(%rcx, %rdx, %rsi, RA1, RB1, RC1, RD1, RA2, RB2, RC2, - RD2, RK0, RK0x, RK1, RK1x, RK2, RK2x, RK3, RK3x, RNOT, - .Lxts_gf128mul_and_shl1_mask_0, - .Lxts_gf128mul_and_shl1_mask_1); - - call __serpent_enc_blk16; - - store_xts_16way(%rsi, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2); - - vzeroupper; - - FRAME_END - ret; -SYM_FUNC_END(serpent_xts_enc_16way) - -SYM_FUNC_START(serpent_xts_dec_16way) - /* input: - * %rdi: ctx, CTX - * %rsi: dst (16 blocks) - * %rdx: src (16 blocks) - * %rcx: iv (t ⊕ α⿠∈ GF(2¹²â¸)) - */ - FRAME_BEGIN - - vzeroupper; - - load_xts_16way(%rcx, %rdx, %rsi, RA1, RB1, RC1, RD1, RA2, RB2, RC2, - RD2, RK0, RK0x, RK1, RK1x, RK2, RK2x, RK3, RK3x, RNOT, - .Lxts_gf128mul_and_shl1_mask_0, - .Lxts_gf128mul_and_shl1_mask_1); - - call __serpent_dec_blk16; - - store_xts_16way(%rsi, RC1, RD1, RB1, RE1, RC2, RD2, RB2, RE2); - - vzeroupper; - - FRAME_END - ret; -SYM_FUNC_END(serpent_xts_dec_16way) diff --git a/arch/x86/include/asm/crypto/serpent-sse2.h b/arch/x86/crypto/serpent-sse2.h similarity index 100% rename from arch/x86/include/asm/crypto/serpent-sse2.h rename to arch/x86/crypto/serpent-sse2.h diff --git a/arch/x86/crypto/serpent_avx2_glue.c b/arch/x86/crypto/serpent_avx2_glue.c index f973ace44ad3580350646cec309167870179ad2a..ccf0b5fa4933066c054c2269a40cbbf6b9d3a08a 100644 --- a/arch/x86/crypto/serpent_avx2_glue.c +++ b/arch/x86/crypto/serpent_avx2_glue.c @@ -12,9 +12,9 @@ #include #include #include -#include -#include -#include + +#include "serpent-avx.h" +#include "ecb_cbc_helpers.h" #define SERPENT_AVX2_PARALLEL_BLOCKS 16 @@ -23,158 +23,44 @@ asmlinkage void serpent_ecb_enc_16way(const void *ctx, u8 *dst, const u8 *src); asmlinkage void serpent_ecb_dec_16way(const void *ctx, u8 *dst, const u8 *src); asmlinkage void serpent_cbc_dec_16way(const void *ctx, u8 *dst, const u8 *src); -asmlinkage void serpent_ctr_16way(const void *ctx, u8 *dst, const u8 *src, - le128 *iv); -asmlinkage void serpent_xts_enc_16way(const void *ctx, u8 *dst, const u8 *src, - le128 *iv); -asmlinkage void serpent_xts_dec_16way(const void *ctx, u8 *dst, const u8 *src, - le128 *iv); - static int serpent_setkey_skcipher(struct crypto_skcipher *tfm, const u8 *key, unsigned int keylen) { return __serpent_setkey(crypto_skcipher_ctx(tfm), key, keylen); } -static const struct common_glue_ctx serpent_enc = { - .num_funcs = 3, - .fpu_blocks_limit = 8, - - .funcs = { { - .num_blocks = 16, - .fn_u = { .ecb = serpent_ecb_enc_16way } - }, { - .num_blocks = 8, - .fn_u = { .ecb = serpent_ecb_enc_8way_avx } - }, { - .num_blocks = 1, - .fn_u = { .ecb = __serpent_encrypt } - } } -}; - -static const struct common_glue_ctx serpent_ctr = { - .num_funcs = 3, - .fpu_blocks_limit = 8, - - .funcs = { { - .num_blocks = 16, - .fn_u = { .ctr = serpent_ctr_16way } - }, { - .num_blocks = 8, - .fn_u = { .ctr = serpent_ctr_8way_avx } - }, { - .num_blocks = 1, - .fn_u = { .ctr = __serpent_crypt_ctr } - } } -}; - -static const struct common_glue_ctx serpent_enc_xts = { - .num_funcs = 3, - .fpu_blocks_limit = 8, - - .funcs = { { - .num_blocks = 16, - .fn_u = { .xts = serpent_xts_enc_16way } - }, { - .num_blocks = 8, - .fn_u = { .xts = serpent_xts_enc_8way_avx } - }, { - .num_blocks = 1, - .fn_u = { .xts = serpent_xts_enc } - } } -}; - -static const struct common_glue_ctx serpent_dec = { - .num_funcs = 3, - .fpu_blocks_limit = 8, - - .funcs = { { - .num_blocks = 16, - .fn_u = { .ecb = serpent_ecb_dec_16way } - }, { - .num_blocks = 8, - .fn_u = { .ecb = serpent_ecb_dec_8way_avx } - }, { - .num_blocks = 1, - .fn_u = { .ecb = __serpent_decrypt } - } } -}; - -static const struct common_glue_ctx serpent_dec_cbc = { - .num_funcs = 3, - .fpu_blocks_limit = 8, - - .funcs = { { - .num_blocks = 16, - .fn_u = { .cbc = serpent_cbc_dec_16way } - }, { - .num_blocks = 8, - .fn_u = { .cbc = serpent_cbc_dec_8way_avx } - }, { - .num_blocks = 1, - .fn_u = { .cbc = __serpent_decrypt } - } } -}; - -static const struct common_glue_ctx serpent_dec_xts = { - .num_funcs = 3, - .fpu_blocks_limit = 8, - - .funcs = { { - .num_blocks = 16, - .fn_u = { .xts = serpent_xts_dec_16way } - }, { - .num_blocks = 8, - .fn_u = { .xts = serpent_xts_dec_8way_avx } - }, { - .num_blocks = 1, - .fn_u = { .xts = serpent_xts_dec } - } } -}; - static int ecb_encrypt(struct skcipher_request *req) { - return glue_ecb_req_128bit(&serpent_enc, req); + ECB_WALK_START(req, SERPENT_BLOCK_SIZE, SERPENT_PARALLEL_BLOCKS); + ECB_BLOCK(SERPENT_AVX2_PARALLEL_BLOCKS, serpent_ecb_enc_16way); + ECB_BLOCK(SERPENT_PARALLEL_BLOCKS, serpent_ecb_enc_8way_avx); + ECB_BLOCK(1, __serpent_encrypt); + ECB_WALK_END(); } static int ecb_decrypt(struct skcipher_request *req) { - return glue_ecb_req_128bit(&serpent_dec, req); + ECB_WALK_START(req, SERPENT_BLOCK_SIZE, SERPENT_PARALLEL_BLOCKS); + ECB_BLOCK(SERPENT_AVX2_PARALLEL_BLOCKS, serpent_ecb_dec_16way); + ECB_BLOCK(SERPENT_PARALLEL_BLOCKS, serpent_ecb_dec_8way_avx); + ECB_BLOCK(1, __serpent_decrypt); + ECB_WALK_END(); } static int cbc_encrypt(struct skcipher_request *req) { - return glue_cbc_encrypt_req_128bit(__serpent_encrypt, req); + CBC_WALK_START(req, SERPENT_BLOCK_SIZE, -1); + CBC_ENC_BLOCK(__serpent_encrypt); + CBC_WALK_END(); } static int cbc_decrypt(struct skcipher_request *req) { - return glue_cbc_decrypt_req_128bit(&serpent_dec_cbc, req); -} - -static int ctr_crypt(struct skcipher_request *req) -{ - return glue_ctr_req_128bit(&serpent_ctr, req); -} - -static int xts_encrypt(struct skcipher_request *req) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct serpent_xts_ctx *ctx = crypto_skcipher_ctx(tfm); - - return glue_xts_req_128bit(&serpent_enc_xts, req, - __serpent_encrypt, &ctx->tweak_ctx, - &ctx->crypt_ctx, false); -} - -static int xts_decrypt(struct skcipher_request *req) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct serpent_xts_ctx *ctx = crypto_skcipher_ctx(tfm); - - return glue_xts_req_128bit(&serpent_dec_xts, req, - __serpent_encrypt, &ctx->tweak_ctx, - &ctx->crypt_ctx, true); + CBC_WALK_START(req, SERPENT_BLOCK_SIZE, SERPENT_PARALLEL_BLOCKS); + CBC_DEC_BLOCK(SERPENT_AVX2_PARALLEL_BLOCKS, serpent_cbc_dec_16way); + CBC_DEC_BLOCK(SERPENT_PARALLEL_BLOCKS, serpent_cbc_dec_8way_avx); + CBC_DEC_BLOCK(1, __serpent_decrypt); + CBC_WALK_END(); } static struct skcipher_alg serpent_algs[] = { @@ -205,35 +91,6 @@ static struct skcipher_alg serpent_algs[] = { .setkey = serpent_setkey_skcipher, .encrypt = cbc_encrypt, .decrypt = cbc_decrypt, - }, { - .base.cra_name = "__ctr(serpent)", - .base.cra_driver_name = "__ctr-serpent-avx2", - .base.cra_priority = 600, - .base.cra_flags = CRYPTO_ALG_INTERNAL, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct serpent_ctx), - .base.cra_module = THIS_MODULE, - .min_keysize = SERPENT_MIN_KEY_SIZE, - .max_keysize = SERPENT_MAX_KEY_SIZE, - .ivsize = SERPENT_BLOCK_SIZE, - .chunksize = SERPENT_BLOCK_SIZE, - .setkey = serpent_setkey_skcipher, - .encrypt = ctr_crypt, - .decrypt = ctr_crypt, - }, { - .base.cra_name = "__xts(serpent)", - .base.cra_driver_name = "__xts-serpent-avx2", - .base.cra_priority = 600, - .base.cra_flags = CRYPTO_ALG_INTERNAL, - .base.cra_blocksize = SERPENT_BLOCK_SIZE, - .base.cra_ctxsize = sizeof(struct serpent_xts_ctx), - .base.cra_module = THIS_MODULE, - .min_keysize = 2 * SERPENT_MIN_KEY_SIZE, - .max_keysize = 2 * SERPENT_MAX_KEY_SIZE, - .ivsize = SERPENT_BLOCK_SIZE, - .setkey = xts_serpent_setkey, - .encrypt = xts_encrypt, - .decrypt = xts_decrypt, }, }; diff --git a/arch/x86/crypto/serpent_avx_glue.c b/arch/x86/crypto/serpent_avx_glue.c index 7806d1cbe8541e8f35936e9cf44814c3ea2a0712..6c248e1ea4ef7dde3a595e5ef0c9d2457be6acdf 100644 --- a/arch/x86/crypto/serpent_avx_glue.c +++ b/arch/x86/crypto/serpent_avx_glue.c @@ -15,9 +15,9 @@ #include #include #include -#include -#include -#include + +#include "serpent-avx.h" +#include "ecb_cbc_helpers.h" /* 8-way parallel cipher functions */ asmlinkage void serpent_ecb_enc_8way_avx(const void *ctx, u8 *dst, @@ -32,191 +32,41 @@ asmlinkage void serpent_cbc_dec_8way_avx(const void *ctx, u8 *dst, const u8 *src); EXPORT_SYMBOL_GPL(serpent_cbc_dec_8way_avx); -asmlinkage void serpent_ctr_8way_avx(const void *ctx, u8 *dst, const u8 *src, - le128 *iv); -EXPORT_SYMBOL_GPL(serpent_ctr_8way_avx); - -asmlinkage void serpent_xts_enc_8way_avx(const void *ctx, u8 *dst, - const u8 *src, le128 *iv); -EXPORT_SYMBOL_GPL(serpent_xts_enc_8way_avx); - -asmlinkage void serpent_xts_dec_8way_avx(const void *ctx, u8 *dst, - const u8 *src, le128 *iv); -EXPORT_SYMBOL_GPL(serpent_xts_dec_8way_avx); - -void __serpent_crypt_ctr(const void *ctx, u8 *d, const u8 *s, le128 *iv) -{ - be128 ctrblk; - u128 *dst = (u128 *)d; - const u128 *src = (const u128 *)s; - - le128_to_be128(&ctrblk, iv); - le128_inc(iv); - - __serpent_encrypt(ctx, (u8 *)&ctrblk, (u8 *)&ctrblk); - u128_xor(dst, src, (u128 *)&ctrblk); -} -EXPORT_SYMBOL_GPL(__serpent_crypt_ctr); - -void serpent_xts_enc(const void *ctx, u8 *dst, const u8 *src, le128 *iv) -{ - glue_xts_crypt_128bit_one(ctx, dst, src, iv, __serpent_encrypt); -} -EXPORT_SYMBOL_GPL(serpent_xts_enc); - -void serpent_xts_dec(const void *ctx, u8 *dst, const u8 *src, le128 *iv) -{ - glue_xts_crypt_128bit_one(ctx, dst, src, iv, __serpent_decrypt); -} -EXPORT_SYMBOL_GPL(serpent_xts_dec); - static int serpent_setkey_skcipher(struct crypto_skcipher *tfm, const u8 *key, unsigned int keylen) { return __serpent_setkey(crypto_skcipher_ctx(tfm), key, keylen); } -int xts_serpent_setkey(struct crypto_skcipher *tfm, const u8 *key, - unsigned int keylen) -{ - struct serpent_xts_ctx *ctx = crypto_skcipher_ctx(tfm); - int err; - - err = xts_verify_key(tfm, key, keylen); - if (err) - return err; - - /* first half of xts-key is for crypt */ - err = __serpent_setkey(&ctx->crypt_ctx, key, keylen / 2); - if (err) - return err; - - /* second half of xts-key is for tweak */ - return __serpent_setkey(&ctx->tweak_ctx, key + keylen / 2, keylen / 2); -} -EXPORT_SYMBOL_GPL(xts_serpent_setkey); - -static const struct common_glue_ctx serpent_enc = { - .num_funcs = 2, - .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS, - - .funcs = { { - .num_blocks = SERPENT_PARALLEL_BLOCKS, - .fn_u = { .ecb = serpent_ecb_enc_8way_avx } - }, { - .num_blocks = 1, - .fn_u = { .ecb = __serpent_encrypt } - } } -}; - -static const struct common_glue_ctx serpent_ctr = { - .num_funcs = 2, - .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS, - - .funcs = { { - .num_blocks = SERPENT_PARALLEL_BLOCKS, - .fn_u = { .ctr = serpent_ctr_8way_avx } - }, { - .num_blocks = 1, - .fn_u = { .ctr = __serpent_crypt_ctr } - } } -}; - -static const struct common_glue_ctx serpent_enc_xts = { - .num_funcs = 2, - .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS, - - .funcs = { { - .num_blocks = SERPENT_PARALLEL_BLOCKS, - .fn_u = { .xts = serpent_xts_enc_8way_avx } - }, { - .num_blocks = 1, - .fn_u = { .xts = serpent_xts_enc } - } } -}; - -static const struct common_glue_ctx serpent_dec = { - .num_funcs = 2, - .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS, - - .funcs = { { - .num_blocks = SERPENT_PARALLEL_BLOCKS, - .fn_u = { .ecb = serpent_ecb_dec_8way_avx } - }, { - .num_blocks = 1, - .fn_u = { .ecb = __serpent_decrypt } - } } -}; - -static const struct common_glue_ctx serpent_dec_cbc = { - .num_funcs = 2, - .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS, - - .funcs = { { - .num_blocks = SERPENT_PARALLEL_BLOCKS, - .fn_u = { .cbc = serpent_cbc_dec_8way_avx } - }, { - .num_blocks = 1, - .fn_u = { .cbc = __serpent_decrypt } - } } -}; - -static const struct common_glue_ctx serpent_dec_xts = { - .num_funcs = 2, - .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS, - - .funcs = { { - .num_blocks = SERPENT_PARALLEL_BLOCKS, - .fn_u = { .xts = serpent_xts_dec_8way_avx } - }, { - .num_blocks = 1, - .fn_u = { .xts = serpent_xts_dec } - } } -}; - static int ecb_encrypt(struct skcipher_request *req) { - return glue_ecb_req_128bit(&serpent_enc, req); + ECB_WALK_START(req, SERPENT_BLOCK_SIZE, SERPENT_PARALLEL_BLOCKS); + ECB_BLOCK(SERPENT_PARALLEL_BLOCKS, serpent_ecb_enc_8way_avx); + ECB_BLOCK(1, __serpent_encrypt); + ECB_WALK_END(); } static int ecb_decrypt(struct skcipher_request *req) { - return glue_ecb_req_128bit(&serpent_dec, req); + ECB_WALK_START(req, SERPENT_BLOCK_SIZE, SERPENT_PARALLEL_BLOCKS); + ECB_BLOCK(SERPENT_PARALLEL_BLOCKS, serpent_ecb_dec_8way_avx); + ECB_BLOCK(1, __serpent_decrypt); + ECB_WALK_END(); } static int cbc_encrypt(struct skcipher_request *req) { - return glue_cbc_encrypt_req_128bit(__serpent_encrypt, req); + CBC_WALK_START(req, SERPENT_BLOCK_SIZE, -1); + CBC_ENC_BLOCK(__serpent_encrypt); + CBC_WALK_END(); } static int cbc_decrypt(struct skcipher_request *req) { - return glue_cbc_decrypt_req_128bit(&serpent_dec_cbc, req); -} - -static int ctr_crypt(struct skcipher_request *req) -{ - return glue_ctr_req_128bit(&serpent_ctr, req); -} - -static int xts_encrypt(struct skcipher_request *req) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct serpent_xts_ctx *ctx = crypto_skcipher_ctx(tfm); - - return glue_xts_req_128bit(&serpent_enc_xts, req, - __serpent_encrypt, &ctx->tweak_ctx, - &ctx->crypt_ctx, false); -} - -static int xts_decrypt(struct skcipher_request *req) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct serpent_xts_ctx *ctx = crypto_skcipher_ctx(tfm); - - return glue_xts_req_128bit(&serpent_dec_xts, req, - __serpent_encrypt, &ctx->tweak_ctx, - &ctx->crypt_ctx, true); + CBC_WALK_START(req, SERPENT_BLOCK_SIZE, SERPENT_PARALLEL_BLOCKS); + CBC_DEC_BLOCK(SERPENT_PARALLEL_BLOCKS, serpent_cbc_dec_8way_avx); + CBC_DEC_BLOCK(1, __serpent_decrypt); + CBC_WALK_END(); } static struct skcipher_alg serpent_algs[] = { @@ -247,35 +97,6 @@ static struct skcipher_alg serpent_algs[] = { .setkey = serpent_setkey_skcipher, .encrypt = cbc_encrypt, .decrypt = cbc_decrypt, - }, { - .base.cra_name = "__ctr(serpent)", - .base.cra_driver_name = "__ctr-serpent-avx", - .base.cra_priority = 500, - .base.cra_flags = CRYPTO_ALG_INTERNAL, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct serpent_ctx), - .base.cra_module = THIS_MODULE, - .min_keysize = SERPENT_MIN_KEY_SIZE, - .max_keysize = SERPENT_MAX_KEY_SIZE, - .ivsize = SERPENT_BLOCK_SIZE, - .chunksize = SERPENT_BLOCK_SIZE, - .setkey = serpent_setkey_skcipher, - .encrypt = ctr_crypt, - .decrypt = ctr_crypt, - }, { - .base.cra_name = "__xts(serpent)", - .base.cra_driver_name = "__xts-serpent-avx", - .base.cra_priority = 500, - .base.cra_flags = CRYPTO_ALG_INTERNAL, - .base.cra_blocksize = SERPENT_BLOCK_SIZE, - .base.cra_ctxsize = sizeof(struct serpent_xts_ctx), - .base.cra_module = THIS_MODULE, - .min_keysize = 2 * SERPENT_MIN_KEY_SIZE, - .max_keysize = 2 * SERPENT_MAX_KEY_SIZE, - .ivsize = SERPENT_BLOCK_SIZE, - .setkey = xts_serpent_setkey, - .encrypt = xts_encrypt, - .decrypt = xts_decrypt, }, }; diff --git a/arch/x86/crypto/serpent_sse2_glue.c b/arch/x86/crypto/serpent_sse2_glue.c index 4fed8d26b91a4eb88f3f87e00e63365c423ef9ca..d78f37e9b2cf71b039048a7efa2d0511d9eba5c9 100644 --- a/arch/x86/crypto/serpent_sse2_glue.c +++ b/arch/x86/crypto/serpent_sse2_glue.c @@ -10,8 +10,6 @@ * * CBC & ECB parts based on code (crypto/cbc.c,ecb.c) by: * Copyright (c) 2006 Herbert Xu - * CTR part based on code (crypto/ctr.c) by: - * (C) Copyright IBM Corp. 2007 - Joy Latten */ #include @@ -22,8 +20,9 @@ #include #include #include -#include -#include + +#include "serpent-sse2.h" +#include "ecb_cbc_helpers.h" static int serpent_setkey_skcipher(struct crypto_skcipher *tfm, const u8 *key, unsigned int keylen) @@ -31,130 +30,46 @@ static int serpent_setkey_skcipher(struct crypto_skcipher *tfm, return __serpent_setkey(crypto_skcipher_ctx(tfm), key, keylen); } -static void serpent_decrypt_cbc_xway(const void *ctx, u8 *d, const u8 *s) -{ - u128 ivs[SERPENT_PARALLEL_BLOCKS - 1]; - u128 *dst = (u128 *)d; - const u128 *src = (const u128 *)s; - unsigned int j; - - for (j = 0; j < SERPENT_PARALLEL_BLOCKS - 1; j++) - ivs[j] = src[j]; - - serpent_dec_blk_xway(ctx, (u8 *)dst, (u8 *)src); - - for (j = 0; j < SERPENT_PARALLEL_BLOCKS - 1; j++) - u128_xor(dst + (j + 1), dst + (j + 1), ivs + j); -} - -static void serpent_crypt_ctr(const void *ctx, u8 *d, const u8 *s, le128 *iv) -{ - be128 ctrblk; - u128 *dst = (u128 *)d; - const u128 *src = (const u128 *)s; - - le128_to_be128(&ctrblk, iv); - le128_inc(iv); - - __serpent_encrypt(ctx, (u8 *)&ctrblk, (u8 *)&ctrblk); - u128_xor(dst, src, (u128 *)&ctrblk); -} - -static void serpent_crypt_ctr_xway(const void *ctx, u8 *d, const u8 *s, - le128 *iv) +static void serpent_decrypt_cbc_xway(const void *ctx, u8 *dst, const u8 *src) { - be128 ctrblks[SERPENT_PARALLEL_BLOCKS]; - u128 *dst = (u128 *)d; - const u128 *src = (const u128 *)s; - unsigned int i; - - for (i = 0; i < SERPENT_PARALLEL_BLOCKS; i++) { - if (dst != src) - dst[i] = src[i]; - - le128_to_be128(&ctrblks[i], iv); - le128_inc(iv); - } + u8 buf[SERPENT_PARALLEL_BLOCKS - 1][SERPENT_BLOCK_SIZE]; + const u8 *s = src; - serpent_enc_blk_xway_xor(ctx, (u8 *)dst, (u8 *)ctrblks); + if (dst == src) + s = memcpy(buf, src, sizeof(buf)); + serpent_dec_blk_xway(ctx, dst, src); + crypto_xor(dst + SERPENT_BLOCK_SIZE, s, sizeof(buf)); } -static const struct common_glue_ctx serpent_enc = { - .num_funcs = 2, - .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS, - - .funcs = { { - .num_blocks = SERPENT_PARALLEL_BLOCKS, - .fn_u = { .ecb = serpent_enc_blk_xway } - }, { - .num_blocks = 1, - .fn_u = { .ecb = __serpent_encrypt } - } } -}; - -static const struct common_glue_ctx serpent_ctr = { - .num_funcs = 2, - .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS, - - .funcs = { { - .num_blocks = SERPENT_PARALLEL_BLOCKS, - .fn_u = { .ctr = serpent_crypt_ctr_xway } - }, { - .num_blocks = 1, - .fn_u = { .ctr = serpent_crypt_ctr } - } } -}; - -static const struct common_glue_ctx serpent_dec = { - .num_funcs = 2, - .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS, - - .funcs = { { - .num_blocks = SERPENT_PARALLEL_BLOCKS, - .fn_u = { .ecb = serpent_dec_blk_xway } - }, { - .num_blocks = 1, - .fn_u = { .ecb = __serpent_decrypt } - } } -}; - -static const struct common_glue_ctx serpent_dec_cbc = { - .num_funcs = 2, - .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS, - - .funcs = { { - .num_blocks = SERPENT_PARALLEL_BLOCKS, - .fn_u = { .cbc = serpent_decrypt_cbc_xway } - }, { - .num_blocks = 1, - .fn_u = { .cbc = __serpent_decrypt } - } } -}; - static int ecb_encrypt(struct skcipher_request *req) { - return glue_ecb_req_128bit(&serpent_enc, req); + ECB_WALK_START(req, SERPENT_BLOCK_SIZE, SERPENT_PARALLEL_BLOCKS); + ECB_BLOCK(SERPENT_PARALLEL_BLOCKS, serpent_enc_blk_xway); + ECB_BLOCK(1, __serpent_encrypt); + ECB_WALK_END(); } static int ecb_decrypt(struct skcipher_request *req) { - return glue_ecb_req_128bit(&serpent_dec, req); + ECB_WALK_START(req, SERPENT_BLOCK_SIZE, SERPENT_PARALLEL_BLOCKS); + ECB_BLOCK(SERPENT_PARALLEL_BLOCKS, serpent_dec_blk_xway); + ECB_BLOCK(1, __serpent_decrypt); + ECB_WALK_END(); } static int cbc_encrypt(struct skcipher_request *req) { - return glue_cbc_encrypt_req_128bit(__serpent_encrypt, - req); + CBC_WALK_START(req, SERPENT_BLOCK_SIZE, -1); + CBC_ENC_BLOCK(__serpent_encrypt); + CBC_WALK_END(); } static int cbc_decrypt(struct skcipher_request *req) { - return glue_cbc_decrypt_req_128bit(&serpent_dec_cbc, req); -} - -static int ctr_crypt(struct skcipher_request *req) -{ - return glue_ctr_req_128bit(&serpent_ctr, req); + CBC_WALK_START(req, SERPENT_BLOCK_SIZE, SERPENT_PARALLEL_BLOCKS); + CBC_DEC_BLOCK(SERPENT_PARALLEL_BLOCKS, serpent_decrypt_cbc_xway); + CBC_DEC_BLOCK(1, __serpent_decrypt); + CBC_WALK_END(); } static struct skcipher_alg serpent_algs[] = { @@ -185,21 +100,6 @@ static struct skcipher_alg serpent_algs[] = { .setkey = serpent_setkey_skcipher, .encrypt = cbc_encrypt, .decrypt = cbc_decrypt, - }, { - .base.cra_name = "__ctr(serpent)", - .base.cra_driver_name = "__ctr-serpent-sse2", - .base.cra_priority = 400, - .base.cra_flags = CRYPTO_ALG_INTERNAL, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct serpent_ctx), - .base.cra_module = THIS_MODULE, - .min_keysize = SERPENT_MIN_KEY_SIZE, - .max_keysize = SERPENT_MAX_KEY_SIZE, - .ivsize = SERPENT_BLOCK_SIZE, - .chunksize = SERPENT_BLOCK_SIZE, - .setkey = serpent_setkey_skcipher, - .encrypt = ctr_crypt, - .decrypt = ctr_crypt, }, }; diff --git a/arch/x86/crypto/twofish-avx-x86_64-asm_64.S b/arch/x86/crypto/twofish-avx-x86_64-asm_64.S index a5151393bb2f328b84a9c8256919eccbef5f2618..37e63b3c664eba8985492bb26d27c654341c22c6 100644 --- a/arch/x86/crypto/twofish-avx-x86_64-asm_64.S +++ b/arch/x86/crypto/twofish-avx-x86_64-asm_64.S @@ -19,11 +19,6 @@ .Lbswap128_mask: .byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 -.section .rodata.cst16.xts_gf128mul_and_shl1_mask, "aM", @progbits, 16 -.align 16 -.Lxts_gf128mul_and_shl1_mask: - .byte 0x87, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0 - .text /* structure of crypto context */ @@ -379,78 +374,3 @@ SYM_FUNC_START(twofish_cbc_dec_8way) FRAME_END ret; SYM_FUNC_END(twofish_cbc_dec_8way) - -SYM_FUNC_START(twofish_ctr_8way) - /* input: - * %rdi: ctx, CTX - * %rsi: dst - * %rdx: src - * %rcx: iv (little endian, 128bit) - */ - FRAME_BEGIN - - pushq %r12; - - movq %rsi, %r11; - movq %rdx, %r12; - - load_ctr_8way(%rcx, .Lbswap128_mask, RA1, RB1, RC1, RD1, RA2, RB2, RC2, - RD2, RX0, RX1, RY0); - - call __twofish_enc_blk8; - - store_ctr_8way(%r12, %r11, RC1, RD1, RA1, RB1, RC2, RD2, RA2, RB2); - - popq %r12; - - FRAME_END - ret; -SYM_FUNC_END(twofish_ctr_8way) - -SYM_FUNC_START(twofish_xts_enc_8way) - /* input: - * %rdi: ctx, CTX - * %rsi: dst - * %rdx: src - * %rcx: iv (t ⊕ α⿠∈ GF(2¹²â¸)) - */ - FRAME_BEGIN - - movq %rsi, %r11; - - /* regs <= src, dst <= IVs, regs <= regs xor IVs */ - load_xts_8way(%rcx, %rdx, %rsi, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2, - RX0, RX1, RY0, .Lxts_gf128mul_and_shl1_mask); - - call __twofish_enc_blk8; - - /* dst <= regs xor IVs(in dst) */ - store_xts_8way(%r11, RC1, RD1, RA1, RB1, RC2, RD2, RA2, RB2); - - FRAME_END - ret; -SYM_FUNC_END(twofish_xts_enc_8way) - -SYM_FUNC_START(twofish_xts_dec_8way) - /* input: - * %rdi: ctx, CTX - * %rsi: dst - * %rdx: src - * %rcx: iv (t ⊕ α⿠∈ GF(2¹²â¸)) - */ - FRAME_BEGIN - - movq %rsi, %r11; - - /* regs <= src, dst <= IVs, regs <= regs xor IVs */ - load_xts_8way(%rcx, %rdx, %rsi, RC1, RD1, RA1, RB1, RC2, RD2, RA2, RB2, - RX0, RX1, RY0, .Lxts_gf128mul_and_shl1_mask); - - call __twofish_dec_blk8; - - /* dst <= regs xor IVs(in dst) */ - store_xts_8way(%r11, RA1, RB1, RC1, RD1, RA2, RB2, RC2, RD2); - - FRAME_END - ret; -SYM_FUNC_END(twofish_xts_dec_8way) diff --git a/arch/x86/include/asm/crypto/twofish.h b/arch/x86/crypto/twofish.h similarity index 80% rename from arch/x86/include/asm/crypto/twofish.h rename to arch/x86/crypto/twofish.h index 2c377a8042e17f3422d9cb4f4dc509ea1ac505bd..12df400e6d533dde1ea03ce1546790d3620ce4a0 100644 --- a/arch/x86/include/asm/crypto/twofish.h +++ b/arch/x86/crypto/twofish.h @@ -17,9 +17,5 @@ asmlinkage void twofish_dec_blk_3way(const void *ctx, u8 *dst, const u8 *src); /* helpers from twofish_x86_64-3way module */ extern void twofish_dec_blk_cbc_3way(const void *ctx, u8 *dst, const u8 *src); -extern void twofish_enc_blk_ctr(const void *ctx, u8 *dst, const u8 *src, - le128 *iv); -extern void twofish_enc_blk_ctr_3way(const void *ctx, u8 *dst, const u8 *src, - le128 *iv); #endif /* ASM_X86_TWOFISH_H */ diff --git a/arch/x86/crypto/twofish_avx_glue.c b/arch/x86/crypto/twofish_avx_glue.c index 2dbc8ce3730e53724b16457f298c6a85c0b2dfa4..3eb3440b477a8e19f7326f3ff17365e421be5164 100644 --- a/arch/x86/crypto/twofish_avx_glue.c +++ b/arch/x86/crypto/twofish_avx_glue.c @@ -15,9 +15,9 @@ #include #include #include -#include -#include -#include + +#include "twofish.h" +#include "ecb_cbc_helpers.h" #define TWOFISH_PARALLEL_BLOCKS 8 @@ -26,13 +26,6 @@ asmlinkage void twofish_ecb_enc_8way(const void *ctx, u8 *dst, const u8 *src); asmlinkage void twofish_ecb_dec_8way(const void *ctx, u8 *dst, const u8 *src); asmlinkage void twofish_cbc_dec_8way(const void *ctx, u8 *dst, const u8 *src); -asmlinkage void twofish_ctr_8way(const void *ctx, u8 *dst, const u8 *src, - le128 *iv); - -asmlinkage void twofish_xts_enc_8way(const void *ctx, u8 *dst, const u8 *src, - le128 *iv); -asmlinkage void twofish_xts_dec_8way(const void *ctx, u8 *dst, const u8 *src, - le128 *iv); static int twofish_setkey_skcipher(struct crypto_skcipher *tfm, const u8 *key, unsigned int keylen) @@ -45,171 +38,38 @@ static inline void twofish_enc_blk_3way(const void *ctx, u8 *dst, const u8 *src) __twofish_enc_blk_3way(ctx, dst, src, false); } -static void twofish_xts_enc(const void *ctx, u8 *dst, const u8 *src, le128 *iv) -{ - glue_xts_crypt_128bit_one(ctx, dst, src, iv, twofish_enc_blk); -} - -static void twofish_xts_dec(const void *ctx, u8 *dst, const u8 *src, le128 *iv) -{ - glue_xts_crypt_128bit_one(ctx, dst, src, iv, twofish_dec_blk); -} - -struct twofish_xts_ctx { - struct twofish_ctx tweak_ctx; - struct twofish_ctx crypt_ctx; -}; - -static int xts_twofish_setkey(struct crypto_skcipher *tfm, const u8 *key, - unsigned int keylen) -{ - struct twofish_xts_ctx *ctx = crypto_skcipher_ctx(tfm); - int err; - - err = xts_verify_key(tfm, key, keylen); - if (err) - return err; - - /* first half of xts-key is for crypt */ - err = __twofish_setkey(&ctx->crypt_ctx, key, keylen / 2); - if (err) - return err; - - /* second half of xts-key is for tweak */ - return __twofish_setkey(&ctx->tweak_ctx, key + keylen / 2, keylen / 2); -} - -static const struct common_glue_ctx twofish_enc = { - .num_funcs = 3, - .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS, - - .funcs = { { - .num_blocks = TWOFISH_PARALLEL_BLOCKS, - .fn_u = { .ecb = twofish_ecb_enc_8way } - }, { - .num_blocks = 3, - .fn_u = { .ecb = twofish_enc_blk_3way } - }, { - .num_blocks = 1, - .fn_u = { .ecb = twofish_enc_blk } - } } -}; - -static const struct common_glue_ctx twofish_ctr = { - .num_funcs = 3, - .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS, - - .funcs = { { - .num_blocks = TWOFISH_PARALLEL_BLOCKS, - .fn_u = { .ctr = twofish_ctr_8way } - }, { - .num_blocks = 3, - .fn_u = { .ctr = twofish_enc_blk_ctr_3way } - }, { - .num_blocks = 1, - .fn_u = { .ctr = twofish_enc_blk_ctr } - } } -}; - -static const struct common_glue_ctx twofish_enc_xts = { - .num_funcs = 2, - .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS, - - .funcs = { { - .num_blocks = TWOFISH_PARALLEL_BLOCKS, - .fn_u = { .xts = twofish_xts_enc_8way } - }, { - .num_blocks = 1, - .fn_u = { .xts = twofish_xts_enc } - } } -}; - -static const struct common_glue_ctx twofish_dec = { - .num_funcs = 3, - .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS, - - .funcs = { { - .num_blocks = TWOFISH_PARALLEL_BLOCKS, - .fn_u = { .ecb = twofish_ecb_dec_8way } - }, { - .num_blocks = 3, - .fn_u = { .ecb = twofish_dec_blk_3way } - }, { - .num_blocks = 1, - .fn_u = { .ecb = twofish_dec_blk } - } } -}; - -static const struct common_glue_ctx twofish_dec_cbc = { - .num_funcs = 3, - .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS, - - .funcs = { { - .num_blocks = TWOFISH_PARALLEL_BLOCKS, - .fn_u = { .cbc = twofish_cbc_dec_8way } - }, { - .num_blocks = 3, - .fn_u = { .cbc = twofish_dec_blk_cbc_3way } - }, { - .num_blocks = 1, - .fn_u = { .cbc = twofish_dec_blk } - } } -}; - -static const struct common_glue_ctx twofish_dec_xts = { - .num_funcs = 2, - .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS, - - .funcs = { { - .num_blocks = TWOFISH_PARALLEL_BLOCKS, - .fn_u = { .xts = twofish_xts_dec_8way } - }, { - .num_blocks = 1, - .fn_u = { .xts = twofish_xts_dec } - } } -}; - static int ecb_encrypt(struct skcipher_request *req) { - return glue_ecb_req_128bit(&twofish_enc, req); + ECB_WALK_START(req, TF_BLOCK_SIZE, TWOFISH_PARALLEL_BLOCKS); + ECB_BLOCK(TWOFISH_PARALLEL_BLOCKS, twofish_ecb_enc_8way); + ECB_BLOCK(3, twofish_enc_blk_3way); + ECB_BLOCK(1, twofish_enc_blk); + ECB_WALK_END(); } static int ecb_decrypt(struct skcipher_request *req) { - return glue_ecb_req_128bit(&twofish_dec, req); + ECB_WALK_START(req, TF_BLOCK_SIZE, TWOFISH_PARALLEL_BLOCKS); + ECB_BLOCK(TWOFISH_PARALLEL_BLOCKS, twofish_ecb_dec_8way); + ECB_BLOCK(3, twofish_dec_blk_3way); + ECB_BLOCK(1, twofish_dec_blk); + ECB_WALK_END(); } static int cbc_encrypt(struct skcipher_request *req) { - return glue_cbc_encrypt_req_128bit(twofish_enc_blk, req); + CBC_WALK_START(req, TF_BLOCK_SIZE, -1); + CBC_ENC_BLOCK(twofish_enc_blk); + CBC_WALK_END(); } static int cbc_decrypt(struct skcipher_request *req) { - return glue_cbc_decrypt_req_128bit(&twofish_dec_cbc, req); -} - -static int ctr_crypt(struct skcipher_request *req) -{ - return glue_ctr_req_128bit(&twofish_ctr, req); -} - -static int xts_encrypt(struct skcipher_request *req) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct twofish_xts_ctx *ctx = crypto_skcipher_ctx(tfm); - - return glue_xts_req_128bit(&twofish_enc_xts, req, twofish_enc_blk, - &ctx->tweak_ctx, &ctx->crypt_ctx, false); -} - -static int xts_decrypt(struct skcipher_request *req) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct twofish_xts_ctx *ctx = crypto_skcipher_ctx(tfm); - - return glue_xts_req_128bit(&twofish_dec_xts, req, twofish_enc_blk, - &ctx->tweak_ctx, &ctx->crypt_ctx, true); + CBC_WALK_START(req, TF_BLOCK_SIZE, TWOFISH_PARALLEL_BLOCKS); + CBC_DEC_BLOCK(TWOFISH_PARALLEL_BLOCKS, twofish_cbc_dec_8way); + CBC_DEC_BLOCK(3, twofish_dec_blk_cbc_3way); + CBC_DEC_BLOCK(1, twofish_dec_blk); + CBC_WALK_END(); } static struct skcipher_alg twofish_algs[] = { @@ -240,35 +100,6 @@ static struct skcipher_alg twofish_algs[] = { .setkey = twofish_setkey_skcipher, .encrypt = cbc_encrypt, .decrypt = cbc_decrypt, - }, { - .base.cra_name = "__ctr(twofish)", - .base.cra_driver_name = "__ctr-twofish-avx", - .base.cra_priority = 400, - .base.cra_flags = CRYPTO_ALG_INTERNAL, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct twofish_ctx), - .base.cra_module = THIS_MODULE, - .min_keysize = TF_MIN_KEY_SIZE, - .max_keysize = TF_MAX_KEY_SIZE, - .ivsize = TF_BLOCK_SIZE, - .chunksize = TF_BLOCK_SIZE, - .setkey = twofish_setkey_skcipher, - .encrypt = ctr_crypt, - .decrypt = ctr_crypt, - }, { - .base.cra_name = "__xts(twofish)", - .base.cra_driver_name = "__xts-twofish-avx", - .base.cra_priority = 400, - .base.cra_flags = CRYPTO_ALG_INTERNAL, - .base.cra_blocksize = TF_BLOCK_SIZE, - .base.cra_ctxsize = sizeof(struct twofish_xts_ctx), - .base.cra_module = THIS_MODULE, - .min_keysize = 2 * TF_MIN_KEY_SIZE, - .max_keysize = 2 * TF_MAX_KEY_SIZE, - .ivsize = TF_BLOCK_SIZE, - .setkey = xts_twofish_setkey, - .encrypt = xts_encrypt, - .decrypt = xts_decrypt, }, }; diff --git a/arch/x86/crypto/twofish_glue_3way.c b/arch/x86/crypto/twofish_glue_3way.c index 768af6075479c1f18e2a257e3ab24f1fbc652d34..03725696397c8e32b80f4735900c5a0355a5bb21 100644 --- a/arch/x86/crypto/twofish_glue_3way.c +++ b/arch/x86/crypto/twofish_glue_3way.c @@ -5,17 +5,16 @@ * Copyright (c) 2011 Jussi Kivilinna */ -#include -#include #include -#include -#include #include #include #include #include #include +#include "twofish.h" +#include "ecb_cbc_helpers.h" + EXPORT_SYMBOL_GPL(__twofish_enc_blk_3way); EXPORT_SYMBOL_GPL(twofish_dec_blk_3way); @@ -30,143 +29,48 @@ static inline void twofish_enc_blk_3way(const void *ctx, u8 *dst, const u8 *src) __twofish_enc_blk_3way(ctx, dst, src, false); } -static inline void twofish_enc_blk_xor_3way(const void *ctx, u8 *dst, - const u8 *src) -{ - __twofish_enc_blk_3way(ctx, dst, src, true); -} - -void twofish_dec_blk_cbc_3way(const void *ctx, u8 *d, const u8 *s) +void twofish_dec_blk_cbc_3way(const void *ctx, u8 *dst, const u8 *src) { - u128 ivs[2]; - u128 *dst = (u128 *)d; - const u128 *src = (const u128 *)s; - - ivs[0] = src[0]; - ivs[1] = src[1]; + u8 buf[2][TF_BLOCK_SIZE]; + const u8 *s = src; - twofish_dec_blk_3way(ctx, (u8 *)dst, (u8 *)src); + if (dst == src) + s = memcpy(buf, src, sizeof(buf)); + twofish_dec_blk_3way(ctx, dst, src); + crypto_xor(dst + TF_BLOCK_SIZE, s, sizeof(buf)); - u128_xor(&dst[1], &dst[1], &ivs[0]); - u128_xor(&dst[2], &dst[2], &ivs[1]); } EXPORT_SYMBOL_GPL(twofish_dec_blk_cbc_3way); -void twofish_enc_blk_ctr(const void *ctx, u8 *d, const u8 *s, le128 *iv) -{ - be128 ctrblk; - u128 *dst = (u128 *)d; - const u128 *src = (const u128 *)s; - - if (dst != src) - *dst = *src; - - le128_to_be128(&ctrblk, iv); - le128_inc(iv); - - twofish_enc_blk(ctx, (u8 *)&ctrblk, (u8 *)&ctrblk); - u128_xor(dst, dst, (u128 *)&ctrblk); -} -EXPORT_SYMBOL_GPL(twofish_enc_blk_ctr); - -void twofish_enc_blk_ctr_3way(const void *ctx, u8 *d, const u8 *s, le128 *iv) -{ - be128 ctrblks[3]; - u128 *dst = (u128 *)d; - const u128 *src = (const u128 *)s; - - if (dst != src) { - dst[0] = src[0]; - dst[1] = src[1]; - dst[2] = src[2]; - } - - le128_to_be128(&ctrblks[0], iv); - le128_inc(iv); - le128_to_be128(&ctrblks[1], iv); - le128_inc(iv); - le128_to_be128(&ctrblks[2], iv); - le128_inc(iv); - - twofish_enc_blk_xor_3way(ctx, (u8 *)dst, (u8 *)ctrblks); -} -EXPORT_SYMBOL_GPL(twofish_enc_blk_ctr_3way); - -static const struct common_glue_ctx twofish_enc = { - .num_funcs = 2, - .fpu_blocks_limit = -1, - - .funcs = { { - .num_blocks = 3, - .fn_u = { .ecb = twofish_enc_blk_3way } - }, { - .num_blocks = 1, - .fn_u = { .ecb = twofish_enc_blk } - } } -}; - -static const struct common_glue_ctx twofish_ctr = { - .num_funcs = 2, - .fpu_blocks_limit = -1, - - .funcs = { { - .num_blocks = 3, - .fn_u = { .ctr = twofish_enc_blk_ctr_3way } - }, { - .num_blocks = 1, - .fn_u = { .ctr = twofish_enc_blk_ctr } - } } -}; - -static const struct common_glue_ctx twofish_dec = { - .num_funcs = 2, - .fpu_blocks_limit = -1, - - .funcs = { { - .num_blocks = 3, - .fn_u = { .ecb = twofish_dec_blk_3way } - }, { - .num_blocks = 1, - .fn_u = { .ecb = twofish_dec_blk } - } } -}; - -static const struct common_glue_ctx twofish_dec_cbc = { - .num_funcs = 2, - .fpu_blocks_limit = -1, - - .funcs = { { - .num_blocks = 3, - .fn_u = { .cbc = twofish_dec_blk_cbc_3way } - }, { - .num_blocks = 1, - .fn_u = { .cbc = twofish_dec_blk } - } } -}; - static int ecb_encrypt(struct skcipher_request *req) { - return glue_ecb_req_128bit(&twofish_enc, req); + ECB_WALK_START(req, TF_BLOCK_SIZE, -1); + ECB_BLOCK(3, twofish_enc_blk_3way); + ECB_BLOCK(1, twofish_enc_blk); + ECB_WALK_END(); } static int ecb_decrypt(struct skcipher_request *req) { - return glue_ecb_req_128bit(&twofish_dec, req); + ECB_WALK_START(req, TF_BLOCK_SIZE, -1); + ECB_BLOCK(3, twofish_dec_blk_3way); + ECB_BLOCK(1, twofish_dec_blk); + ECB_WALK_END(); } static int cbc_encrypt(struct skcipher_request *req) { - return glue_cbc_encrypt_req_128bit(twofish_enc_blk, req); + CBC_WALK_START(req, TF_BLOCK_SIZE, -1); + CBC_ENC_BLOCK(twofish_enc_blk); + CBC_WALK_END(); } static int cbc_decrypt(struct skcipher_request *req) { - return glue_cbc_decrypt_req_128bit(&twofish_dec_cbc, req); -} - -static int ctr_crypt(struct skcipher_request *req) -{ - return glue_ctr_req_128bit(&twofish_ctr, req); + CBC_WALK_START(req, TF_BLOCK_SIZE, -1); + CBC_DEC_BLOCK(3, twofish_dec_blk_cbc_3way); + CBC_DEC_BLOCK(1, twofish_dec_blk); + CBC_WALK_END(); } static struct skcipher_alg tf_skciphers[] = { @@ -195,20 +99,6 @@ static struct skcipher_alg tf_skciphers[] = { .setkey = twofish_setkey_skcipher, .encrypt = cbc_encrypt, .decrypt = cbc_decrypt, - }, { - .base.cra_name = "ctr(twofish)", - .base.cra_driver_name = "ctr-twofish-3way", - .base.cra_priority = 300, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct twofish_ctx), - .base.cra_module = THIS_MODULE, - .min_keysize = TF_MIN_KEY_SIZE, - .max_keysize = TF_MAX_KEY_SIZE, - .ivsize = TF_BLOCK_SIZE, - .chunksize = TF_BLOCK_SIZE, - .setkey = twofish_setkey_skcipher, - .encrypt = ctr_crypt, - .decrypt = ctr_crypt, }, }; diff --git a/arch/x86/entry/common.c b/arch/x86/entry/common.c index 0904f5676e4d8880562ad9d00532c12d2fb79b8c..a2433ae8a65e764028e37e9d87bc0b0a7e93cbd0 100644 --- a/arch/x86/entry/common.c +++ b/arch/x86/entry/common.c @@ -249,30 +249,23 @@ static __always_inline bool get_and_clear_inhcall(void) { return false; } static __always_inline void restore_inhcall(bool inhcall) { } #endif -static void __xen_pv_evtchn_do_upcall(void) +static void __xen_pv_evtchn_do_upcall(struct pt_regs *regs) { - irq_enter_rcu(); + struct pt_regs *old_regs = set_irq_regs(regs); + inc_irq_stat(irq_hv_callback_count); xen_hvm_evtchn_do_upcall(); - irq_exit_rcu(); + set_irq_regs(old_regs); } __visible noinstr void xen_pv_evtchn_do_upcall(struct pt_regs *regs) { - struct pt_regs *old_regs; + irqentry_state_t state = irqentry_enter(regs); bool inhcall; - irqentry_state_t state; - state = irqentry_enter(regs); - old_regs = set_irq_regs(regs); - - instrumentation_begin(); - run_on_irqstack_cond(__xen_pv_evtchn_do_upcall, regs); - instrumentation_begin(); - - set_irq_regs(old_regs); + run_sysvec_on_irqstack_cond(__xen_pv_evtchn_do_upcall, regs); inhcall = get_and_clear_inhcall(); if (inhcall && !WARN_ON_ONCE(state.exit_rcu)) { diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S index ce0464d630a23b08fc1db899427aaed3024c1b2b..400908dff42ebfebf0f0dda3063b9b41ff7ffd2c 100644 --- a/arch/x86/entry/entry_64.S +++ b/arch/x86/entry/entry_64.S @@ -754,47 +754,6 @@ SYM_CODE_START_LOCAL_NOALIGN(.Lbad_gs) SYM_CODE_END(.Lbad_gs) .previous -/* - * rdi: New stack pointer points to the top word of the stack - * rsi: Function pointer - * rdx: Function argument (can be NULL if none) - */ -SYM_FUNC_START(asm_call_on_stack) -SYM_INNER_LABEL(asm_call_sysvec_on_stack, SYM_L_GLOBAL) -SYM_INNER_LABEL(asm_call_irq_on_stack, SYM_L_GLOBAL) - /* - * Save the frame pointer unconditionally. This allows the ORC - * unwinder to handle the stack switch. - */ - pushq %rbp - mov %rsp, %rbp - - /* - * The unwinder relies on the word at the top of the new stack - * page linking back to the previous RSP. - */ - mov %rsp, (%rdi) - mov %rdi, %rsp - /* Move the argument to the right place */ - mov %rdx, %rdi - -1: - .pushsection .discard.instr_begin - .long 1b - . - .popsection - - CALL_NOSPEC rsi - -2: - .pushsection .discard.instr_end - .long 2b - . - .popsection - - /* Restore the previous stack pointer from RBP. */ - leaveq - ret -SYM_FUNC_END(asm_call_on_stack) - #ifdef CONFIG_XEN_PV /* * A note on the "critical region" in our callback handler. diff --git a/arch/x86/entry/syscalls/Makefile b/arch/x86/entry/syscalls/Makefile index 6fb9b57ed5ba05dcc5247b26ee8b7b0575cf38e9..d8c4f6c9eadc070e1af04df4a358b3d8e7495a8d 100644 --- a/arch/x86/entry/syscalls/Makefile +++ b/arch/x86/entry/syscalls/Makefile @@ -6,8 +6,8 @@ uapi := arch/$(SRCARCH)/include/generated/uapi/asm _dummy := $(shell [ -d '$(out)' ] || mkdir -p '$(out)') \ $(shell [ -d '$(uapi)' ] || mkdir -p '$(uapi)') -syscall32 := $(srctree)/$(src)/syscall_32.tbl -syscall64 := $(srctree)/$(src)/syscall_64.tbl +syscall32 := $(src)/syscall_32.tbl +syscall64 := $(src)/syscall_64.tbl syshdr := $(srctree)/$(src)/syscallhdr.sh systbl := $(srctree)/$(src)/syscalltbl.sh @@ -21,37 +21,37 @@ quiet_cmd_systbl = SYSTBL $@ cmd_systbl = $(CONFIG_SHELL) '$(systbl)' $< $@ quiet_cmd_hypercalls = HYPERCALLS $@ - cmd_hypercalls = $(CONFIG_SHELL) '$<' $@ $(filter-out $<,$^) + cmd_hypercalls = $(CONFIG_SHELL) '$<' $@ $(filter-out $<, $(real-prereqs)) syshdr_abi_unistd_32 := i386 -$(uapi)/unistd_32.h: $(syscall32) $(syshdr) +$(uapi)/unistd_32.h: $(syscall32) $(syshdr) FORCE $(call if_changed,syshdr) syshdr_abi_unistd_32_ia32 := i386 syshdr_pfx_unistd_32_ia32 := ia32_ -$(out)/unistd_32_ia32.h: $(syscall32) $(syshdr) +$(out)/unistd_32_ia32.h: $(syscall32) $(syshdr) FORCE $(call if_changed,syshdr) syshdr_abi_unistd_x32 := common,x32 syshdr_offset_unistd_x32 := __X32_SYSCALL_BIT -$(uapi)/unistd_x32.h: $(syscall64) $(syshdr) +$(uapi)/unistd_x32.h: $(syscall64) $(syshdr) FORCE $(call if_changed,syshdr) syshdr_abi_unistd_64 := common,64 -$(uapi)/unistd_64.h: $(syscall64) $(syshdr) +$(uapi)/unistd_64.h: $(syscall64) $(syshdr) FORCE $(call if_changed,syshdr) syshdr_abi_unistd_64_x32 := x32 syshdr_pfx_unistd_64_x32 := x32_ -$(out)/unistd_64_x32.h: $(syscall64) $(syshdr) +$(out)/unistd_64_x32.h: $(syscall64) $(syshdr) FORCE $(call if_changed,syshdr) -$(out)/syscalls_32.h: $(syscall32) $(systbl) +$(out)/syscalls_32.h: $(syscall32) $(systbl) FORCE $(call if_changed,systbl) -$(out)/syscalls_64.h: $(syscall64) $(systbl) +$(out)/syscalls_64.h: $(syscall64) $(systbl) FORCE $(call if_changed,systbl) -$(out)/xen-hypercalls.h: $(srctree)/scripts/xen-hypercalls.sh +$(out)/xen-hypercalls.h: $(srctree)/scripts/xen-hypercalls.sh FORCE $(call if_changed,hypercalls) $(out)/xen-hypercalls.h: $(srctree)/include/xen/interface/xen*.h @@ -62,9 +62,10 @@ syshdr-$(CONFIG_X86_64) += unistd_32_ia32.h unistd_64_x32.h syshdr-$(CONFIG_X86_64) += syscalls_64.h syshdr-$(CONFIG_XEN) += xen-hypercalls.h -targets += $(uapisyshdr-y) $(syshdr-y) +uapisyshdr-y := $(addprefix $(uapi)/, $(uapisyshdr-y)) +syshdr-y := $(addprefix $(out)/, $(syshdr-y)) +targets += $(addprefix ../../../../, $(uapisyshdr-y) $(syshdr-y)) PHONY += all -all: $(addprefix $(uapi)/,$(uapisyshdr-y)) -all: $(addprefix $(out)/,$(syshdr-y)) +all: $(uapisyshdr-y) $(syshdr-y) @: diff --git a/arch/x86/entry/syscalls/syscall_32.tbl b/arch/x86/entry/syscalls/syscall_32.tbl index 874aeacde2dd157cd1d8f41e34f74598a4ef3f5a..a1c9f496fca6a2bfb44e761b2c5a18199b57d1c7 100644 --- a/arch/x86/entry/syscalls/syscall_32.tbl +++ b/arch/x86/entry/syscalls/syscall_32.tbl @@ -446,3 +446,4 @@ 439 i386 faccessat2 sys_faccessat2 440 i386 process_madvise sys_process_madvise 441 i386 epoll_pwait2 sys_epoll_pwait2 compat_sys_epoll_pwait2 +442 i386 mount_setattr sys_mount_setattr diff --git a/arch/x86/entry/syscalls/syscall_64.tbl b/arch/x86/entry/syscalls/syscall_64.tbl index 78672124d28be0da465300ee9a301bab59c1ac1d..7bf01cbe582f03bc26adb4092bb9023384ec217f 100644 --- a/arch/x86/entry/syscalls/syscall_64.tbl +++ b/arch/x86/entry/syscalls/syscall_64.tbl @@ -363,6 +363,7 @@ 439 common faccessat2 sys_faccessat2 440 common process_madvise sys_process_madvise 441 common epoll_pwait2 sys_epoll_pwait2 +442 common mount_setattr sys_mount_setattr # # Due to a historical design error, certain syscalls are numbered differently diff --git a/arch/x86/entry/vdso/Makefile b/arch/x86/entry/vdso/Makefile index 02e3e42f380bd442cd82ef50caa10c41fffa028d..05c4abc2fdfdeeed590c3c475e085558d96c4104 100644 --- a/arch/x86/entry/vdso/Makefile +++ b/arch/x86/entry/vdso/Makefile @@ -91,7 +91,7 @@ ifneq ($(RETPOLINE_VDSO_CFLAGS),) endif endif -$(vobjs): KBUILD_CFLAGS := $(filter-out $(GCC_PLUGINS_CFLAGS) $(RETPOLINE_CFLAGS),$(KBUILD_CFLAGS)) $(CFL) +$(vobjs): KBUILD_CFLAGS := $(filter-out $(CC_FLAGS_LTO) $(GCC_PLUGINS_CFLAGS) $(RETPOLINE_CFLAGS),$(KBUILD_CFLAGS)) $(CFL) # # vDSO code runs in userspace and -pg doesn't help with profiling anyway. @@ -150,6 +150,7 @@ KBUILD_CFLAGS_32 := $(filter-out -fno-pic,$(KBUILD_CFLAGS_32)) KBUILD_CFLAGS_32 := $(filter-out -mfentry,$(KBUILD_CFLAGS_32)) KBUILD_CFLAGS_32 := $(filter-out $(GCC_PLUGINS_CFLAGS),$(KBUILD_CFLAGS_32)) KBUILD_CFLAGS_32 := $(filter-out $(RETPOLINE_CFLAGS),$(KBUILD_CFLAGS_32)) +KBUILD_CFLAGS_32 := $(filter-out $(CC_FLAGS_LTO),$(KBUILD_CFLAGS_32)) KBUILD_CFLAGS_32 += -m32 -msoft-float -mregparm=0 -fpic KBUILD_CFLAGS_32 += -fno-stack-protector KBUILD_CFLAGS_32 += $(call cc-option, -foptimize-sibling-calls) diff --git a/arch/x86/include/asm/acrn.h b/arch/x86/include/asm/acrn.h new file mode 100644 index 0000000000000000000000000000000000000000..e003a01b7c6721f6dfcddd148e4be6847b3d67ce --- /dev/null +++ b/arch/x86/include/asm/acrn.h @@ -0,0 +1,78 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_X86_ACRN_H +#define _ASM_X86_ACRN_H + +/* + * This CPUID returns feature bitmaps in EAX. + * Guest VM uses this to detect the appropriate feature bit. + */ +#define ACRN_CPUID_FEATURES 0x40000001 +/* Bit 0 indicates whether guest VM is privileged */ +#define ACRN_FEATURE_PRIVILEGED_VM BIT(0) + +void acrn_setup_intr_handler(void (*handler)(void)); +void acrn_remove_intr_handler(void); + +static inline u32 acrn_cpuid_base(void) +{ + if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) + return hypervisor_cpuid_base("ACRNACRNACRN", 0); + + return 0; +} + +/* + * Hypercalls for ACRN + * + * - VMCALL instruction is used to implement ACRN hypercalls. + * - ACRN hypercall ABI: + * - Hypercall number is passed in R8 register. + * - Up to 2 arguments are passed in RDI, RSI. + * - Return value will be placed in RAX. + * + * Because GCC doesn't support R8 register as direct register constraints, use + * supported constraint as input with a explicit MOV to R8 in beginning of asm. + */ +static inline long acrn_hypercall0(unsigned long hcall_id) +{ + long result; + + asm volatile("movl %1, %%r8d\n\t" + "vmcall\n\t" + : "=a" (result) + : "g" (hcall_id) + : "r8", "memory"); + + return result; +} + +static inline long acrn_hypercall1(unsigned long hcall_id, + unsigned long param1) +{ + long result; + + asm volatile("movl %1, %%r8d\n\t" + "vmcall\n\t" + : "=a" (result) + : "g" (hcall_id), "D" (param1) + : "r8", "memory"); + + return result; +} + +static inline long acrn_hypercall2(unsigned long hcall_id, + unsigned long param1, + unsigned long param2) +{ + long result; + + asm volatile("movl %1, %%r8d\n\t" + "vmcall\n\t" + : "=a" (result) + : "g" (hcall_id), "D" (param1), "S" (param2) + : "r8", "memory"); + + return result; +} + +#endif /* _ASM_X86_ACRN_H */ diff --git a/arch/x86/include/asm/apb_timer.h b/arch/x86/include/asm/apb_timer.h deleted file mode 100644 index 87ce8e963215797de269ab39f3c01ac4c112e910..0000000000000000000000000000000000000000 --- a/arch/x86/include/asm/apb_timer.h +++ /dev/null @@ -1,40 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * apb_timer.h: Driver for Langwell APB timer based on Synopsis DesignWare - * - * (C) Copyright 2009 Intel Corporation - * Author: Jacob Pan (jacob.jun.pan@intel.com) - * - * Note: - */ - -#ifndef ASM_X86_APBT_H -#define ASM_X86_APBT_H -#include - -#ifdef CONFIG_APB_TIMER - -/* default memory mapped register base */ -#define LNW_SCU_ADDR 0xFF100000 -#define LNW_EXT_TIMER_OFFSET 0x1B800 -#define APBT_DEFAULT_BASE (LNW_SCU_ADDR+LNW_EXT_TIMER_OFFSET) -#define LNW_EXT_TIMER_PGOFFSET 0x800 - -/* APBT clock speed range from PCLK to fabric base, 25-100MHz */ -#define APBT_MAX_FREQ 50000000 -#define APBT_MIN_FREQ 1000000 -#define APBT_MMAP_SIZE 1024 - -extern void apbt_time_init(void); -extern void apbt_setup_secondary_clock(void); - -extern struct sfi_timer_table_entry *sfi_get_mtmr(int hint); -extern void sfi_free_mtmr(struct sfi_timer_table_entry *mtmr); -extern int sfi_mtimer_num; - -#else /* CONFIG_APB_TIMER */ - -static inline void apbt_time_init(void) { } - -#endif -#endif /* ASM_X86_APBT_H */ diff --git a/arch/x86/include/asm/crypto/glue_helper.h b/arch/x86/include/asm/crypto/glue_helper.h deleted file mode 100644 index 777c0f63418c89dee90d4391a902537232fd0947..0000000000000000000000000000000000000000 --- a/arch/x86/include/asm/crypto/glue_helper.h +++ /dev/null @@ -1,118 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * Shared glue code for 128bit block ciphers - */ - -#ifndef _CRYPTO_GLUE_HELPER_H -#define _CRYPTO_GLUE_HELPER_H - -#include -#include -#include -#include - -typedef void (*common_glue_func_t)(const void *ctx, u8 *dst, const u8 *src); -typedef void (*common_glue_cbc_func_t)(const void *ctx, u8 *dst, const u8 *src); -typedef void (*common_glue_ctr_func_t)(const void *ctx, u8 *dst, const u8 *src, - le128 *iv); -typedef void (*common_glue_xts_func_t)(const void *ctx, u8 *dst, const u8 *src, - le128 *iv); - -struct common_glue_func_entry { - unsigned int num_blocks; /* number of blocks that @fn will process */ - union { - common_glue_func_t ecb; - common_glue_cbc_func_t cbc; - common_glue_ctr_func_t ctr; - common_glue_xts_func_t xts; - } fn_u; -}; - -struct common_glue_ctx { - unsigned int num_funcs; - int fpu_blocks_limit; /* -1 means fpu not needed at all */ - - /* - * First funcs entry must have largest num_blocks and last funcs entry - * must have num_blocks == 1! - */ - struct common_glue_func_entry funcs[]; -}; - -static inline bool glue_fpu_begin(unsigned int bsize, int fpu_blocks_limit, - struct skcipher_walk *walk, - bool fpu_enabled, unsigned int nbytes) -{ - if (likely(fpu_blocks_limit < 0)) - return false; - - if (fpu_enabled) - return true; - - /* - * Vector-registers are only used when chunk to be processed is large - * enough, so do not enable FPU until it is necessary. - */ - if (nbytes < bsize * (unsigned int)fpu_blocks_limit) - return false; - - /* prevent sleeping if FPU is in use */ - skcipher_walk_atomise(walk); - - kernel_fpu_begin(); - return true; -} - -static inline void glue_fpu_end(bool fpu_enabled) -{ - if (fpu_enabled) - kernel_fpu_end(); -} - -static inline void le128_to_be128(be128 *dst, const le128 *src) -{ - dst->a = cpu_to_be64(le64_to_cpu(src->a)); - dst->b = cpu_to_be64(le64_to_cpu(src->b)); -} - -static inline void be128_to_le128(le128 *dst, const be128 *src) -{ - dst->a = cpu_to_le64(be64_to_cpu(src->a)); - dst->b = cpu_to_le64(be64_to_cpu(src->b)); -} - -static inline void le128_inc(le128 *i) -{ - u64 a = le64_to_cpu(i->a); - u64 b = le64_to_cpu(i->b); - - b++; - if (!b) - a++; - - i->a = cpu_to_le64(a); - i->b = cpu_to_le64(b); -} - -extern int glue_ecb_req_128bit(const struct common_glue_ctx *gctx, - struct skcipher_request *req); - -extern int glue_cbc_encrypt_req_128bit(const common_glue_func_t fn, - struct skcipher_request *req); - -extern int glue_cbc_decrypt_req_128bit(const struct common_glue_ctx *gctx, - struct skcipher_request *req); - -extern int glue_ctr_req_128bit(const struct common_glue_ctx *gctx, - struct skcipher_request *req); - -extern int glue_xts_req_128bit(const struct common_glue_ctx *gctx, - struct skcipher_request *req, - common_glue_func_t tweak_fn, void *tweak_ctx, - void *crypt_ctx, bool decrypt); - -extern void glue_xts_crypt_128bit_one(const void *ctx, u8 *dst, - const u8 *src, le128 *iv, - common_glue_func_t fn); - -#endif /* _CRYPTO_GLUE_HELPER_H */ diff --git a/arch/x86/include/asm/crypto/serpent-avx.h b/arch/x86/include/asm/crypto/serpent-avx.h deleted file mode 100644 index 251c2c89d7cfe9ffceee74e50c6f492a5b6da064..0000000000000000000000000000000000000000 --- a/arch/x86/include/asm/crypto/serpent-avx.h +++ /dev/null @@ -1,42 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef ASM_X86_SERPENT_AVX_H -#define ASM_X86_SERPENT_AVX_H - -#include -#include -#include - -struct crypto_skcipher; - -#define SERPENT_PARALLEL_BLOCKS 8 - -struct serpent_xts_ctx { - struct serpent_ctx tweak_ctx; - struct serpent_ctx crypt_ctx; -}; - -asmlinkage void serpent_ecb_enc_8way_avx(const void *ctx, u8 *dst, - const u8 *src); -asmlinkage void serpent_ecb_dec_8way_avx(const void *ctx, u8 *dst, - const u8 *src); - -asmlinkage void serpent_cbc_dec_8way_avx(const void *ctx, u8 *dst, - const u8 *src); -asmlinkage void serpent_ctr_8way_avx(const void *ctx, u8 *dst, const u8 *src, - le128 *iv); - -asmlinkage void serpent_xts_enc_8way_avx(const void *ctx, u8 *dst, - const u8 *src, le128 *iv); -asmlinkage void serpent_xts_dec_8way_avx(const void *ctx, u8 *dst, - const u8 *src, le128 *iv); - -extern void __serpent_crypt_ctr(const void *ctx, u8 *dst, const u8 *src, - le128 *iv); - -extern void serpent_xts_enc(const void *ctx, u8 *dst, const u8 *src, le128 *iv); -extern void serpent_xts_dec(const void *ctx, u8 *dst, const u8 *src, le128 *iv); - -extern int xts_serpent_setkey(struct crypto_skcipher *tfm, const u8 *key, - unsigned int keylen); - -#endif diff --git a/arch/x86/include/asm/fixmap.h b/arch/x86/include/asm/fixmap.h index 9f1a0a987e5e5bf69ccad03aa26910bccc892e0c..d0dcefb5cc59d35a68b449427e79f8ca29fa40f4 100644 --- a/arch/x86/include/asm/fixmap.h +++ b/arch/x86/include/asm/fixmap.h @@ -108,9 +108,6 @@ enum fixed_addresses { #ifdef CONFIG_PARAVIRT_XXL FIX_PARAVIRT_BOOTMAP, #endif -#ifdef CONFIG_X86_INTEL_MID - FIX_LNW_VRTC, -#endif #ifdef CONFIG_ACPI_APEI_GHES /* Used for GHES mapping from assorted contexts */ diff --git a/arch/x86/include/asm/idtentry.h b/arch/x86/include/asm/idtentry.h index 41e2e2e1b439986f672b754490c33011e05efe9a..5eb3bdf36a41984811cd4da8c3f1c9370912b2c6 100644 --- a/arch/x86/include/asm/idtentry.h +++ b/arch/x86/include/asm/idtentry.h @@ -187,23 +187,22 @@ __visible noinstr void func(struct pt_regs *regs, unsigned long error_code) * has to be done in the function body if necessary. */ #define DEFINE_IDTENTRY_IRQ(func) \ -static __always_inline void __##func(struct pt_regs *regs, u8 vector); \ +static void __##func(struct pt_regs *regs, u32 vector); \ \ __visible noinstr void func(struct pt_regs *regs, \ unsigned long error_code) \ { \ irqentry_state_t state = irqentry_enter(regs); \ + u32 vector = (u32)(u8)error_code; \ \ instrumentation_begin(); \ - irq_enter_rcu(); \ kvm_set_cpu_l1tf_flush_l1d(); \ - __##func (regs, (u8)error_code); \ - irq_exit_rcu(); \ + run_irq_on_irqstack_cond(__##func, regs, vector); \ instrumentation_end(); \ irqentry_exit(regs, state); \ } \ \ -static __always_inline void __##func(struct pt_regs *regs, u8 vector) +static noinline void __##func(struct pt_regs *regs, u32 vector) /** * DECLARE_IDTENTRY_SYSVEC - Declare functions for system vector entry points @@ -237,10 +236,8 @@ __visible noinstr void func(struct pt_regs *regs) \ irqentry_state_t state = irqentry_enter(regs); \ \ instrumentation_begin(); \ - irq_enter_rcu(); \ kvm_set_cpu_l1tf_flush_l1d(); \ run_sysvec_on_irqstack_cond(__##func, regs); \ - irq_exit_rcu(); \ instrumentation_end(); \ irqentry_exit(regs, state); \ } \ diff --git a/arch/x86/include/asm/insn.h b/arch/x86/include/asm/insn.h index a8c3d284fa46c4ad3f2dd4b8116d7ee6008d7f81..95a448fbb44c9fae9ce7b5b2888d6ecc7058f926 100644 --- a/arch/x86/include/asm/insn.h +++ b/arch/x86/include/asm/insn.h @@ -7,9 +7,12 @@ * Copyright (C) IBM Corporation, 2009 */ +#include /* insn_attr_t is defined in inat.h */ #include +#if defined(__BYTE_ORDER) ? __BYTE_ORDER == __LITTLE_ENDIAN : defined(__LITTLE_ENDIAN) + struct insn_field { union { insn_value_t value; @@ -20,6 +23,48 @@ struct insn_field { unsigned char nbytes; }; +static inline void insn_field_set(struct insn_field *p, insn_value_t v, + unsigned char n) +{ + p->value = v; + p->nbytes = n; +} + +static inline void insn_set_byte(struct insn_field *p, unsigned char n, + insn_byte_t v) +{ + p->bytes[n] = v; +} + +#else + +struct insn_field { + insn_value_t value; + union { + insn_value_t little; + insn_byte_t bytes[4]; + }; + /* !0 if we've run insn_get_xxx() for this field */ + unsigned char got; + unsigned char nbytes; +}; + +static inline void insn_field_set(struct insn_field *p, insn_value_t v, + unsigned char n) +{ + p->value = v; + p->little = __cpu_to_le32(v); + p->nbytes = n; +} + +static inline void insn_set_byte(struct insn_field *p, unsigned char n, + insn_byte_t v) +{ + p->bytes[n] = v; + p->value = __le32_to_cpu(p->little); +} +#endif + struct insn { struct insn_field prefixes; /* * Prefixes diff --git a/arch/x86/include/asm/intel-mid.h b/arch/x86/include/asm/intel-mid.h index cf0e25f45422bb0104d77beabb528a57a363131f..c201083b34f699db5840ee470c3837ee4add0e6e 100644 --- a/arch/x86/include/asm/intel-mid.h +++ b/arch/x86/include/asm/intel-mid.h @@ -1,15 +1,13 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* - * intel-mid.h: Intel MID specific setup code + * Intel MID specific setup code * - * (C) Copyright 2009 Intel Corporation + * (C) Copyright 2009, 2021 Intel Corporation */ #ifndef _ASM_X86_INTEL_MID_H #define _ASM_X86_INTEL_MID_H -#include #include -#include extern int intel_mid_pci_init(void); extern int intel_mid_pci_set_power_state(struct pci_dev *pdev, pci_power_t state); @@ -22,93 +20,18 @@ extern void intel_mid_pwr_power_off(void); extern int intel_mid_pwr_get_lss_id(struct pci_dev *pdev); -extern int get_gpio_by_name(const char *name); -extern int __init sfi_parse_mrtc(struct sfi_table_header *table); -extern int __init sfi_parse_mtmr(struct sfi_table_header *table); -extern int sfi_mrtc_num; -extern struct sfi_rtc_table_entry sfi_mrtc_array[]; - -/* - * Here defines the array of devices platform data that IAFW would export - * through SFI "DEVS" table, we use name and type to match the device and - * its platform data. - */ -struct devs_id { - char name[SFI_NAME_LEN + 1]; - u8 type; - u8 delay; - u8 msic; - void *(*get_platform_data)(void *info); -}; - -#define sfi_device(i) \ - static const struct devs_id *const __intel_mid_sfi_##i##_dev __used \ - __section(".x86_intel_mid_dev.init") = &i - -/** -* struct mid_sd_board_info - template for SD device creation -* @name: identifies the driver -* @bus_num: board-specific identifier for a given SD controller -* @max_clk: the maximum frequency device supports -* @platform_data: the particular data stored there is driver-specific -*/ -struct mid_sd_board_info { - char name[SFI_NAME_LEN]; - int bus_num; - unsigned short addr; - u32 max_clk; - void *platform_data; -}; - -/* - * Medfield is the follow-up of Moorestown, it combines two chip solution into - * one. Other than that it also added always-on and constant tsc and lapic - * timers. Medfield is the platform name, and the chip name is called Penwell - * we treat Medfield/Penwell as a variant of Moorestown. Penwell can be - * identified via MSRs. - */ -enum intel_mid_cpu_type { - /* 1 was Moorestown */ - INTEL_MID_CPU_CHIP_PENWELL = 2, - INTEL_MID_CPU_CHIP_CLOVERVIEW, - INTEL_MID_CPU_CHIP_TANGIER, -}; - -extern enum intel_mid_cpu_type __intel_mid_cpu_chip; - #ifdef CONFIG_X86_INTEL_MID -static inline enum intel_mid_cpu_type intel_mid_identify_cpu(void) -{ - return __intel_mid_cpu_chip; -} - -static inline bool intel_mid_has_msic(void) -{ - return (intel_mid_identify_cpu() == INTEL_MID_CPU_CHIP_PENWELL); -} - extern void intel_scu_devices_create(void); extern void intel_scu_devices_destroy(void); #else /* !CONFIG_X86_INTEL_MID */ -#define intel_mid_identify_cpu() 0 -#define intel_mid_has_msic() 0 - static inline void intel_scu_devices_create(void) { } static inline void intel_scu_devices_destroy(void) { } #endif /* !CONFIG_X86_INTEL_MID */ -enum intel_mid_timer_options { - INTEL_MID_TIMER_DEFAULT, - INTEL_MID_TIMER_APBT_ONLY, - INTEL_MID_TIMER_LAPIC_APBT, -}; - -extern enum intel_mid_timer_options intel_mid_timer_options; - /* Bus Select SoC Fuse value */ #define BSEL_SOC_FUSE_MASK 0x7 /* FSB 133MHz */ @@ -118,16 +41,4 @@ extern enum intel_mid_timer_options intel_mid_timer_options; /* FSB 83MHz */ #define BSEL_SOC_FUSE_111 0x7 -#define SFI_MTMR_MAX_NUM 8 -#define SFI_MRTC_MAX 8 - -/* VRTC timer */ -#define MRST_VRTC_MAP_SZ 1024 -/* #define MRST_VRTC_PGOFFSET 0xc00 */ - -extern void intel_mid_rtc_init(void); - -/* The offset for the mapping of global gpio pin to irq */ -#define INTEL_MID_IRQ_OFFSET 0x100 - #endif /* _ASM_X86_INTEL_MID_H */ diff --git a/arch/x86/include/asm/intel_mid_vrtc.h b/arch/x86/include/asm/intel_mid_vrtc.h deleted file mode 100644 index 0b44b1abe4d9cb931ddcf543f25507920741625b..0000000000000000000000000000000000000000 --- a/arch/x86/include/asm/intel_mid_vrtc.h +++ /dev/null @@ -1,10 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _INTEL_MID_VRTC_H -#define _INTEL_MID_VRTC_H - -extern unsigned char vrtc_cmos_read(unsigned char reg); -extern void vrtc_cmos_write(unsigned char val, unsigned char reg); -extern void vrtc_get_time(struct timespec64 *now); -extern int vrtc_set_mmss(const struct timespec64 *now); - -#endif diff --git a/arch/x86/include/asm/intel_scu_ipc.h b/arch/x86/include/asm/intel_scu_ipc.h index 11d457af68c5420bb0bf84149ac1fdd7e4f91611..8537f597d20ac5e52c40c2b2aa5d3e05f40af42e 100644 --- a/arch/x86/include/asm/intel_scu_ipc.h +++ b/arch/x86/include/asm/intel_scu_ipc.h @@ -65,6 +65,4 @@ static inline int intel_scu_ipc_dev_command(struct intel_scu_ipc_dev *scu, int c inlen, out, outlen); } -#include - #endif diff --git a/arch/x86/include/asm/intel_scu_ipc_legacy.h b/arch/x86/include/asm/intel_scu_ipc_legacy.h deleted file mode 100644 index 4cf13fecb673effb7641fb1cbedbf78b151cd884..0000000000000000000000000000000000000000 --- a/arch/x86/include/asm/intel_scu_ipc_legacy.h +++ /dev/null @@ -1,91 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _ASM_X86_INTEL_SCU_IPC_LEGACY_H_ -#define _ASM_X86_INTEL_SCU_IPC_LEGACY_H_ - -#include - -#define IPCMSG_INDIRECT_READ 0x02 -#define IPCMSG_INDIRECT_WRITE 0x05 - -#define IPCMSG_COLD_OFF 0x80 /* Only for Tangier */ - -#define IPCMSG_WARM_RESET 0xF0 -#define IPCMSG_COLD_RESET 0xF1 -#define IPCMSG_SOFT_RESET 0xF2 -#define IPCMSG_COLD_BOOT 0xF3 - -#define IPCMSG_VRTC 0xFA /* Set vRTC device */ -/* Command id associated with message IPCMSG_VRTC */ -#define IPC_CMD_VRTC_SETTIME 1 /* Set time */ -#define IPC_CMD_VRTC_SETALARM 2 /* Set alarm */ - -/* Don't call these in new code - they will be removed eventually */ - -/* Read single register */ -static inline int intel_scu_ipc_ioread8(u16 addr, u8 *data) -{ - return intel_scu_ipc_dev_ioread8(NULL, addr, data); -} - -/* Read a vector */ -static inline int intel_scu_ipc_readv(u16 *addr, u8 *data, int len) -{ - return intel_scu_ipc_dev_readv(NULL, addr, data, len); -} - -/* Write single register */ -static inline int intel_scu_ipc_iowrite8(u16 addr, u8 data) -{ - return intel_scu_ipc_dev_iowrite8(NULL, addr, data); -} - -/* Write a vector */ -static inline int intel_scu_ipc_writev(u16 *addr, u8 *data, int len) -{ - return intel_scu_ipc_dev_writev(NULL, addr, data, len); -} - -/* Update single register based on the mask */ -static inline int intel_scu_ipc_update_register(u16 addr, u8 data, u8 mask) -{ - return intel_scu_ipc_dev_update(NULL, addr, data, mask); -} - -/* Issue commands to the SCU with or without data */ -static inline int intel_scu_ipc_simple_command(int cmd, int sub) -{ - return intel_scu_ipc_dev_simple_command(NULL, cmd, sub); -} - -static inline int intel_scu_ipc_command(int cmd, int sub, u32 *in, int inlen, - u32 *out, int outlen) -{ - /* New API takes both inlen and outlen as bytes so convert here */ - size_t inbytes = inlen * sizeof(u32); - size_t outbytes = outlen * sizeof(u32); - - return intel_scu_ipc_dev_command_with_size(NULL, cmd, sub, in, inbytes, - inlen, out, outbytes); -} - -extern struct blocking_notifier_head intel_scu_notifier; - -static inline void intel_scu_notifier_add(struct notifier_block *nb) -{ - blocking_notifier_chain_register(&intel_scu_notifier, nb); -} - -static inline void intel_scu_notifier_remove(struct notifier_block *nb) -{ - blocking_notifier_chain_unregister(&intel_scu_notifier, nb); -} - -static inline int intel_scu_notifier_post(unsigned long v, void *p) -{ - return blocking_notifier_call_chain(&intel_scu_notifier, v, p); -} - -#define SCU_AVAILABLE 1 -#define SCU_DOWN 2 - -#endif diff --git a/arch/x86/include/asm/irq.h b/arch/x86/include/asm/irq.h index 76d389691b5b6240dd3d28e466b41cf18d6358ca..768aa234cbb4a6c2fb3c6362156825e11ee046d1 100644 --- a/arch/x86/include/asm/irq.h +++ b/arch/x86/include/asm/irq.h @@ -25,8 +25,6 @@ static inline int irq_canonicalize(int irq) extern int irq_init_percpu_irqstack(unsigned int cpu); -#define __ARCH_HAS_DO_SOFTIRQ - struct irq_desc; extern void fixup_irqs(void); diff --git a/arch/x86/include/asm/irq_stack.h b/arch/x86/include/asm/irq_stack.h index 775816965c6ae098f249dcf83c61d131da0ced9a..9b2a0ff76c73e215ca788300835f74462ded371e 100644 --- a/arch/x86/include/asm/irq_stack.h +++ b/arch/x86/include/asm/irq_stack.h @@ -7,100 +7,217 @@ #include #ifdef CONFIG_X86_64 -static __always_inline bool irqstack_active(void) -{ - return __this_cpu_read(irq_count) != -1; -} - -void asm_call_on_stack(void *sp, void (*func)(void), void *arg); -void asm_call_sysvec_on_stack(void *sp, void (*func)(struct pt_regs *regs), - struct pt_regs *regs); -void asm_call_irq_on_stack(void *sp, void (*func)(struct irq_desc *desc), - struct irq_desc *desc); -static __always_inline void __run_on_irqstack(void (*func)(void)) -{ - void *tos = __this_cpu_read(hardirq_stack_ptr); - - __this_cpu_add(irq_count, 1); - asm_call_on_stack(tos - 8, func, NULL); - __this_cpu_sub(irq_count, 1); +/* + * Macro to inline switching to an interrupt stack and invoking function + * calls from there. The following rules apply: + * + * - Ordering: + * + * 1. Write the stack pointer into the top most place of the irq + * stack. This ensures that the various unwinders can link back to the + * original stack. + * + * 2. Switch the stack pointer to the top of the irq stack. + * + * 3. Invoke whatever needs to be done (@asm_call argument) + * + * 4. Pop the original stack pointer from the top of the irq stack + * which brings it back to the original stack where it left off. + * + * - Function invocation: + * + * To allow flexible usage of the macro, the actual function code including + * the store of the arguments in the call ABI registers is handed in via + * the @asm_call argument. + * + * - Local variables: + * + * @tos: + * The @tos variable holds a pointer to the top of the irq stack and + * _must_ be allocated in a non-callee saved register as this is a + * restriction coming from objtool. + * + * Note, that (tos) is both in input and output constraints to ensure + * that the compiler does not assume that R11 is left untouched in + * case this macro is used in some place where the per cpu interrupt + * stack pointer is used again afterwards + * + * - Function arguments: + * The function argument(s), if any, have to be defined in register + * variables at the place where this is invoked. Storing the + * argument(s) in the proper register(s) is part of the @asm_call + * + * - Constraints: + * + * The constraints have to be done very carefully because the compiler + * does not know about the assembly call. + * + * output: + * As documented already above the @tos variable is required to be in + * the output constraints to make the compiler aware that R11 cannot be + * reused after the asm() statement. + * + * For builds with CONFIG_UNWIND_FRAME_POINTER ASM_CALL_CONSTRAINT is + * required as well as this prevents certain creative GCC variants from + * misplacing the ASM code. + * + * input: + * - func: + * Immediate, which tells the compiler that the function is referenced. + * + * - tos: + * Register. The actual register is defined by the variable declaration. + * + * - function arguments: + * The constraints are handed in via the 'argconstr' argument list. They + * describe the register arguments which are used in @asm_call. + * + * clobbers: + * Function calls can clobber anything except the callee-saved + * registers. Tell the compiler. + */ +#define call_on_irqstack(func, asm_call, argconstr...) \ +{ \ + register void *tos asm("r11"); \ + \ + tos = ((void *)__this_cpu_read(hardirq_stack_ptr)); \ + \ + asm_inline volatile( \ + "movq %%rsp, (%[tos]) \n" \ + "movq %[tos], %%rsp \n" \ + \ + asm_call \ + \ + "popq %%rsp \n" \ + \ + : "+r" (tos), ASM_CALL_CONSTRAINT \ + : [__func] "i" (func), [tos] "r" (tos) argconstr \ + : "cc", "rax", "rcx", "rdx", "rsi", "rdi", "r8", "r9", "r10", \ + "memory" \ + ); \ } -static __always_inline void -__run_sysvec_on_irqstack(void (*func)(struct pt_regs *regs), - struct pt_regs *regs) -{ - void *tos = __this_cpu_read(hardirq_stack_ptr); - - __this_cpu_add(irq_count, 1); - asm_call_sysvec_on_stack(tos - 8, func, regs); - __this_cpu_sub(irq_count, 1); +/* Macros to assert type correctness for run_*_on_irqstack macros */ +#define assert_function_type(func, proto) \ + static_assert(__builtin_types_compatible_p(typeof(&func), proto)) + +#define assert_arg_type(arg, proto) \ + static_assert(__builtin_types_compatible_p(typeof(arg), proto)) + +/* + * Macro to invoke system vector and device interrupt C handlers. + */ +#define call_on_irqstack_cond(func, regs, asm_call, constr, c_args...) \ +{ \ + /* \ + * User mode entry and interrupt on the irq stack do not \ + * switch stacks. If from user mode the task stack is empty. \ + */ \ + if (user_mode(regs) || __this_cpu_read(hardirq_stack_inuse)) { \ + irq_enter_rcu(); \ + func(c_args); \ + irq_exit_rcu(); \ + } else { \ + /* \ + * Mark the irq stack inuse _before_ and unmark _after_ \ + * switching stacks. Interrupts are disabled in both \ + * places. Invoke the stack switch macro with the call \ + * sequence which matches the above direct invocation. \ + */ \ + __this_cpu_write(hardirq_stack_inuse, true); \ + call_on_irqstack(func, asm_call, constr); \ + __this_cpu_write(hardirq_stack_inuse, false); \ + } \ } -static __always_inline void -__run_irq_on_irqstack(void (*func)(struct irq_desc *desc), - struct irq_desc *desc) -{ - void *tos = __this_cpu_read(hardirq_stack_ptr); - - __this_cpu_add(irq_count, 1); - asm_call_irq_on_stack(tos - 8, func, desc); - __this_cpu_sub(irq_count, 1); +/* + * Function call sequence for __call_on_irqstack() for system vectors. + * + * Note that irq_enter_rcu() and irq_exit_rcu() do not use the input + * mechanism because these functions are global and cannot be optimized out + * when compiling a particular source file which uses one of these macros. + * + * The argument (regs) does not need to be pushed or stashed in a callee + * saved register to be safe vs. the irq_enter_rcu() call because the + * clobbers already prevent the compiler from storing it in a callee + * clobbered register. As the compiler has to preserve @regs for the final + * call to idtentry_exit() anyway, it's likely that it does not cause extra + * effort for this asm magic. + */ +#define ASM_CALL_SYSVEC \ + "call irq_enter_rcu \n" \ + "movq %[arg1], %%rdi \n" \ + "call %P[__func] \n" \ + "call irq_exit_rcu \n" + +#define SYSVEC_CONSTRAINTS , [arg1] "r" (regs) + +#define run_sysvec_on_irqstack_cond(func, regs) \ +{ \ + assert_function_type(func, void (*)(struct pt_regs *)); \ + assert_arg_type(regs, struct pt_regs *); \ + \ + call_on_irqstack_cond(func, regs, ASM_CALL_SYSVEC, \ + SYSVEC_CONSTRAINTS, regs); \ } -#else /* CONFIG_X86_64 */ -static inline bool irqstack_active(void) { return false; } -static inline void __run_on_irqstack(void (*func)(void)) { } -static inline void __run_sysvec_on_irqstack(void (*func)(struct pt_regs *regs), - struct pt_regs *regs) { } -static inline void __run_irq_on_irqstack(void (*func)(struct irq_desc *desc), - struct irq_desc *desc) { } -#endif /* !CONFIG_X86_64 */ - -static __always_inline bool irq_needs_irq_stack(struct pt_regs *regs) -{ - if (IS_ENABLED(CONFIG_X86_32)) - return false; - if (!regs) - return !irqstack_active(); - return !user_mode(regs) && !irqstack_active(); +/* + * As in ASM_CALL_SYSVEC above the clobbers force the compiler to store + * @regs and @vector in callee saved registers. + */ +#define ASM_CALL_IRQ \ + "call irq_enter_rcu \n" \ + "movq %[arg1], %%rdi \n" \ + "movl %[arg2], %%esi \n" \ + "call %P[__func] \n" \ + "call irq_exit_rcu \n" + +#define IRQ_CONSTRAINTS , [arg1] "r" (regs), [arg2] "r" (vector) + +#define run_irq_on_irqstack_cond(func, regs, vector) \ +{ \ + assert_function_type(func, void (*)(struct pt_regs *, u32)); \ + assert_arg_type(regs, struct pt_regs *); \ + assert_arg_type(vector, u32); \ + \ + call_on_irqstack_cond(func, regs, ASM_CALL_IRQ, \ + IRQ_CONSTRAINTS, regs, vector); \ } - -static __always_inline void run_on_irqstack_cond(void (*func)(void), - struct pt_regs *regs) -{ - lockdep_assert_irqs_disabled(); - - if (irq_needs_irq_stack(regs)) - __run_on_irqstack(func); - else - func(); +#define ASM_CALL_SOFTIRQ \ + "call %P[__func] \n" + +/* + * Macro to invoke __do_softirq on the irq stack. This is only called from + * task context when bottom halfs are about to be reenabled and soft + * interrupts are pending to be processed. The interrupt stack cannot be in + * use here. + */ +#define do_softirq_own_stack() \ +{ \ + __this_cpu_write(hardirq_stack_inuse, true); \ + call_on_irqstack(__do_softirq, ASM_CALL_SOFTIRQ); \ + __this_cpu_write(hardirq_stack_inuse, false); \ } -static __always_inline void -run_sysvec_on_irqstack_cond(void (*func)(struct pt_regs *regs), - struct pt_regs *regs) -{ - lockdep_assert_irqs_disabled(); - - if (irq_needs_irq_stack(regs)) - __run_sysvec_on_irqstack(func, regs); - else - func(regs); +#else /* CONFIG_X86_64 */ +/* System vector handlers always run on the stack they interrupted. */ +#define run_sysvec_on_irqstack_cond(func, regs) \ +{ \ + irq_enter_rcu(); \ + func(regs); \ + irq_exit_rcu(); \ } -static __always_inline void -run_irq_on_irqstack_cond(void (*func)(struct irq_desc *desc), struct irq_desc *desc, - struct pt_regs *regs) -{ - lockdep_assert_irqs_disabled(); - - if (irq_needs_irq_stack(regs)) - __run_irq_on_irqstack(func, desc); - else - func(desc); +/* Switches to the irq stack within func() */ +#define run_irq_on_irqstack_cond(func, regs, vector) \ +{ \ + irq_enter_rcu(); \ + func(regs, vector); \ + irq_exit_rcu(); \ } +#endif /* !CONFIG_X86_64 */ + #endif diff --git a/arch/x86/include/asm/kfence.h b/arch/x86/include/asm/kfence.h new file mode 100644 index 0000000000000000000000000000000000000000..97bbb4a9083a76cce625d5255d2e25ce86aeb2a5 --- /dev/null +++ b/arch/x86/include/asm/kfence.h @@ -0,0 +1,64 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * x86 KFENCE support. + * + * Copyright (C) 2020, Google LLC. + */ + +#ifndef _ASM_X86_KFENCE_H +#define _ASM_X86_KFENCE_H + +#include +#include + +#include +#include +#include +#include + +/* Force 4K pages for __kfence_pool. */ +static inline bool arch_kfence_init_pool(void) +{ + unsigned long addr; + + for (addr = (unsigned long)__kfence_pool; is_kfence_address((void *)addr); + addr += PAGE_SIZE) { + unsigned int level; + + if (!lookup_address(addr, &level)) + return false; + + if (level != PG_LEVEL_4K) + set_memory_4k(addr, 1); + } + + return true; +} + +/* Protect the given page and flush TLB. */ +static inline bool kfence_protect_page(unsigned long addr, bool protect) +{ + unsigned int level; + pte_t *pte = lookup_address(addr, &level); + + if (WARN_ON(!pte || level != PG_LEVEL_4K)) + return false; + + /* + * We need to avoid IPIs, as we may get KFENCE allocations or faults + * with interrupts disabled. Therefore, the below is best-effort, and + * does not flush TLBs on all CPUs. We can tolerate some inaccuracy; + * lazy fault handling takes care of faults after the page is PRESENT. + */ + + if (protect) + set_pte(pte, __pte(pte_val(*pte) & ~_PAGE_PRESENT)); + else + set_pte(pte, __pte(pte_val(*pte) | _PAGE_PRESENT)); + + /* Flush this CPU's TLB. */ + flush_tlb_one_kernel(addr); + return true; +} + +#endif /* _ASM_X86_KFENCE_H */ diff --git a/arch/x86/include/asm/kvm-x86-ops.h b/arch/x86/include/asm/kvm-x86-ops.h index 355a2ab8fc090ed6da86d3dd3ae32d4d8242bbb0..323641097f63aa697bcc6aefb621252cce61849c 100644 --- a/arch/x86/include/asm/kvm-x86-ops.h +++ b/arch/x86/include/asm/kvm-x86-ops.h @@ -93,11 +93,7 @@ KVM_X86_OP(check_intercept) KVM_X86_OP(handle_exit_irqoff) KVM_X86_OP_NULL(request_immediate_exit) KVM_X86_OP(sched_in) -KVM_X86_OP_NULL(slot_enable_log_dirty) -KVM_X86_OP_NULL(slot_disable_log_dirty) -KVM_X86_OP_NULL(flush_log_dirty) -KVM_X86_OP_NULL(enable_log_dirty_pt_masked) -KVM_X86_OP_NULL(cpu_dirty_log_size) +KVM_X86_OP_NULL(update_cpu_dirty_logging) KVM_X86_OP_NULL(pre_block) KVM_X86_OP_NULL(post_block) KVM_X86_OP_NULL(vcpu_blocking) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 84499aad01a468820a961568ee58682d6726938b..0cf71ff2b2e5f6763c15742af685a83f82c1ebab 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -89,6 +89,8 @@ KVM_ARCH_REQ_FLAGS(27, KVM_REQUEST_NO_WAKEUP) #define KVM_REQ_APF_READY KVM_ARCH_REQ(28) #define KVM_REQ_MSR_FILTER_CHANGED KVM_ARCH_REQ(29) +#define KVM_REQ_UPDATE_CPU_DIRTY_LOGGING \ + KVM_ARCH_REQ_FLAGS(30, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) #define CR0_RESERVED_BITS \ (~(unsigned long)(X86_CR0_PE | X86_CR0_MP | X86_CR0_EM | X86_CR0_TS \ @@ -1007,6 +1009,7 @@ struct kvm_arch { u32 bsp_vcpu_id; u64 disabled_quirks; + int cpu_dirty_logging_count; enum kvm_irqchip_mode irqchip_mode; u8 nr_reserved_ioapic_pins; @@ -1271,30 +1274,11 @@ struct kvm_x86_ops { void (*sched_in)(struct kvm_vcpu *kvm, int cpu); /* - * Arch-specific dirty logging hooks. These hooks are only supposed to - * be valid if the specific arch has hardware-accelerated dirty logging - * mechanism. Currently only for PML on VMX. - * - * - slot_enable_log_dirty: - * called when enabling log dirty mode for the slot. - * - slot_disable_log_dirty: - * called when disabling log dirty mode for the slot. - * also called when slot is created with log dirty disabled. - * - flush_log_dirty: - * called before reporting dirty_bitmap to userspace. - * - enable_log_dirty_pt_masked: - * called when reenabling log dirty for the GFNs in the mask after - * corresponding bits are cleared in slot->dirty_bitmap. + * Size of the CPU's dirty log buffer, i.e. VMX's PML buffer. A zero + * value indicates CPU dirty logging is unsupported or disabled. */ - void (*slot_enable_log_dirty)(struct kvm *kvm, - struct kvm_memory_slot *slot); - void (*slot_disable_log_dirty)(struct kvm *kvm, - struct kvm_memory_slot *slot); - void (*flush_log_dirty)(struct kvm *kvm); - void (*enable_log_dirty_pt_masked)(struct kvm *kvm, - struct kvm_memory_slot *slot, - gfn_t offset, unsigned long mask); - int (*cpu_dirty_log_size)(void); + int cpu_dirty_log_size; + void (*update_cpu_dirty_logging)(struct kvm_vcpu *vcpu); /* pmu operations of sub-arch */ const struct kvm_pmu_ops *pmu_ops; @@ -1437,11 +1421,6 @@ void kvm_mmu_slot_leaf_clear_dirty(struct kvm *kvm, struct kvm_memory_slot *memslot); void kvm_mmu_slot_largepage_remove_write_access(struct kvm *kvm, struct kvm_memory_slot *memslot); -void kvm_mmu_slot_set_dirty(struct kvm *kvm, - struct kvm_memory_slot *memslot); -void kvm_mmu_clear_dirty_pt_masked(struct kvm *kvm, - struct kvm_memory_slot *slot, - gfn_t gfn_offset, unsigned long mask); void kvm_mmu_zap_all(struct kvm *kvm); void kvm_mmu_invalidate_mmio_sptes(struct kvm *kvm, u64 gen); unsigned long kvm_mmu_calculate_default_mmu_pages(struct kvm *kvm); @@ -1613,7 +1592,6 @@ void kvm_inject_nmi(struct kvm_vcpu *vcpu); void kvm_update_dr7(struct kvm_vcpu *vcpu); int kvm_mmu_unprotect_page(struct kvm *kvm, gfn_t gfn); -int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva); void __kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu); int kvm_mmu_load(struct kvm_vcpu *vcpu); void kvm_mmu_unload(struct kvm_vcpu *vcpu); diff --git a/arch/x86/include/asm/orc_types.h b/arch/x86/include/asm/orc_types.h index fdbffec4cfdeaba82e7fe6c1c76b169316cbb308..5a2baf28a1dcdaf546cd213e4d0eef389a43ad50 100644 --- a/arch/x86/include/asm/orc_types.h +++ b/arch/x86/include/asm/orc_types.h @@ -40,6 +40,8 @@ #define ORC_REG_MAX 15 #ifndef __ASSEMBLY__ +#include + /* * This struct is more or less a vastly simplified version of the DWARF Call * Frame Information standard. It contains only the necessary parts of DWARF @@ -51,10 +53,18 @@ struct orc_entry { s16 sp_offset; s16 bp_offset; +#if defined(__LITTLE_ENDIAN_BITFIELD) unsigned sp_reg:4; unsigned bp_reg:4; unsigned type:2; unsigned end:1; +#elif defined(__BIG_ENDIAN_BITFIELD) + unsigned bp_reg:4; + unsigned sp_reg:4; + unsigned unused:5; + unsigned end:1; + unsigned type:2; +#endif } __packed; #endif /* __ASSEMBLY__ */ diff --git a/arch/x86/include/asm/platform_sst_audio.h b/arch/x86/include/asm/platform_sst_audio.h index 16b9f220bdebd3b898c353c1cf57ea6178ae3c44..40f92270515bc40de049b99e69916803d8c309a4 100644 --- a/arch/x86/include/asm/platform_sst_audio.h +++ b/arch/x86/include/asm/platform_sst_audio.h @@ -10,8 +10,6 @@ #ifndef _PLATFORM_SST_AUDIO_H_ #define _PLATFORM_SST_AUDIO_H_ -#include - #define MAX_NUM_STREAMS_MRFLD 25 #define MAX_NUM_STREAMS MAX_NUM_STREAMS_MRFLD diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h index c20a52b5534b4ba949e583f220b3301d0a131dd8..dc6d149bf851fa911c7c7915fbe68e39f1016b4c 100644 --- a/arch/x86/include/asm/processor.h +++ b/arch/x86/include/asm/processor.h @@ -426,8 +426,6 @@ struct irq_stack { char stack[IRQ_STACK_SIZE]; } __aligned(IRQ_STACK_SIZE); -DECLARE_PER_CPU(struct irq_stack *, hardirq_stack_ptr); - #ifdef CONFIG_X86_32 DECLARE_PER_CPU(unsigned long, cpu_current_top_of_stack); #else @@ -454,7 +452,8 @@ static inline unsigned long cpu_kernelmode_gs_base(int cpu) return (unsigned long)per_cpu(fixed_percpu_data.gs_base, cpu); } -DECLARE_PER_CPU(unsigned int, irq_count); +DECLARE_PER_CPU(void *, hardirq_stack_ptr); +DECLARE_PER_CPU(bool, hardirq_stack_inuse); extern asmlinkage void ignore_sysret(void); /* Save actual FS/GS selectors and bases to current->thread */ @@ -473,9 +472,9 @@ struct stack_canary { }; DECLARE_PER_CPU_ALIGNED(struct stack_canary, stack_canary); #endif -/* Per CPU softirq stack pointer */ +DECLARE_PER_CPU(struct irq_stack *, hardirq_stack_ptr); DECLARE_PER_CPU(struct irq_stack *, softirq_stack_ptr); -#endif /* X86_64 */ +#endif /* !X86_64 */ extern unsigned int fpu_kernel_xstate_size; extern unsigned int fpu_user_xstate_size; diff --git a/arch/x86/include/asm/softirq_stack.h b/arch/x86/include/asm/softirq_stack.h new file mode 100644 index 0000000000000000000000000000000000000000..889d53d6a0e1215a377d3f5371f5b0e6edd6265a --- /dev/null +++ b/arch/x86/include/asm/softirq_stack.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_X86_SOFTIRQ_STACK_H +#define _ASM_X86_SOFTIRQ_STACK_H + +#ifdef CONFIG_X86_64 +# include +#else +# include +#endif + +#endif diff --git a/arch/x86/include/asm/unwind_hints.h b/arch/x86/include/asm/unwind_hints.h index 664d4610d700e35f4b5cd81823333656e6260380..8e574c0afef80dfc49303eae60b5704a8dc0977f 100644 --- a/arch/x86/include/asm/unwind_hints.h +++ b/arch/x86/include/asm/unwind_hints.h @@ -48,17 +48,8 @@ UNWIND_HINT_REGS base=\base offset=\offset partial=1 .endm -.macro UNWIND_HINT_FUNC sp_offset=8 - UNWIND_HINT sp_reg=ORC_REG_SP sp_offset=\sp_offset type=UNWIND_HINT_TYPE_CALL -.endm - -/* - * RET_OFFSET: Used on instructions that terminate a function; mostly RETURN - * and sibling calls. On these, sp_offset denotes the expected offset from - * initial_func_cfi. - */ -.macro UNWIND_HINT_RET_OFFSET sp_offset=8 - UNWIND_HINT sp_reg=ORC_REG_SP type=UNWIND_HINT_TYPE_RET_OFFSET sp_offset=\sp_offset +.macro UNWIND_HINT_FUNC + UNWIND_HINT sp_reg=ORC_REG_SP sp_offset=8 type=UNWIND_HINT_TYPE_FUNC .endm #endif /* __ASSEMBLY__ */ diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile index 5eeb808eb02424c7ee2aa9c9b2a23f7b49225c79..2ddf08351f0bca0b5f3911a1855e7fa91d748ef0 100644 --- a/arch/x86/kernel/Makefile +++ b/arch/x86/kernel/Makefile @@ -116,7 +116,6 @@ obj-$(CONFIG_VM86) += vm86_32.o obj-$(CONFIG_EARLY_PRINTK) += early_printk.o obj-$(CONFIG_HPET_TIMER) += hpet.o -obj-$(CONFIG_APB_TIMER) += apb_timer.o obj-$(CONFIG_AMD_NB) += amd_nb.o obj-$(CONFIG_DEBUG_NMI_SELFTEST) += nmi_selftest.o diff --git a/arch/x86/kernel/acpi/Makefile b/arch/x86/kernel/acpi/Makefile index f1bb57b0e41ea780754e4fb495222a413a2e9bf8..cf340d85946a89d28c28f4599c2c8922b804b1f9 100644 --- a/arch/x86/kernel/acpi/Makefile +++ b/arch/x86/kernel/acpi/Makefile @@ -1,5 +1,4 @@ # SPDX-License-Identifier: GPL-2.0 -OBJECT_FILES_NON_STANDARD_wakeup_$(BITS).o := y obj-$(CONFIG_ACPI) += boot.o obj-$(CONFIG_ACPI_SLEEP) += sleep.o wakeup_$(BITS).o diff --git a/arch/x86/kernel/acpi/wakeup_64.S b/arch/x86/kernel/acpi/wakeup_64.S index 5d3a0b8fd37981563530a2c8c2b86f6957693c9a..56b6865afb2ac68cadf7798209c875576cf6f90b 100644 --- a/arch/x86/kernel/acpi/wakeup_64.S +++ b/arch/x86/kernel/acpi/wakeup_64.S @@ -1,12 +1,14 @@ /* SPDX-License-Identifier: GPL-2.0-only */ .text #include +#include #include #include #include #include #include #include +#include # Copyright 2003 Pavel Machek -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -#define APBT_CLOCKEVENT_RATING 110 -#define APBT_CLOCKSOURCE_RATING 250 - -#define APBT_CLOCKEVENT0_NUM (0) -#define APBT_CLOCKSOURCE_NUM (2) - -static phys_addr_t apbt_address; -static int apb_timer_block_enabled; -static void __iomem *apbt_virt_address; - -/* - * Common DW APB timer info - */ -static unsigned long apbt_freq; - -struct apbt_dev { - struct dw_apb_clock_event_device *timer; - unsigned int num; - int cpu; - unsigned int irq; - char name[10]; -}; - -static struct dw_apb_clocksource *clocksource_apbt; - -static inline void __iomem *adev_virt_addr(struct apbt_dev *adev) -{ - return apbt_virt_address + adev->num * APBTMRS_REG_SIZE; -} - -static DEFINE_PER_CPU(struct apbt_dev, cpu_apbt_dev); - -#ifdef CONFIG_SMP -static unsigned int apbt_num_timers_used; -#endif - -static inline void apbt_set_mapping(void) -{ - struct sfi_timer_table_entry *mtmr; - int phy_cs_timer_id = 0; - - if (apbt_virt_address) { - pr_debug("APBT base already mapped\n"); - return; - } - mtmr = sfi_get_mtmr(APBT_CLOCKEVENT0_NUM); - if (mtmr == NULL) { - printk(KERN_ERR "Failed to get MTMR %d from SFI\n", - APBT_CLOCKEVENT0_NUM); - return; - } - apbt_address = (phys_addr_t)mtmr->phys_addr; - if (!apbt_address) { - printk(KERN_WARNING "No timer base from SFI, use default\n"); - apbt_address = APBT_DEFAULT_BASE; - } - apbt_virt_address = ioremap(apbt_address, APBT_MMAP_SIZE); - if (!apbt_virt_address) { - pr_debug("Failed mapping APBT phy address at %lu\n",\ - (unsigned long)apbt_address); - goto panic_noapbt; - } - apbt_freq = mtmr->freq_hz; - sfi_free_mtmr(mtmr); - - /* Now figure out the physical timer id for clocksource device */ - mtmr = sfi_get_mtmr(APBT_CLOCKSOURCE_NUM); - if (mtmr == NULL) - goto panic_noapbt; - - /* Now figure out the physical timer id */ - pr_debug("Use timer %d for clocksource\n", - (int)(mtmr->phys_addr & 0xff) / APBTMRS_REG_SIZE); - phy_cs_timer_id = (unsigned int)(mtmr->phys_addr & 0xff) / - APBTMRS_REG_SIZE; - - clocksource_apbt = dw_apb_clocksource_init(APBT_CLOCKSOURCE_RATING, - "apbt0", apbt_virt_address + phy_cs_timer_id * - APBTMRS_REG_SIZE, apbt_freq); - return; - -panic_noapbt: - panic("Failed to setup APB system timer\n"); - -} - -static inline void apbt_clear_mapping(void) -{ - iounmap(apbt_virt_address); - apbt_virt_address = NULL; -} - -static int __init apbt_clockevent_register(void) -{ - struct sfi_timer_table_entry *mtmr; - struct apbt_dev *adev = this_cpu_ptr(&cpu_apbt_dev); - - mtmr = sfi_get_mtmr(APBT_CLOCKEVENT0_NUM); - if (mtmr == NULL) { - printk(KERN_ERR "Failed to get MTMR %d from SFI\n", - APBT_CLOCKEVENT0_NUM); - return -ENODEV; - } - - adev->num = smp_processor_id(); - adev->timer = dw_apb_clockevent_init(smp_processor_id(), "apbt0", - intel_mid_timer_options == INTEL_MID_TIMER_LAPIC_APBT ? - APBT_CLOCKEVENT_RATING - 100 : APBT_CLOCKEVENT_RATING, - adev_virt_addr(adev), 0, apbt_freq); - /* Firmware does EOI handling for us. */ - adev->timer->eoi = NULL; - - if (intel_mid_timer_options == INTEL_MID_TIMER_LAPIC_APBT) { - global_clock_event = &adev->timer->ced; - printk(KERN_DEBUG "%s clockevent registered as global\n", - global_clock_event->name); - } - - dw_apb_clockevent_register(adev->timer); - - sfi_free_mtmr(mtmr); - return 0; -} - -#ifdef CONFIG_SMP - -static void apbt_setup_irq(struct apbt_dev *adev) -{ - irq_modify_status(adev->irq, 0, IRQ_MOVE_PCNTXT); - irq_set_affinity(adev->irq, cpumask_of(adev->cpu)); -} - -/* Should be called with per cpu */ -void apbt_setup_secondary_clock(void) -{ - struct apbt_dev *adev; - int cpu; - - /* Don't register boot CPU clockevent */ - cpu = smp_processor_id(); - if (!cpu) - return; - - adev = this_cpu_ptr(&cpu_apbt_dev); - if (!adev->timer) { - adev->timer = dw_apb_clockevent_init(cpu, adev->name, - APBT_CLOCKEVENT_RATING, adev_virt_addr(adev), - adev->irq, apbt_freq); - adev->timer->eoi = NULL; - } else { - dw_apb_clockevent_resume(adev->timer); - } - - printk(KERN_INFO "Registering CPU %d clockevent device %s, cpu %08x\n", - cpu, adev->name, adev->cpu); - - apbt_setup_irq(adev); - dw_apb_clockevent_register(adev->timer); - - return; -} - -/* - * this notify handler process CPU hotplug events. in case of S0i3, nonboot - * cpus are disabled/enabled frequently, for performance reasons, we keep the - * per cpu timer irq registered so that we do need to do free_irq/request_irq. - * - * TODO: it might be more reliable to directly disable percpu clockevent device - * without the notifier chain. currently, cpu 0 may get interrupts from other - * cpu timers during the offline process due to the ordering of notification. - * the extra interrupt is harmless. - */ -static int apbt_cpu_dead(unsigned int cpu) -{ - struct apbt_dev *adev = &per_cpu(cpu_apbt_dev, cpu); - - dw_apb_clockevent_pause(adev->timer); - if (system_state == SYSTEM_RUNNING) { - pr_debug("skipping APBT CPU %u offline\n", cpu); - } else { - pr_debug("APBT clockevent for cpu %u offline\n", cpu); - dw_apb_clockevent_stop(adev->timer); - } - return 0; -} - -static __init int apbt_late_init(void) -{ - if (intel_mid_timer_options == INTEL_MID_TIMER_LAPIC_APBT || - !apb_timer_block_enabled) - return 0; - return cpuhp_setup_state(CPUHP_X86_APB_DEAD, "x86/apb:dead", NULL, - apbt_cpu_dead); -} -fs_initcall(apbt_late_init); -#else - -void apbt_setup_secondary_clock(void) {} - -#endif /* CONFIG_SMP */ - -static int apbt_clocksource_register(void) -{ - u64 start, now; - u64 t1; - - /* Start the counter, use timer 2 as source, timer 0/1 for event */ - dw_apb_clocksource_start(clocksource_apbt); - - /* Verify whether apbt counter works */ - t1 = dw_apb_clocksource_read(clocksource_apbt); - start = rdtsc(); - - /* - * We don't know the TSC frequency yet, but waiting for - * 200000 TSC cycles is safe: - * 4 GHz == 50us - * 1 GHz == 200us - */ - do { - rep_nop(); - now = rdtsc(); - } while ((now - start) < 200000UL); - - /* APBT is the only always on clocksource, it has to work! */ - if (t1 == dw_apb_clocksource_read(clocksource_apbt)) - panic("APBT counter not counting. APBT disabled\n"); - - dw_apb_clocksource_register(clocksource_apbt); - - return 0; -} - -/* - * Early setup the APBT timer, only use timer 0 for booting then switch to - * per CPU timer if possible. - * returns 1 if per cpu apbt is setup - * returns 0 if no per cpu apbt is chosen - * panic if set up failed, this is the only platform timer on Moorestown. - */ -void __init apbt_time_init(void) -{ -#ifdef CONFIG_SMP - int i; - struct sfi_timer_table_entry *p_mtmr; - struct apbt_dev *adev; -#endif - - if (apb_timer_block_enabled) - return; - apbt_set_mapping(); - if (!apbt_virt_address) - goto out_noapbt; - /* - * Read the frequency and check for a sane value, for ESL model - * we extend the possible clock range to allow time scaling. - */ - - if (apbt_freq < APBT_MIN_FREQ || apbt_freq > APBT_MAX_FREQ) { - pr_debug("APBT has invalid freq 0x%lx\n", apbt_freq); - goto out_noapbt; - } - if (apbt_clocksource_register()) { - pr_debug("APBT has failed to register clocksource\n"); - goto out_noapbt; - } - if (!apbt_clockevent_register()) - apb_timer_block_enabled = 1; - else { - pr_debug("APBT has failed to register clockevent\n"); - goto out_noapbt; - } -#ifdef CONFIG_SMP - /* kernel cmdline disable apb timer, so we will use lapic timers */ - if (intel_mid_timer_options == INTEL_MID_TIMER_LAPIC_APBT) { - printk(KERN_INFO "apbt: disabled per cpu timer\n"); - return; - } - pr_debug("%s: %d CPUs online\n", __func__, num_online_cpus()); - if (num_possible_cpus() <= sfi_mtimer_num) - apbt_num_timers_used = num_possible_cpus(); - else - apbt_num_timers_used = 1; - pr_debug("%s: %d APB timers used\n", __func__, apbt_num_timers_used); - - /* here we set up per CPU timer data structure */ - for (i = 0; i < apbt_num_timers_used; i++) { - adev = &per_cpu(cpu_apbt_dev, i); - adev->num = i; - adev->cpu = i; - p_mtmr = sfi_get_mtmr(i); - if (p_mtmr) - adev->irq = p_mtmr->irq; - else - printk(KERN_ERR "Failed to get timer for cpu %d\n", i); - snprintf(adev->name, sizeof(adev->name) - 1, "apbt%d", i); - } -#endif - - return; - -out_noapbt: - apbt_clear_mapping(); - apb_timer_block_enabled = 0; - panic("failed to enable APB timer\n"); -} diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c index 819db00c9388d4746d1bf2ec89dd37002d722c0d..bda4f2a36868da2bbf8e7788ff2dadae94f3f312 100644 --- a/arch/x86/kernel/apic/apic.c +++ b/arch/x86/kernel/apic/apic.c @@ -2138,18 +2138,11 @@ void __init register_lapic_address(unsigned long address) * Local APIC interrupts */ -/** - * spurious_interrupt - Catch all for interrupts raised on unused vectors - * @regs: Pointer to pt_regs on stack - * @vector: The vector number - * - * This is invoked from ASM entry code to catch all interrupts which - * trigger on an entry which is routed to the common_spurious idtentry - * point. - * - * Also called from sysvec_spurious_apic_interrupt(). +/* + * Common handling code for spurious_interrupt and spurious_vector entry + * points below. No point in allowing the compiler to inline it twice. */ -DEFINE_IDTENTRY_IRQ(spurious_interrupt) +static noinline void handle_spurious_interrupt(u8 vector) { u32 v; @@ -2184,9 +2177,23 @@ DEFINE_IDTENTRY_IRQ(spurious_interrupt) trace_spurious_apic_exit(vector); } +/** + * spurious_interrupt - Catch all for interrupts raised on unused vectors + * @regs: Pointer to pt_regs on stack + * @vector: The vector number + * + * This is invoked from ASM entry code to catch all interrupts which + * trigger on an entry which is routed to the common_spurious idtentry + * point. + */ +DEFINE_IDTENTRY_IRQ(spurious_interrupt) +{ + handle_spurious_interrupt(vector); +} + DEFINE_IDTENTRY_SYSVEC(sysvec_spurious_apic_interrupt) { - __spurious_interrupt(regs, SPURIOUS_APIC_VECTOR); + handle_spurious_interrupt(SPURIOUS_APIC_VECTOR); } /* diff --git a/arch/x86/kernel/apic/io_apic.c b/arch/x86/kernel/apic/io_apic.c index e4ab4804b20df43cd1e1cb3e00ff6f61216a47cc..c3b60c37c728066e2713e49769c35277b6657075 100644 --- a/arch/x86/kernel/apic/io_apic.c +++ b/arch/x86/kernel/apic/io_apic.c @@ -198,7 +198,7 @@ static int __init parse_noapic(char *str) } early_param("noapic", parse_noapic); -/* Will be called in mpparse/acpi/sfi codes for saving IRQ info */ +/* Will be called in mpparse/ACPI codes for saving IRQ info */ void mp_save_irq(struct mpc_intsrc *m) { int i; @@ -2863,7 +2863,7 @@ int mp_register_ioapic(int id, u32 address, u32 gsi_base, /* * If mp_register_ioapic() is called during early boot stage when - * walking ACPI/SFI/DT tables, it's too early to create irqdomain, + * walking ACPI/DT tables, it's too early to create irqdomain, * we are still using bootmem allocator. So delay it to setup_IO_APIC(). */ if (hotplug) { diff --git a/arch/x86/kernel/cpu/acrn.c b/arch/x86/kernel/cpu/acrn.c index 0b2c03943ac6b31fea3d742e5db5b379f70018c7..23f5f27b5a024865054b19b43c766c3f6afa8bfa 100644 --- a/arch/x86/kernel/cpu/acrn.c +++ b/arch/x86/kernel/cpu/acrn.c @@ -10,6 +10,8 @@ */ #include + +#include #include #include #include @@ -19,7 +21,7 @@ static u32 __init acrn_detect(void) { - return hypervisor_cpuid_base("ACRNACRNACRN", 0); + return acrn_cpuid_base(); } static void __init acrn_init_platform(void) @@ -55,6 +57,18 @@ DEFINE_IDTENTRY_SYSVEC(sysvec_acrn_hv_callback) set_irq_regs(old_regs); } +void acrn_setup_intr_handler(void (*handler)(void)) +{ + acrn_intr_handler = handler; +} +EXPORT_SYMBOL_GPL(acrn_setup_intr_handler); + +void acrn_remove_intr_handler(void) +{ + acrn_intr_handler = NULL; +} +EXPORT_SYMBOL_GPL(acrn_remove_intr_handler); + const __initconst struct hypervisor_x86 x86_hyper_acrn = { .name = "ACRN", .detect = acrn_detect, diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index 9215b91bc044bc4671c722fe0a10f4d6fa62d68a..ab640abe26b686a57e89f95dafa6b7a158db1fd9 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -1742,8 +1742,8 @@ DEFINE_PER_CPU(struct task_struct *, current_task) ____cacheline_aligned = &init_task; EXPORT_PER_CPU_SYMBOL(current_task); -DEFINE_PER_CPU(struct irq_stack *, hardirq_stack_ptr); -DEFINE_PER_CPU(unsigned int, irq_count) __visible = -1; +DEFINE_PER_CPU(void *, hardirq_stack_ptr); +DEFINE_PER_CPU(bool, hardirq_stack_inuse); DEFINE_PER_CPU(int, __preempt_count) = INIT_PREEMPT_COUNT; EXPORT_PER_CPU_SYMBOL(__preempt_count); diff --git a/arch/x86/kernel/dumpstack_64.c b/arch/x86/kernel/dumpstack_64.c index 1dd851397bd909a915447d75975d2ab6db52413b..5601b95944faedef320f6fb980df21f4fbe58cd6 100644 --- a/arch/x86/kernel/dumpstack_64.c +++ b/arch/x86/kernel/dumpstack_64.c @@ -128,12 +128,21 @@ static __always_inline bool in_exception_stack(unsigned long *stack, struct stac static __always_inline bool in_irq_stack(unsigned long *stack, struct stack_info *info) { - unsigned long *end = (unsigned long *)this_cpu_read(hardirq_stack_ptr); - unsigned long *begin = end - (IRQ_STACK_SIZE / sizeof(long)); + unsigned long *end = (unsigned long *)this_cpu_read(hardirq_stack_ptr); + unsigned long *begin; /* - * This is a software stack, so 'end' can be a valid stack pointer. - * It just means the stack is empty. + * @end points directly to the top most stack entry to avoid a -8 + * adjustment in the stack switch hotpath. Adjust it back before + * calculating @begin. + */ + end++; + begin = end - (IRQ_STACK_SIZE / sizeof(long)); + + /* + * Due to the switching logic RSP can never be == @end because the + * final operation is 'popq %rsp' which means after that RSP points + * to the original stack and not to @end. */ if (stack < begin || stack >= end) return false; @@ -143,8 +152,9 @@ static __always_inline bool in_irq_stack(unsigned long *stack, struct stack_info info->end = end; /* - * The next stack pointer is the first thing pushed by the entry code - * after switching to the irq stack. + * The next stack pointer is stored at the top of the irq stack + * before switching to the irq stack. Actual stack entries are all + * below that. */ info->next_sp = (unsigned long *)*(end - 1); diff --git a/arch/x86/kernel/ftrace_64.S b/arch/x86/kernel/ftrace_64.S index 0d54099c2a3a3156b4a18520091eb59966ef9cfe..7c273846c68774471f7deefe9630b30bc7ef6f7d 100644 --- a/arch/x86/kernel/ftrace_64.S +++ b/arch/x86/kernel/ftrace_64.S @@ -184,6 +184,7 @@ SYM_INNER_LABEL(ftrace_graph_call, SYM_L_GLOBAL) * It is also used to copy the retq for trampolines. */ SYM_INNER_LABEL_ALIGN(ftrace_stub, SYM_L_WEAK) + UNWIND_HINT_FUNC retq SYM_FUNC_END(ftrace_epilogue) @@ -276,7 +277,7 @@ SYM_INNER_LABEL(ftrace_regs_caller_end, SYM_L_GLOBAL) restore_mcount_regs 8 /* Restore flags */ popfq - UNWIND_HINT_RET_OFFSET + UNWIND_HINT_FUNC jmp ftrace_epilogue SYM_FUNC_END(ftrace_regs_caller) @@ -333,8 +334,7 @@ SYM_FUNC_START(ftrace_graph_caller) retq SYM_FUNC_END(ftrace_graph_caller) -SYM_CODE_START(return_to_handler) - UNWIND_HINT_EMPTY +SYM_FUNC_START(return_to_handler) subq $24, %rsp /* Save the return values */ @@ -349,5 +349,5 @@ SYM_CODE_START(return_to_handler) movq (%rsp), %rax addq $24, %rsp JMP_NOSPEC rdi -SYM_CODE_END(return_to_handler) +SYM_FUNC_END(return_to_handler) #endif diff --git a/arch/x86/kernel/irq.c b/arch/x86/kernel/irq.c index d4ad344e80bf1101ccd3da52c5067e90d92d77ba..58aa712973ac80608652378785a3255c9a88dfb7 100644 --- a/arch/x86/kernel/irq.c +++ b/arch/x86/kernel/irq.c @@ -228,7 +228,7 @@ static __always_inline void handle_irq(struct irq_desc *desc, struct pt_regs *regs) { if (IS_ENABLED(CONFIG_X86_64)) - run_irq_on_irqstack_cond(desc->handle_irq, desc, regs); + generic_handle_irq_desc(desc); else __handle_irq(desc, regs); } diff --git a/arch/x86/kernel/irq_32.c b/arch/x86/kernel/irq_32.c index 0b79efc87be527a08b91ffb8e6849ec9743d7a95..044902d5a3c4afc37c0a422a9a980b5dbf1f4bc9 100644 --- a/arch/x86/kernel/irq_32.c +++ b/arch/x86/kernel/irq_32.c @@ -22,6 +22,7 @@ #include #include +#include #ifdef CONFIG_DEBUG_STACKOVERFLOW diff --git a/arch/x86/kernel/irq_64.c b/arch/x86/kernel/irq_64.c index 440eed558558d9891547ee1bd09a4570bc398156..1c0fb96b9e39069d4d246f32f5e5c782e8e7be38 100644 --- a/arch/x86/kernel/irq_64.c +++ b/arch/x86/kernel/irq_64.c @@ -20,6 +20,7 @@ #include #include +#include #include #include #include @@ -48,7 +49,8 @@ static int map_irq_stack(unsigned int cpu) if (!va) return -ENOMEM; - per_cpu(hardirq_stack_ptr, cpu) = va + IRQ_STACK_SIZE; + /* Store actual TOS to avoid adjustment in the hotpath */ + per_cpu(hardirq_stack_ptr, cpu) = va + IRQ_STACK_SIZE - 8; return 0; } #else @@ -60,7 +62,8 @@ static int map_irq_stack(unsigned int cpu) { void *va = per_cpu_ptr(&irq_stack_backing_store, cpu); - per_cpu(hardirq_stack_ptr, cpu) = va + IRQ_STACK_SIZE; + /* Store actual TOS to avoid adjustment in the hotpath */ + per_cpu(hardirq_stack_ptr, cpu) = va + IRQ_STACK_SIZE - 8; return 0; } #endif @@ -71,8 +74,3 @@ int irq_init_percpu_irqstack(unsigned int cpu) return 0; return map_irq_stack(cpu); } - -void do_softirq_own_stack(void) -{ - run_on_irqstack_cond(__do_softirq, NULL); -} diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c index 145a7ac0c19aa1f9a6e72d47d65a257029ec434a..9c214d7085a489a2c18c093619e6ef6b1210b01d 100644 --- a/arch/x86/kernel/process.c +++ b/arch/x86/kernel/process.c @@ -161,7 +161,7 @@ int copy_thread(unsigned long clone_flags, unsigned long sp, unsigned long arg, #endif /* Kernel thread ? */ - if (unlikely(p->flags & PF_KTHREAD)) { + if (unlikely(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { memset(childregs, 0, sizeof(struct pt_regs)); kthread_frame_init(frame, sp, arg); return 0; diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c index ad582f9ac5a6f3f29392d6af5b26d619c85d8a61..d08307df69ad5aea129ee7f78547f6ffde281310 100644 --- a/arch/x86/kernel/process_64.c +++ b/arch/x86/kernel/process_64.c @@ -539,7 +539,7 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p) int cpu = smp_processor_id(); WARN_ON_ONCE(IS_ENABLED(CONFIG_DEBUG_ENTRY) && - this_cpu_read(irq_count) != -1); + this_cpu_read(hardirq_stack_inuse)); if (!test_thread_flag(TIF_NEED_FPU_LOAD)) switch_fpu_prepare(prev_fpu, cpu); diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c index 740f3bdb3f61939d28da1e8bdd47552a6912b298..d883176ef2ce04c1f86465b433d4a3b8ec74549e 100644 --- a/arch/x86/kernel/setup.c +++ b/arch/x86/kernel/setup.c @@ -16,7 +16,6 @@ #include #include #include -#include #include #include #include @@ -1185,7 +1184,6 @@ void __init setup_arch(char **cmdline_p) * Read APIC and some other early information from ACPI tables. */ acpi_boot_init(); - sfi_init(); x86_dtb_init(); /* diff --git a/arch/x86/kernel/unwind_orc.c b/arch/x86/kernel/unwind_orc.c index 73f8001000669e2595fe8a4a9eeb7bf9108e9c62..2a1d47f47eee2b3d1f5034d1828d1faa2ee9e4c0 100644 --- a/arch/x86/kernel/unwind_orc.c +++ b/arch/x86/kernel/unwind_orc.c @@ -471,7 +471,7 @@ bool unwind_next_frame(struct unwind_state *state) break; case ORC_REG_SP_INDIRECT: - sp = state->sp + orc->sp_offset; + sp = state->sp; indirect = true; break; @@ -521,6 +521,9 @@ bool unwind_next_frame(struct unwind_state *state) if (indirect) { if (!deref_stack_reg(state, sp, &sp)) goto err; + + if (orc->sp_reg == ORC_REG_SP_INDIRECT) + sp += orc->sp_offset; } /* Find IP, SP and possibly regs: */ diff --git a/arch/x86/kvm/cpuid.c b/arch/x86/kvm/cpuid.c index c8f2592ccc999780c4c97e7a0fa8c391d7e2466e..6bd2f8b830e49ff115ea9058bb6a2c51ebcdfaf5 100644 --- a/arch/x86/kvm/cpuid.c +++ b/arch/x86/kvm/cpuid.c @@ -408,7 +408,7 @@ void kvm_set_cpu_caps(void) kvm_cpu_cap_mask(CPUID_7_0_EBX, F(FSGSBASE) | F(BMI1) | F(HLE) | F(AVX2) | F(SMEP) | - F(BMI2) | F(ERMS) | 0 /*INVPCID*/ | F(RTM) | 0 /*MPX*/ | F(RDSEED) | + F(BMI2) | F(ERMS) | F(INVPCID) | F(RTM) | 0 /*MPX*/ | F(RDSEED) | F(ADX) | F(SMAP) | F(AVX512IFMA) | F(AVX512F) | F(AVX512PF) | F(AVX512ER) | F(AVX512CD) | F(CLFLUSHOPT) | F(CLWB) | F(AVX512DQ) | F(SHA_NI) | F(AVX512BW) | F(AVX512VL) | 0 /*INTEL_PT*/ diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index e507568cd55d69d9ab50b6ed3f193f26f8e01ce5..d75524bc84234ecea48b0b6606238fead77c6e04 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -1165,7 +1165,8 @@ static bool spte_wrprot_for_clear_dirty(u64 *sptep) * - W bit on ad-disabled SPTEs. * Returns true iff any D or W bits were cleared. */ -static bool __rmap_clear_dirty(struct kvm *kvm, struct kvm_rmap_head *rmap_head) +static bool __rmap_clear_dirty(struct kvm *kvm, struct kvm_rmap_head *rmap_head, + struct kvm_memory_slot *slot) { u64 *sptep; struct rmap_iterator iter; @@ -1180,35 +1181,6 @@ static bool __rmap_clear_dirty(struct kvm *kvm, struct kvm_rmap_head *rmap_head) return flush; } -static bool spte_set_dirty(u64 *sptep) -{ - u64 spte = *sptep; - - rmap_printk("spte %p %llx\n", sptep, *sptep); - - /* - * Similar to the !kvm_x86_ops.slot_disable_log_dirty case, - * do not bother adding back write access to pages marked - * SPTE_AD_WRPROT_ONLY_MASK. - */ - spte |= shadow_dirty_mask; - - return mmu_spte_update(sptep, spte); -} - -static bool __rmap_set_dirty(struct kvm *kvm, struct kvm_rmap_head *rmap_head) -{ - u64 *sptep; - struct rmap_iterator iter; - bool flush = false; - - for_each_rmap_spte(rmap_head, &iter, sptep) - if (spte_ad_enabled(*sptep)) - flush |= spte_set_dirty(sptep); - - return flush; -} - /** * kvm_mmu_write_protect_pt_masked - write protect selected PT level pages * @kvm: kvm instance @@ -1248,9 +1220,9 @@ static void kvm_mmu_write_protect_pt_masked(struct kvm *kvm, * * Used for PML to re-log the dirty GPAs after userspace querying dirty_bitmap. */ -void kvm_mmu_clear_dirty_pt_masked(struct kvm *kvm, - struct kvm_memory_slot *slot, - gfn_t gfn_offset, unsigned long mask) +static void kvm_mmu_clear_dirty_pt_masked(struct kvm *kvm, + struct kvm_memory_slot *slot, + gfn_t gfn_offset, unsigned long mask) { struct kvm_rmap_head *rmap_head; @@ -1260,13 +1232,12 @@ void kvm_mmu_clear_dirty_pt_masked(struct kvm *kvm, while (mask) { rmap_head = __gfn_to_rmap(slot->base_gfn + gfn_offset + __ffs(mask), PG_LEVEL_4K, slot); - __rmap_clear_dirty(kvm, rmap_head); + __rmap_clear_dirty(kvm, rmap_head, slot); /* clear the first set bit */ mask &= mask - 1; } } -EXPORT_SYMBOL_GPL(kvm_mmu_clear_dirty_pt_masked); /** * kvm_arch_mmu_enable_log_dirty_pt_masked - enable dirty logging for selected @@ -1282,20 +1253,15 @@ void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm, struct kvm_memory_slot *slot, gfn_t gfn_offset, unsigned long mask) { - if (kvm_x86_ops.enable_log_dirty_pt_masked) - static_call(kvm_x86_enable_log_dirty_pt_masked)(kvm, slot, - gfn_offset, - mask); + if (kvm_x86_ops.cpu_dirty_log_size) + kvm_mmu_clear_dirty_pt_masked(kvm, slot, gfn_offset, mask); else kvm_mmu_write_protect_pt_masked(kvm, slot, gfn_offset, mask); } int kvm_cpu_dirty_log_size(void) { - if (kvm_x86_ops.cpu_dirty_log_size) - return static_call(kvm_x86_cpu_dirty_log_size)(); - - return 0; + return kvm_x86_ops.cpu_dirty_log_size; } bool kvm_mmu_slot_gfn_write_protect(struct kvm *kvm, @@ -1325,7 +1291,8 @@ static bool rmap_write_protect(struct kvm_vcpu *vcpu, u64 gfn) return kvm_mmu_slot_gfn_write_protect(vcpu->kvm, slot, gfn); } -static bool kvm_zap_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head) +static bool kvm_zap_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head, + struct kvm_memory_slot *slot) { u64 *sptep; struct rmap_iterator iter; @@ -1345,7 +1312,7 @@ static int kvm_unmap_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head, struct kvm_memory_slot *slot, gfn_t gfn, int level, unsigned long data) { - return kvm_zap_rmapp(kvm, rmap_head); + return kvm_zap_rmapp(kvm, rmap_head, slot); } static int kvm_set_pte_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head, @@ -2499,7 +2466,21 @@ int kvm_mmu_unprotect_page(struct kvm *kvm, gfn_t gfn) return r; } -EXPORT_SYMBOL_GPL(kvm_mmu_unprotect_page); + +static int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva) +{ + gpa_t gpa; + int r; + + if (vcpu->arch.mmu->direct_map) + return 0; + + gpa = kvm_mmu_gva_to_gpa_read(vcpu, gva, NULL); + + r = kvm_mmu_unprotect_page(vcpu->kvm, gpa >> PAGE_SHIFT); + + return r; +} static void kvm_unsync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp) { @@ -2753,11 +2734,18 @@ static void direct_pte_prefetch(struct kvm_vcpu *vcpu, u64 *sptep) if (sp->role.level > PG_LEVEL_4K) return; + /* + * If addresses are being invalidated, skip prefetching to avoid + * accidentally prefetching those addresses. + */ + if (unlikely(vcpu->kvm->mmu_notifier_count)) + return; + __direct_pte_prefetch(vcpu, sp, sptep); } -static int host_pfn_mapping_level(struct kvm_vcpu *vcpu, gfn_t gfn, - kvm_pfn_t pfn, struct kvm_memory_slot *slot) +static int host_pfn_mapping_level(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, + struct kvm_memory_slot *slot) { unsigned long hva; pte_t *pte; @@ -2776,19 +2764,36 @@ static int host_pfn_mapping_level(struct kvm_vcpu *vcpu, gfn_t gfn, */ hva = __gfn_to_hva_memslot(slot, gfn); - pte = lookup_address_in_mm(vcpu->kvm->mm, hva, &level); + pte = lookup_address_in_mm(kvm->mm, hva, &level); if (unlikely(!pte)) return PG_LEVEL_4K; return level; } +int kvm_mmu_max_mapping_level(struct kvm *kvm, struct kvm_memory_slot *slot, + gfn_t gfn, kvm_pfn_t pfn, int max_level) +{ + struct kvm_lpage_info *linfo; + + max_level = min(max_level, max_huge_page_level); + for ( ; max_level > PG_LEVEL_4K; max_level--) { + linfo = lpage_info_slot(gfn, slot, max_level); + if (!linfo->disallow_lpage) + break; + } + + if (max_level == PG_LEVEL_4K) + return PG_LEVEL_4K; + + return host_pfn_mapping_level(kvm, gfn, pfn, slot); +} + int kvm_mmu_hugepage_adjust(struct kvm_vcpu *vcpu, gfn_t gfn, int max_level, kvm_pfn_t *pfnp, bool huge_page_disallowed, int *req_level) { struct kvm_memory_slot *slot; - struct kvm_lpage_info *linfo; kvm_pfn_t pfn = *pfnp; kvm_pfn_t mask; int level; @@ -2805,17 +2810,7 @@ int kvm_mmu_hugepage_adjust(struct kvm_vcpu *vcpu, gfn_t gfn, if (!slot) return PG_LEVEL_4K; - max_level = min(max_level, max_huge_page_level); - for ( ; max_level > PG_LEVEL_4K; max_level--) { - linfo = lpage_info_slot(gfn, slot, max_level); - if (!linfo->disallow_lpage) - break; - } - - if (max_level == PG_LEVEL_4K) - return PG_LEVEL_4K; - - level = host_pfn_mapping_level(vcpu, gfn, pfn, slot); + level = kvm_mmu_max_mapping_level(vcpu->kvm, slot, gfn, pfn, max_level); if (level == PG_LEVEL_4K) return level; @@ -3437,7 +3432,6 @@ void kvm_mmu_sync_roots(struct kvm_vcpu *vcpu) kvm_mmu_audit(vcpu, AUDIT_POST_SYNC); write_unlock(&vcpu->kvm->mmu_lock); } -EXPORT_SYMBOL_GPL(kvm_mmu_sync_roots); static gpa_t nonpaging_gva_to_gpa(struct kvm_vcpu *vcpu, gpa_t vaddr, u32 access, struct x86_exception *exception) @@ -3653,8 +3647,8 @@ static bool kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, } static bool try_async_pf(struct kvm_vcpu *vcpu, bool prefault, gfn_t gfn, - gpa_t cr2_or_gpa, kvm_pfn_t *pfn, bool write, - bool *writable) + gpa_t cr2_or_gpa, kvm_pfn_t *pfn, hva_t *hva, + bool write, bool *writable) { struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn); bool async; @@ -3667,7 +3661,8 @@ static bool try_async_pf(struct kvm_vcpu *vcpu, bool prefault, gfn_t gfn, } async = false; - *pfn = __gfn_to_pfn_memslot(slot, gfn, false, &async, write, writable); + *pfn = __gfn_to_pfn_memslot(slot, gfn, false, &async, + write, writable, hva); if (!async) return false; /* *pfn has correct page already */ @@ -3681,7 +3676,8 @@ static bool try_async_pf(struct kvm_vcpu *vcpu, bool prefault, gfn_t gfn, return true; } - *pfn = __gfn_to_pfn_memslot(slot, gfn, false, NULL, write, writable); + *pfn = __gfn_to_pfn_memslot(slot, gfn, false, NULL, + write, writable, hva); return false; } @@ -3694,6 +3690,7 @@ static int direct_page_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u32 error_code, gfn_t gfn = gpa >> PAGE_SHIFT; unsigned long mmu_seq; kvm_pfn_t pfn; + hva_t hva; int r; if (page_fault_handle_page_track(vcpu, error_code, gfn)) @@ -3712,7 +3709,8 @@ static int direct_page_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u32 error_code, mmu_seq = vcpu->kvm->mmu_notifier_seq; smp_rmb(); - if (try_async_pf(vcpu, prefault, gfn, gpa, &pfn, write, &map_writable)) + if (try_async_pf(vcpu, prefault, gfn, gpa, &pfn, &hva, + write, &map_writable)) return RET_PF_RETRY; if (handle_abnormal_pfn(vcpu, is_tdp ? 0 : gpa, gfn, pfn, ACC_ALL, &r)) @@ -3725,7 +3723,7 @@ static int direct_page_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u32 error_code, else write_lock(&vcpu->kvm->mmu_lock); - if (mmu_notifier_retry(vcpu->kvm, mmu_seq)) + if (!is_noslot_pfn(pfn) && mmu_notifier_retry_hva(vcpu->kvm, mmu_seq, hva)) goto out_unlock; r = make_mmu_pages_available(vcpu); if (r) @@ -5003,22 +5001,6 @@ static void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa, write_unlock(&vcpu->kvm->mmu_lock); } -int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva) -{ - gpa_t gpa; - int r; - - if (vcpu->arch.mmu->direct_map) - return 0; - - gpa = kvm_mmu_gva_to_gpa_read(vcpu, gva, NULL); - - r = kvm_mmu_unprotect_page(vcpu->kvm, gpa >> PAGE_SHIFT); - - return r; -} -EXPORT_SYMBOL_GPL(kvm_mmu_unprotect_page_virt); - int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, u64 error_code, void *insn, int insn_len) { @@ -5117,7 +5099,6 @@ void kvm_mmu_invalidate_gva(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, mmu->invlpg(vcpu, gva, root_hpa); } } -EXPORT_SYMBOL_GPL(kvm_mmu_invalidate_gva); void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva) { @@ -5157,7 +5138,6 @@ void kvm_mmu_invpcid_gva(struct kvm_vcpu *vcpu, gva_t gva, unsigned long pcid) * for them. */ } -EXPORT_SYMBOL_GPL(kvm_mmu_invpcid_gva); void kvm_configure_mmu(bool enable_tdp, int tdp_max_root_level, int tdp_huge_page_level) @@ -5182,7 +5162,8 @@ void kvm_configure_mmu(bool enable_tdp, int tdp_max_root_level, EXPORT_SYMBOL_GPL(kvm_configure_mmu); /* The return value indicates if tlb flush on all vcpus is needed. */ -typedef bool (*slot_level_handler) (struct kvm *kvm, struct kvm_rmap_head *rmap_head); +typedef bool (*slot_level_handler) (struct kvm *kvm, struct kvm_rmap_head *rmap_head, + struct kvm_memory_slot *slot); /* The caller should hold mmu-lock before calling this function. */ static __always_inline bool @@ -5196,7 +5177,7 @@ slot_handle_level_range(struct kvm *kvm, struct kvm_memory_slot *memslot, for_each_slot_rmap_range(memslot, start_level, end_level, start_gfn, end_gfn, &iterator) { if (iterator.rmap) - flush |= fn(kvm, iterator.rmap); + flush |= fn(kvm, iterator.rmap, memslot); if (need_resched() || rwlock_needbreak(&kvm->mmu_lock)) { if (flush && lock_flush_tlb) { @@ -5229,22 +5210,6 @@ slot_handle_level(struct kvm *kvm, struct kvm_memory_slot *memslot, lock_flush_tlb); } -static __always_inline bool -slot_handle_all_level(struct kvm *kvm, struct kvm_memory_slot *memslot, - slot_level_handler fn, bool lock_flush_tlb) -{ - return slot_handle_level(kvm, memslot, fn, PG_LEVEL_4K, - KVM_MAX_HUGEPAGE_LEVEL, lock_flush_tlb); -} - -static __always_inline bool -slot_handle_large_level(struct kvm *kvm, struct kvm_memory_slot *memslot, - slot_level_handler fn, bool lock_flush_tlb) -{ - return slot_handle_level(kvm, memslot, fn, PG_LEVEL_4K + 1, - KVM_MAX_HUGEPAGE_LEVEL, lock_flush_tlb); -} - static __always_inline bool slot_handle_leaf(struct kvm *kvm, struct kvm_memory_slot *memslot, slot_level_handler fn, bool lock_flush_tlb) @@ -5485,7 +5450,8 @@ void kvm_zap_gfn_range(struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_end) } static bool slot_rmap_write_protect(struct kvm *kvm, - struct kvm_rmap_head *rmap_head) + struct kvm_rmap_head *rmap_head, + struct kvm_memory_slot *slot) { return __rmap_write_protect(kvm, rmap_head, false); } @@ -5519,7 +5485,8 @@ void kvm_mmu_slot_remove_write_access(struct kvm *kvm, } static bool kvm_mmu_zap_collapsible_spte(struct kvm *kvm, - struct kvm_rmap_head *rmap_head) + struct kvm_rmap_head *rmap_head, + struct kvm_memory_slot *slot) { u64 *sptep; struct rmap_iterator iter; @@ -5540,8 +5507,8 @@ static bool kvm_mmu_zap_collapsible_spte(struct kvm *kvm, * mapping if the indirect sp has level = 1. */ if (sp->role.direct && !kvm_is_reserved_pfn(pfn) && - (kvm_is_zone_device_pfn(pfn) || - PageCompound(pfn_to_page(pfn)))) { + sp->role.level < kvm_mmu_max_mapping_level(kvm, slot, sp->gfn, + pfn, PG_LEVEL_NUM)) { pte_list_remove(rmap_head, sptep); if (kvm_available_flush_tlb_with_range()) @@ -5561,12 +5528,13 @@ void kvm_mmu_zap_collapsible_sptes(struct kvm *kvm, const struct kvm_memory_slot *memslot) { /* FIXME: const-ify all uses of struct kvm_memory_slot. */ + struct kvm_memory_slot *slot = (struct kvm_memory_slot *)memslot; + write_lock(&kvm->mmu_lock); - slot_handle_leaf(kvm, (struct kvm_memory_slot *)memslot, - kvm_mmu_zap_collapsible_spte, true); + slot_handle_leaf(kvm, slot, kvm_mmu_zap_collapsible_spte, true); if (is_tdp_mmu_enabled(kvm)) - kvm_tdp_mmu_zap_collapsible_sptes(kvm, memslot); + kvm_tdp_mmu_zap_collapsible_sptes(kvm, slot); write_unlock(&kvm->mmu_lock); } @@ -5605,40 +5573,6 @@ void kvm_mmu_slot_leaf_clear_dirty(struct kvm *kvm, if (flush) kvm_arch_flush_remote_tlbs_memslot(kvm, memslot); } -EXPORT_SYMBOL_GPL(kvm_mmu_slot_leaf_clear_dirty); - -void kvm_mmu_slot_largepage_remove_write_access(struct kvm *kvm, - struct kvm_memory_slot *memslot) -{ - bool flush; - - write_lock(&kvm->mmu_lock); - flush = slot_handle_large_level(kvm, memslot, slot_rmap_write_protect, - false); - if (is_tdp_mmu_enabled(kvm)) - flush |= kvm_tdp_mmu_wrprot_slot(kvm, memslot, PG_LEVEL_2M); - write_unlock(&kvm->mmu_lock); - - if (flush) - kvm_arch_flush_remote_tlbs_memslot(kvm, memslot); -} -EXPORT_SYMBOL_GPL(kvm_mmu_slot_largepage_remove_write_access); - -void kvm_mmu_slot_set_dirty(struct kvm *kvm, - struct kvm_memory_slot *memslot) -{ - bool flush; - - write_lock(&kvm->mmu_lock); - flush = slot_handle_all_level(kvm, memslot, __rmap_set_dirty, false); - if (is_tdp_mmu_enabled(kvm)) - flush |= kvm_tdp_mmu_slot_set_dirty(kvm, memslot); - write_unlock(&kvm->mmu_lock); - - if (flush) - kvm_arch_flush_remote_tlbs_memslot(kvm, memslot); -} -EXPORT_SYMBOL_GPL(kvm_mmu_slot_set_dirty); void kvm_mmu_zap_all(struct kvm *kvm) { diff --git a/arch/x86/kvm/mmu/mmu_internal.h b/arch/x86/kvm/mmu/mmu_internal.h index 9e38d3c5daad8c0a11f06322e1bf4248d1865d38..72b0928f2b2d96c54cb2d75bbff84494c150b627 100644 --- a/arch/x86/kvm/mmu/mmu_internal.h +++ b/arch/x86/kvm/mmu/mmu_internal.h @@ -84,7 +84,10 @@ static inline bool kvm_vcpu_ad_need_write_protect(struct kvm_vcpu *vcpu) * When using the EPT page-modification log, the GPAs in the log * would come from L2 rather than L1. Therefore, we need to rely * on write protection to record dirty pages. This also bypasses - * PML, since writes now result in a vmexit. + * PML, since writes now result in a vmexit. Note, this helper will + * tag SPTEs as needing write-protection even if PML is disabled or + * unsupported, but that's ok because the tag is consumed if and only + * if PML is enabled. Omit the PML check to save a few uops. */ return vcpu->arch.mmu == &vcpu->arch.guest_mmu; } @@ -138,6 +141,8 @@ enum { #define SET_SPTE_NEED_REMOTE_TLB_FLUSH BIT(1) #define SET_SPTE_SPURIOUS BIT(2) +int kvm_mmu_max_mapping_level(struct kvm *kvm, struct kvm_memory_slot *slot, + gfn_t gfn, kvm_pfn_t pfn, int max_level); int kvm_mmu_hugepage_adjust(struct kvm_vcpu *vcpu, gfn_t gfn, int max_level, kvm_pfn_t *pfnp, bool huge_page_disallowed, int *req_level); diff --git a/arch/x86/kvm/mmu/paging_tmpl.h b/arch/x86/kvm/mmu/paging_tmpl.h index d9f66cc459e84993b08752529d45c514f0435e8d..55d7b473ac447de3189071b89b89f823605a69b0 100644 --- a/arch/x86/kvm/mmu/paging_tmpl.h +++ b/arch/x86/kvm/mmu/paging_tmpl.h @@ -601,6 +601,13 @@ static void FNAME(pte_prefetch)(struct kvm_vcpu *vcpu, struct guest_walker *gw, if (sp->role.level > PG_LEVEL_4K) return; + /* + * If addresses are being invalidated, skip prefetching to avoid + * accidentally prefetching those addresses. + */ + if (unlikely(vcpu->kvm->mmu_notifier_count)) + return; + if (sp->role.direct) return __direct_pte_prefetch(vcpu, sp, sptep); @@ -790,6 +797,7 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gpa_t addr, u32 error_code, struct guest_walker walker; int r; kvm_pfn_t pfn; + hva_t hva; unsigned long mmu_seq; bool map_writable, is_self_change_mapping; int max_level; @@ -840,8 +848,8 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gpa_t addr, u32 error_code, mmu_seq = vcpu->kvm->mmu_notifier_seq; smp_rmb(); - if (try_async_pf(vcpu, prefault, walker.gfn, addr, &pfn, write_fault, - &map_writable)) + if (try_async_pf(vcpu, prefault, walker.gfn, addr, &pfn, &hva, + write_fault, &map_writable)) return RET_PF_RETRY; if (handle_abnormal_pfn(vcpu, addr, walker.gfn, pfn, walker.pte_access, &r)) @@ -869,7 +877,7 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gpa_t addr, u32 error_code, r = RET_PF_RETRY; write_lock(&vcpu->kvm->mmu_lock); - if (mmu_notifier_retry(vcpu->kvm, mmu_seq)) + if (!is_noslot_pfn(pfn) && mmu_notifier_retry_hva(vcpu->kvm, mmu_seq, hva)) goto out_unlock; kvm_mmu_audit(vcpu, AUDIT_PRE_PAGE_FAULT); diff --git a/arch/x86/kvm/mmu/tdp_mmu.c b/arch/x86/kvm/mmu/tdp_mmu.c index 71e100a5670f91bc37126438929cf9fd36a15071..c926c6b899a106c704a68b25997209aff05ff3b1 100644 --- a/arch/x86/kvm/mmu/tdp_mmu.c +++ b/arch/x86/kvm/mmu/tdp_mmu.c @@ -1268,68 +1268,16 @@ void kvm_tdp_mmu_clear_dirty_pt_masked(struct kvm *kvm, } } -/* - * Set the dirty status of all the SPTEs mapping GFNs in the memslot. This is - * only used for PML, and so will involve setting the dirty bit on each SPTE. - * Returns true if an SPTE has been changed and the TLBs need to be flushed. - */ -static bool set_dirty_gfn_range(struct kvm *kvm, struct kvm_mmu_page *root, - gfn_t start, gfn_t end) -{ - struct tdp_iter iter; - u64 new_spte; - bool spte_set = false; - - rcu_read_lock(); - - tdp_root_for_each_pte(iter, root, start, end) { - if (tdp_mmu_iter_cond_resched(kvm, &iter, false)) - continue; - - if (!is_shadow_present_pte(iter.old_spte) || - iter.old_spte & shadow_dirty_mask) - continue; - - new_spte = iter.old_spte | shadow_dirty_mask; - - tdp_mmu_set_spte(kvm, &iter, new_spte); - spte_set = true; - } - - rcu_read_unlock(); - return spte_set; -} - -/* - * Set the dirty status of all the SPTEs mapping GFNs in the memslot. This is - * only used for PML, and so will involve setting the dirty bit on each SPTE. - * Returns true if an SPTE has been changed and the TLBs need to be flushed. - */ -bool kvm_tdp_mmu_slot_set_dirty(struct kvm *kvm, struct kvm_memory_slot *slot) -{ - struct kvm_mmu_page *root; - int root_as_id; - bool spte_set = false; - - for_each_tdp_mmu_root_yield_safe(kvm, root) { - root_as_id = kvm_mmu_page_as_id(root); - if (root_as_id != slot->as_id) - continue; - - spte_set |= set_dirty_gfn_range(kvm, root, slot->base_gfn, - slot->base_gfn + slot->npages); - } - return spte_set; -} - /* * Clear leaf entries which could be replaced by large mappings, for * GFNs within the slot. */ static void zap_collapsible_spte_range(struct kvm *kvm, struct kvm_mmu_page *root, - gfn_t start, gfn_t end) + struct kvm_memory_slot *slot) { + gfn_t start = slot->base_gfn; + gfn_t end = start + slot->npages; struct tdp_iter iter; kvm_pfn_t pfn; bool spte_set = false; @@ -1348,7 +1296,8 @@ static void zap_collapsible_spte_range(struct kvm *kvm, pfn = spte_to_pfn(iter.old_spte); if (kvm_is_reserved_pfn(pfn) || - !PageTransCompoundMap(pfn_to_page(pfn))) + iter.level >= kvm_mmu_max_mapping_level(kvm, slot, iter.gfn, + pfn, PG_LEVEL_NUM)) continue; tdp_mmu_set_spte(kvm, &iter, 0); @@ -1366,7 +1315,7 @@ static void zap_collapsible_spte_range(struct kvm *kvm, * be replaced by large mappings, for GFNs within the slot. */ void kvm_tdp_mmu_zap_collapsible_sptes(struct kvm *kvm, - const struct kvm_memory_slot *slot) + struct kvm_memory_slot *slot) { struct kvm_mmu_page *root; int root_as_id; @@ -1376,8 +1325,7 @@ void kvm_tdp_mmu_zap_collapsible_sptes(struct kvm *kvm, if (root_as_id != slot->as_id) continue; - zap_collapsible_spte_range(kvm, root, slot->base_gfn, - slot->base_gfn + slot->npages); + zap_collapsible_spte_range(kvm, root, slot); } } diff --git a/arch/x86/kvm/mmu/tdp_mmu.h b/arch/x86/kvm/mmu/tdp_mmu.h index b4b65e3699b36dcfe1d6f6ea2990f83a33c93df1..3b761c111bff13bf9d09531ebeaf1b21962ed42a 100644 --- a/arch/x86/kvm/mmu/tdp_mmu.h +++ b/arch/x86/kvm/mmu/tdp_mmu.h @@ -33,9 +33,8 @@ void kvm_tdp_mmu_clear_dirty_pt_masked(struct kvm *kvm, struct kvm_memory_slot *slot, gfn_t gfn, unsigned long mask, bool wrprot); -bool kvm_tdp_mmu_slot_set_dirty(struct kvm *kvm, struct kvm_memory_slot *slot); void kvm_tdp_mmu_zap_collapsible_sptes(struct kvm *kvm, - const struct kvm_memory_slot *slot); + struct kvm_memory_slot *slot); bool kvm_tdp_mmu_write_protect_gfn(struct kvm *kvm, struct kvm_memory_slot *slot, gfn_t gfn); diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index cc91738ab445cf8fd7d759458ba33d21eba03727..35891d9a1099b3f54d146e0abb26b89c74075b2e 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -51,6 +51,23 @@ static void nested_svm_inject_npf_exit(struct kvm_vcpu *vcpu, nested_svm_vmexit(svm); } +static void svm_inject_page_fault_nested(struct kvm_vcpu *vcpu, struct x86_exception *fault) +{ + struct vcpu_svm *svm = to_svm(vcpu); + WARN_ON(!is_guest_mode(vcpu)); + + if (vmcb_is_intercept(&svm->nested.ctl, INTERCEPT_EXCEPTION_OFFSET + PF_VECTOR) && + !svm->nested.nested_run_pending) { + svm->vmcb->control.exit_code = SVM_EXIT_EXCP_BASE + PF_VECTOR; + svm->vmcb->control.exit_code_hi = 0; + svm->vmcb->control.exit_info_1 = fault->error_code; + svm->vmcb->control.exit_info_2 = fault->address; + nested_svm_vmexit(svm); + } else { + kvm_inject_page_fault(vcpu, fault); + } +} + static u64 nested_svm_get_tdp_pdptr(struct kvm_vcpu *vcpu, int index) { struct vcpu_svm *svm = to_svm(vcpu); @@ -436,16 +453,33 @@ int enter_svm_guest_mode(struct vcpu_svm *svm, u64 vmcb12_gpa, { int ret; + trace_kvm_nested_vmrun(svm->vmcb->save.rip, vmcb12_gpa, + vmcb12->save.rip, + vmcb12->control.int_ctl, + vmcb12->control.event_inj, + vmcb12->control.nested_ctl); + + trace_kvm_nested_intercepts(vmcb12->control.intercepts[INTERCEPT_CR] & 0xffff, + vmcb12->control.intercepts[INTERCEPT_CR] >> 16, + vmcb12->control.intercepts[INTERCEPT_EXCEPTION], + vmcb12->control.intercepts[INTERCEPT_WORD3], + vmcb12->control.intercepts[INTERCEPT_WORD4], + vmcb12->control.intercepts[INTERCEPT_WORD5]); + + svm->nested.vmcb12_gpa = vmcb12_gpa; load_nested_vmcb_control(svm, &vmcb12->control); - nested_prepare_vmcb_save(svm, vmcb12); nested_prepare_vmcb_control(svm); + nested_prepare_vmcb_save(svm, vmcb12); ret = nested_svm_load_cr3(&svm->vcpu, vmcb12->save.cr3, nested_npt_enabled(svm)); if (ret) return ret; + if (!npt_enabled) + svm->vcpu.arch.mmu->inject_page_fault = svm_inject_page_fault_nested; + svm_set_gif(svm, true); return 0; @@ -489,18 +523,6 @@ int nested_svm_vmrun(struct vcpu_svm *svm) goto out; } - trace_kvm_nested_vmrun(svm->vmcb->save.rip, vmcb12_gpa, - vmcb12->save.rip, - vmcb12->control.int_ctl, - vmcb12->control.event_inj, - vmcb12->control.nested_ctl); - - trace_kvm_nested_intercepts(vmcb12->control.intercepts[INTERCEPT_CR] & 0xffff, - vmcb12->control.intercepts[INTERCEPT_CR] >> 16, - vmcb12->control.intercepts[INTERCEPT_EXCEPTION], - vmcb12->control.intercepts[INTERCEPT_WORD3], - vmcb12->control.intercepts[INTERCEPT_WORD4], - vmcb12->control.intercepts[INTERCEPT_WORD5]); /* Clear internal status */ kvm_clear_exception_queue(&svm->vcpu); diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index adb3619a3c1696e9ece09971f5c4b77b34ebfe11..c636021b066b0b1cf4b9fc36e95c6ce9d0c80b4f 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -926,9 +926,6 @@ static __init void svm_set_cpu_caps(void) if (boot_cpu_has(X86_FEATURE_LS_CFG_SSBD) || boot_cpu_has(X86_FEATURE_AMD_SSBD)) kvm_cpu_cap_set(X86_FEATURE_VIRT_SSBD); - - /* Enable INVPCID feature */ - kvm_cpu_cap_check_and_set(X86_FEATURE_INVPCID); } static __init int svm_hardware_setup(void) @@ -1103,12 +1100,12 @@ static u64 svm_write_l1_tsc_offset(struct kvm_vcpu *vcpu, u64 offset) static void svm_check_invpcid(struct vcpu_svm *svm) { /* - * Intercept INVPCID instruction only if shadow page table is - * enabled. Interception is not required with nested page table - * enabled. + * Intercept INVPCID if shadow paging is enabled to sync/free shadow + * roots, or if INVPCID is disabled in the guest to inject #UD. */ if (kvm_cpu_cap_has(X86_FEATURE_INVPCID)) { - if (!npt_enabled) + if (!npt_enabled || + !guest_cpuid_has(&svm->vcpu, X86_FEATURE_INVPCID)) svm_set_intercept(svm, INTERCEPT_INVPCID); else svm_clr_intercept(svm, INTERCEPT_INVPCID); @@ -2214,15 +2211,20 @@ static int emulate_svm_instr(struct kvm_vcpu *vcpu, int opcode) [SVM_INSTR_VMSAVE] = vmsave_interception, }; struct vcpu_svm *svm = to_svm(vcpu); + int ret; if (is_guest_mode(vcpu)) { svm->vmcb->control.exit_code = guest_mode_exit_codes[opcode]; svm->vmcb->control.exit_info_1 = 0; svm->vmcb->control.exit_info_2 = 0; - return nested_svm_vmexit(svm); - } else - return svm_instr_handlers[opcode](svm); + /* Returns '1' or -errno on failure, '0' on success. */ + ret = nested_svm_vmexit(svm); + if (ret) + return ret; + return 1; + } + return svm_instr_handlers[opcode](svm); } /* diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index b2f0b5e9cd638e99b7ed8b7b4459e8530558d8ba..bcca0b80e0d040be4ce9a4f9a2ee05d77e0c8257 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -2167,15 +2167,13 @@ static void prepare_vmcs02_constant_state(struct vcpu_vmx *vmx) vmcs_write64(MSR_BITMAP, __pa(vmx->nested.vmcs02.msr_bitmap)); /* - * The PML address never changes, so it is constant in vmcs02. - * Conceptually we want to copy the PML index from vmcs01 here, - * and then back to vmcs01 on nested vmexit. But since we flush - * the log and reset GUEST_PML_INDEX on each vmexit, the PML - * index is also effectively constant in vmcs02. + * PML is emulated for L2, but never enabled in hardware as the MMU + * handles A/D emulation. Disabling PML for L2 also avoids having to + * deal with filtering out L2 GPAs from the buffer. */ if (enable_pml) { - vmcs_write64(PML_ADDRESS, page_to_phys(vmx->pml_pg)); - vmcs_write16(GUEST_PML_INDEX, PML_ENTITY_NUM - 1); + vmcs_write64(PML_ADDRESS, 0); + vmcs_write16(GUEST_PML_INDEX, -1); } if (cpu_has_vmx_encls_vmexit()) @@ -2210,7 +2208,7 @@ static void prepare_vmcs02_early_rare(struct vcpu_vmx *vmx, static void prepare_vmcs02_early(struct vcpu_vmx *vmx, struct vmcs12 *vmcs12) { - u32 exec_control, vmcs12_exec_ctrl; + u32 exec_control; u64 guest_efer = nested_vmx_calc_efer(vmx, vmcs12); if (vmx->nested.dirty_vmcs12 || vmx->nested.hv_evmcs) @@ -2284,11 +2282,11 @@ static void prepare_vmcs02_early(struct vcpu_vmx *vmx, struct vmcs12 *vmcs12) SECONDARY_EXEC_APIC_REGISTER_VIRT | SECONDARY_EXEC_ENABLE_VMFUNC); if (nested_cpu_has(vmcs12, - CPU_BASED_ACTIVATE_SECONDARY_CONTROLS)) { - vmcs12_exec_ctrl = vmcs12->secondary_vm_exec_control & - ~SECONDARY_EXEC_ENABLE_PML; - exec_control |= vmcs12_exec_ctrl; - } + CPU_BASED_ACTIVATE_SECONDARY_CONTROLS)) + exec_control |= vmcs12->secondary_vm_exec_control; + + /* PML is emulated and never enabled in hardware for L2. */ + exec_control &= ~SECONDARY_EXEC_ENABLE_PML; /* VMCS shadowing for L2 is emulated for now */ exec_control &= ~SECONDARY_EXEC_SHADOW_VMCS; @@ -4200,9 +4198,6 @@ static void load_vmcs12_host_state(struct kvm_vcpu *vcpu, if (nested_vmx_load_cr3(vcpu, vmcs12->host_cr3, false, &ignored)) nested_vmx_abort(vcpu, VMX_ABORT_LOAD_HOST_PDPTE_FAIL); - if (!enable_ept) - vcpu->arch.walk_mmu->inject_page_fault = kvm_inject_page_fault; - nested_vmx_transition_tlb_flush(vcpu, vmcs12, false); vmcs_write32(GUEST_SYSENTER_CS, vmcs12->host_ia32_sysenter_cs); @@ -4495,6 +4490,11 @@ void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 vm_exit_reason, vmx_set_virtual_apic_mode(vcpu); } + if (vmx->nested.update_vmcs01_cpu_dirty_logging) { + vmx->nested.update_vmcs01_cpu_dirty_logging = false; + vmx_update_cpu_dirty_logging(vcpu); + } + /* Unpin physical memory we referred to in vmcs02 */ if (vmx->nested.apic_access_page) { kvm_release_page_clean(vmx->nested.apic_access_page); @@ -5793,7 +5793,10 @@ static bool nested_vmx_l0_wants_exit(struct kvm_vcpu *vcpu, case EXIT_REASON_PREEMPTION_TIMER: return true; case EXIT_REASON_PML_FULL: - /* We emulate PML support to L1. */ + /* + * PML is emulated for an L1 VMM and should never be enabled in + * vmcs02, always "handle" PML_FULL by exiting to userspace. + */ return true; case EXIT_REASON_VMFUNC: /* VM functions are emulated through L2->L0 vmexits. */ diff --git a/arch/x86/kvm/vmx/pmu_intel.c b/arch/x86/kvm/vmx/pmu_intel.c index d1df618cb7deb47e535c5686b5860bc014405ab1..9efc1a6b86930ad5879faa487270c8ab459a2da6 100644 --- a/arch/x86/kvm/vmx/pmu_intel.c +++ b/arch/x86/kvm/vmx/pmu_intel.c @@ -298,7 +298,7 @@ int intel_pmu_create_guest_lbr_event(struct kvm_vcpu *vcpu) if (IS_ERR(event)) { pr_debug_ratelimited("%s: failed %ld\n", __func__, PTR_ERR(event)); - return -ENOENT; + return PTR_ERR(event); } lbr_desc->event = event; pmu->event_count++; @@ -320,7 +320,7 @@ static bool intel_pmu_handle_lbr_msrs_access(struct kvm_vcpu *vcpu, if (!intel_pmu_is_valid_lbr_msr(vcpu, index)) return false; - if (!lbr_desc->event && !intel_pmu_create_guest_lbr_event(vcpu)) + if (!lbr_desc->event && intel_pmu_create_guest_lbr_event(vcpu) < 0) goto dummy; /* diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index e0a3a9be654bac8f4de80d1e673c652e4180854c..50810d4714628c2f3978847e9068c6106529797a 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -4277,7 +4277,12 @@ static void vmx_compute_secondary_exec_control(struct vcpu_vmx *vmx) */ exec_control &= ~SECONDARY_EXEC_SHADOW_VMCS; - if (!enable_pml) + /* + * PML is enabled/disabled when dirty logging of memsmlots changes, but + * it needs to be set here when dirty logging is already active, e.g. + * if this vCPU was created after dirty logging was enabled. + */ + if (!vcpu->kvm->arch.cpu_dirty_logging_count) exec_control &= ~SECONDARY_EXEC_ENABLE_PML; if (cpu_has_vmx_xsaves()) { @@ -4295,18 +4300,8 @@ static void vmx_compute_secondary_exec_control(struct vcpu_vmx *vmx) } vmx_adjust_sec_exec_feature(vmx, &exec_control, rdtscp, RDTSCP); - - /* - * Expose INVPCID if and only if PCID is also exposed to the guest. - * INVPCID takes a #UD when it's disabled in the VMCS, but a #GP or #PF - * if CR4.PCIDE=0. Enumerating CPUID.INVPCID=1 would lead to incorrect - * behavior from the guest perspective (it would expect #GP or #PF). - */ - if (!guest_cpuid_has(vcpu, X86_FEATURE_PCID)) - guest_cpuid_clear(vcpu, X86_FEATURE_INVPCID); vmx_adjust_sec_exec_feature(vmx, &exec_control, invpcid, INVPCID); - vmx_adjust_sec_exec_exiting(vmx, &exec_control, rdrand, RDRAND); vmx_adjust_sec_exec_exiting(vmx, &exec_control, rdseed, RDSEED); @@ -5776,24 +5771,6 @@ static void vmx_flush_pml_buffer(struct kvm_vcpu *vcpu) vmcs_write16(GUEST_PML_INDEX, PML_ENTITY_NUM - 1); } -/* - * Flush all vcpus' PML buffer and update logged GPAs to dirty_bitmap. - * Called before reporting dirty_bitmap to userspace. - */ -static void kvm_flush_pml_buffers(struct kvm *kvm) -{ - int i; - struct kvm_vcpu *vcpu; - /* - * We only need to kick vcpu out of guest mode here, as PML buffer - * is flushed at beginning of all VMEXITs, and it's obvious that only - * vcpus running in guest are possible to have unflushed GPAs in PML - * buffer. - */ - kvm_for_each_vcpu(i, vcpu, kvm) - kvm_vcpu_kick(vcpu); -} - static void vmx_dump_sel(char *name, uint32_t sel) { pr_err("%s sel=0x%04x, attr=0x%05x, limit=0x%08x, base=0x%016lx\n", @@ -5976,9 +5953,10 @@ static int __vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath) * updated. Another good is, in kvm_vm_ioctl_get_dirty_log, before * querying dirty_bitmap, we only need to kick all vcpus out of guest * mode as if vcpus is in root mode, the PML buffer must has been - * flushed already. + * flushed already. Note, PML is never enabled in hardware while + * running L2. */ - if (enable_pml) + if (enable_pml && !is_guest_mode(vcpu)) vmx_flush_pml_buffer(vcpu); /* @@ -5994,6 +5972,13 @@ static int __vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath) return handle_invalid_guest_state(vcpu); if (is_guest_mode(vcpu)) { + /* + * PML is never enabled when running L2, bail immediately if a + * PML full exit occurs as something is horribly wrong. + */ + if (exit_reason.basic == EXIT_REASON_PML_FULL) + goto unexpected_vmexit; + /* * The host physical addresses of some pages of guest memory * are loaded into the vmcs02 (e.g. vmcs12's Virtual APIC @@ -6851,13 +6836,15 @@ static fastpath_t vmx_vcpu_run(struct kvm_vcpu *vcpu) if (unlikely((u16)vmx->exit_reason.basic == EXIT_REASON_MCE_DURING_VMENTRY)) kvm_machine_check(); + if (likely(!vmx->exit_reason.failed_vmentry)) + vmx->idt_vectoring_info = vmcs_read32(IDT_VECTORING_INFO_FIELD); + trace_kvm_exit(vmx->exit_reason.full, vcpu, KVM_ISA_VMX); if (unlikely(vmx->exit_reason.failed_vmentry)) return EXIT_FASTPATH_NONE; vmx->loaded_vmcs->launched = 1; - vmx->idt_vectoring_info = vmcs_read32(IDT_VECTORING_INFO_FIELD); vmx_recover_nmi_blocking(vmx); vmx_complete_interrupts(vmx); @@ -7330,8 +7317,8 @@ static __init void vmx_set_cpu_caps(void) /* CPUID 0x7 */ if (kvm_mpx_supported()) kvm_cpu_cap_check_and_set(X86_FEATURE_MPX); - if (cpu_has_vmx_invpcid()) - kvm_cpu_cap_check_and_set(X86_FEATURE_INVPCID); + if (!cpu_has_vmx_invpcid()) + kvm_cpu_cap_clear(X86_FEATURE_INVPCID); if (vmx_pt_mode_is_host_guest()) kvm_cpu_cap_check_and_set(X86_FEATURE_INTEL_PT); @@ -7509,30 +7496,24 @@ static void vmx_sched_in(struct kvm_vcpu *vcpu, int cpu) shrink_ple_window(vcpu); } -static void vmx_slot_enable_log_dirty(struct kvm *kvm, - struct kvm_memory_slot *slot) +void vmx_update_cpu_dirty_logging(struct kvm_vcpu *vcpu) { - if (!kvm_dirty_log_manual_protect_and_init_set(kvm)) - kvm_mmu_slot_leaf_clear_dirty(kvm, slot); - kvm_mmu_slot_largepage_remove_write_access(kvm, slot); -} - -static void vmx_slot_disable_log_dirty(struct kvm *kvm, - struct kvm_memory_slot *slot) -{ - kvm_mmu_slot_set_dirty(kvm, slot); -} + struct vcpu_vmx *vmx = to_vmx(vcpu); -static void vmx_flush_log_dirty(struct kvm *kvm) -{ - kvm_flush_pml_buffers(kvm); -} + if (is_guest_mode(vcpu)) { + vmx->nested.update_vmcs01_cpu_dirty_logging = true; + return; + } -static void vmx_enable_log_dirty_pt_masked(struct kvm *kvm, - struct kvm_memory_slot *memslot, - gfn_t offset, unsigned long mask) -{ - kvm_mmu_clear_dirty_pt_masked(kvm, memslot, offset, mask); + /* + * Note, cpu_dirty_logging_count can be changed concurrent with this + * code, but in that case another update request will be made and so + * the guest will never run with a stale PML value. + */ + if (vcpu->kvm->arch.cpu_dirty_logging_count) + secondary_exec_controls_setbit(vmx, SECONDARY_EXEC_ENABLE_PML); + else + secondary_exec_controls_clearbit(vmx, SECONDARY_EXEC_ENABLE_PML); } static int vmx_pre_block(struct kvm_vcpu *vcpu) @@ -7642,11 +7623,6 @@ static bool vmx_check_apicv_inhibit_reasons(ulong bit) return supported & BIT(bit); } -static int vmx_cpu_dirty_log_size(void) -{ - return enable_pml ? PML_ENTITY_NUM : 0; -} - static struct kvm_x86_ops vmx_x86_ops __initdata = { .hardware_unsetup = hardware_unsetup, @@ -7746,10 +7722,8 @@ static struct kvm_x86_ops vmx_x86_ops __initdata = { .sched_in = vmx_sched_in, - .slot_enable_log_dirty = vmx_slot_enable_log_dirty, - .slot_disable_log_dirty = vmx_slot_disable_log_dirty, - .flush_log_dirty = vmx_flush_log_dirty, - .enable_log_dirty_pt_masked = vmx_enable_log_dirty_pt_masked, + .cpu_dirty_log_size = PML_ENTITY_NUM, + .update_cpu_dirty_logging = vmx_update_cpu_dirty_logging, .pre_block = vmx_pre_block, .post_block = vmx_post_block, @@ -7777,7 +7751,6 @@ static struct kvm_x86_ops vmx_x86_ops __initdata = { .msr_filter_changed = vmx_msr_filter_changed, .complete_emulated_msr = kvm_complete_insn_gp, - .cpu_dirty_log_size = vmx_cpu_dirty_log_size, .vcpu_deliver_sipi_vector = kvm_vcpu_deliver_sipi_vector, }; @@ -7894,13 +7867,8 @@ static __init int hardware_setup(void) if (!enable_ept || !enable_ept_ad_bits || !cpu_has_vmx_pml()) enable_pml = 0; - if (!enable_pml) { - vmx_x86_ops.slot_enable_log_dirty = NULL; - vmx_x86_ops.slot_disable_log_dirty = NULL; - vmx_x86_ops.flush_log_dirty = NULL; - vmx_x86_ops.enable_log_dirty_pt_masked = NULL; - vmx_x86_ops.cpu_dirty_log_size = NULL; - } + if (!enable_pml) + vmx_x86_ops.cpu_dirty_log_size = 0; if (!cpu_has_vmx_preemption_timer()) enable_preemption_timer = false; diff --git a/arch/x86/kvm/vmx/vmx.h b/arch/x86/kvm/vmx/vmx.h index 12c53d05a902becd08fa8f07904cba623c32cbd0..89da5e1251f18c115c8436aca484b3656151a0cf 100644 --- a/arch/x86/kvm/vmx/vmx.h +++ b/arch/x86/kvm/vmx/vmx.h @@ -165,6 +165,7 @@ struct nested_vmx { bool change_vmcs01_virtual_apic_mode; bool reload_vmcs01_apic_access_page; + bool update_vmcs01_cpu_dirty_logging; /* * Enlightened VMCS has been enabled. It does not mean that L1 has to @@ -393,6 +394,7 @@ int vmx_find_loadstore_msr_slot(struct vmx_msrs *m, u32 msr); void vmx_ept_load_pdptrs(struct kvm_vcpu *vcpu); void vmx_set_intercept_for_msr(struct kvm_vcpu *vcpu, u32 msr, int type, bool value); +void vmx_update_cpu_dirty_logging(struct kvm_vcpu *vcpu); static inline u8 vmx_get_rvi(void) { diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 884e5b3838c735db2e772c80638667fcdb390866..3712bb5245eb9a206b872bb69e476d172ef343ed 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -5215,10 +5215,18 @@ static int kvm_vm_ioctl_reinject(struct kvm *kvm, void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot) { + /* - * Flush potentially hardware-cached dirty pages to dirty_bitmap. + * Flush all CPUs' dirty log buffers to the dirty_bitmap. Called + * before reporting dirty_bitmap to userspace. KVM flushes the buffers + * on all VM-Exits, thus we only need to kick running vCPUs to force a + * VM-Exit. */ - static_call_cond(kvm_x86_flush_log_dirty)(kvm); + struct kvm_vcpu *vcpu; + int i; + + kvm_for_each_vcpu(i, vcpu, kvm) + kvm_vcpu_kick(vcpu); } int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_event, @@ -8980,6 +8988,9 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) kvm_check_async_pf_completion(vcpu); if (kvm_check_request(KVM_REQ_MSR_FILTER_CHANGED, vcpu)) static_call(kvm_x86_msr_filter_changed)(vcpu); + + if (kvm_check_request(KVM_REQ_UPDATE_CPU_DIRTY_LOGGING, vcpu)) + static_call(kvm_x86_update_cpu_dirty_logging)(vcpu); } if (kvm_check_request(KVM_REQ_EVENT, vcpu) || req_int_win || @@ -10748,75 +10759,96 @@ int kvm_arch_prepare_memory_region(struct kvm *kvm, return 0; } + +static void kvm_mmu_update_cpu_dirty_logging(struct kvm *kvm, bool enable) +{ + struct kvm_arch *ka = &kvm->arch; + + if (!kvm_x86_ops.cpu_dirty_log_size) + return; + + if ((enable && ++ka->cpu_dirty_logging_count == 1) || + (!enable && --ka->cpu_dirty_logging_count == 0)) + kvm_make_all_cpus_request(kvm, KVM_REQ_UPDATE_CPU_DIRTY_LOGGING); + + WARN_ON_ONCE(ka->cpu_dirty_logging_count < 0); +} + static void kvm_mmu_slot_apply_flags(struct kvm *kvm, struct kvm_memory_slot *old, struct kvm_memory_slot *new, enum kvm_mr_change change) { + bool log_dirty_pages = new->flags & KVM_MEM_LOG_DIRTY_PAGES; + /* - * Nothing to do for RO slots or CREATE/MOVE/DELETE of a slot. - * See comments below. + * Update CPU dirty logging if dirty logging is being toggled. This + * applies to all operations. */ - if ((change != KVM_MR_FLAGS_ONLY) || (new->flags & KVM_MEM_READONLY)) - return; + if ((old->flags ^ new->flags) & KVM_MEM_LOG_DIRTY_PAGES) + kvm_mmu_update_cpu_dirty_logging(kvm, log_dirty_pages); /* - * Dirty logging tracks sptes in 4k granularity, meaning that large - * sptes have to be split. If live migration is successful, the guest - * in the source machine will be destroyed and large sptes will be - * created in the destination. However, if the guest continues to run - * in the source machine (for example if live migration fails), small - * sptes will remain around and cause bad performance. - * - * Scan sptes if dirty logging has been stopped, dropping those - * which can be collapsed into a single large-page spte. Later - * page faults will create the large-page sptes. + * Nothing more to do for RO slots (which can't be dirtied and can't be + * made writable) or CREATE/MOVE/DELETE of a slot. * - * There is no need to do this in any of the following cases: + * For a memslot with dirty logging disabled: * CREATE: No dirty mappings will already exist. * MOVE/DELETE: The old mappings will already have been cleaned up by * kvm_arch_flush_shadow_memslot() + * + * For a memslot with dirty logging enabled: + * CREATE: No shadow pages exist, thus nothing to write-protect + * and no dirty bits to clear. + * MOVE/DELETE: The old mappings will already have been cleaned up by + * kvm_arch_flush_shadow_memslot(). */ - if ((old->flags & KVM_MEM_LOG_DIRTY_PAGES) && - !(new->flags & KVM_MEM_LOG_DIRTY_PAGES)) - kvm_mmu_zap_collapsible_sptes(kvm, new); + if ((change != KVM_MR_FLAGS_ONLY) || (new->flags & KVM_MEM_READONLY)) + return; /* - * Enable or disable dirty logging for the slot. - * - * For KVM_MR_DELETE and KVM_MR_MOVE, the shadow pages of the old - * slot have been zapped so no dirty logging updates are needed for - * the old slot. - * For KVM_MR_CREATE and KVM_MR_MOVE, once the new slot is visible - * any mappings that might be created in it will consume the - * properties of the new slot and do not need to be updated here. - * - * When PML is enabled, the kvm_x86_ops dirty logging hooks are - * called to enable/disable dirty logging. - * - * When disabling dirty logging with PML enabled, the D-bit is set - * for sptes in the slot in order to prevent unnecessary GPA - * logging in the PML buffer (and potential PML buffer full VMEXIT). - * This guarantees leaving PML enabled for the guest's lifetime - * won't have any additional overhead from PML when the guest is - * running with dirty logging disabled. - * - * When enabling dirty logging, large sptes are write-protected - * so they can be split on first write. New large sptes cannot - * be created for this slot until the end of the logging. - * See the comments in fast_page_fault(). - * For small sptes, nothing is done if the dirty log is in the - * initial-all-set state. Otherwise, depending on whether pml - * is enabled the D-bit or the W-bit will be cleared. + * READONLY and non-flags changes were filtered out above, and the only + * other flag is LOG_DIRTY_PAGES, i.e. something is wrong if dirty + * logging isn't being toggled on or off. */ - if (new->flags & KVM_MEM_LOG_DIRTY_PAGES) { - if (kvm_x86_ops.slot_enable_log_dirty) { - static_call(kvm_x86_slot_enable_log_dirty)(kvm, new); - } else { - int level = - kvm_dirty_log_manual_protect_and_init_set(kvm) ? - PG_LEVEL_2M : PG_LEVEL_4K; + if (WARN_ON_ONCE(!((old->flags ^ new->flags) & KVM_MEM_LOG_DIRTY_PAGES))) + return; + + if (!log_dirty_pages) { + /* + * Dirty logging tracks sptes in 4k granularity, meaning that + * large sptes have to be split. If live migration succeeds, + * the guest in the source machine will be destroyed and large + * sptes will be created in the destination. However, if the + * guest continues to run in the source machine (for example if + * live migration fails), small sptes will remain around and + * cause bad performance. + * + * Scan sptes if dirty logging has been stopped, dropping those + * which can be collapsed into a single large-page spte. Later + * page faults will create the large-page sptes. + */ + kvm_mmu_zap_collapsible_sptes(kvm, new); + } else { + /* By default, write-protect everything to log writes. */ + int level = PG_LEVEL_4K; + + if (kvm_x86_ops.cpu_dirty_log_size) { + /* + * Clear all dirty bits, unless pages are treated as + * dirty from the get-go. + */ + if (!kvm_dirty_log_manual_protect_and_init_set(kvm)) + kvm_mmu_slot_leaf_clear_dirty(kvm, new); + /* + * Write-protect large pages on write so that dirty + * logging happens at 4k granularity. No need to + * write-protect small SPTEs since write accesses are + * logged by the CPU via dirty bits. + */ + level = PG_LEVEL_2M; + } else if (kvm_dirty_log_manual_protect_and_init_set(kvm)) { /* * If we're with initial-all-set, we don't need * to write protect any small page because @@ -10825,10 +10857,9 @@ static void kvm_mmu_slot_apply_flags(struct kvm *kvm, * so that the page split can happen lazily on * the first write to the huge page. */ - kvm_mmu_slot_remove_write_access(kvm, new, level); + level = PG_LEVEL_2M; } - } else { - static_call_cond(kvm_x86_slot_disable_log_dirty)(kvm, new); + kvm_mmu_slot_remove_write_access(kvm, new, level); } } diff --git a/arch/x86/lib/insn.c b/arch/x86/lib/insn.c index 404279563891ac4db6e3f0909baa3af4f839ba54..435630a6ec9709babbe05a898825d1bdb6350b53 100644 --- a/arch/x86/lib/insn.c +++ b/arch/x86/lib/insn.c @@ -5,6 +5,7 @@ * Copyright (C) IBM Corporation, 2002, 2004, 2009 */ +#include #ifdef __KERNEL__ #include #else @@ -15,15 +16,28 @@ #include +#define leXX_to_cpu(t, r) \ +({ \ + __typeof__(t) v; \ + switch (sizeof(t)) { \ + case 4: v = le32_to_cpu(r); break; \ + case 2: v = le16_to_cpu(r); break; \ + case 1: v = r; break; \ + default: \ + BUILD_BUG(); break; \ + } \ + v; \ +}) + /* Verify next sizeof(t) bytes can be on the same instruction */ #define validate_next(t, insn, n) \ ((insn)->next_byte + sizeof(t) + n <= (insn)->end_kaddr) #define __get_next(t, insn) \ - ({ t r = *(t*)insn->next_byte; insn->next_byte += sizeof(t); r; }) + ({ t r = *(t*)insn->next_byte; insn->next_byte += sizeof(t); leXX_to_cpu(t, r); }) #define __peek_nbyte_next(t, insn, n) \ - ({ t r = *(t*)((insn)->next_byte + n); r; }) + ({ t r = *(t*)((insn)->next_byte + n); leXX_to_cpu(t, r); }) #define get_next(t, insn) \ ({ if (unlikely(!validate_next(t, insn, 0))) goto err_out; __get_next(t, insn); }) @@ -147,9 +161,9 @@ void insn_get_prefixes(struct insn *insn) b = insn->prefixes.bytes[3]; for (i = 0; i < nb; i++) if (prefixes->bytes[i] == lb) - prefixes->bytes[i] = b; + insn_set_byte(prefixes, i, b); } - insn->prefixes.bytes[3] = lb; + insn_set_byte(&insn->prefixes, 3, lb); } /* Decode REX prefix */ @@ -157,8 +171,7 @@ void insn_get_prefixes(struct insn *insn) b = peek_next(insn_byte_t, insn); attr = inat_get_opcode_attribute(b); if (inat_is_rex_prefix(attr)) { - insn->rex_prefix.value = b; - insn->rex_prefix.nbytes = 1; + insn_field_set(&insn->rex_prefix, b, 1); insn->next_byte++; if (X86_REX_W(b)) /* REX.W overrides opnd_size */ @@ -181,13 +194,13 @@ void insn_get_prefixes(struct insn *insn) if (X86_MODRM_MOD(b2) != 3) goto vex_end; } - insn->vex_prefix.bytes[0] = b; - insn->vex_prefix.bytes[1] = b2; + insn_set_byte(&insn->vex_prefix, 0, b); + insn_set_byte(&insn->vex_prefix, 1, b2); if (inat_is_evex_prefix(attr)) { b2 = peek_nbyte_next(insn_byte_t, insn, 2); - insn->vex_prefix.bytes[2] = b2; + insn_set_byte(&insn->vex_prefix, 2, b2); b2 = peek_nbyte_next(insn_byte_t, insn, 3); - insn->vex_prefix.bytes[3] = b2; + insn_set_byte(&insn->vex_prefix, 3, b2); insn->vex_prefix.nbytes = 4; insn->next_byte += 4; if (insn->x86_64 && X86_VEX_W(b2)) @@ -195,7 +208,7 @@ void insn_get_prefixes(struct insn *insn) insn->opnd_bytes = 8; } else if (inat_is_vex3_prefix(attr)) { b2 = peek_nbyte_next(insn_byte_t, insn, 2); - insn->vex_prefix.bytes[2] = b2; + insn_set_byte(&insn->vex_prefix, 2, b2); insn->vex_prefix.nbytes = 3; insn->next_byte += 3; if (insn->x86_64 && X86_VEX_W(b2)) @@ -207,7 +220,7 @@ void insn_get_prefixes(struct insn *insn) * Makes it easier to decode vex.W, vex.vvvv, * vex.L and vex.pp. Masking with 0x7f sets vex.W == 0. */ - insn->vex_prefix.bytes[2] = b2 & 0x7f; + insn_set_byte(&insn->vex_prefix, 2, b2 & 0x7f); insn->vex_prefix.nbytes = 2; insn->next_byte += 2; } @@ -243,7 +256,7 @@ void insn_get_opcode(struct insn *insn) /* Get first opcode */ op = get_next(insn_byte_t, insn); - opcode->bytes[0] = op; + insn_set_byte(opcode, 0, op); opcode->nbytes = 1; /* Check if there is VEX prefix or not */ @@ -295,8 +308,7 @@ void insn_get_modrm(struct insn *insn) if (inat_has_modrm(insn->attr)) { mod = get_next(insn_byte_t, insn); - modrm->value = mod; - modrm->nbytes = 1; + insn_field_set(modrm, mod, 1); if (inat_is_group(insn->attr)) { pfx_id = insn_last_prefix_id(insn); insn->attr = inat_get_group_attribute(mod, pfx_id, @@ -334,7 +346,7 @@ int insn_rip_relative(struct insn *insn) * For rip-relative instructions, the mod field (top 2 bits) * is zero and the r/m field (bottom 3 bits) is 0x5. */ - return (modrm->nbytes && (modrm->value & 0xc7) == 0x5); + return (modrm->nbytes && (modrm->bytes[0] & 0xc7) == 0x5); } /** @@ -353,11 +365,11 @@ void insn_get_sib(struct insn *insn) if (!insn->modrm.got) insn_get_modrm(insn); if (insn->modrm.nbytes) { - modrm = (insn_byte_t)insn->modrm.value; + modrm = insn->modrm.bytes[0]; if (insn->addr_bytes != 2 && X86_MODRM_MOD(modrm) != 3 && X86_MODRM_RM(modrm) == 4) { - insn->sib.value = get_next(insn_byte_t, insn); - insn->sib.nbytes = 1; + insn_field_set(&insn->sib, + get_next(insn_byte_t, insn), 1); } } insn->sib.got = 1; @@ -407,19 +419,18 @@ void insn_get_displacement(struct insn *insn) if (mod == 3) goto out; if (mod == 1) { - insn->displacement.value = get_next(signed char, insn); - insn->displacement.nbytes = 1; + insn_field_set(&insn->displacement, + get_next(signed char, insn), 1); } else if (insn->addr_bytes == 2) { if ((mod == 0 && rm == 6) || mod == 2) { - insn->displacement.value = - get_next(short, insn); - insn->displacement.nbytes = 2; + insn_field_set(&insn->displacement, + get_next(short, insn), 2); } } else { if ((mod == 0 && rm == 5) || mod == 2 || (mod == 0 && base == 5)) { - insn->displacement.value = get_next(int, insn); - insn->displacement.nbytes = 4; + insn_field_set(&insn->displacement, + get_next(int, insn), 4); } } } @@ -435,18 +446,14 @@ static int __get_moffset(struct insn *insn) { switch (insn->addr_bytes) { case 2: - insn->moffset1.value = get_next(short, insn); - insn->moffset1.nbytes = 2; + insn_field_set(&insn->moffset1, get_next(short, insn), 2); break; case 4: - insn->moffset1.value = get_next(int, insn); - insn->moffset1.nbytes = 4; + insn_field_set(&insn->moffset1, get_next(int, insn), 4); break; case 8: - insn->moffset1.value = get_next(int, insn); - insn->moffset1.nbytes = 4; - insn->moffset2.value = get_next(int, insn); - insn->moffset2.nbytes = 4; + insn_field_set(&insn->moffset1, get_next(int, insn), 4); + insn_field_set(&insn->moffset2, get_next(int, insn), 4); break; default: /* opnd_bytes must be modified manually */ goto err_out; @@ -464,13 +471,11 @@ static int __get_immv32(struct insn *insn) { switch (insn->opnd_bytes) { case 2: - insn->immediate.value = get_next(short, insn); - insn->immediate.nbytes = 2; + insn_field_set(&insn->immediate, get_next(short, insn), 2); break; case 4: case 8: - insn->immediate.value = get_next(int, insn); - insn->immediate.nbytes = 4; + insn_field_set(&insn->immediate, get_next(int, insn), 4); break; default: /* opnd_bytes must be modified manually */ goto err_out; @@ -487,18 +492,15 @@ static int __get_immv(struct insn *insn) { switch (insn->opnd_bytes) { case 2: - insn->immediate1.value = get_next(short, insn); - insn->immediate1.nbytes = 2; + insn_field_set(&insn->immediate1, get_next(short, insn), 2); break; case 4: - insn->immediate1.value = get_next(int, insn); + insn_field_set(&insn->immediate1, get_next(int, insn), 4); insn->immediate1.nbytes = 4; break; case 8: - insn->immediate1.value = get_next(int, insn); - insn->immediate1.nbytes = 4; - insn->immediate2.value = get_next(int, insn); - insn->immediate2.nbytes = 4; + insn_field_set(&insn->immediate1, get_next(int, insn), 4); + insn_field_set(&insn->immediate2, get_next(int, insn), 4); break; default: /* opnd_bytes must be modified manually */ goto err_out; @@ -515,12 +517,10 @@ static int __get_immptr(struct insn *insn) { switch (insn->opnd_bytes) { case 2: - insn->immediate1.value = get_next(short, insn); - insn->immediate1.nbytes = 2; + insn_field_set(&insn->immediate1, get_next(short, insn), 2); break; case 4: - insn->immediate1.value = get_next(int, insn); - insn->immediate1.nbytes = 4; + insn_field_set(&insn->immediate1, get_next(int, insn), 4); break; case 8: /* ptr16:64 is not exist (no segment) */ @@ -528,8 +528,7 @@ static int __get_immptr(struct insn *insn) default: /* opnd_bytes must be modified manually */ goto err_out; } - insn->immediate2.value = get_next(unsigned short, insn); - insn->immediate2.nbytes = 2; + insn_field_set(&insn->immediate2, get_next(unsigned short, insn), 2); insn->immediate1.got = insn->immediate2.got = 1; return 1; @@ -565,22 +564,17 @@ void insn_get_immediate(struct insn *insn) switch (inat_immediate_size(insn->attr)) { case INAT_IMM_BYTE: - insn->immediate.value = get_next(signed char, insn); - insn->immediate.nbytes = 1; + insn_field_set(&insn->immediate, get_next(signed char, insn), 1); break; case INAT_IMM_WORD: - insn->immediate.value = get_next(short, insn); - insn->immediate.nbytes = 2; + insn_field_set(&insn->immediate, get_next(short, insn), 2); break; case INAT_IMM_DWORD: - insn->immediate.value = get_next(int, insn); - insn->immediate.nbytes = 4; + insn_field_set(&insn->immediate, get_next(int, insn), 4); break; case INAT_IMM_QWORD: - insn->immediate1.value = get_next(int, insn); - insn->immediate1.nbytes = 4; - insn->immediate2.value = get_next(int, insn); - insn->immediate2.nbytes = 4; + insn_field_set(&insn->immediate1, get_next(int, insn), 4); + insn_field_set(&insn->immediate2, get_next(int, insn), 4); break; case INAT_IMM_PTR: if (!__get_immptr(insn)) @@ -599,8 +593,7 @@ void insn_get_immediate(struct insn *insn) goto err_out; } if (inat_has_second_immediate(insn->attr)) { - insn->immediate2.value = get_next(signed char, insn); - insn->immediate2.nbytes = 1; + insn_field_set(&insn->immediate2, get_next(signed char, insn), 1); } done: insn->immediate.got = 1; diff --git a/arch/x86/lib/retpoline.S b/arch/x86/lib/retpoline.S index b4c43a9b148362052a9c94f76dac3362ef20e225..f6fb1d218dccf7fe84de55591c6ed22f5ddc528f 100644 --- a/arch/x86/lib/retpoline.S +++ b/arch/x86/lib/retpoline.S @@ -28,7 +28,7 @@ SYM_FUNC_START_NOALIGN(__x86_retpoline_\reg) jmp .Lspec_trap_\@ .Ldo_rop_\@: mov %\reg, (%_ASM_SP) - UNWIND_HINT_RET_OFFSET + UNWIND_HINT_FUNC ret SYM_FUNC_END(__x86_retpoline_\reg) diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c index 525197381baa8107c513a713e867a52967e7295d..a73347e2cdfc569e7ef2dbd6a3b12ea1af5f6702 100644 --- a/arch/x86/mm/fault.c +++ b/arch/x86/mm/fault.c @@ -9,6 +9,7 @@ #include /* oops_begin/end, ... */ #include /* search_exception_tables */ #include /* max_low_pfn */ +#include /* kfence_handle_page_fault */ #include /* NOKPROBE_SYMBOL, ... */ #include /* kmmio_handler, ... */ #include /* perf_sw_event */ @@ -680,6 +681,11 @@ page_fault_oops(struct pt_regs *regs, unsigned long error_code, if (IS_ENABLED(CONFIG_EFI)) efi_crash_gracefully_on_page_fault(address); + /* Only not-present faults should be handled by KFENCE. */ + if (!(error_code & X86_PF_PROT) && + kfence_handle_page_fault(address, error_code & X86_PF_WRITE, regs)) + return; + oops: /* * Oops. The kernel tried to access some bad page. We'll have to diff --git a/arch/x86/mm/pat/memtype.c b/arch/x86/mm/pat/memtype.c index 8f665c352bf0d3a62603fce86c450e08b7029cd2..ca311aaa67b889b6985eeefdad379569432b76c8 100644 --- a/arch/x86/mm/pat/memtype.c +++ b/arch/x86/mm/pat/memtype.c @@ -1164,12 +1164,14 @@ static void *memtype_seq_start(struct seq_file *seq, loff_t *pos) static void *memtype_seq_next(struct seq_file *seq, void *v, loff_t *pos) { + kfree(v); ++*pos; return memtype_get_idx(*pos); } static void memtype_seq_stop(struct seq_file *seq, void *v) { + kfree(v); } static int memtype_seq_show(struct seq_file *seq, void *v) @@ -1181,8 +1183,6 @@ static int memtype_seq_show(struct seq_file *seq, void *v) entry_print->end, cattr_name(entry_print->type)); - kfree(entry_print); - return 0; } diff --git a/arch/x86/pci/intel_mid_pci.c b/arch/x86/pci/intel_mid_pci.c index 95e2e6bd8d8c4df78a95fe47d6613b91e55beebf..8edd6220660446842d28d9eb9ab0a495a89ccc66 100644 --- a/arch/x86/pci/intel_mid_pci.c +++ b/arch/x86/pci/intel_mid_pci.c @@ -28,10 +28,12 @@ #include #include +#include #include #include #include #include +#include #include #include @@ -140,6 +142,7 @@ static int pci_device_update_fixed(struct pci_bus *bus, unsigned int devfn, * type1_access_ok - check whether to use type 1 * @bus: bus number * @devfn: device & function in question + * @reg: configuration register offset * * If the bus is on a Lincroft chip and it exists, or is not on a Lincroft at * all, the we can go ahead with any reads & writes. If it's on a Lincroft, @@ -212,10 +215,17 @@ static int pci_write(struct pci_bus *bus, unsigned int devfn, int where, where, size, value); } +static const struct x86_cpu_id intel_mid_cpu_ids[] = { + X86_MATCH_INTEL_FAM6_MODEL(ATOM_SILVERMONT_MID, NULL), + {} +}; + static int intel_mid_pci_irq_enable(struct pci_dev *dev) { + const struct x86_cpu_id *id; struct irq_alloc_info info; bool polarity_low; + u16 model = 0; int ret; u8 gsi; @@ -228,8 +238,12 @@ static int intel_mid_pci_irq_enable(struct pci_dev *dev) return ret; } - switch (intel_mid_identify_cpu()) { - case INTEL_MID_CPU_CHIP_TANGIER: + id = x86_match_cpu(intel_mid_cpu_ids); + if (id) + model = id->model; + + switch (model) { + case INTEL_FAM6_ATOM_SILVERMONT_MID: polarity_low = false; /* Special treatment for IRQ0 */ diff --git a/arch/x86/pci/mmconfig-shared.c b/arch/x86/pci/mmconfig-shared.c index 234998f196d4def219895040d10d371d97b0dda4..de6bf0e7e8f84e12665155e430d57eb562f2dacc 100644 --- a/arch/x86/pci/mmconfig-shared.c +++ b/arch/x86/pci/mmconfig-shared.c @@ -11,9 +11,9 @@ * themselves. */ +#include #include #include -#include #include #include #include @@ -665,7 +665,7 @@ void __init pci_mmcfg_early_init(void) if (pci_mmcfg_check_hostbridge()) known_bridge = 1; else - acpi_sfi_table_parse(ACPI_SIG_MCFG, pci_parse_mcfg); + acpi_table_parse(ACPI_SIG_MCFG, pci_parse_mcfg); __pci_mmcfg_init(1); set_apei_filter(); @@ -683,7 +683,7 @@ void __init pci_mmcfg_late_init(void) /* MMCONFIG hasn't been enabled yet, try again */ if (pci_probe & PCI_PROBE_MASK & ~PCI_PROBE_MMCONF) { - acpi_sfi_table_parse(ACPI_SIG_MCFG, pci_parse_mcfg); + acpi_table_parse(ACPI_SIG_MCFG, pci_parse_mcfg); __pci_mmcfg_init(0); } } diff --git a/arch/x86/platform/Makefile b/arch/x86/platform/Makefile index b2f90a1a89f1039888decc8200b8eceaf28a382e..3ed03a2552d01294990208a17b8f1236644c43c9 100644 --- a/arch/x86/platform/Makefile +++ b/arch/x86/platform/Makefile @@ -10,6 +10,5 @@ obj-y += intel-mid/ obj-y += intel-quark/ obj-y += olpc/ obj-y += scx200/ -obj-y += sfi/ obj-y += ts5500/ obj-y += uv/ diff --git a/arch/x86/platform/intel-mid/Makefile b/arch/x86/platform/intel-mid/Makefile index cc2549f0ccb1a04653d9464b8b51933d1020cf4a..ddfc08783fb8fefdcea9157e9d893594242200f0 100644 --- a/arch/x86/platform/intel-mid/Makefile +++ b/arch/x86/platform/intel-mid/Makefile @@ -1,7 +1,2 @@ # SPDX-License-Identifier: GPL-2.0-only -obj-$(CONFIG_X86_INTEL_MID) += intel-mid.o intel_mid_vrtc.o pwr.o - -# SFI specific code -ifdef CONFIG_X86_INTEL_MID -obj-$(CONFIG_SFI) += sfi.o device_libs/ -endif +obj-$(CONFIG_X86_INTEL_MID) += intel-mid.o pwr.o diff --git a/arch/x86/platform/intel-mid/device_libs/Makefile b/arch/x86/platform/intel-mid/device_libs/Makefile deleted file mode 100644 index 480fed21cc7d41a8188135b441ae2f8bb69c3d69..0000000000000000000000000000000000000000 --- a/arch/x86/platform/intel-mid/device_libs/Makefile +++ /dev/null @@ -1,33 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 -# Family-Level Interface Shim (FLIS) -obj-$(subst m,y,$(CONFIG_PINCTRL_MERRIFIELD)) += platform_mrfld_pinctrl.o -# SDHCI Devices -obj-$(subst m,y,$(CONFIG_MMC_SDHCI_PCI)) += platform_mrfld_sd.o -# WiFi + BT -obj-$(subst m,y,$(CONFIG_BRCMFMAC_SDIO)) += platform_bcm43xx.o -obj-$(subst m,y,$(CONFIG_BT_HCIUART_BCM)) += platform_bt.o -# IPC Devices -obj-$(subst m,y,$(CONFIG_MFD_INTEL_MSIC)) += platform_msic.o -obj-$(subst m,y,$(CONFIG_SND_MFLD_MACHINE)) += platform_msic_audio.o -obj-$(subst m,y,$(CONFIG_GPIO_MSIC)) += platform_msic_gpio.o -obj-$(subst m,y,$(CONFIG_MFD_INTEL_MSIC)) += platform_msic_ocd.o -obj-$(subst m,y,$(CONFIG_MFD_INTEL_MSIC)) += platform_msic_battery.o -obj-$(subst m,y,$(CONFIG_INTEL_MID_POWER_BUTTON)) += platform_msic_power_btn.o -obj-$(subst m,y,$(CONFIG_INTEL_MFLD_THERMAL)) += platform_msic_thermal.o -# SPI Devices -obj-$(subst m,y,$(CONFIG_SPI_SPIDEV)) += platform_mrfld_spidev.o -# I2C Devices -obj-$(subst m,y,$(CONFIG_SENSORS_EMC1403)) += platform_emc1403.o -obj-$(subst m,y,$(CONFIG_SENSORS_LIS3LV02D)) += platform_lis331.o -obj-$(subst m,y,$(CONFIG_MPU3050_I2C)) += platform_mpu3050.o -obj-$(subst m,y,$(CONFIG_INPUT_BMA150)) += platform_bma023.o -obj-$(subst m,y,$(CONFIG_DRM_MEDFIELD)) += platform_tc35876x.o -# I2C GPIO Expanders -obj-$(subst m,y,$(CONFIG_GPIO_PCA953X)) += platform_max7315.o -obj-$(subst m,y,$(CONFIG_GPIO_PCA953X)) += platform_pcal9555a.o -obj-$(subst m,y,$(CONFIG_GPIO_PCA953X)) += platform_tca6416.o -# MISC Devices -obj-$(subst m,y,$(CONFIG_KEYBOARD_GPIO)) += platform_gpio_keys.o -obj-$(subst m,y,$(CONFIG_INTEL_MID_POWER_BUTTON)) += platform_mrfld_power_btn.o -obj-$(subst m,y,$(CONFIG_RTC_DRV_CMOS)) += platform_mrfld_rtc.o -obj-$(subst m,y,$(CONFIG_INTEL_MID_WATCHDOG)) += platform_mrfld_wdt.o diff --git a/arch/x86/platform/intel-mid/device_libs/platform_bcm43xx.c b/arch/x86/platform/intel-mid/device_libs/platform_bcm43xx.c deleted file mode 100644 index 564c47c53f3a406d4dd67a12ed49d845b4d042fe..0000000000000000000000000000000000000000 --- a/arch/x86/platform/intel-mid/device_libs/platform_bcm43xx.c +++ /dev/null @@ -1,101 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * platform_bcm43xx.c: bcm43xx platform data initialization file - * - * (C) Copyright 2016 Intel Corporation - * Author: Andy Shevchenko - */ - -#include -#include -#include -#include -#include - -#include - -#define WLAN_SFI_GPIO_IRQ_NAME "WLAN-interrupt" -#define WLAN_SFI_GPIO_ENABLE_NAME "WLAN-enable" - -#define WLAN_DEV_NAME "0000:00:01.3" - -static struct regulator_consumer_supply bcm43xx_vmmc_supply = { - .dev_name = WLAN_DEV_NAME, - .supply = "vmmc", -}; - -static struct regulator_init_data bcm43xx_vmmc_data = { - .constraints = { - .valid_ops_mask = REGULATOR_CHANGE_STATUS, - }, - .num_consumer_supplies = 1, - .consumer_supplies = &bcm43xx_vmmc_supply, -}; - -static struct fixed_voltage_config bcm43xx_vmmc = { - .supply_name = "bcm43xx-vmmc-regulator", - /* - * Announce 2.0V here to be compatible with SDIO specification. The - * real voltage and signaling are still 1.8V. - */ - .microvolts = 2000000, /* 1.8V */ - .startup_delay = 250 * 1000, /* 250ms */ - .enabled_at_boot = 0, /* disabled at boot */ - .init_data = &bcm43xx_vmmc_data, -}; - -static struct platform_device bcm43xx_vmmc_regulator = { - .name = "reg-fixed-voltage", - .id = PLATFORM_DEVID_AUTO, - .dev = { - .platform_data = &bcm43xx_vmmc, - }, -}; - -static struct gpiod_lookup_table bcm43xx_vmmc_gpio_table = { - .dev_id = "reg-fixed-voltage.0", - .table = { - GPIO_LOOKUP("0000:00:0c.0", -1, NULL, GPIO_ACTIVE_LOW), - {} - }, -}; - -static int __init bcm43xx_regulator_register(void) -{ - struct gpiod_lookup_table *table = &bcm43xx_vmmc_gpio_table; - struct gpiod_lookup *lookup = table->table; - int ret; - - lookup[0].chip_hwnum = get_gpio_by_name(WLAN_SFI_GPIO_ENABLE_NAME); - gpiod_add_lookup_table(table); - - ret = platform_device_register(&bcm43xx_vmmc_regulator); - if (ret) { - pr_err("%s: vmmc regulator register failed\n", __func__); - return ret; - } - - return 0; -} - -static void __init *bcm43xx_platform_data(void *info) -{ - int ret; - - ret = bcm43xx_regulator_register(); - if (ret) - return NULL; - - pr_info("Using generic wifi platform data\n"); - - /* For now it's empty */ - return NULL; -} - -static const struct devs_id bcm43xx_clk_vmmc_dev_id __initconst = { - .name = "bcm43xx_clk_vmmc", - .type = SFI_DEV_TYPE_SD, - .get_platform_data = &bcm43xx_platform_data, -}; - -sfi_device(bcm43xx_clk_vmmc_dev_id); diff --git a/arch/x86/platform/intel-mid/device_libs/platform_bma023.c b/arch/x86/platform/intel-mid/device_libs/platform_bma023.c deleted file mode 100644 index 32912a17f68e478ee11becea191a97a75facd6f7..0000000000000000000000000000000000000000 --- a/arch/x86/platform/intel-mid/device_libs/platform_bma023.c +++ /dev/null @@ -1,16 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * platform_bma023.c: bma023 platform data initialization file - * - * (C) Copyright 2013 Intel Corporation - */ - -#include - -static const struct devs_id bma023_dev_id __initconst = { - .name = "bma023", - .type = SFI_DEV_TYPE_I2C, - .delay = 1, -}; - -sfi_device(bma023_dev_id); diff --git a/arch/x86/platform/intel-mid/device_libs/platform_bt.c b/arch/x86/platform/intel-mid/device_libs/platform_bt.c deleted file mode 100644 index 2930b6e9473e12793a8225c88d69927b80f55dc5..0000000000000000000000000000000000000000 --- a/arch/x86/platform/intel-mid/device_libs/platform_bt.c +++ /dev/null @@ -1,101 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Bluetooth platform data initialization file - * - * (C) Copyright 2017 Intel Corporation - * Author: Andy Shevchenko - */ - -#include -#include -#include - -#include -#include -#include - -struct bt_sfi_data { - struct device *dev; - const char *name; - int (*setup)(struct bt_sfi_data *ddata); -}; - -static struct gpiod_lookup_table tng_bt_sfi_gpio_table = { - .dev_id = "hci_bcm", - .table = { - GPIO_LOOKUP("0000:00:0c.0", -1, "device-wakeup", GPIO_ACTIVE_HIGH), - GPIO_LOOKUP("0000:00:0c.0", -1, "shutdown", GPIO_ACTIVE_HIGH), - GPIO_LOOKUP("0000:00:0c.0", -1, "host-wakeup", GPIO_ACTIVE_HIGH), - { }, - }, -}; - -#define TNG_BT_SFI_GPIO_DEVICE_WAKEUP "bt_wakeup" -#define TNG_BT_SFI_GPIO_SHUTDOWN "BT-reset" -#define TNG_BT_SFI_GPIO_HOST_WAKEUP "bt_uart_enable" - -static int __init tng_bt_sfi_setup(struct bt_sfi_data *ddata) -{ - struct gpiod_lookup_table *table = &tng_bt_sfi_gpio_table; - struct gpiod_lookup *lookup = table->table; - struct pci_dev *pdev; - - /* Connected to /dev/ttyS0 */ - pdev = pci_get_domain_bus_and_slot(0, 0, PCI_DEVFN(4, 1)); - if (!pdev) - return -ENODEV; - - ddata->dev = &pdev->dev; - ddata->name = table->dev_id; - - lookup[0].chip_hwnum = get_gpio_by_name(TNG_BT_SFI_GPIO_DEVICE_WAKEUP); - lookup[1].chip_hwnum = get_gpio_by_name(TNG_BT_SFI_GPIO_SHUTDOWN); - lookup[2].chip_hwnum = get_gpio_by_name(TNG_BT_SFI_GPIO_HOST_WAKEUP); - - gpiod_add_lookup_table(table); - return 0; -} - -static struct bt_sfi_data tng_bt_sfi_data __initdata = { - .setup = tng_bt_sfi_setup, -}; - -static const struct x86_cpu_id bt_sfi_cpu_ids[] = { - X86_MATCH_INTEL_FAM6_MODEL(ATOM_SILVERMONT_MID, &tng_bt_sfi_data), - {} -}; - -static int __init bt_sfi_init(void) -{ - struct platform_device_info info; - struct platform_device *pdev; - const struct x86_cpu_id *id; - struct bt_sfi_data *ddata; - int ret; - - id = x86_match_cpu(bt_sfi_cpu_ids); - if (!id) - return -ENODEV; - - ddata = (struct bt_sfi_data *)id->driver_data; - if (!ddata) - return -ENODEV; - - ret = ddata->setup(ddata); - if (ret) - return ret; - - memset(&info, 0, sizeof(info)); - info.fwnode = ddata->dev->fwnode; - info.parent = ddata->dev; - info.name = ddata->name; - info.id = PLATFORM_DEVID_NONE; - - pdev = platform_device_register_full(&info); - if (IS_ERR(pdev)) - return PTR_ERR(pdev); - - dev_info(ddata->dev, "Registered Bluetooth device: %s\n", ddata->name); - return 0; -} -device_initcall(bt_sfi_init); diff --git a/arch/x86/platform/intel-mid/device_libs/platform_emc1403.c b/arch/x86/platform/intel-mid/device_libs/platform_emc1403.c deleted file mode 100644 index a2508582a0b1146b8d878367bcdecb935f095760..0000000000000000000000000000000000000000 --- a/arch/x86/platform/intel-mid/device_libs/platform_emc1403.c +++ /dev/null @@ -1,39 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * platform_emc1403.c: emc1403 platform data initialization file - * - * (C) Copyright 2013 Intel Corporation - * Author: Sathyanarayanan Kuppuswamy - */ - -#include -#include -#include -#include - -static void __init *emc1403_platform_data(void *info) -{ - static short intr2nd_pdata; - struct i2c_board_info *i2c_info = info; - int intr = get_gpio_by_name("thermal_int"); - int intr2nd = get_gpio_by_name("thermal_alert"); - - if (intr < 0) - return NULL; - if (intr2nd < 0) - return NULL; - - i2c_info->irq = intr + INTEL_MID_IRQ_OFFSET; - intr2nd_pdata = intr2nd + INTEL_MID_IRQ_OFFSET; - - return &intr2nd_pdata; -} - -static const struct devs_id emc1403_dev_id __initconst = { - .name = "emc1403", - .type = SFI_DEV_TYPE_I2C, - .delay = 1, - .get_platform_data = &emc1403_platform_data, -}; - -sfi_device(emc1403_dev_id); diff --git a/arch/x86/platform/intel-mid/device_libs/platform_gpio_keys.c b/arch/x86/platform/intel-mid/device_libs/platform_gpio_keys.c deleted file mode 100644 index d9435d2196a41e3917b55d9851eb90b6300082ec..0000000000000000000000000000000000000000 --- a/arch/x86/platform/intel-mid/device_libs/platform_gpio_keys.c +++ /dev/null @@ -1,81 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * platform_gpio_keys.c: gpio_keys platform data initialization file - * - * (C) Copyright 2013 Intel Corporation - * Author: Sathyanarayanan Kuppuswamy - */ - -#include -#include -#include -#include -#include -#include -#include - -#define DEVICE_NAME "gpio-keys" - -/* - * we will search these buttons in SFI GPIO table (by name) - * and register them dynamically. Please add all possible - * buttons here, we will shrink them if no GPIO found. - */ -static struct gpio_keys_button gpio_button[] = { - {KEY_POWER, -1, 1, "power_btn", EV_KEY, 0, 3000}, - {KEY_PROG1, -1, 1, "prog_btn1", EV_KEY, 0, 20}, - {KEY_PROG2, -1, 1, "prog_btn2", EV_KEY, 0, 20}, - {SW_LID, -1, 1, "lid_switch", EV_SW, 0, 20}, - {KEY_VOLUMEUP, -1, 1, "vol_up", EV_KEY, 0, 20}, - {KEY_VOLUMEDOWN, -1, 1, "vol_down", EV_KEY, 0, 20}, - {KEY_MUTE, -1, 1, "mute_enable", EV_KEY, 0, 20}, - {KEY_VOLUMEUP, -1, 1, "volume_up", EV_KEY, 0, 20}, - {KEY_VOLUMEDOWN, -1, 1, "volume_down", EV_KEY, 0, 20}, - {KEY_CAMERA, -1, 1, "camera_full", EV_KEY, 0, 20}, - {KEY_CAMERA_FOCUS, -1, 1, "camera_half", EV_KEY, 0, 20}, - {SW_KEYPAD_SLIDE, -1, 1, "MagSw1", EV_SW, 0, 20}, - {SW_KEYPAD_SLIDE, -1, 1, "MagSw2", EV_SW, 0, 20}, -}; - -static struct gpio_keys_platform_data gpio_keys = { - .buttons = gpio_button, - .rep = 1, - .nbuttons = -1, /* will fill it after search */ -}; - -static struct platform_device pb_device = { - .name = DEVICE_NAME, - .id = -1, - .dev = { - .platform_data = &gpio_keys, - }, -}; - -/* - * Shrink the non-existent buttons, register the gpio button - * device if there is some - */ -static int __init pb_keys_init(void) -{ - struct gpio_keys_button *gb = gpio_button; - int i, good = 0; - - for (i = 0; i < ARRAY_SIZE(gpio_button); i++) { - gb[i].gpio = get_gpio_by_name(gb[i].desc); - pr_debug("info[%2d]: name = %s, gpio = %d\n", i, gb[i].desc, - gb[i].gpio); - if (gb[i].gpio < 0) - continue; - - if (i != good) - gb[good] = gb[i]; - good++; - } - - if (good) { - gpio_keys.nbuttons = good; - return platform_device_register(&pb_device); - } - return 0; -} -late_initcall(pb_keys_init); diff --git a/arch/x86/platform/intel-mid/device_libs/platform_lis331.c b/arch/x86/platform/intel-mid/device_libs/platform_lis331.c deleted file mode 100644 index a4485cd638c690f6717400962523ace5b35f0012..0000000000000000000000000000000000000000 --- a/arch/x86/platform/intel-mid/device_libs/platform_lis331.c +++ /dev/null @@ -1,37 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * platform_lis331.c: lis331 platform data initialization file - * - * (C) Copyright 2013 Intel Corporation - * Author: Sathyanarayanan Kuppuswamy - */ - -#include -#include -#include - -static void __init *lis331dl_platform_data(void *info) -{ - static short intr2nd_pdata; - struct i2c_board_info *i2c_info = info; - int intr = get_gpio_by_name("accel_int"); - int intr2nd = get_gpio_by_name("accel_2"); - - if (intr < 0) - return NULL; - if (intr2nd < 0) - return NULL; - - i2c_info->irq = intr + INTEL_MID_IRQ_OFFSET; - intr2nd_pdata = intr2nd + INTEL_MID_IRQ_OFFSET; - - return &intr2nd_pdata; -} - -static const struct devs_id lis331dl_dev_id __initconst = { - .name = "i2c_accel", - .type = SFI_DEV_TYPE_I2C, - .get_platform_data = &lis331dl_platform_data, -}; - -sfi_device(lis331dl_dev_id); diff --git a/arch/x86/platform/intel-mid/device_libs/platform_max7315.c b/arch/x86/platform/intel-mid/device_libs/platform_max7315.c deleted file mode 100644 index e9287c3184da45f7a42cb748d5b252f168828b6c..0000000000000000000000000000000000000000 --- a/arch/x86/platform/intel-mid/device_libs/platform_max7315.c +++ /dev/null @@ -1,77 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * platform_max7315.c: max7315 platform data initialization file - * - * (C) Copyright 2013 Intel Corporation - * Author: Sathyanarayanan Kuppuswamy - */ - -#include -#include -#include -#include -#include - -#define MAX7315_NUM 2 - -static void __init *max7315_platform_data(void *info) -{ - static struct pca953x_platform_data max7315_pdata[MAX7315_NUM]; - static int nr; - struct pca953x_platform_data *max7315 = &max7315_pdata[nr]; - struct i2c_board_info *i2c_info = info; - int gpio_base, intr; - char base_pin_name[SFI_NAME_LEN + 1]; - char intr_pin_name[SFI_NAME_LEN + 1]; - - if (nr == MAX7315_NUM) { - pr_err("too many max7315s, we only support %d\n", - MAX7315_NUM); - return NULL; - } - /* we have several max7315 on the board, we only need load several - * instances of the same pca953x driver to cover them - */ - strcpy(i2c_info->type, "max7315"); - if (nr++) { - snprintf(base_pin_name, sizeof(base_pin_name), - "max7315_%d_base", nr); - snprintf(intr_pin_name, sizeof(intr_pin_name), - "max7315_%d_int", nr); - } else { - strcpy(base_pin_name, "max7315_base"); - strcpy(intr_pin_name, "max7315_int"); - } - - gpio_base = get_gpio_by_name(base_pin_name); - intr = get_gpio_by_name(intr_pin_name); - - if (gpio_base < 0) - return NULL; - max7315->gpio_base = gpio_base; - if (intr != -1) { - i2c_info->irq = intr + INTEL_MID_IRQ_OFFSET; - max7315->irq_base = gpio_base + INTEL_MID_IRQ_OFFSET; - } else { - i2c_info->irq = -1; - max7315->irq_base = -1; - } - return max7315; -} - -static const struct devs_id max7315_dev_id __initconst = { - .name = "i2c_max7315", - .type = SFI_DEV_TYPE_I2C, - .delay = 1, - .get_platform_data = &max7315_platform_data, -}; - -static const struct devs_id max7315_2_dev_id __initconst = { - .name = "i2c_max7315_2", - .type = SFI_DEV_TYPE_I2C, - .delay = 1, - .get_platform_data = &max7315_platform_data, -}; - -sfi_device(max7315_dev_id); -sfi_device(max7315_2_dev_id); diff --git a/arch/x86/platform/intel-mid/device_libs/platform_mpu3050.c b/arch/x86/platform/intel-mid/device_libs/platform_mpu3050.c deleted file mode 100644 index 28a1827139340f72b342c609befb0eb47533a1da..0000000000000000000000000000000000000000 --- a/arch/x86/platform/intel-mid/device_libs/platform_mpu3050.c +++ /dev/null @@ -1,32 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * platform_mpu3050.c: mpu3050 platform data initialization file - * - * (C) Copyright 2013 Intel Corporation - * Author: Sathyanarayanan Kuppuswamy - */ - -#include -#include -#include - -static void *mpu3050_platform_data(void *info) -{ - struct i2c_board_info *i2c_info = info; - int intr = get_gpio_by_name("mpu3050_int"); - - if (intr < 0) - return NULL; - - i2c_info->irq = intr + INTEL_MID_IRQ_OFFSET; - return NULL; -} - -static const struct devs_id mpu3050_dev_id __initconst = { - .name = "mpu3050", - .type = SFI_DEV_TYPE_I2C, - .delay = 1, - .get_platform_data = &mpu3050_platform_data, -}; - -sfi_device(mpu3050_dev_id); diff --git a/arch/x86/platform/intel-mid/device_libs/platform_mrfld_pinctrl.c b/arch/x86/platform/intel-mid/device_libs/platform_mrfld_pinctrl.c deleted file mode 100644 index 605e1f94ad895f4b21ff43ff00ecfb0adb6dcacf..0000000000000000000000000000000000000000 --- a/arch/x86/platform/intel-mid/device_libs/platform_mrfld_pinctrl.c +++ /dev/null @@ -1,39 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Intel Merrifield FLIS platform device initialization file - * - * Copyright (C) 2016, Intel Corporation - * - * Author: Andy Shevchenko - */ - -#include -#include -#include - -#include - -#define FLIS_BASE_ADDR 0xff0c0000 -#define FLIS_LENGTH 0x8000 - -static struct resource mrfld_pinctrl_mmio_resource = { - .start = FLIS_BASE_ADDR, - .end = FLIS_BASE_ADDR + FLIS_LENGTH - 1, - .flags = IORESOURCE_MEM, -}; - -static struct platform_device mrfld_pinctrl_device = { - .name = "pinctrl-merrifield", - .id = PLATFORM_DEVID_NONE, - .resource = &mrfld_pinctrl_mmio_resource, - .num_resources = 1, -}; - -static int __init mrfld_pinctrl_init(void) -{ - if (intel_mid_identify_cpu() == INTEL_MID_CPU_CHIP_TANGIER) - return platform_device_register(&mrfld_pinctrl_device); - - return -ENODEV; -} -arch_initcall(mrfld_pinctrl_init); diff --git a/arch/x86/platform/intel-mid/device_libs/platform_mrfld_power_btn.c b/arch/x86/platform/intel-mid/device_libs/platform_mrfld_power_btn.c deleted file mode 100644 index ec2afb41b34a9e03c9d618b4677ed81e0d027b2a..0000000000000000000000000000000000000000 --- a/arch/x86/platform/intel-mid/device_libs/platform_mrfld_power_btn.c +++ /dev/null @@ -1,78 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Intel Merrifield power button support - * - * (C) Copyright 2017 Intel Corporation - * - * Author: Andy Shevchenko - */ - -#include -#include -#include -#include - -#include -#include - -static struct resource mrfld_power_btn_resources[] = { - { - .flags = IORESOURCE_IRQ, - }, -}; - -static struct platform_device mrfld_power_btn_dev = { - .name = "msic_power_btn", - .id = PLATFORM_DEVID_NONE, - .num_resources = ARRAY_SIZE(mrfld_power_btn_resources), - .resource = mrfld_power_btn_resources, -}; - -static int mrfld_power_btn_scu_status_change(struct notifier_block *nb, - unsigned long code, void *data) -{ - if (code == SCU_DOWN) { - platform_device_unregister(&mrfld_power_btn_dev); - return 0; - } - - return platform_device_register(&mrfld_power_btn_dev); -} - -static struct notifier_block mrfld_power_btn_scu_notifier = { - .notifier_call = mrfld_power_btn_scu_status_change, -}; - -static int __init register_mrfld_power_btn(void) -{ - if (intel_mid_identify_cpu() != INTEL_MID_CPU_CHIP_TANGIER) - return -ENODEV; - - /* - * We need to be sure that the SCU IPC is ready before - * PMIC power button device can be registered: - */ - intel_scu_notifier_add(&mrfld_power_btn_scu_notifier); - - return 0; -} -arch_initcall(register_mrfld_power_btn); - -static void __init *mrfld_power_btn_platform_data(void *info) -{ - struct resource *res = mrfld_power_btn_resources; - struct sfi_device_table_entry *pentry = info; - - res->start = res->end = pentry->irq; - return NULL; -} - -static const struct devs_id mrfld_power_btn_dev_id __initconst = { - .name = "bcove_power_btn", - .type = SFI_DEV_TYPE_IPC, - .delay = 1, - .msic = 1, - .get_platform_data = &mrfld_power_btn_platform_data, -}; - -sfi_device(mrfld_power_btn_dev_id); diff --git a/arch/x86/platform/intel-mid/device_libs/platform_mrfld_rtc.c b/arch/x86/platform/intel-mid/device_libs/platform_mrfld_rtc.c deleted file mode 100644 index 40e9808a9634a466fce80698a2a409248fe55cf3..0000000000000000000000000000000000000000 --- a/arch/x86/platform/intel-mid/device_libs/platform_mrfld_rtc.c +++ /dev/null @@ -1,44 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Intel Merrifield legacy RTC initialization file - * - * (C) Copyright 2017 Intel Corporation - * - * Author: Andy Shevchenko - */ - -#include - -#include -#include -#include -#include -#include - -static int __init mrfld_legacy_rtc_alloc_irq(void) -{ - struct irq_alloc_info info; - int ret; - - if (!x86_platform.legacy.rtc) - return -ENODEV; - - ioapic_set_alloc_attr(&info, NUMA_NO_NODE, 1, 0); - ret = mp_map_gsi_to_irq(RTC_IRQ, IOAPIC_MAP_ALLOC, &info); - if (ret < 0) { - pr_info("Failed to allocate RTC interrupt. Disabling RTC\n"); - x86_platform.legacy.rtc = 0; - return ret; - } - - return 0; -} - -static int __init mrfld_legacy_rtc_init(void) -{ - if (intel_mid_identify_cpu() != INTEL_MID_CPU_CHIP_TANGIER) - return -ENODEV; - - return mrfld_legacy_rtc_alloc_irq(); -} -arch_initcall(mrfld_legacy_rtc_init); diff --git a/arch/x86/platform/intel-mid/device_libs/platform_mrfld_sd.c b/arch/x86/platform/intel-mid/device_libs/platform_mrfld_sd.c deleted file mode 100644 index fe3b7ff975f37a5bcadd064131934e29ed9450bc..0000000000000000000000000000000000000000 --- a/arch/x86/platform/intel-mid/device_libs/platform_mrfld_sd.c +++ /dev/null @@ -1,43 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * SDHCI platform data initilisation file - * - * (C) Copyright 2016 Intel Corporation - * Author: Andy Shevchenko - */ - -#include -#include - -#include - -#include - -#define INTEL_MRFLD_SD 2 -#define INTEL_MRFLD_SD_CD_GPIO 77 - -static struct sdhci_pci_data mrfld_sdhci_pci_data = { - .rst_n_gpio = -EINVAL, - .cd_gpio = INTEL_MRFLD_SD_CD_GPIO, -}; - -static struct sdhci_pci_data * -mrfld_sdhci_pci_get_data(struct pci_dev *pdev, int slotno) -{ - unsigned int func = PCI_FUNC(pdev->devfn); - - if (func == INTEL_MRFLD_SD) - return &mrfld_sdhci_pci_data; - - return NULL; -} - -static int __init mrfld_sd_init(void) -{ - if (intel_mid_identify_cpu() != INTEL_MID_CPU_CHIP_TANGIER) - return -ENODEV; - - sdhci_pci_get_data = mrfld_sdhci_pci_get_data; - return 0; -} -arch_initcall(mrfld_sd_init); diff --git a/arch/x86/platform/intel-mid/device_libs/platform_mrfld_spidev.c b/arch/x86/platform/intel-mid/device_libs/platform_mrfld_spidev.c deleted file mode 100644 index b828f4fd40beb8455a4ad8d9bdfbc4898b286e3d..0000000000000000000000000000000000000000 --- a/arch/x86/platform/intel-mid/device_libs/platform_mrfld_spidev.c +++ /dev/null @@ -1,50 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * spidev platform data initialization file - * - * (C) Copyright 2014, 2016 Intel Corporation - * Authors: Andy Shevchenko - * Dan O'Donovan - */ - -#include -#include -#include -#include -#include - -#include - -#define MRFLD_SPI_DEFAULT_DMA_BURST 8 -#define MRFLD_SPI_DEFAULT_TIMEOUT 500 - -/* GPIO pin for spidev chipselect */ -#define MRFLD_SPIDEV_GPIO_CS 111 - -static struct pxa2xx_spi_chip spidev_spi_chip = { - .dma_burst_size = MRFLD_SPI_DEFAULT_DMA_BURST, - .timeout = MRFLD_SPI_DEFAULT_TIMEOUT, - .gpio_cs = MRFLD_SPIDEV_GPIO_CS, -}; - -static void __init *spidev_platform_data(void *info) -{ - struct spi_board_info *spi_info = info; - - if (intel_mid_identify_cpu() != INTEL_MID_CPU_CHIP_TANGIER) - return ERR_PTR(-ENODEV); - - spi_info->mode = SPI_MODE_0; - spi_info->controller_data = &spidev_spi_chip; - - return NULL; -} - -static const struct devs_id spidev_dev_id __initconst = { - .name = "spidev", - .type = SFI_DEV_TYPE_SPI, - .delay = 0, - .get_platform_data = &spidev_platform_data, -}; - -sfi_device(spidev_dev_id); diff --git a/arch/x86/platform/intel-mid/device_libs/platform_msic.c b/arch/x86/platform/intel-mid/device_libs/platform_msic.c deleted file mode 100644 index b17783d0d4e7de9bbc7a744618d768fe0ff3fbac..0000000000000000000000000000000000000000 --- a/arch/x86/platform/intel-mid/device_libs/platform_msic.c +++ /dev/null @@ -1,83 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * platform_msic.c: MSIC platform data initialization file - * - * (C) Copyright 2013 Intel Corporation - * Author: Sathyanarayanan Kuppuswamy - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include "platform_msic.h" - -struct intel_msic_platform_data msic_pdata; - -static struct resource msic_resources[] = { - { - .start = INTEL_MSIC_IRQ_PHYS_BASE, - .end = INTEL_MSIC_IRQ_PHYS_BASE + 64 - 1, - .flags = IORESOURCE_MEM, - }, -}; - -static struct platform_device msic_device = { - .name = "intel_msic", - .id = -1, - .dev = { - .platform_data = &msic_pdata, - }, - .num_resources = ARRAY_SIZE(msic_resources), - .resource = msic_resources, -}; - -static int msic_scu_status_change(struct notifier_block *nb, - unsigned long code, void *data) -{ - if (code == SCU_DOWN) { - platform_device_unregister(&msic_device); - return 0; - } - - return platform_device_register(&msic_device); -} - -static int __init msic_init(void) -{ - static struct notifier_block msic_scu_notifier = { - .notifier_call = msic_scu_status_change, - }; - - /* - * We need to be sure that the SCU IPC is ready before MSIC device - * can be registered. - */ - if (intel_mid_has_msic()) - intel_scu_notifier_add(&msic_scu_notifier); - - return 0; -} -arch_initcall(msic_init); - -/* - * msic_generic_platform_data - sets generic platform data for the block - * @info: pointer to the SFI device table entry for this block - * @block: MSIC block - * - * Function sets IRQ number from the SFI table entry for given device to - * the MSIC platform data. - */ -void *msic_generic_platform_data(void *info, enum intel_msic_block block) -{ - struct sfi_device_table_entry *entry = info; - - BUG_ON(block < 0 || block >= INTEL_MSIC_BLOCK_LAST); - msic_pdata.irq[block] = entry->irq; - - return NULL; -} diff --git a/arch/x86/platform/intel-mid/device_libs/platform_msic.h b/arch/x86/platform/intel-mid/device_libs/platform_msic.h deleted file mode 100644 index 91deb2e65b0e312bc5cf84bb5d472ce313045c8e..0000000000000000000000000000000000000000 --- a/arch/x86/platform/intel-mid/device_libs/platform_msic.h +++ /dev/null @@ -1,15 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * platform_msic.h: MSIC platform data header file - * - * (C) Copyright 2013 Intel Corporation - * Author: Sathyanarayanan Kuppuswamy - */ -#ifndef _PLATFORM_MSIC_H_ -#define _PLATFORM_MSIC_H_ - -extern struct intel_msic_platform_data msic_pdata; - -void *msic_generic_platform_data(void *info, enum intel_msic_block block); - -#endif diff --git a/arch/x86/platform/intel-mid/device_libs/platform_msic_audio.c b/arch/x86/platform/intel-mid/device_libs/platform_msic_audio.c deleted file mode 100644 index e765da78ad8c0f9248f91792ef15bb416936a069..0000000000000000000000000000000000000000 --- a/arch/x86/platform/intel-mid/device_libs/platform_msic_audio.c +++ /dev/null @@ -1,42 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * platform_msic_audio.c: MSIC audio platform data initialization file - * - * (C) Copyright 2013 Intel Corporation - * Author: Sathyanarayanan Kuppuswamy - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "platform_msic.h" - -static void *msic_audio_platform_data(void *info) -{ - struct platform_device *pdev; - - pdev = platform_device_register_simple("sst-platform", -1, NULL, 0); - - if (IS_ERR(pdev)) { - pr_err("failed to create audio platform device\n"); - return NULL; - } - - return msic_generic_platform_data(info, INTEL_MSIC_BLOCK_AUDIO); -} - -static const struct devs_id msic_audio_dev_id __initconst = { - .name = "msic_audio", - .type = SFI_DEV_TYPE_IPC, - .delay = 1, - .msic = 1, - .get_platform_data = &msic_audio_platform_data, -}; - -sfi_device(msic_audio_dev_id); diff --git a/arch/x86/platform/intel-mid/device_libs/platform_msic_battery.c b/arch/x86/platform/intel-mid/device_libs/platform_msic_battery.c deleted file mode 100644 index f461f84903f8b0d4620aceeaba49d363f9933c10..0000000000000000000000000000000000000000 --- a/arch/x86/platform/intel-mid/device_libs/platform_msic_battery.c +++ /dev/null @@ -1,32 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * platform_msic_battery.c: MSIC battery platform data initialization file - * - * (C) Copyright 2013 Intel Corporation - * Author: Sathyanarayanan Kuppuswamy - */ - -#include -#include -#include -#include -#include -#include -#include - -#include "platform_msic.h" - -static void __init *msic_battery_platform_data(void *info) -{ - return msic_generic_platform_data(info, INTEL_MSIC_BLOCK_BATTERY); -} - -static const struct devs_id msic_battery_dev_id __initconst = { - .name = "msic_battery", - .type = SFI_DEV_TYPE_IPC, - .delay = 1, - .msic = 1, - .get_platform_data = &msic_battery_platform_data, -}; - -sfi_device(msic_battery_dev_id); diff --git a/arch/x86/platform/intel-mid/device_libs/platform_msic_gpio.c b/arch/x86/platform/intel-mid/device_libs/platform_msic_gpio.c deleted file mode 100644 index 71a7d6db38787932e35a34a6da1f1e7c4e37c20e..0000000000000000000000000000000000000000 --- a/arch/x86/platform/intel-mid/device_libs/platform_msic_gpio.c +++ /dev/null @@ -1,43 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * platform_msic_gpio.c: MSIC GPIO platform data initialization file - * - * (C) Copyright 2013 Intel Corporation - * Author: Sathyanarayanan Kuppuswamy - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "platform_msic.h" - -static void __init *msic_gpio_platform_data(void *info) -{ - static struct intel_msic_gpio_pdata msic_gpio_pdata; - - int gpio = get_gpio_by_name("msic_gpio_base"); - - if (gpio < 0) - return NULL; - - msic_gpio_pdata.gpio_base = gpio; - msic_pdata.gpio = &msic_gpio_pdata; - - return msic_generic_platform_data(info, INTEL_MSIC_BLOCK_GPIO); -} - -static const struct devs_id msic_gpio_dev_id __initconst = { - .name = "msic_gpio", - .type = SFI_DEV_TYPE_IPC, - .delay = 1, - .msic = 1, - .get_platform_data = &msic_gpio_platform_data, -}; - -sfi_device(msic_gpio_dev_id); diff --git a/arch/x86/platform/intel-mid/device_libs/platform_msic_ocd.c b/arch/x86/platform/intel-mid/device_libs/platform_msic_ocd.c deleted file mode 100644 index 558c0d97443071e9462c6582232176395300848a..0000000000000000000000000000000000000000 --- a/arch/x86/platform/intel-mid/device_libs/platform_msic_ocd.c +++ /dev/null @@ -1,44 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * platform_msic_ocd.c: MSIC OCD platform data initialization file - * - * (C) Copyright 2013 Intel Corporation - * Author: Sathyanarayanan Kuppuswamy - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "platform_msic.h" - -static void __init *msic_ocd_platform_data(void *info) -{ - static struct intel_msic_ocd_pdata msic_ocd_pdata; - int gpio; - - gpio = get_gpio_by_name("ocd_gpio"); - - if (gpio < 0) - return NULL; - - msic_ocd_pdata.gpio = gpio; - msic_pdata.ocd = &msic_ocd_pdata; - - return msic_generic_platform_data(info, INTEL_MSIC_BLOCK_OCD); -} - -static const struct devs_id msic_ocd_dev_id __initconst = { - .name = "msic_ocd", - .type = SFI_DEV_TYPE_IPC, - .delay = 1, - .msic = 1, - .get_platform_data = &msic_ocd_platform_data, -}; - -sfi_device(msic_ocd_dev_id); diff --git a/arch/x86/platform/intel-mid/device_libs/platform_msic_power_btn.c b/arch/x86/platform/intel-mid/device_libs/platform_msic_power_btn.c deleted file mode 100644 index 3d3de2d5972689b640e1e959e3433b14c5f83e37..0000000000000000000000000000000000000000 --- a/arch/x86/platform/intel-mid/device_libs/platform_msic_power_btn.c +++ /dev/null @@ -1,31 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * platform_msic_power_btn.c: MSIC power btn platform data initialization file - * - * (C) Copyright 2013 Intel Corporation - * Author: Sathyanarayanan Kuppuswamy - */ -#include -#include -#include -#include -#include -#include -#include - -#include "platform_msic.h" - -static void __init *msic_power_btn_platform_data(void *info) -{ - return msic_generic_platform_data(info, INTEL_MSIC_BLOCK_POWER_BTN); -} - -static const struct devs_id msic_power_btn_dev_id __initconst = { - .name = "msic_power_btn", - .type = SFI_DEV_TYPE_IPC, - .delay = 1, - .msic = 1, - .get_platform_data = &msic_power_btn_platform_data, -}; - -sfi_device(msic_power_btn_dev_id); diff --git a/arch/x86/platform/intel-mid/device_libs/platform_msic_thermal.c b/arch/x86/platform/intel-mid/device_libs/platform_msic_thermal.c deleted file mode 100644 index 4858da1d78c67fd6ca6931c5f3944eb0e15424fa..0000000000000000000000000000000000000000 --- a/arch/x86/platform/intel-mid/device_libs/platform_msic_thermal.c +++ /dev/null @@ -1,32 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * platform_msic_thermal.c: msic_thermal platform data initialization file - * - * (C) Copyright 2013 Intel Corporation - * Author: Sathyanarayanan Kuppuswamy - */ - -#include -#include -#include -#include -#include -#include -#include - -#include "platform_msic.h" - -static void __init *msic_thermal_platform_data(void *info) -{ - return msic_generic_platform_data(info, INTEL_MSIC_BLOCK_THERMAL); -} - -static const struct devs_id msic_thermal_dev_id __initconst = { - .name = "msic_thermal", - .type = SFI_DEV_TYPE_IPC, - .delay = 1, - .msic = 1, - .get_platform_data = &msic_thermal_platform_data, -}; - -sfi_device(msic_thermal_dev_id); diff --git a/arch/x86/platform/intel-mid/device_libs/platform_pcal9555a.c b/arch/x86/platform/intel-mid/device_libs/platform_pcal9555a.c deleted file mode 100644 index 5609d8da3978ef39c51340468bbdaac50f79c57e..0000000000000000000000000000000000000000 --- a/arch/x86/platform/intel-mid/device_libs/platform_pcal9555a.c +++ /dev/null @@ -1,95 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * PCAL9555a platform data initialization file - * - * Copyright (C) 2016, Intel Corporation - * - * Authors: Andy Shevchenko - * Dan O'Donovan - */ - -#include -#include -#include -#include -#include - -#include - -#define PCAL9555A_NUM 4 - -static struct pca953x_platform_data pcal9555a_pdata[PCAL9555A_NUM]; -static int nr; - -static void __init *pcal9555a_platform_data(void *info) -{ - struct i2c_board_info *i2c_info = info; - char *type = i2c_info->type; - struct pca953x_platform_data *pcal9555a; - char base_pin_name[SFI_NAME_LEN + 1]; - char intr_pin_name[SFI_NAME_LEN + 1]; - int gpio_base, intr; - - snprintf(base_pin_name, sizeof(base_pin_name), "%s_base", type); - snprintf(intr_pin_name, sizeof(intr_pin_name), "%s_int", type); - - gpio_base = get_gpio_by_name(base_pin_name); - intr = get_gpio_by_name(intr_pin_name); - - /* Check if the SFI record valid */ - if (gpio_base == -1) - return NULL; - - if (nr >= PCAL9555A_NUM) { - pr_err("%s: Too many instances, only %d supported\n", __func__, - PCAL9555A_NUM); - return NULL; - } - - pcal9555a = &pcal9555a_pdata[nr++]; - pcal9555a->gpio_base = gpio_base; - - if (intr >= 0) { - i2c_info->irq = intr + INTEL_MID_IRQ_OFFSET; - pcal9555a->irq_base = gpio_base + INTEL_MID_IRQ_OFFSET; - } else { - i2c_info->irq = -1; - pcal9555a->irq_base = -1; - } - - strcpy(type, "pcal9555a"); - return pcal9555a; -} - -static const struct devs_id pcal9555a_1_dev_id __initconst = { - .name = "pcal9555a-1", - .type = SFI_DEV_TYPE_I2C, - .delay = 1, - .get_platform_data = &pcal9555a_platform_data, -}; - -static const struct devs_id pcal9555a_2_dev_id __initconst = { - .name = "pcal9555a-2", - .type = SFI_DEV_TYPE_I2C, - .delay = 1, - .get_platform_data = &pcal9555a_platform_data, -}; - -static const struct devs_id pcal9555a_3_dev_id __initconst = { - .name = "pcal9555a-3", - .type = SFI_DEV_TYPE_I2C, - .delay = 1, - .get_platform_data = &pcal9555a_platform_data, -}; - -static const struct devs_id pcal9555a_4_dev_id __initconst = { - .name = "pcal9555a-4", - .type = SFI_DEV_TYPE_I2C, - .delay = 1, - .get_platform_data = &pcal9555a_platform_data, -}; - -sfi_device(pcal9555a_1_dev_id); -sfi_device(pcal9555a_2_dev_id); -sfi_device(pcal9555a_3_dev_id); -sfi_device(pcal9555a_4_dev_id); diff --git a/arch/x86/platform/intel-mid/device_libs/platform_tc35876x.c b/arch/x86/platform/intel-mid/device_libs/platform_tc35876x.c deleted file mode 100644 index 139738bbdd36e50ef660ae247aceb2c7a897b904..0000000000000000000000000000000000000000 --- a/arch/x86/platform/intel-mid/device_libs/platform_tc35876x.c +++ /dev/null @@ -1,42 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * platform_tc35876x.c: tc35876x platform data initialization file - * - * (C) Copyright 2013 Intel Corporation - * Author: Sathyanarayanan Kuppuswamy - */ - -#include -#include - -static struct gpiod_lookup_table tc35876x_gpio_table = { - .dev_id = "i2c_disp_brig", - .table = { - GPIO_LOOKUP("0000:00:0c.0", -1, "bridge-reset", GPIO_ACTIVE_HIGH), - GPIO_LOOKUP("0000:00:0c.0", -1, "bl-en", GPIO_ACTIVE_HIGH), - GPIO_LOOKUP("0000:00:0c.0", -1, "vadd", GPIO_ACTIVE_HIGH), - { }, - }, -}; - -/*tc35876x DSI_LVDS bridge chip and panel platform data*/ -static void *tc35876x_platform_data(void *data) -{ - struct gpiod_lookup_table *table = &tc35876x_gpio_table; - struct gpiod_lookup *lookup = table->table; - - lookup[0].chip_hwnum = get_gpio_by_name("LCMB_RXEN"); - lookup[1].chip_hwnum = get_gpio_by_name("6S6P_BL_EN"); - lookup[2].chip_hwnum = get_gpio_by_name("EN_VREG_LCD_V3P3"); - gpiod_add_lookup_table(table); - - return NULL; -} - -static const struct devs_id tc35876x_dev_id __initconst = { - .name = "i2c_disp_brig", - .type = SFI_DEV_TYPE_I2C, - .get_platform_data = &tc35876x_platform_data, -}; - -sfi_device(tc35876x_dev_id); diff --git a/arch/x86/platform/intel-mid/device_libs/platform_tca6416.c b/arch/x86/platform/intel-mid/device_libs/platform_tca6416.c deleted file mode 100644 index e689d8f61059e6518657eee506d5229def17e5a7..0000000000000000000000000000000000000000 --- a/arch/x86/platform/intel-mid/device_libs/platform_tca6416.c +++ /dev/null @@ -1,53 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * platform_tca6416.c: tca6416 platform data initialization file - * - * (C) Copyright 2013 Intel Corporation - * Author: Sathyanarayanan Kuppuswamy - */ - -#include -#include -#include -#include - -#define TCA6416_NAME "tca6416" -#define TCA6416_BASE "tca6416_base" -#define TCA6416_INTR "tca6416_int" - -static void *tca6416_platform_data(void *info) -{ - static struct pca953x_platform_data tca6416; - struct i2c_board_info *i2c_info = info; - int gpio_base, intr; - char base_pin_name[SFI_NAME_LEN + 1]; - char intr_pin_name[SFI_NAME_LEN + 1]; - - strcpy(i2c_info->type, TCA6416_NAME); - strcpy(base_pin_name, TCA6416_BASE); - strcpy(intr_pin_name, TCA6416_INTR); - - gpio_base = get_gpio_by_name(base_pin_name); - intr = get_gpio_by_name(intr_pin_name); - - if (gpio_base < 0) - return NULL; - tca6416.gpio_base = gpio_base; - if (intr >= 0) { - i2c_info->irq = intr + INTEL_MID_IRQ_OFFSET; - tca6416.irq_base = gpio_base + INTEL_MID_IRQ_OFFSET; - } else { - i2c_info->irq = -1; - tca6416.irq_base = -1; - } - return &tca6416; -} - -static const struct devs_id tca6416_dev_id __initconst = { - .name = "tca6416", - .type = SFI_DEV_TYPE_I2C, - .delay = 1, - .get_platform_data = &tca6416_platform_data, -}; - -sfi_device(tca6416_dev_id); diff --git a/arch/x86/platform/intel-mid/intel-mid.c b/arch/x86/platform/intel-mid/intel-mid.c index 780728161f7dcde31ffee9ba5a42e08b5b6e4f3d..f4592dc7a1c193b4cbfad2a83ca237a6061b2f67 100644 --- a/arch/x86/platform/intel-mid/intel-mid.c +++ b/arch/x86/platform/intel-mid/intel-mid.c @@ -1,8 +1,8 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * intel-mid.c: Intel MID platform setup code + * Intel MID platform setup code * - * (C) Copyright 2008, 2012 Intel Corporation + * (C) Copyright 2008, 2012, 2021 Intel Corporation * Author: Jacob Pan (jacob.jun.pan@intel.com) * Author: Sathyanarayanan Kuppuswamy */ @@ -14,7 +14,6 @@ #include #include #include -#include #include #include #include @@ -25,38 +24,13 @@ #include #include #include -#include #include #include #include -#include #include -/* - * the clockevent devices on Moorestown/Medfield can be APBT or LAPIC clock, - * cmdline option x86_intel_mid_timer can be used to override the configuration - * to prefer one or the other. - * at runtime, there are basically three timer configurations: - * 1. per cpu apbt clock only - * 2. per cpu always-on lapic clocks only, this is Penwell/Medfield only - * 3. per cpu lapic clock (C3STOP) and one apbt clock, with broadcast. - * - * by default (without cmdline option), platform code first detects cpu type - * to see if we are on lincroft or penwell, then set up both lapic or apbt - * clocks accordingly. - * i.e. by default, medfield uses configuration #2, moorestown uses #1. - * config #3 is supported but not recommended on medfield. - * - * rating and feature summary: - * lapic (with C3STOP) --------- 100 - * apbt (always-on) ------------ 110 - * lapic (always-on,ARAT) ------ 150 - */ - -enum intel_mid_timer_options intel_mid_timer_options; - -enum intel_mid_cpu_type __intel_mid_cpu_chip; -EXPORT_SYMBOL_GPL(__intel_mid_cpu_chip); +#define IPCMSG_COLD_OFF 0x80 /* Only for Tangier */ +#define IPCMSG_COLD_RESET 0xF1 static void intel_mid_power_off(void) { @@ -64,69 +38,32 @@ static void intel_mid_power_off(void) intel_mid_pwr_power_off(); /* Only for Tangier, the rest will ignore this command */ - intel_scu_ipc_simple_command(IPCMSG_COLD_OFF, 1); + intel_scu_ipc_dev_simple_command(NULL, IPCMSG_COLD_OFF, 1); }; static void intel_mid_reboot(void) { - intel_scu_ipc_simple_command(IPCMSG_COLD_RESET, 0); -} - -static void __init intel_mid_setup_bp_timer(void) -{ - apbt_time_init(); - setup_boot_APIC_clock(); + intel_scu_ipc_dev_simple_command(NULL, IPCMSG_COLD_RESET, 0); } static void __init intel_mid_time_init(void) { - sfi_table_parse(SFI_SIG_MTMR, NULL, NULL, sfi_parse_mtmr); - - switch (intel_mid_timer_options) { - case INTEL_MID_TIMER_APBT_ONLY: - break; - case INTEL_MID_TIMER_LAPIC_APBT: - /* Use apbt and local apic */ - x86_init.timers.setup_percpu_clockev = intel_mid_setup_bp_timer; - x86_cpuinit.setup_percpu_clockev = setup_secondary_APIC_clock; - return; - default: - if (!boot_cpu_has(X86_FEATURE_ARAT)) - break; - /* Lapic only, no apbt */ - x86_init.timers.setup_percpu_clockev = setup_boot_APIC_clock; - x86_cpuinit.setup_percpu_clockev = setup_secondary_APIC_clock; - return; - } - - x86_init.timers.setup_percpu_clockev = apbt_time_init; + /* Lapic only, no apbt */ + x86_init.timers.setup_percpu_clockev = setup_boot_APIC_clock; + x86_cpuinit.setup_percpu_clockev = setup_secondary_APIC_clock; } static void intel_mid_arch_setup(void) { - if (boot_cpu_data.x86 != 6) { - pr_err("Unknown Intel MID CPU (%d:%d), default to Penwell\n", - boot_cpu_data.x86, boot_cpu_data.x86_model); - __intel_mid_cpu_chip = INTEL_MID_CPU_CHIP_PENWELL; - goto out; - } - switch (boot_cpu_data.x86_model) { - case 0x35: - __intel_mid_cpu_chip = INTEL_MID_CPU_CHIP_CLOVERVIEW; - break; case 0x3C: case 0x4A: - __intel_mid_cpu_chip = INTEL_MID_CPU_CHIP_TANGIER; x86_platform.legacy.rtc = 1; break; - case 0x27: default: - __intel_mid_cpu_chip = INTEL_MID_CPU_CHIP_PENWELL; break; } -out: /* * Intel MID platforms are using explicitly defined regulators. * @@ -159,14 +96,11 @@ void __init x86_intel_mid_early_setup(void) x86_init.timers.timer_init = intel_mid_time_init; x86_init.timers.setup_percpu_clockev = x86_init_noop; - x86_init.timers.wallclock_init = intel_mid_rtc_init; x86_init.irqs.pre_vector_init = x86_init_noop; x86_init.oem.arch_setup = intel_mid_arch_setup; - x86_cpuinit.setup_percpu_clockev = apbt_setup_secondary_clock; - x86_platform.get_nmi_reason = intel_mid_get_nmi_reason; x86_init.pci.arch_init = intel_mid_pci_init; @@ -188,25 +122,3 @@ void __init x86_intel_mid_early_setup(void) x86_init.mpparse.get_smp_config = x86_init_uint_noop; set_bit(MP_BUS_ISA, mp_bus_not_pci); } - -/* - * if user does not want to use per CPU apb timer, just give it a lower rating - * than local apic timer and skip the late per cpu timer init. - */ -static inline int __init setup_x86_intel_mid_timer(char *arg) -{ - if (!arg) - return -EINVAL; - - if (strcmp("apbt_only", arg) == 0) - intel_mid_timer_options = INTEL_MID_TIMER_APBT_ONLY; - else if (strcmp("lapic_and_apbt", arg) == 0) - intel_mid_timer_options = INTEL_MID_TIMER_LAPIC_APBT; - else { - pr_warn("X86 INTEL_MID timer option %s not recognised use x86_intel_mid_timer=apbt_only or lapic_and_apbt\n", - arg); - return -EINVAL; - } - return 0; -} -__setup("x86_intel_mid_timer=", setup_x86_intel_mid_timer); diff --git a/arch/x86/platform/intel-mid/intel_mid_vrtc.c b/arch/x86/platform/intel-mid/intel_mid_vrtc.c deleted file mode 100644 index 2226da4f437a52eb1fa6be1086a814790f1061a0..0000000000000000000000000000000000000000 --- a/arch/x86/platform/intel-mid/intel_mid_vrtc.c +++ /dev/null @@ -1,173 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * intel_mid_vrtc.c: Driver for virtual RTC device on Intel MID platform - * - * (C) Copyright 2009 Intel Corporation - * - * Note: - * VRTC is emulated by system controller firmware, the real HW - * RTC is located in the PMIC device. SCU FW shadows PMIC RTC - * in a memory mapped IO space that is visible to the host IA - * processor. - * - * This driver is based on RTC CMOS driver. - */ - -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -static unsigned char __iomem *vrtc_virt_base; - -unsigned char vrtc_cmos_read(unsigned char reg) -{ - unsigned char retval; - - /* vRTC's registers range from 0x0 to 0xD */ - if (reg > 0xd || !vrtc_virt_base) - return 0xff; - - lock_cmos_prefix(reg); - retval = __raw_readb(vrtc_virt_base + (reg << 2)); - lock_cmos_suffix(reg); - return retval; -} -EXPORT_SYMBOL_GPL(vrtc_cmos_read); - -void vrtc_cmos_write(unsigned char val, unsigned char reg) -{ - if (reg > 0xd || !vrtc_virt_base) - return; - - lock_cmos_prefix(reg); - __raw_writeb(val, vrtc_virt_base + (reg << 2)); - lock_cmos_suffix(reg); -} -EXPORT_SYMBOL_GPL(vrtc_cmos_write); - -void vrtc_get_time(struct timespec64 *now) -{ - u8 sec, min, hour, mday, mon; - unsigned long flags; - u32 year; - - spin_lock_irqsave(&rtc_lock, flags); - - while ((vrtc_cmos_read(RTC_FREQ_SELECT) & RTC_UIP)) - cpu_relax(); - - sec = vrtc_cmos_read(RTC_SECONDS); - min = vrtc_cmos_read(RTC_MINUTES); - hour = vrtc_cmos_read(RTC_HOURS); - mday = vrtc_cmos_read(RTC_DAY_OF_MONTH); - mon = vrtc_cmos_read(RTC_MONTH); - year = vrtc_cmos_read(RTC_YEAR); - - spin_unlock_irqrestore(&rtc_lock, flags); - - /* vRTC YEAR reg contains the offset to 1972 */ - year += 1972; - - pr_info("vRTC: sec: %d min: %d hour: %d day: %d " - "mon: %d year: %d\n", sec, min, hour, mday, mon, year); - - now->tv_sec = mktime64(year, mon, mday, hour, min, sec); - now->tv_nsec = 0; -} - -int vrtc_set_mmss(const struct timespec64 *now) -{ - unsigned long flags; - struct rtc_time tm; - int year; - int retval = 0; - - rtc_time64_to_tm(now->tv_sec, &tm); - if (!rtc_valid_tm(&tm) && tm.tm_year >= 72) { - /* - * tm.year is the number of years since 1900, and the - * vrtc need the years since 1972. - */ - year = tm.tm_year - 72; - spin_lock_irqsave(&rtc_lock, flags); - vrtc_cmos_write(year, RTC_YEAR); - vrtc_cmos_write(tm.tm_mon, RTC_MONTH); - vrtc_cmos_write(tm.tm_mday, RTC_DAY_OF_MONTH); - vrtc_cmos_write(tm.tm_hour, RTC_HOURS); - vrtc_cmos_write(tm.tm_min, RTC_MINUTES); - vrtc_cmos_write(tm.tm_sec, RTC_SECONDS); - spin_unlock_irqrestore(&rtc_lock, flags); - } else { - pr_err("%s: Invalid vRTC value: write of %llx to vRTC failed\n", - __func__, (s64)now->tv_sec); - retval = -EINVAL; - } - return retval; -} - -void __init intel_mid_rtc_init(void) -{ - unsigned long vrtc_paddr; - - sfi_table_parse(SFI_SIG_MRTC, NULL, NULL, sfi_parse_mrtc); - - vrtc_paddr = sfi_mrtc_array[0].phys_addr; - if (!sfi_mrtc_num || !vrtc_paddr) - return; - - vrtc_virt_base = (void __iomem *)set_fixmap_offset_nocache(FIX_LNW_VRTC, - vrtc_paddr); - x86_platform.get_wallclock = vrtc_get_time; - x86_platform.set_wallclock = vrtc_set_mmss; -} - -/* - * The Moorestown platform has a memory mapped virtual RTC device that emulates - * the programming interface of the RTC. - */ - -static struct resource vrtc_resources[] = { - [0] = { - .flags = IORESOURCE_MEM, - }, - [1] = { - .flags = IORESOURCE_IRQ, - } -}; - -static struct platform_device vrtc_device = { - .name = "rtc_mrst", - .id = -1, - .resource = vrtc_resources, - .num_resources = ARRAY_SIZE(vrtc_resources), -}; - -/* Register the RTC device if appropriate */ -static int __init intel_mid_device_create(void) -{ - /* No Moorestown, no device */ - if (!intel_mid_identify_cpu()) - return -ENODEV; - /* No timer, no device */ - if (!sfi_mrtc_num) - return -ENODEV; - - /* iomem resource */ - vrtc_resources[0].start = sfi_mrtc_array[0].phys_addr; - vrtc_resources[0].end = sfi_mrtc_array[0].phys_addr + - MRST_VRTC_MAP_SZ; - /* irq resource */ - vrtc_resources[1].start = sfi_mrtc_array[0].irq; - vrtc_resources[1].end = sfi_mrtc_array[0].irq; - - return platform_device_register(&vrtc_device); -} -device_initcall(intel_mid_device_create); diff --git a/arch/x86/platform/intel-mid/sfi.c b/arch/x86/platform/intel-mid/sfi.c deleted file mode 100644 index 30bd5714a3d4736a5603020617355423a981ddde..0000000000000000000000000000000000000000 --- a/arch/x86/platform/intel-mid/sfi.c +++ /dev/null @@ -1,543 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * intel_mid_sfi.c: Intel MID SFI initialization code - * - * (C) Copyright 2013 Intel Corporation - * Author: Sathyanarayanan Kuppuswamy - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define SFI_SIG_OEM0 "OEM0" -#define MAX_IPCDEVS 24 -#define MAX_SCU_SPI 24 -#define MAX_SCU_I2C 24 - -static struct platform_device *ipc_devs[MAX_IPCDEVS]; -static struct spi_board_info *spi_devs[MAX_SCU_SPI]; -static struct i2c_board_info *i2c_devs[MAX_SCU_I2C]; -static struct sfi_gpio_table_entry *gpio_table; -static struct sfi_timer_table_entry sfi_mtimer_array[SFI_MTMR_MAX_NUM]; -static int ipc_next_dev; -static int spi_next_dev; -static int i2c_next_dev; -static int i2c_bus[MAX_SCU_I2C]; -static int gpio_num_entry; -static u32 sfi_mtimer_usage[SFI_MTMR_MAX_NUM]; -int sfi_mrtc_num; -int sfi_mtimer_num; - -struct sfi_rtc_table_entry sfi_mrtc_array[SFI_MRTC_MAX]; -EXPORT_SYMBOL_GPL(sfi_mrtc_array); - -struct blocking_notifier_head intel_scu_notifier = - BLOCKING_NOTIFIER_INIT(intel_scu_notifier); -EXPORT_SYMBOL_GPL(intel_scu_notifier); - -#define intel_mid_sfi_get_pdata(dev, priv) \ - ((dev)->get_platform_data ? (dev)->get_platform_data(priv) : NULL) - -/* parse all the mtimer info to a static mtimer array */ -int __init sfi_parse_mtmr(struct sfi_table_header *table) -{ - struct sfi_table_simple *sb; - struct sfi_timer_table_entry *pentry; - struct mpc_intsrc mp_irq; - int totallen; - - sb = (struct sfi_table_simple *)table; - if (!sfi_mtimer_num) { - sfi_mtimer_num = SFI_GET_NUM_ENTRIES(sb, - struct sfi_timer_table_entry); - pentry = (struct sfi_timer_table_entry *) sb->pentry; - totallen = sfi_mtimer_num * sizeof(*pentry); - memcpy(sfi_mtimer_array, pentry, totallen); - } - - pr_debug("SFI MTIMER info (num = %d):\n", sfi_mtimer_num); - pentry = sfi_mtimer_array; - for (totallen = 0; totallen < sfi_mtimer_num; totallen++, pentry++) { - pr_debug("timer[%d]: paddr = 0x%08x, freq = %dHz, irq = %d\n", - totallen, (u32)pentry->phys_addr, - pentry->freq_hz, pentry->irq); - mp_irq.type = MP_INTSRC; - mp_irq.irqtype = mp_INT; - mp_irq.irqflag = MP_IRQTRIG_EDGE | MP_IRQPOL_ACTIVE_HIGH; - mp_irq.srcbus = MP_BUS_ISA; - mp_irq.srcbusirq = pentry->irq; /* IRQ */ - mp_irq.dstapic = MP_APIC_ALL; - mp_irq.dstirq = pentry->irq; - mp_save_irq(&mp_irq); - mp_map_gsi_to_irq(pentry->irq, IOAPIC_MAP_ALLOC, NULL); - } - - return 0; -} - -struct sfi_timer_table_entry *sfi_get_mtmr(int hint) -{ - int i; - if (hint < sfi_mtimer_num) { - if (!sfi_mtimer_usage[hint]) { - pr_debug("hint taken for timer %d irq %d\n", - hint, sfi_mtimer_array[hint].irq); - sfi_mtimer_usage[hint] = 1; - return &sfi_mtimer_array[hint]; - } - } - /* take the first timer available */ - for (i = 0; i < sfi_mtimer_num;) { - if (!sfi_mtimer_usage[i]) { - sfi_mtimer_usage[i] = 1; - return &sfi_mtimer_array[i]; - } - i++; - } - return NULL; -} - -void sfi_free_mtmr(struct sfi_timer_table_entry *mtmr) -{ - int i; - for (i = 0; i < sfi_mtimer_num;) { - if (mtmr->irq == sfi_mtimer_array[i].irq) { - sfi_mtimer_usage[i] = 0; - return; - } - i++; - } -} - -/* parse all the mrtc info to a global mrtc array */ -int __init sfi_parse_mrtc(struct sfi_table_header *table) -{ - struct sfi_table_simple *sb; - struct sfi_rtc_table_entry *pentry; - struct mpc_intsrc mp_irq; - - int totallen; - - sb = (struct sfi_table_simple *)table; - if (!sfi_mrtc_num) { - sfi_mrtc_num = SFI_GET_NUM_ENTRIES(sb, - struct sfi_rtc_table_entry); - pentry = (struct sfi_rtc_table_entry *)sb->pentry; - totallen = sfi_mrtc_num * sizeof(*pentry); - memcpy(sfi_mrtc_array, pentry, totallen); - } - - pr_debug("SFI RTC info (num = %d):\n", sfi_mrtc_num); - pentry = sfi_mrtc_array; - for (totallen = 0; totallen < sfi_mrtc_num; totallen++, pentry++) { - pr_debug("RTC[%d]: paddr = 0x%08x, irq = %d\n", - totallen, (u32)pentry->phys_addr, pentry->irq); - mp_irq.type = MP_INTSRC; - mp_irq.irqtype = mp_INT; - mp_irq.irqflag = MP_IRQTRIG_LEVEL | MP_IRQPOL_ACTIVE_LOW; - mp_irq.srcbus = MP_BUS_ISA; - mp_irq.srcbusirq = pentry->irq; /* IRQ */ - mp_irq.dstapic = MP_APIC_ALL; - mp_irq.dstirq = pentry->irq; - mp_save_irq(&mp_irq); - mp_map_gsi_to_irq(pentry->irq, IOAPIC_MAP_ALLOC, NULL); - } - return 0; -} - - -/* - * Parsing GPIO table first, since the DEVS table will need this table - * to map the pin name to the actual pin. - */ -static int __init sfi_parse_gpio(struct sfi_table_header *table) -{ - struct sfi_table_simple *sb; - struct sfi_gpio_table_entry *pentry; - int num, i; - - if (gpio_table) - return 0; - sb = (struct sfi_table_simple *)table; - num = SFI_GET_NUM_ENTRIES(sb, struct sfi_gpio_table_entry); - pentry = (struct sfi_gpio_table_entry *)sb->pentry; - - gpio_table = kmemdup(pentry, num * sizeof(*pentry), GFP_KERNEL); - if (!gpio_table) - return -1; - gpio_num_entry = num; - - pr_debug("GPIO pin info:\n"); - for (i = 0; i < num; i++, pentry++) - pr_debug("info[%2d]: controller = %16.16s, pin_name = %16.16s," - " pin = %d\n", i, - pentry->controller_name, - pentry->pin_name, - pentry->pin_no); - return 0; -} - -int get_gpio_by_name(const char *name) -{ - struct sfi_gpio_table_entry *pentry = gpio_table; - int i; - - if (!pentry) - return -1; - for (i = 0; i < gpio_num_entry; i++, pentry++) { - if (!strncmp(name, pentry->pin_name, SFI_NAME_LEN)) - return pentry->pin_no; - } - return -EINVAL; -} - -static void __init intel_scu_ipc_device_register(struct platform_device *pdev) -{ - if (ipc_next_dev == MAX_IPCDEVS) - pr_err("too many SCU IPC devices"); - else - ipc_devs[ipc_next_dev++] = pdev; -} - -static void __init intel_scu_spi_device_register(struct spi_board_info *sdev) -{ - struct spi_board_info *new_dev; - - if (spi_next_dev == MAX_SCU_SPI) { - pr_err("too many SCU SPI devices"); - return; - } - - new_dev = kzalloc(sizeof(*sdev), GFP_KERNEL); - if (!new_dev) { - pr_err("failed to alloc mem for delayed spi dev %s\n", - sdev->modalias); - return; - } - *new_dev = *sdev; - - spi_devs[spi_next_dev++] = new_dev; -} - -static void __init intel_scu_i2c_device_register(int bus, - struct i2c_board_info *idev) -{ - struct i2c_board_info *new_dev; - - if (i2c_next_dev == MAX_SCU_I2C) { - pr_err("too many SCU I2C devices"); - return; - } - - new_dev = kzalloc(sizeof(*idev), GFP_KERNEL); - if (!new_dev) { - pr_err("failed to alloc mem for delayed i2c dev %s\n", - idev->type); - return; - } - *new_dev = *idev; - - i2c_bus[i2c_next_dev] = bus; - i2c_devs[i2c_next_dev++] = new_dev; -} - -/* Called by IPC driver */ -void intel_scu_devices_create(void) -{ - int i; - - for (i = 0; i < ipc_next_dev; i++) - platform_device_add(ipc_devs[i]); - - for (i = 0; i < spi_next_dev; i++) - spi_register_board_info(spi_devs[i], 1); - - for (i = 0; i < i2c_next_dev; i++) { - struct i2c_adapter *adapter; - struct i2c_client *client; - - adapter = i2c_get_adapter(i2c_bus[i]); - if (adapter) { - client = i2c_new_client_device(adapter, i2c_devs[i]); - if (IS_ERR(client)) - pr_err("can't create i2c device %s\n", - i2c_devs[i]->type); - } else - i2c_register_board_info(i2c_bus[i], i2c_devs[i], 1); - } - intel_scu_notifier_post(SCU_AVAILABLE, NULL); -} -EXPORT_SYMBOL_GPL(intel_scu_devices_create); - -/* Called by IPC driver */ -void intel_scu_devices_destroy(void) -{ - int i; - - intel_scu_notifier_post(SCU_DOWN, NULL); - - for (i = 0; i < ipc_next_dev; i++) - platform_device_del(ipc_devs[i]); -} -EXPORT_SYMBOL_GPL(intel_scu_devices_destroy); - -static void __init install_irq_resource(struct platform_device *pdev, int irq) -{ - /* Single threaded */ - static struct resource res __initdata = { - .name = "IRQ", - .flags = IORESOURCE_IRQ, - }; - res.start = irq; - platform_device_add_resources(pdev, &res, 1); -} - -static void __init sfi_handle_ipc_dev(struct sfi_device_table_entry *pentry, - struct devs_id *dev) -{ - struct platform_device *pdev; - void *pdata = NULL; - - pr_debug("IPC bus, name = %16.16s, irq = 0x%2x\n", - pentry->name, pentry->irq); - - /* - * We need to call platform init of IPC devices to fill misc_pdata - * structure. It will be used in msic_init for initialization. - */ - pdata = intel_mid_sfi_get_pdata(dev, pentry); - if (IS_ERR(pdata)) - return; - - /* - * On Medfield the platform device creation is handled by the MSIC - * MFD driver so we don't need to do it here. - */ - if (dev->msic && intel_mid_has_msic()) - return; - - pdev = platform_device_alloc(pentry->name, 0); - if (pdev == NULL) { - pr_err("out of memory for SFI platform device '%s'.\n", - pentry->name); - return; - } - install_irq_resource(pdev, pentry->irq); - - pdev->dev.platform_data = pdata; - if (dev->delay) - intel_scu_ipc_device_register(pdev); - else - platform_device_add(pdev); -} - -static void __init sfi_handle_spi_dev(struct sfi_device_table_entry *pentry, - struct devs_id *dev) -{ - struct spi_board_info spi_info; - void *pdata = NULL; - - memset(&spi_info, 0, sizeof(spi_info)); - strncpy(spi_info.modalias, pentry->name, SFI_NAME_LEN); - spi_info.irq = ((pentry->irq == (u8)0xff) ? 0 : pentry->irq); - spi_info.bus_num = pentry->host_num; - spi_info.chip_select = pentry->addr; - spi_info.max_speed_hz = pentry->max_freq; - pr_debug("SPI bus=%d, name=%16.16s, irq=0x%2x, max_freq=%d, cs=%d\n", - spi_info.bus_num, - spi_info.modalias, - spi_info.irq, - spi_info.max_speed_hz, - spi_info.chip_select); - - pdata = intel_mid_sfi_get_pdata(dev, &spi_info); - if (IS_ERR(pdata)) - return; - - spi_info.platform_data = pdata; - if (dev->delay) - intel_scu_spi_device_register(&spi_info); - else - spi_register_board_info(&spi_info, 1); -} - -static void __init sfi_handle_i2c_dev(struct sfi_device_table_entry *pentry, - struct devs_id *dev) -{ - struct i2c_board_info i2c_info; - void *pdata = NULL; - - memset(&i2c_info, 0, sizeof(i2c_info)); - strncpy(i2c_info.type, pentry->name, SFI_NAME_LEN); - i2c_info.irq = ((pentry->irq == (u8)0xff) ? 0 : pentry->irq); - i2c_info.addr = pentry->addr; - pr_debug("I2C bus = %d, name = %16.16s, irq = 0x%2x, addr = 0x%x\n", - pentry->host_num, - i2c_info.type, - i2c_info.irq, - i2c_info.addr); - pdata = intel_mid_sfi_get_pdata(dev, &i2c_info); - i2c_info.platform_data = pdata; - if (IS_ERR(pdata)) - return; - - if (dev->delay) - intel_scu_i2c_device_register(pentry->host_num, &i2c_info); - else - i2c_register_board_info(pentry->host_num, &i2c_info, 1); -} - -static void __init sfi_handle_sd_dev(struct sfi_device_table_entry *pentry, - struct devs_id *dev) -{ - struct mid_sd_board_info sd_info; - void *pdata; - - memset(&sd_info, 0, sizeof(sd_info)); - strncpy(sd_info.name, pentry->name, SFI_NAME_LEN); - sd_info.bus_num = pentry->host_num; - sd_info.max_clk = pentry->max_freq; - sd_info.addr = pentry->addr; - pr_debug("SD bus = %d, name = %16.16s, max_clk = %d, addr = 0x%x\n", - sd_info.bus_num, - sd_info.name, - sd_info.max_clk, - sd_info.addr); - pdata = intel_mid_sfi_get_pdata(dev, &sd_info); - if (IS_ERR(pdata)) - return; - - /* Nothing we can do with this for now */ - sd_info.platform_data = pdata; - - pr_debug("Successfully registered %16.16s", sd_info.name); -} - -extern struct devs_id *const __x86_intel_mid_dev_start[], - *const __x86_intel_mid_dev_end[]; - -static struct devs_id __init *get_device_id(u8 type, char *name) -{ - struct devs_id *const *dev_table; - - for (dev_table = __x86_intel_mid_dev_start; - dev_table < __x86_intel_mid_dev_end; dev_table++) { - struct devs_id *dev = *dev_table; - if (dev->type == type && - !strncmp(dev->name, name, SFI_NAME_LEN)) { - return dev; - } - } - - return NULL; -} - -static int __init sfi_parse_devs(struct sfi_table_header *table) -{ - struct sfi_table_simple *sb; - struct sfi_device_table_entry *pentry; - struct devs_id *dev = NULL; - int num, i, ret; - int polarity; - struct irq_alloc_info info; - - sb = (struct sfi_table_simple *)table; - num = SFI_GET_NUM_ENTRIES(sb, struct sfi_device_table_entry); - pentry = (struct sfi_device_table_entry *)sb->pentry; - - for (i = 0; i < num; i++, pentry++) { - int irq = pentry->irq; - - if (irq != (u8)0xff) { /* native RTE case */ - /* these SPI2 devices are not exposed to system as PCI - * devices, but they have separate RTE entry in IOAPIC - * so we have to enable them one by one here - */ - if (intel_mid_identify_cpu() == - INTEL_MID_CPU_CHIP_TANGIER) { - if (!strncmp(pentry->name, "r69001-ts-i2c", 13)) - /* active low */ - polarity = 1; - else if (!strncmp(pentry->name, - "synaptics_3202", 14)) - /* active low */ - polarity = 1; - else if (irq == 41) - /* fast_int_1 */ - polarity = 1; - else - /* active high */ - polarity = 0; - } else { - /* PNW and CLV go with active low */ - polarity = 1; - } - - ioapic_set_alloc_attr(&info, NUMA_NO_NODE, 1, polarity); - ret = mp_map_gsi_to_irq(irq, IOAPIC_MAP_ALLOC, &info); - WARN_ON(ret < 0); - } - - dev = get_device_id(pentry->type, pentry->name); - - if (!dev) - continue; - - switch (pentry->type) { - case SFI_DEV_TYPE_IPC: - sfi_handle_ipc_dev(pentry, dev); - break; - case SFI_DEV_TYPE_SPI: - sfi_handle_spi_dev(pentry, dev); - break; - case SFI_DEV_TYPE_I2C: - sfi_handle_i2c_dev(pentry, dev); - break; - case SFI_DEV_TYPE_SD: - sfi_handle_sd_dev(pentry, dev); - break; - case SFI_DEV_TYPE_UART: - case SFI_DEV_TYPE_HSI: - default: - break; - } - } - return 0; -} - -static int __init intel_mid_platform_init(void) -{ - sfi_table_parse(SFI_SIG_GPIO, NULL, NULL, sfi_parse_gpio); - sfi_table_parse(SFI_SIG_DEVS, NULL, NULL, sfi_parse_devs); - return 0; -} -arch_initcall(intel_mid_platform_init); diff --git a/arch/x86/platform/pvh/head.S b/arch/x86/platform/pvh/head.S index 43b4d864817ecc37ab7b4257187c5dec4405884c..d2ccadc247e6f9a51abcf09184eafc2c8b59346a 100644 --- a/arch/x86/platform/pvh/head.S +++ b/arch/x86/platform/pvh/head.S @@ -16,6 +16,7 @@ #include #include #include +#include #include __HEAD @@ -105,6 +106,7 @@ SYM_CODE_START_LOCAL(pvh_start_xen) /* startup_64 expects boot_params in %rsi. */ mov $_pa(pvh_bootparams), %rsi mov $_pa(startup_64), %rax + ANNOTATE_RETPOLINE_SAFE jmp *%rax #else /* CONFIG_X86_64 */ diff --git a/arch/x86/platform/sfi/sfi.c b/arch/x86/platform/sfi/sfi.c deleted file mode 100644 index 6259563760f99e6be165f6895c9cdac5fab19821..0000000000000000000000000000000000000000 --- a/arch/x86/platform/sfi/sfi.c +++ /dev/null @@ -1,100 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * sfi.c - x86 architecture SFI support. - * - * Copyright (c) 2009, Intel Corporation. - */ - -#define KMSG_COMPONENT "SFI" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt - -#include -#include -#include -#include - -#include -#include -#include -#include -#include - -#ifdef CONFIG_X86_LOCAL_APIC -static unsigned long sfi_lapic_addr __initdata = APIC_DEFAULT_PHYS_BASE; - -/* All CPUs enumerated by SFI must be present and enabled */ -static void __init mp_sfi_register_lapic(u8 id) -{ - if (MAX_LOCAL_APIC - id <= 0) { - pr_warn("Processor #%d invalid (max %d)\n", id, MAX_LOCAL_APIC); - return; - } - - pr_info("registering lapic[%d]\n", id); - - generic_processor_info(id, GET_APIC_VERSION(apic_read(APIC_LVR))); -} - -static int __init sfi_parse_cpus(struct sfi_table_header *table) -{ - struct sfi_table_simple *sb; - struct sfi_cpu_table_entry *pentry; - int i; - int cpu_num; - - sb = (struct sfi_table_simple *)table; - cpu_num = SFI_GET_NUM_ENTRIES(sb, struct sfi_cpu_table_entry); - pentry = (struct sfi_cpu_table_entry *)sb->pentry; - - for (i = 0; i < cpu_num; i++) { - mp_sfi_register_lapic(pentry->apic_id); - pentry++; - } - - smp_found_config = 1; - return 0; -} -#endif /* CONFIG_X86_LOCAL_APIC */ - -#ifdef CONFIG_X86_IO_APIC - -static int __init sfi_parse_ioapic(struct sfi_table_header *table) -{ - struct sfi_table_simple *sb; - struct sfi_apic_table_entry *pentry; - int i, num; - struct ioapic_domain_cfg cfg = { - .type = IOAPIC_DOMAIN_STRICT, - .ops = &mp_ioapic_irqdomain_ops, - }; - - sb = (struct sfi_table_simple *)table; - num = SFI_GET_NUM_ENTRIES(sb, struct sfi_apic_table_entry); - pentry = (struct sfi_apic_table_entry *)sb->pentry; - - for (i = 0; i < num; i++) { - mp_register_ioapic(i, pentry->phys_addr, gsi_top, &cfg); - pentry++; - } - - WARN(pic_mode, KERN_WARNING - "SFI: pic_mod shouldn't be 1 when IOAPIC table is present\n"); - pic_mode = 0; - return 0; -} -#endif /* CONFIG_X86_IO_APIC */ - -/* - * sfi_platform_init(): register lapics & io-apics - */ -int __init sfi_platform_init(void) -{ -#ifdef CONFIG_X86_LOCAL_APIC - register_lapic_address(sfi_lapic_addr); - sfi_table_parse(SFI_SIG_CPUS, NULL, NULL, sfi_parse_cpus); -#endif -#ifdef CONFIG_X86_IO_APIC - sfi_table_parse(SFI_SIG_APIC, NULL, NULL, sfi_parse_ioapic); -#endif - return 0; -} diff --git a/arch/x86/power/Makefile b/arch/x86/power/Makefile index 6907b523e856b9c9dab6e34aad821f2d49ecee65..379777572bc9fe665b845e829fbe86f5cd9c853b 100644 --- a/arch/x86/power/Makefile +++ b/arch/x86/power/Makefile @@ -1,9 +1,12 @@ # SPDX-License-Identifier: GPL-2.0 -OBJECT_FILES_NON_STANDARD_hibernate_asm_$(BITS).o := y # __restore_processor_state() restores %gs after S3 resume and so should not # itself be stack-protected CFLAGS_cpu.o := -fno-stack-protector +# Clang may incorrectly inline functions with stack protector enabled into +# __restore_processor_state(): https://bugs.llvm.org/show_bug.cgi?id=47479 +CFLAGS_REMOVE_cpu.o := $(CC_FLAGS_LTO) + obj-$(CONFIG_PM_SLEEP) += cpu.o obj-$(CONFIG_HIBERNATION) += hibernate_$(BITS).o hibernate_asm_$(BITS).o hibernate.o diff --git a/arch/x86/power/hibernate_asm_64.S b/arch/x86/power/hibernate_asm_64.S index 7918b8415f132dbb6a8f9fc6f9d5f48b7bef7891..d9bed596d849c4b28a789b582f89a41c0ad8998a 100644 --- a/arch/x86/power/hibernate_asm_64.S +++ b/arch/x86/power/hibernate_asm_64.S @@ -21,6 +21,53 @@ #include #include #include +#include + + /* code below belongs to the image kernel */ + .align PAGE_SIZE +SYM_FUNC_START(restore_registers) + /* go back to the original page tables */ + movq %r9, %cr3 + + /* Flush TLB, including "global" things (vmalloc) */ + movq mmu_cr4_features(%rip), %rax + movq %rax, %rdx + andq $~(X86_CR4_PGE), %rdx + movq %rdx, %cr4; # turn off PGE + movq %cr3, %rcx; # flush TLB + movq %rcx, %cr3 + movq %rax, %cr4; # turn PGE back on + + /* We don't restore %rax, it must be 0 anyway */ + movq $saved_context, %rax + movq pt_regs_sp(%rax), %rsp + movq pt_regs_bp(%rax), %rbp + movq pt_regs_si(%rax), %rsi + movq pt_regs_di(%rax), %rdi + movq pt_regs_bx(%rax), %rbx + movq pt_regs_cx(%rax), %rcx + movq pt_regs_dx(%rax), %rdx + movq pt_regs_r8(%rax), %r8 + movq pt_regs_r9(%rax), %r9 + movq pt_regs_r10(%rax), %r10 + movq pt_regs_r11(%rax), %r11 + movq pt_regs_r12(%rax), %r12 + movq pt_regs_r13(%rax), %r13 + movq pt_regs_r14(%rax), %r14 + movq pt_regs_r15(%rax), %r15 + pushq pt_regs_flags(%rax) + popfq + + /* Saved in save_processor_state. */ + lgdt saved_context_gdt_desc(%rax) + + xorl %eax, %eax + + /* tell the hibernation core that we've just restored the memory */ + movq %rax, in_suspend(%rip) + + ret +SYM_FUNC_END(restore_registers) SYM_FUNC_START(swsusp_arch_suspend) movq $saved_context, %rax @@ -52,7 +99,7 @@ SYM_FUNC_START(swsusp_arch_suspend) ret SYM_FUNC_END(swsusp_arch_suspend) -SYM_CODE_START(restore_image) +SYM_FUNC_START(restore_image) /* prepare to jump to the image kernel */ movq restore_jump_address(%rip), %r8 movq restore_cr3(%rip), %r9 @@ -66,11 +113,12 @@ SYM_CODE_START(restore_image) /* jump to relocated restore code */ movq relocated_restore_code(%rip), %rcx + ANNOTATE_RETPOLINE_SAFE jmpq *%rcx -SYM_CODE_END(restore_image) +SYM_FUNC_END(restore_image) /* code below has been relocated to a safe page */ -SYM_CODE_START(core_restore_code) +SYM_FUNC_START(core_restore_code) /* switch to temporary page tables */ movq %rax, %cr3 /* flush TLB */ @@ -97,51 +145,6 @@ SYM_CODE_START(core_restore_code) .Ldone: /* jump to the restore_registers address from the image header */ + ANNOTATE_RETPOLINE_SAFE jmpq *%r8 -SYM_CODE_END(core_restore_code) - - /* code below belongs to the image kernel */ - .align PAGE_SIZE -SYM_FUNC_START(restore_registers) - /* go back to the original page tables */ - movq %r9, %cr3 - - /* Flush TLB, including "global" things (vmalloc) */ - movq mmu_cr4_features(%rip), %rax - movq %rax, %rdx - andq $~(X86_CR4_PGE), %rdx - movq %rdx, %cr4; # turn off PGE - movq %cr3, %rcx; # flush TLB - movq %rcx, %cr3 - movq %rax, %cr4; # turn PGE back on - - /* We don't restore %rax, it must be 0 anyway */ - movq $saved_context, %rax - movq pt_regs_sp(%rax), %rsp - movq pt_regs_bp(%rax), %rbp - movq pt_regs_si(%rax), %rsi - movq pt_regs_di(%rax), %rdi - movq pt_regs_bx(%rax), %rbx - movq pt_regs_cx(%rax), %rcx - movq pt_regs_dx(%rax), %rdx - movq pt_regs_r8(%rax), %r8 - movq pt_regs_r9(%rax), %r9 - movq pt_regs_r10(%rax), %r10 - movq pt_regs_r11(%rax), %r11 - movq pt_regs_r12(%rax), %r12 - movq pt_regs_r13(%rax), %r13 - movq pt_regs_r14(%rax), %r14 - movq pt_regs_r15(%rax), %r15 - pushq pt_regs_flags(%rax) - popfq - - /* Saved in save_processor_state. */ - lgdt saved_context_gdt_desc(%rax) - - xorl %eax, %eax - - /* tell the hibernation core that we've just restored the memory */ - movq %rax, in_suspend(%rip) - - ret -SYM_FUNC_END(restore_registers) +SYM_FUNC_END(core_restore_code) diff --git a/arch/x86/tools/Makefile b/arch/x86/tools/Makefile index 55b1ab378974ba062780c004436e56f3dc9c7d19..bddfc9a466453126d00d9f19e40fefc0fff1b3b1 100644 --- a/arch/x86/tools/Makefile +++ b/arch/x86/tools/Makefile @@ -29,14 +29,14 @@ posttest: $(obj)/insn_decoder_test vmlinux $(obj)/insn_sanity hostprogs += insn_decoder_test insn_sanity # -I needed for generated C source and C source which in the kernel tree. -HOSTCFLAGS_insn_decoder_test.o := -Wall -I$(objtree)/arch/x86/lib/ -I$(srctree)/arch/x86/include/uapi/ -I$(srctree)/arch/x86/include/ -I$(srctree)/arch/x86/lib/ -I$(srctree)/include/uapi/ +HOSTCFLAGS_insn_decoder_test.o := -Wall -I$(srctree)/tools/arch/x86/lib/ -I$(srctree)/tools/arch/x86/include/ -I$(objtree)/arch/x86/lib/ -HOSTCFLAGS_insn_sanity.o := -Wall -I$(objtree)/arch/x86/lib/ -I$(srctree)/arch/x86/include/ -I$(srctree)/arch/x86/lib/ -I$(srctree)/include/ +HOSTCFLAGS_insn_sanity.o := -Wall -I$(srctree)/tools/arch/x86/lib/ -I$(srctree)/tools/arch/x86/include/ -I$(objtree)/arch/x86/lib/ # Dependencies are also needed. -$(obj)/insn_decoder_test.o: $(srctree)/arch/x86/lib/insn.c $(srctree)/arch/x86/lib/inat.c $(srctree)/arch/x86/include/asm/inat_types.h $(srctree)/arch/x86/include/asm/inat.h $(srctree)/arch/x86/include/asm/insn.h $(objtree)/arch/x86/lib/inat-tables.c +$(obj)/insn_decoder_test.o: $(srctree)/tools/arch/x86/lib/insn.c $(srctree)/tools/arch/x86/lib/inat.c $(srctree)/tools/arch/x86/include/asm/inat_types.h $(srctree)/tools/arch/x86/include/asm/inat.h $(srctree)/tools/arch/x86/include/asm/insn.h $(objtree)/arch/x86/lib/inat-tables.c -$(obj)/insn_sanity.o: $(srctree)/arch/x86/lib/insn.c $(srctree)/arch/x86/lib/inat.c $(srctree)/arch/x86/include/asm/inat_types.h $(srctree)/arch/x86/include/asm/inat.h $(srctree)/arch/x86/include/asm/insn.h $(objtree)/arch/x86/lib/inat-tables.c +$(obj)/insn_sanity.o: $(srctree)/tools/arch/x86/lib/insn.c $(srctree)/tools/arch/x86/lib/inat.c $(srctree)/tools/arch/x86/include/asm/inat_types.h $(srctree)/tools/arch/x86/include/asm/inat.h $(srctree)/tools/arch/x86/include/asm/insn.h $(objtree)/arch/x86/lib/inat-tables.c HOST_EXTRACFLAGS += -I$(srctree)/tools/include hostprogs += relocs diff --git a/arch/x86/tools/insn_sanity.c b/arch/x86/tools/insn_sanity.c index 185ceba9d289bd5de91019cebc25502c9a385a2f..c6a0000ae635f34d02984272235fda6f70d8a22f 100644 --- a/arch/x86/tools/insn_sanity.c +++ b/arch/x86/tools/insn_sanity.c @@ -14,10 +14,6 @@ #include #include #include - -#define unlikely(cond) (cond) -#define ARRAY_SIZE(a) (sizeof(a)/sizeof(a[0])) - #include #include #include diff --git a/arch/x86/tools/relocs.c b/arch/x86/tools/relocs.c index 1c3a1962cade65db5fac4951743c39fb962b2362..04c5a44b968276ec3975acbe035acce8a438572d 100644 --- a/arch/x86/tools/relocs.c +++ b/arch/x86/tools/relocs.c @@ -61,8 +61,8 @@ static const char * const sym_regex_kernel[S_NSYMTYPES] = { "(__iommu_table|__apicdrivers|__smp_locks)(|_end)|" "__(start|end)_pci_.*|" "__(start|end)_builtin_fw|" - "__(start|stop)___ksymtab(|_gpl|_unused|_unused_gpl|_gpl_future)|" - "__(start|stop)___kcrctab(|_gpl|_unused|_unused_gpl|_gpl_future)|" + "__(start|stop)___ksymtab(|_gpl)|" + "__(start|stop)___kcrctab(|_gpl)|" "__(start|stop)___param|" "__(start|stop)___modver|" "__(start|stop)___bug_table|" diff --git a/arch/x86/xen/Makefile b/arch/x86/xen/Makefile index fc5c5ba4aacbac869ef76c406f04395a9ffced70..40b5779fce21cf85be58b626ecb4e7245ff75a4f 100644 --- a/arch/x86/xen/Makefile +++ b/arch/x86/xen/Makefile @@ -1,5 +1,4 @@ # SPDX-License-Identifier: GPL-2.0 -OBJECT_FILES_NON_STANDARD_xen-asm.o := y ifdef CONFIG_FUNCTION_TRACER # Do not profile debug and lowlevel utilities diff --git a/arch/x86/xen/xen-asm.S b/arch/x86/xen/xen-asm.S index 02f31341e435ed37f0fa23dd1196bb6128fc8c1c..1e626444712bedb40aa8ae214f0003b1eee37232 100644 --- a/arch/x86/xen/xen-asm.S +++ b/arch/x86/xen/xen-asm.S @@ -14,6 +14,7 @@ #include #include #include +#include #include @@ -118,6 +119,7 @@ SYM_FUNC_END(xen_read_cr2_direct); .macro xen_pv_trap name SYM_CODE_START(xen_\name) + UNWIND_HINT_EMPTY pop %rcx pop %r11 jmp \name @@ -157,6 +159,7 @@ xen_pv_trap asm_exc_xen_hypervisor_callback SYM_CODE_START(xen_early_idt_handler_array) i = 0 .rept NUM_EXCEPTION_VECTORS + UNWIND_HINT_EMPTY pop %rcx pop %r11 jmp early_idt_handler_array + i*EARLY_IDT_HANDLER_SIZE @@ -183,6 +186,7 @@ hypercall_iret = hypercall_page + __HYPERVISOR_iret * 32 * rsp->rax } */ SYM_CODE_START(xen_iret) + UNWIND_HINT_EMPTY pushq $0 jmp hypercall_iret SYM_CODE_END(xen_iret) @@ -203,7 +207,8 @@ SYM_CODE_END(xen_iret) */ /* Normal 64-bit system call target */ -SYM_FUNC_START(xen_syscall_target) +SYM_CODE_START(xen_syscall_target) + UNWIND_HINT_EMPTY popq %rcx popq %r11 @@ -216,12 +221,13 @@ SYM_FUNC_START(xen_syscall_target) movq $__USER_CS, 1*8(%rsp) jmp entry_SYSCALL_64_after_hwframe -SYM_FUNC_END(xen_syscall_target) +SYM_CODE_END(xen_syscall_target) #ifdef CONFIG_IA32_EMULATION /* 32-bit compat syscall target */ -SYM_FUNC_START(xen_syscall32_target) +SYM_CODE_START(xen_syscall32_target) + UNWIND_HINT_EMPTY popq %rcx popq %r11 @@ -234,10 +240,11 @@ SYM_FUNC_START(xen_syscall32_target) movq $__USER32_CS, 1*8(%rsp) jmp entry_SYSCALL_compat_after_hwframe -SYM_FUNC_END(xen_syscall32_target) +SYM_CODE_END(xen_syscall32_target) /* 32-bit compat sysenter target */ -SYM_FUNC_START(xen_sysenter_target) +SYM_CODE_START(xen_sysenter_target) + UNWIND_HINT_EMPTY /* * NB: Xen is polite and clears TF from EFLAGS for us. This means * that we don't need to guard against single step exceptions here. @@ -254,17 +261,18 @@ SYM_FUNC_START(xen_sysenter_target) movq $__USER32_CS, 1*8(%rsp) jmp entry_SYSENTER_compat_after_hwframe -SYM_FUNC_END(xen_sysenter_target) +SYM_CODE_END(xen_sysenter_target) #else /* !CONFIG_IA32_EMULATION */ -SYM_FUNC_START_ALIAS(xen_syscall32_target) -SYM_FUNC_START(xen_sysenter_target) +SYM_CODE_START(xen_syscall32_target) +SYM_CODE_START(xen_sysenter_target) + UNWIND_HINT_EMPTY lea 16(%rsp), %rsp /* strip %rcx, %r11 */ mov $-ENOSYS, %rax pushq $0 jmp hypercall_iret -SYM_FUNC_END(xen_sysenter_target) -SYM_FUNC_END_ALIAS(xen_syscall32_target) +SYM_CODE_END(xen_sysenter_target) +SYM_CODE_END(xen_syscall32_target) #endif /* CONFIG_IA32_EMULATION */ diff --git a/arch/x86/xen/xen-head.S b/arch/x86/xen/xen-head.S index 2d7c8f34f56c75a4ea76ead1b4e74147bfd84621..cb6538ae2fe07f084595f4d38c256fdbca45abcd 100644 --- a/arch/x86/xen/xen-head.S +++ b/arch/x86/xen/xen-head.S @@ -68,8 +68,9 @@ SYM_CODE_END(asm_cpu_bringup_and_idle) .balign PAGE_SIZE SYM_CODE_START(hypercall_page) .rept (PAGE_SIZE / 32) - UNWIND_HINT_EMPTY - .skip 32 + UNWIND_HINT_FUNC + .skip 31, 0x90 + ret .endr #define HYPERCALL(n) \ diff --git a/arch/xtensa/kernel/process.c b/arch/xtensa/kernel/process.c index 397a7de563776acbc5c8a17b27abae1cf1564d43..9534ef515d7487d9a82cdbb1aa626f2f4b9b99f7 100644 --- a/arch/xtensa/kernel/process.c +++ b/arch/xtensa/kernel/process.c @@ -217,7 +217,7 @@ int copy_thread(unsigned long clone_flags, unsigned long usp_thread_fn, p->thread.sp = (unsigned long)childregs; - if (!(p->flags & PF_KTHREAD)) { + if (!(p->flags & (PF_KTHREAD | PF_IO_WORKER))) { struct pt_regs *regs = current_pt_regs(); unsigned long usp = usp_thread_fn ? usp_thread_fn : regs->areg[1]; diff --git a/arch/xtensa/kernel/syscalls/Makefile b/arch/xtensa/kernel/syscalls/Makefile index 659faefdcb1da5f0ffbafc40fd598dd98d9a5fde..285aaba832d93936c3f2b7edb2938dbfe13fc21f 100644 --- a/arch/xtensa/kernel/syscalls/Makefile +++ b/arch/xtensa/kernel/syscalls/Makefile @@ -5,7 +5,7 @@ uapi := arch/$(SRCARCH)/include/generated/uapi/asm _dummy := $(shell [ -d '$(uapi)' ] || mkdir -p '$(uapi)') \ $(shell [ -d '$(kapi)' ] || mkdir -p '$(kapi)') -syscall := $(srctree)/$(src)/syscall.tbl +syscall := $(src)/syscall.tbl syshdr := $(srctree)/$(src)/syscallhdr.sh systbl := $(srctree)/$(src)/syscalltbl.sh @@ -21,18 +21,19 @@ quiet_cmd_systbl = SYSTBL $@ '$(systbl_abi_$(basetarget))' \ '$(systbl_offset_$(basetarget))' -$(uapi)/unistd_32.h: $(syscall) $(syshdr) +$(uapi)/unistd_32.h: $(syscall) $(syshdr) FORCE $(call if_changed,syshdr) -$(kapi)/syscall_table.h: $(syscall) $(systbl) +$(kapi)/syscall_table.h: $(syscall) $(systbl) FORCE $(call if_changed,systbl) uapisyshdr-y += unistd_32.h kapisyshdr-y += syscall_table.h -targets += $(uapisyshdr-y) $(kapisyshdr-y) +uapisyshdr-y := $(addprefix $(uapi)/, $(uapisyshdr-y)) +kapisyshdr-y := $(addprefix $(kapi)/, $(kapisyshdr-y)) +targets += $(addprefix ../../../../, $(uapisyshdr-y) $(kapisyshdr-y)) PHONY += all -all: $(addprefix $(uapi)/,$(uapisyshdr-y)) -all: $(addprefix $(kapi)/,$(kapisyshdr-y)) +all: $(uapisyshdr-y) $(kapisyshdr-y) @: diff --git a/arch/xtensa/kernel/syscalls/syscall.tbl b/arch/xtensa/kernel/syscalls/syscall.tbl index 46116a28eeed3bd0b250ab06874f6238103720a4..365a9b84922482f194962e6744be37a9a82990bd 100644 --- a/arch/xtensa/kernel/syscalls/syscall.tbl +++ b/arch/xtensa/kernel/syscalls/syscall.tbl @@ -412,3 +412,4 @@ 439 common faccessat2 sys_faccessat2 440 common process_madvise sys_process_madvise 441 common epoll_pwait2 sys_epoll_pwait2 +442 common mount_setattr sys_mount_setattr diff --git a/block/bfq-iosched.c b/block/bfq-iosched.c index b398dde53af9b420a6b42e73572e2774e70f5f6e..ec482e6641ff514bd859d8c7c2c87286d1604dfe 100644 --- a/block/bfq-iosched.c +++ b/block/bfq-iosched.c @@ -125,6 +125,8 @@ #include #include +#include + #include "blk.h" #include "blk-mq.h" #include "blk-mq-tag.h" @@ -5621,7 +5623,7 @@ static void bfq_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq, spin_unlock_irq(&bfqd->lock); - blk_mq_sched_request_inserted(rq); + trace_block_rq_insert(rq); spin_lock_irq(&bfqd->lock); bfqq = bfq_init_rq(rq); diff --git a/block/blk-core.c b/block/blk-core.c index 5e752840b41a1fa80b7146fc7b6f2b9c0a336ec3..fc60ff2084973811b7316da72436bab365af9260 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -59,6 +59,7 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(block_rq_remap); EXPORT_TRACEPOINT_SYMBOL_GPL(block_bio_complete); EXPORT_TRACEPOINT_SYMBOL_GPL(block_split); EXPORT_TRACEPOINT_SYMBOL_GPL(block_unplug); +EXPORT_TRACEPOINT_SYMBOL_GPL(block_rq_insert); DEFINE_IDA(blk_queue_ida); diff --git a/block/blk-crypto-fallback.c b/block/blk-crypto-fallback.c index e8327c50d7c9f4ddcf6ace26dc97f227d2dcbcba..c176b7af56a7a57742f65757a3d9901ac9e14497 100644 --- a/block/blk-crypto-fallback.c +++ b/block/blk-crypto-fallback.c @@ -80,6 +80,7 @@ static struct blk_crypto_keyslot { static struct blk_keyslot_manager blk_crypto_ksm; static struct workqueue_struct *blk_crypto_wq; static mempool_t *blk_crypto_bounce_page_pool; +static struct bio_set crypto_bio_split; /* * This is the key we set when evicting a keyslot. This *should* be the all 0's @@ -224,7 +225,8 @@ static bool blk_crypto_split_bio_if_needed(struct bio **bio_ptr) if (num_sectors < bio_sectors(bio)) { struct bio *split_bio; - split_bio = bio_split(bio, num_sectors, GFP_NOIO, NULL); + split_bio = bio_split(bio, num_sectors, GFP_NOIO, + &crypto_bio_split); if (!split_bio) { bio->bi_status = BLK_STS_RESOURCE; return false; @@ -538,9 +540,13 @@ static int blk_crypto_fallback_init(void) prandom_bytes(blank_key, BLK_CRYPTO_MAX_KEY_SIZE); - err = blk_ksm_init(&blk_crypto_ksm, blk_crypto_num_keyslots); + err = bioset_init(&crypto_bio_split, 64, 0, 0); if (err) goto out; + + err = blk_ksm_init(&blk_crypto_ksm, blk_crypto_num_keyslots); + if (err) + goto fail_free_bioset; err = -ENOMEM; blk_crypto_ksm.ksm_ll_ops = blk_crypto_ksm_ll_ops; @@ -591,6 +597,8 @@ static int blk_crypto_fallback_init(void) destroy_workqueue(blk_crypto_wq); fail_free_ksm: blk_ksm_destroy(&blk_crypto_ksm); +fail_free_bioset: + bioset_exit(&crypto_bio_split); out: return err; } diff --git a/block/blk-crypto.c b/block/blk-crypto.c index 09fcb18fa778fad699cc4ac52894814d22adfd0a..c5bdaafffa29f09db48813a0784a7e3b3603fdf9 100644 --- a/block/blk-crypto.c +++ b/block/blk-crypto.c @@ -409,3 +409,4 @@ int blk_crypto_evict_key(struct request_queue *q, */ return blk_crypto_fallback_evict_key(key); } +EXPORT_SYMBOL_GPL(blk_crypto_evict_key); diff --git a/block/blk-map.c b/block/blk-map.c index 21630dccac628c3fac3131c5d900fc144ea962e1..369e204d14d013f66e3422444d19361d09a10863 100644 --- a/block/blk-map.c +++ b/block/blk-map.c @@ -150,9 +150,7 @@ static int bio_copy_user_iov(struct request *rq, struct rq_map_data *map_data, bmd->is_our_pages = !map_data; bmd->is_null_mapped = (map_data && map_data->null_mapped); - nr_pages = DIV_ROUND_UP(offset + len, PAGE_SIZE); - if (nr_pages > BIO_MAX_PAGES) - nr_pages = BIO_MAX_PAGES; + nr_pages = bio_max_segs(DIV_ROUND_UP(offset + len, PAGE_SIZE)); ret = -ENOMEM; bio = bio_kmalloc(gfp_mask, nr_pages); diff --git a/block/blk-mq-sched.c b/block/blk-mq-sched.c index deff4e826e234dea5ed72dbb041ca7d3e1768a1d..ddb65e9e6fd9d814ec828ef5af569d368b13087d 100644 --- a/block/blk-mq-sched.c +++ b/block/blk-mq-sched.c @@ -384,12 +384,6 @@ bool blk_mq_sched_try_insert_merge(struct request_queue *q, struct request *rq) } EXPORT_SYMBOL_GPL(blk_mq_sched_try_insert_merge); -void blk_mq_sched_request_inserted(struct request *rq) -{ - trace_block_rq_insert(rq); -} -EXPORT_SYMBOL_GPL(blk_mq_sched_request_inserted); - static bool blk_mq_sched_bypass_insert(struct blk_mq_hw_ctx *hctx, bool has_sched, struct request *rq) diff --git a/block/blk-mq-sched.h b/block/blk-mq-sched.h index 0476360f05f18a23c01bfc92c0ba2d891c88ce2e..5b18ab915c652fad4774216481d07d0567b4d63c 100644 --- a/block/blk-mq-sched.h +++ b/block/blk-mq-sched.h @@ -7,7 +7,6 @@ void blk_mq_sched_assign_ioc(struct request *rq); -void blk_mq_sched_request_inserted(struct request *rq); bool blk_mq_sched_try_merge(struct request_queue *q, struct bio *bio, unsigned int nr_segs, struct request **merged_request); bool __blk_mq_sched_bio_merge(struct request_queue *q, struct bio *bio, diff --git a/block/blk-mq.c b/block/blk-mq.c index f21d922ecfaf0b92e09810a2df423a5927f52648..d4d7c1caa439666f212c63df86a2a61a81a68ca9 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -41,7 +41,7 @@ #include "blk-mq-sched.h" #include "blk-rq-qos.h" -static DEFINE_PER_CPU(struct list_head, blk_cpu_done); +static DEFINE_PER_CPU(struct llist_head, blk_cpu_done); static void blk_mq_poll_stats_start(struct request_queue *q); static void blk_mq_poll_stats_fn(struct blk_stat_callback *cb); @@ -567,80 +567,29 @@ void blk_mq_end_request(struct request *rq, blk_status_t error) } EXPORT_SYMBOL(blk_mq_end_request); -/* - * Softirq action handler - move entries to local list and loop over them - * while passing them to the queue registered handler. - */ -static __latent_entropy void blk_done_softirq(struct softirq_action *h) +static void blk_complete_reqs(struct llist_head *list) { - struct list_head *cpu_list, local_list; - - local_irq_disable(); - cpu_list = this_cpu_ptr(&blk_cpu_done); - list_replace_init(cpu_list, &local_list); - local_irq_enable(); - - while (!list_empty(&local_list)) { - struct request *rq; + struct llist_node *entry = llist_reverse_order(llist_del_all(list)); + struct request *rq, *next; - rq = list_entry(local_list.next, struct request, ipi_list); - list_del_init(&rq->ipi_list); + llist_for_each_entry_safe(rq, next, entry, ipi_list) rq->q->mq_ops->complete(rq); - } } -static void blk_mq_trigger_softirq(struct request *rq) +static __latent_entropy void blk_done_softirq(struct softirq_action *h) { - struct list_head *list; - unsigned long flags; - - local_irq_save(flags); - list = this_cpu_ptr(&blk_cpu_done); - list_add_tail(&rq->ipi_list, list); - - /* - * If the list only contains our just added request, signal a raise of - * the softirq. If there are already entries there, someone already - * raised the irq but it hasn't run yet. - */ - if (list->next == &rq->ipi_list) - raise_softirq_irqoff(BLOCK_SOFTIRQ); - local_irq_restore(flags); + blk_complete_reqs(this_cpu_ptr(&blk_cpu_done)); } static int blk_softirq_cpu_dead(unsigned int cpu) { - /* - * If a CPU goes away, splice its entries to the current CPU - * and trigger a run of the softirq - */ - local_irq_disable(); - list_splice_init(&per_cpu(blk_cpu_done, cpu), - this_cpu_ptr(&blk_cpu_done)); - raise_softirq_irqoff(BLOCK_SOFTIRQ); - local_irq_enable(); - + blk_complete_reqs(&per_cpu(blk_cpu_done, cpu)); return 0; } - static void __blk_mq_complete_request_remote(void *data) { - struct request *rq = data; - - /* - * For most of single queue controllers, there is only one irq vector - * for handling I/O completion, and the only irq's affinity is set - * to all possible CPUs. On most of ARCHs, this affinity means the irq - * is handled on one specific CPU. - * - * So complete I/O requests in softirq context in case of single queue - * devices to avoid degrading I/O performance due to irqsoff latency. - */ - if (rq->q->nr_hw_queues == 1) - blk_mq_trigger_softirq(rq); - else - rq->q->mq_ops->complete(rq); + __raise_softirq_irqoff(BLOCK_SOFTIRQ); } static inline bool blk_mq_complete_need_ipi(struct request *rq) @@ -669,6 +618,30 @@ static inline bool blk_mq_complete_need_ipi(struct request *rq) return cpu_online(rq->mq_ctx->cpu); } +static void blk_mq_complete_send_ipi(struct request *rq) +{ + struct llist_head *list; + unsigned int cpu; + + cpu = rq->mq_ctx->cpu; + list = &per_cpu(blk_cpu_done, cpu); + if (llist_add(&rq->ipi_list, list)) { + INIT_CSD(&rq->csd, __blk_mq_complete_request_remote, rq); + smp_call_function_single_async(cpu, &rq->csd); + } +} + +static void blk_mq_raise_softirq(struct request *rq) +{ + struct llist_head *list; + + preempt_disable(); + list = this_cpu_ptr(&blk_cpu_done); + if (llist_add(&rq->ipi_list, list)) + raise_softirq(BLOCK_SOFTIRQ); + preempt_enable(); +} + bool blk_mq_complete_request_remote(struct request *rq) { WRITE_ONCE(rq->state, MQ_RQ_COMPLETE); @@ -681,15 +654,15 @@ bool blk_mq_complete_request_remote(struct request *rq) return false; if (blk_mq_complete_need_ipi(rq)) { - INIT_CSD(&rq->csd, __blk_mq_complete_request_remote, rq); - smp_call_function_single_async(rq->mq_ctx->cpu, &rq->csd); - } else { - if (rq->q->nr_hw_queues > 1) - return false; - blk_mq_trigger_softirq(rq); + blk_mq_complete_send_ipi(rq); + return true; } - return true; + if (rq->q->nr_hw_queues == 1) { + blk_mq_raise_softirq(rq); + return true; + } + return false; } EXPORT_SYMBOL_GPL(blk_mq_complete_request_remote); @@ -3957,7 +3930,7 @@ static int __init blk_mq_init(void) int i; for_each_possible_cpu(i) - INIT_LIST_HEAD(&per_cpu(blk_cpu_done, i)); + init_llist_head(&per_cpu(blk_cpu_done, i)); open_softirq(BLOCK_SOFTIRQ, blk_done_softirq); cpuhp_setup_state_nocalls(CPUHP_BLOCK_SOFTIRQ_DEAD, diff --git a/block/blk-pm.h b/block/blk-pm.h index a2283cc9f716dc89622a5966fd6e894d8ddefee9..8a5a0d4b357fe3f70573ff1041df403c5cc40cb4 100644 --- a/block/blk-pm.h +++ b/block/blk-pm.h @@ -21,31 +21,6 @@ static inline void blk_pm_mark_last_busy(struct request *rq) if (rq->q->dev && !(rq->rq_flags & RQF_PM)) pm_runtime_mark_last_busy(rq->q->dev); } - -static inline void blk_pm_requeue_request(struct request *rq) -{ - lockdep_assert_held(&rq->q->queue_lock); - - if (rq->q->dev && !(rq->rq_flags & RQF_PM)) - rq->q->nr_pending--; -} - -static inline void blk_pm_add_request(struct request_queue *q, - struct request *rq) -{ - lockdep_assert_held(&q->queue_lock); - - if (q->dev && !(rq->rq_flags & RQF_PM)) - q->nr_pending++; -} - -static inline void blk_pm_put_request(struct request *rq) -{ - lockdep_assert_held(&rq->q->queue_lock); - - if (rq->q->dev && !(rq->rq_flags & RQF_PM)) - --rq->q->nr_pending; -} #else static inline int blk_pm_resume_queue(const bool pm, struct request_queue *q) { @@ -55,19 +30,6 @@ static inline int blk_pm_resume_queue(const bool pm, struct request_queue *q) static inline void blk_pm_mark_last_busy(struct request *rq) { } - -static inline void blk_pm_requeue_request(struct request *rq) -{ -} - -static inline void blk_pm_add_request(struct request_queue *q, - struct request *rq) -{ -} - -static inline void blk_pm_put_request(struct request *rq) -{ -} #endif #endif /* _BLOCK_BLK_PM_H_ */ diff --git a/block/blk-settings.c b/block/blk-settings.c index 7dd8be314ac65cdcdae77f02019c1db3b82a7347..b4aa2f37fab6f589aab68a1a08c0adf5cbb8cd0d 100644 --- a/block/blk-settings.c +++ b/block/blk-settings.c @@ -504,6 +504,14 @@ void blk_queue_io_opt(struct request_queue *q, unsigned int opt) } EXPORT_SYMBOL(blk_queue_io_opt); +static unsigned int blk_round_down_sectors(unsigned int sectors, unsigned int lbs) +{ + sectors = round_down(sectors, lbs >> SECTOR_SHIFT); + if (sectors < PAGE_SIZE >> SECTOR_SHIFT) + sectors = PAGE_SIZE >> SECTOR_SHIFT; + return sectors; +} + /** * blk_stack_limits - adjust queue_limits for stacked devices * @t: the stacking driver limits (top device) @@ -630,6 +638,10 @@ int blk_stack_limits(struct queue_limits *t, struct queue_limits *b, ret = -1; } + t->max_sectors = blk_round_down_sectors(t->max_sectors, t->logical_block_size); + t->max_hw_sectors = blk_round_down_sectors(t->max_hw_sectors, t->logical_block_size); + t->max_dev_sectors = blk_round_down_sectors(t->max_dev_sectors, t->logical_block_size); + /* Discard alignment and granularity */ if (b->discard_granularity) { alignment = queue_limit_discard_alignment(b, start); diff --git a/block/blk-sysfs.c b/block/blk-sysfs.c index ae39c7f3d83dc02e437c7590caf2e948d7f599a0..0f4f0c8a7825637a93ee43e74b68ae0b3dad6f9f 100644 --- a/block/blk-sysfs.c +++ b/block/blk-sysfs.c @@ -434,10 +434,13 @@ static ssize_t queue_poll_store(struct request_queue *q, const char *page, if (ret < 0) return ret; - if (poll_on) + if (poll_on) { blk_queue_flag_set(QUEUE_FLAG_POLL, q); - else + } else { + blk_mq_freeze_queue(q); blk_queue_flag_clear(QUEUE_FLAG_POLL, q); + blk_mq_unfreeze_queue(q); + } return ret; } diff --git a/block/bounce.c b/block/bounce.c index fc55314aa4269ab41fefa0928413011e7932d87a..87983a35079c22b821e7aa21bf2a1a3b5a75164b 100644 --- a/block/bounce.c +++ b/block/bounce.c @@ -214,8 +214,7 @@ static void bounce_end_io_read_isa(struct bio *bio) __bounce_end_io_read(bio, &isa_page_pool); } -static struct bio *bounce_clone_bio(struct bio *bio_src, gfp_t gfp_mask, - struct bio_set *bs) +static struct bio *bounce_clone_bio(struct bio *bio_src) { struct bvec_iter iter; struct bio_vec bv; @@ -242,10 +241,12 @@ static struct bio *bounce_clone_bio(struct bio *bio_src, gfp_t gfp_mask, * asking for trouble and would force extra work on * __bio_clone_fast() anyways. */ - - bio = bio_alloc_bioset(gfp_mask, bio_segments(bio_src), bs); - if (!bio) - return NULL; + if (bio_is_passthrough(bio_src)) + bio = bio_kmalloc(GFP_NOIO | __GFP_NOFAIL, + bio_segments(bio_src)); + else + bio = bio_alloc_bioset(GFP_NOIO, bio_segments(bio_src), + &bounce_bio_set); bio->bi_bdev = bio_src->bi_bdev; if (bio_flagged(bio_src, BIO_REMAPPED)) bio_set_flag(bio, BIO_REMAPPED); @@ -269,11 +270,11 @@ static struct bio *bounce_clone_bio(struct bio *bio_src, gfp_t gfp_mask, break; } - if (bio_crypt_clone(bio, bio_src, gfp_mask) < 0) + if (bio_crypt_clone(bio, bio_src, GFP_NOIO) < 0) goto err_put; if (bio_integrity(bio_src) && - bio_integrity_clone(bio, bio_src, gfp_mask) < 0) + bio_integrity_clone(bio, bio_src, GFP_NOIO) < 0) goto err_put; bio_clone_blkg_association(bio, bio_src); @@ -296,7 +297,6 @@ static void __blk_queue_bounce(struct request_queue *q, struct bio **bio_orig, unsigned i = 0; bool bounce = false; int sectors = 0; - bool passthrough = bio_is_passthrough(*bio_orig); bio_for_each_segment(from, *bio_orig, iter) { if (i++ < BIO_MAX_PAGES) @@ -307,14 +307,14 @@ static void __blk_queue_bounce(struct request_queue *q, struct bio **bio_orig, if (!bounce) return; - if (!passthrough && sectors < bio_sectors(*bio_orig)) { + if (!bio_is_passthrough(*bio_orig) && + sectors < bio_sectors(*bio_orig)) { bio = bio_split(*bio_orig, sectors, GFP_NOIO, &bounce_bio_split); bio_chain(bio, *bio_orig); submit_bio_noacct(*bio_orig); *bio_orig = bio; } - bio = bounce_clone_bio(*bio_orig, GFP_NOIO, passthrough ? NULL : - &bounce_bio_set); + bio = bounce_clone_bio(*bio_orig); /* * Bvec table can't be updated by bio_for_each_segment_all(), diff --git a/block/genhd.c b/block/genhd.c index 36ff45bbaaaf0c5784065b86ea5538ec689bd3ee..fcc530164b5ab4d06dac4175f03b79949e92d280 100644 --- a/block/genhd.c +++ b/block/genhd.c @@ -74,7 +74,7 @@ bool set_capacity_and_notify(struct gendisk *disk, sector_t size) return false; pr_info("%s: detected capacity change from %lld to %lld\n", - disk->disk_name, size, capacity); + disk->disk_name, capacity, size); /* * Historically we did not send a uevent for changes to/from an empty @@ -476,7 +476,7 @@ void disk_uevent(struct gendisk *disk, enum kobject_action action) struct disk_part_iter piter; struct block_device *part; - disk_part_iter_init(&piter, disk, DISK_PITER_INCL_PART0); + disk_part_iter_init(&piter, disk, DISK_PITER_INCL_EMPTY_PART0); while ((part = disk_part_iter_next(&piter))) kobject_uevent(bdev_kobj(part), action); disk_part_iter_exit(&piter); diff --git a/block/ioctl.c b/block/ioctl.c index d61d652078f41c583792f85d03fbeea74c502b50..ff241e663c018feae6f102065dbcce475eb3ae29 100644 --- a/block/ioctl.c +++ b/block/ioctl.c @@ -81,20 +81,27 @@ static int compat_blkpg_ioctl(struct block_device *bdev, } #endif -static int blkdev_reread_part(struct block_device *bdev) +static int blkdev_reread_part(struct block_device *bdev, fmode_t mode) { - int ret; + struct block_device *tmp; if (!disk_part_scan_enabled(bdev->bd_disk) || bdev_is_partition(bdev)) return -EINVAL; if (!capable(CAP_SYS_ADMIN)) return -EACCES; - mutex_lock(&bdev->bd_mutex); - ret = bdev_disk_changed(bdev, false); - mutex_unlock(&bdev->bd_mutex); + /* + * Reopen the device to revalidate the driver state and force a + * partition rescan. + */ + mode &= ~FMODE_EXCL; + set_bit(GD_NEED_PART_SCAN, &bdev->bd_disk->state); - return ret; + tmp = blkdev_get_by_dev(bdev->bd_dev, mode, NULL); + if (IS_ERR(tmp)) + return PTR_ERR(tmp); + blkdev_put(tmp, mode); + return 0; } static int blk_ioctl_discard(struct block_device *bdev, fmode_t mode, @@ -498,7 +505,7 @@ static int blkdev_common_ioctl(struct block_device *bdev, fmode_t mode, bdev->bd_bdi->ra_pages = (arg * 512) / PAGE_SIZE; return 0; case BLKRRPART: - return blkdev_reread_part(bdev); + return blkdev_reread_part(bdev, mode); case BLKTRACESTART: case BLKTRACESTOP: case BLKTRACETEARDOWN: diff --git a/block/keyslot-manager.c b/block/keyslot-manager.c index 86f8195d8039e5da6e5465277a9e22a692524ac8..2c4a55bea6ca1e6a4edfbc49154e0f9f52bf86b1 100644 --- a/block/keyslot-manager.c +++ b/block/keyslot-manager.c @@ -29,6 +29,7 @@ #define pr_fmt(fmt) "blk-crypto: " fmt #include +#include #include #include #include @@ -62,6 +63,11 @@ static inline void blk_ksm_hw_exit(struct blk_keyslot_manager *ksm) pm_runtime_put_sync(ksm->dev); } +static inline bool blk_ksm_is_passthrough(struct blk_keyslot_manager *ksm) +{ + return ksm->num_slots == 0; +} + /** * blk_ksm_init() - Initialize a keyslot manager * @ksm: The keyslot_manager to initialize. @@ -127,6 +133,34 @@ int blk_ksm_init(struct blk_keyslot_manager *ksm, unsigned int num_slots) } EXPORT_SYMBOL_GPL(blk_ksm_init); +static void blk_ksm_destroy_callback(void *ksm) +{ + blk_ksm_destroy(ksm); +} + +/** + * devm_blk_ksm_init() - Resource-managed blk_ksm_init() + * @dev: The device which owns the blk_keyslot_manager. + * @ksm: The blk_keyslot_manager to initialize. + * @num_slots: The number of key slots to manage. + * + * Like blk_ksm_init(), but causes blk_ksm_destroy() to be called automatically + * on driver detach. + * + * Return: 0 on success, or else a negative error code. + */ +int devm_blk_ksm_init(struct device *dev, struct blk_keyslot_manager *ksm, + unsigned int num_slots) +{ + int err = blk_ksm_init(ksm, num_slots); + + if (err) + return err; + + return devm_add_action_or_reset(dev, blk_ksm_destroy_callback, ksm); +} +EXPORT_SYMBOL_GPL(devm_blk_ksm_init); + static inline struct hlist_head * blk_ksm_hash_bucket_for_key(struct blk_keyslot_manager *ksm, const struct blk_crypto_key *key) @@ -205,6 +239,10 @@ blk_status_t blk_ksm_get_slot_for_key(struct blk_keyslot_manager *ksm, int err; *slot_ptr = NULL; + + if (blk_ksm_is_passthrough(ksm)) + return BLK_STS_OK; + down_read(&ksm->lock); slot = blk_ksm_find_and_grab_keyslot(ksm, key); up_read(&ksm->lock); @@ -325,6 +363,16 @@ int blk_ksm_evict_key(struct blk_keyslot_manager *ksm, struct blk_ksm_keyslot *slot; int err = 0; + if (blk_ksm_is_passthrough(ksm)) { + if (ksm->ksm_ll_ops.keyslot_evict) { + blk_ksm_hw_enter(ksm); + err = ksm->ksm_ll_ops.keyslot_evict(ksm, key, -1); + blk_ksm_hw_exit(ksm); + return err; + } + return 0; + } + blk_ksm_hw_enter(ksm); slot = blk_ksm_find_keyslot(ksm, key); if (!slot) @@ -360,6 +408,9 @@ void blk_ksm_reprogram_all_keys(struct blk_keyslot_manager *ksm) { unsigned int slot; + if (blk_ksm_is_passthrough(ksm)) + return; + /* This is for device initialization, so don't resume the device */ down_write(&ksm->lock); for (slot = 0; slot < ksm->num_slots; slot++) { @@ -401,3 +452,127 @@ void blk_ksm_unregister(struct request_queue *q) { q->ksm = NULL; } + +/** + * blk_ksm_intersect_modes() - restrict supported modes by child device + * @parent: The keyslot manager for parent device + * @child: The keyslot manager for child device, or NULL + * + * Clear any crypto mode support bits in @parent that aren't set in @child. + * If @child is NULL, then all parent bits are cleared. + * + * Only use this when setting up the keyslot manager for a layered device, + * before it's been exposed yet. + */ +void blk_ksm_intersect_modes(struct blk_keyslot_manager *parent, + const struct blk_keyslot_manager *child) +{ + if (child) { + unsigned int i; + + parent->max_dun_bytes_supported = + min(parent->max_dun_bytes_supported, + child->max_dun_bytes_supported); + for (i = 0; i < ARRAY_SIZE(child->crypto_modes_supported); + i++) { + parent->crypto_modes_supported[i] &= + child->crypto_modes_supported[i]; + } + } else { + parent->max_dun_bytes_supported = 0; + memset(parent->crypto_modes_supported, 0, + sizeof(parent->crypto_modes_supported)); + } +} +EXPORT_SYMBOL_GPL(blk_ksm_intersect_modes); + +/** + * blk_ksm_is_superset() - Check if a KSM supports a superset of crypto modes + * and DUN bytes that another KSM supports. Here, + * "superset" refers to the mathematical meaning of the + * word - i.e. if two KSMs have the *same* capabilities, + * they *are* considered supersets of each other. + * @ksm_superset: The KSM that we want to verify is a superset + * @ksm_subset: The KSM that we want to verify is a subset + * + * Return: True if @ksm_superset supports a superset of the crypto modes and DUN + * bytes that @ksm_subset supports. + */ +bool blk_ksm_is_superset(struct blk_keyslot_manager *ksm_superset, + struct blk_keyslot_manager *ksm_subset) +{ + int i; + + if (!ksm_subset) + return true; + + if (!ksm_superset) + return false; + + for (i = 0; i < ARRAY_SIZE(ksm_superset->crypto_modes_supported); i++) { + if (ksm_subset->crypto_modes_supported[i] & + (~ksm_superset->crypto_modes_supported[i])) { + return false; + } + } + + if (ksm_subset->max_dun_bytes_supported > + ksm_superset->max_dun_bytes_supported) { + return false; + } + + return true; +} +EXPORT_SYMBOL_GPL(blk_ksm_is_superset); + +/** + * blk_ksm_update_capabilities() - Update the restrictions of a KSM to those of + * another KSM + * @target_ksm: The KSM whose restrictions to update. + * @reference_ksm: The KSM to whose restrictions this function will update + * @target_ksm's restrictions to. + * + * Blk-crypto requires that crypto capabilities that were + * advertised when a bio was created continue to be supported by the + * device until that bio is ended. This is turn means that a device cannot + * shrink its advertised crypto capabilities without any explicit + * synchronization with upper layers. So if there's no such explicit + * synchronization, @reference_ksm must support all the crypto capabilities that + * @target_ksm does + * (i.e. we need blk_ksm_is_superset(@reference_ksm, @target_ksm) == true). + * + * Note also that as long as the crypto capabilities are being expanded, the + * order of updates becoming visible is not important because it's alright + * for blk-crypto to see stale values - they only cause blk-crypto to + * believe that a crypto capability isn't supported when it actually is (which + * might result in blk-crypto-fallback being used if available, or the bio being + * failed). + */ +void blk_ksm_update_capabilities(struct blk_keyslot_manager *target_ksm, + struct blk_keyslot_manager *reference_ksm) +{ + memcpy(target_ksm->crypto_modes_supported, + reference_ksm->crypto_modes_supported, + sizeof(target_ksm->crypto_modes_supported)); + + target_ksm->max_dun_bytes_supported = + reference_ksm->max_dun_bytes_supported; +} +EXPORT_SYMBOL_GPL(blk_ksm_update_capabilities); + +/** + * blk_ksm_init_passthrough() - Init a passthrough keyslot manager + * @ksm: The keyslot manager to init + * + * Initialize a passthrough keyslot manager. + * Called by e.g. storage drivers to set up a keyslot manager in their + * request_queue, when the storage driver wants to manage its keys by itself. + * This is useful for inline encryption hardware that doesn't have the concept + * of keyslots, and for layered devices. + */ +void blk_ksm_init_passthrough(struct blk_keyslot_manager *ksm) +{ + memset(ksm, 0, sizeof(*ksm)); + init_rwsem(&ksm->lock); +} +EXPORT_SYMBOL_GPL(blk_ksm_init_passthrough); diff --git a/block/kyber-iosched.c b/block/kyber-iosched.c index c25c41d0d061c0f7840677231e1a9c382e23f5dc..33d34d69cade2c03491040c0e1255cba516de83c 100644 --- a/block/kyber-iosched.c +++ b/block/kyber-iosched.c @@ -13,6 +13,8 @@ #include #include +#include + #include "blk.h" #include "blk-mq.h" #include "blk-mq-debugfs.h" @@ -353,19 +355,9 @@ static void kyber_timer_fn(struct timer_list *t) } } -static unsigned int kyber_sched_tags_shift(struct request_queue *q) -{ - /* - * All of the hardware queues have the same depth, so we can just grab - * the shift of the first one. - */ - return q->queue_hw_ctx[0]->sched_tags->bitmap_tags->sb.shift; -} - static struct kyber_queue_data *kyber_queue_data_alloc(struct request_queue *q) { struct kyber_queue_data *kqd; - unsigned int shift; int ret = -ENOMEM; int i; @@ -400,9 +392,6 @@ static struct kyber_queue_data *kyber_queue_data_alloc(struct request_queue *q) kqd->latency_targets[i] = kyber_latency_targets[i]; } - shift = kyber_sched_tags_shift(q); - kqd->async_depth = (1U << shift) * KYBER_ASYNC_PERCENT / 100U; - return kqd; err_buckets: @@ -458,9 +447,19 @@ static void kyber_ctx_queue_init(struct kyber_ctx_queue *kcq) INIT_LIST_HEAD(&kcq->rq_list[i]); } -static int kyber_init_hctx(struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx) +static void kyber_depth_updated(struct blk_mq_hw_ctx *hctx) { struct kyber_queue_data *kqd = hctx->queue->elevator->elevator_data; + struct blk_mq_tags *tags = hctx->sched_tags; + unsigned int shift = tags->bitmap_tags->sb.shift; + + kqd->async_depth = (1U << shift) * KYBER_ASYNC_PERCENT / 100U; + + sbitmap_queue_min_shallow_depth(tags->bitmap_tags, kqd->async_depth); +} + +static int kyber_init_hctx(struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx) +{ struct kyber_hctx_data *khd; int i; @@ -502,8 +501,7 @@ static int kyber_init_hctx(struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx) khd->batching = 0; hctx->sched_data = khd; - sbitmap_queue_min_shallow_depth(hctx->sched_tags->bitmap_tags, - kqd->async_depth); + kyber_depth_updated(hctx); return 0; @@ -602,7 +600,7 @@ static void kyber_insert_requests(struct blk_mq_hw_ctx *hctx, list_move_tail(&rq->queuelist, head); sbitmap_set_bit(&khd->kcq_map[sched_domain], rq->mq_ctx->index_hw[hctx->type]); - blk_mq_sched_request_inserted(rq); + trace_block_rq_insert(rq); spin_unlock(&kcq->lock); } } @@ -1022,6 +1020,7 @@ static struct elevator_type kyber_sched = { .completed_request = kyber_completed_request, .dispatch_request = kyber_dispatch_request, .has_work = kyber_has_work, + .depth_updated = kyber_depth_updated, }, #ifdef CONFIG_BLK_DEBUG_FS .queue_debugfs_attrs = kyber_queue_debugfs_attrs, diff --git a/block/mq-deadline.c b/block/mq-deadline.c index b57470e154c8f599584beb6170da013b87b7fc18..f3631a28746676ec1f2aa7261501a9f79ae84e78 100644 --- a/block/mq-deadline.c +++ b/block/mq-deadline.c @@ -18,6 +18,8 @@ #include #include +#include + #include "blk.h" #include "blk-mq.h" #include "blk-mq-debugfs.h" @@ -496,7 +498,7 @@ static void dd_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq, if (blk_mq_sched_try_insert_merge(q, rq)) return; - blk_mq_sched_request_inserted(rq); + trace_block_rq_insert(rq); if (at_head || blk_rq_is_passthrough(rq)) { if (at_head) diff --git a/certs/blacklist.c b/certs/blacklist.c index 6514f9ebc943f4e7e8cd2f3ceba36da0762c269a..bffe4c6f4a9e204d215119278d419bb491d2d7c8 100644 --- a/certs/blacklist.c +++ b/certs/blacklist.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include "blacklist.h" @@ -37,7 +38,7 @@ static int blacklist_vet_description(const char *desc) found_colon: desc++; for (; *desc; desc++) { - if (!isxdigit(*desc)) + if (!isxdigit(*desc) || isupper(*desc)) return -EINVAL; n++; } @@ -78,7 +79,7 @@ static struct key_type key_type_blacklist = { /** * mark_hash_blacklisted - Add a hash to the system blacklist - * @hash - The hash as a hex string with a type prefix (eg. "tbs:23aa429783") + * @hash: The hash as a hex string with a type prefix (eg. "tbs:23aa429783") */ int mark_hash_blacklisted(const char *hash) { @@ -156,13 +157,12 @@ static int __init blacklist_init(void) blacklist_keyring = keyring_alloc(".blacklist", - KUIDT_INIT(0), KGIDT_INIT(0), - current_cred(), + GLOBAL_ROOT_UID, GLOBAL_ROOT_GID, current_cred(), (KEY_POS_ALL & ~KEY_POS_SETATTR) | KEY_USR_VIEW | KEY_USR_READ | KEY_USR_SEARCH, KEY_ALLOC_NOT_IN_QUOTA | - KEY_FLAG_KEEP, + KEY_ALLOC_SET_KEEP, NULL, NULL); if (IS_ERR(blacklist_keyring)) panic("Can't allocate system blacklist keyring\n"); diff --git a/certs/system_keyring.c b/certs/system_keyring.c index 798291177186c361ffdb6797096bdb0f219228ae..4b693da488f14bcc66b452789f2eab92da959f9f 100644 --- a/certs/system_keyring.c +++ b/certs/system_keyring.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -98,7 +99,7 @@ static __init int system_trusted_keyring_init(void) builtin_trusted_keys = keyring_alloc(".builtin_trusted_keys", - KUIDT_INIT(0), KGIDT_INIT(0), current_cred(), + GLOBAL_ROOT_UID, GLOBAL_ROOT_GID, current_cred(), ((KEY_POS_ALL & ~KEY_POS_SETATTR) | KEY_USR_VIEW | KEY_USR_READ | KEY_USR_SEARCH), KEY_ALLOC_NOT_IN_QUOTA, @@ -109,7 +110,7 @@ static __init int system_trusted_keyring_init(void) #ifdef CONFIG_SECONDARY_TRUSTED_KEYRING secondary_trusted_keys = keyring_alloc(".secondary_trusted_keys", - KUIDT_INIT(0), KGIDT_INIT(0), current_cred(), + GLOBAL_ROOT_UID, GLOBAL_ROOT_GID, current_cred(), ((KEY_POS_ALL & ~KEY_POS_SETATTR) | KEY_USR_VIEW | KEY_USR_READ | KEY_USR_SEARCH | KEY_USR_WRITE), diff --git a/crypto/Kconfig b/crypto/Kconfig index a367fcfeb5d4528c8565e32ef0b03cfbf8ea1653..15c9c28d9f5333866cd179accc1bb030741aca2d 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -210,11 +210,6 @@ config CRYPTO_SIMD tristate select CRYPTO_CRYPTD -config CRYPTO_GLUE_HELPER_X86 - tristate - depends on X86 - select CRYPTO_SKCIPHER - config CRYPTO_ENGINE tristate @@ -822,19 +817,6 @@ config CRYPTO_MICHAEL_MIC should not be used for other purposes because of the weakness of the algorithm. -config CRYPTO_RMD128 - tristate "RIPEMD-128 digest algorithm" - select CRYPTO_HASH - help - RIPEMD-128 (ISO/IEC 10118-3:2004). - - RIPEMD-128 is a 128-bit cryptographic hash function. It should only - be used as a secure replacement for RIPEMD. For other use cases, - RIPEMD-160 should be used. - - Developed by Hans Dobbertin, Antoon Bosselaers and Bart Preneel. - See - config CRYPTO_RMD160 tristate "RIPEMD-160 digest algorithm" select CRYPTO_HASH @@ -852,30 +834,6 @@ config CRYPTO_RMD160 Developed by Hans Dobbertin, Antoon Bosselaers and Bart Preneel. See -config CRYPTO_RMD256 - tristate "RIPEMD-256 digest algorithm" - select CRYPTO_HASH - help - RIPEMD-256 is an optional extension of RIPEMD-128 with a - 256 bit hash. It is intended for applications that require - longer hash-results, without needing a larger security level - (than RIPEMD-128). - - Developed by Hans Dobbertin, Antoon Bosselaers and Bart Preneel. - See - -config CRYPTO_RMD320 - tristate "RIPEMD-320 digest algorithm" - select CRYPTO_HASH - help - RIPEMD-320 is an optional extension of RIPEMD-160 with a - 320 bit hash. It is intended for applications that require - longer hash-results, without needing a larger security level - (than RIPEMD-160). - - Developed by Hans Dobbertin, Antoon Bosselaers and Bart Preneel. - See - config CRYPTO_SHA1 tristate "SHA1 digest algorithm" select CRYPTO_HASH @@ -1051,19 +1009,6 @@ config CRYPTO_STREEBOG https://tc26.ru/upload/iblock/fed/feddbb4d26b685903faa2ba11aea43f6.pdf https://tools.ietf.org/html/rfc6986 -config CRYPTO_TGR192 - tristate "Tiger digest algorithms" - select CRYPTO_HASH - help - Tiger hash algorithm 192, 160 and 128-bit hashes - - Tiger is a hash function optimized for 64-bit processors while - still having decent performance on 32-bit processors. - Tiger was developed by Ross Anderson and Eli Biham. - - See also: - . - config CRYPTO_WP512 tristate "Whirlpool digest algorithms" select CRYPTO_HASH @@ -1133,7 +1078,6 @@ config CRYPTO_AES_NI_INTEL select CRYPTO_LIB_AES select CRYPTO_ALGAPI select CRYPTO_SKCIPHER - select CRYPTO_GLUE_HELPER_X86 if 64BIT select CRYPTO_SIMD help Use Intel AES-NI instructions for AES algorithm. @@ -1256,6 +1200,7 @@ config CRYPTO_BLOWFISH_X86_64 depends on X86 && 64BIT select CRYPTO_SKCIPHER select CRYPTO_BLOWFISH_COMMON + imply CRYPTO_CTR help Blowfish cipher algorithm (x86_64), by Bruce Schneier. @@ -1286,7 +1231,7 @@ config CRYPTO_CAMELLIA_X86_64 depends on X86 && 64BIT depends on CRYPTO select CRYPTO_SKCIPHER - select CRYPTO_GLUE_HELPER_X86 + imply CRYPTO_CTR help Camellia cipher algorithm module (x86_64). @@ -1304,9 +1249,8 @@ config CRYPTO_CAMELLIA_AESNI_AVX_X86_64 depends on CRYPTO select CRYPTO_SKCIPHER select CRYPTO_CAMELLIA_X86_64 - select CRYPTO_GLUE_HELPER_X86 select CRYPTO_SIMD - select CRYPTO_XTS + imply CRYPTO_XTS help Camellia cipher algorithm module (x86_64/AES-NI/AVX). @@ -1372,6 +1316,7 @@ config CRYPTO_CAST5_AVX_X86_64 select CRYPTO_CAST5 select CRYPTO_CAST_COMMON select CRYPTO_SIMD + imply CRYPTO_CTR help The CAST5 encryption algorithm (synonymous with CAST-128) is described in RFC2144. @@ -1393,9 +1338,9 @@ config CRYPTO_CAST6_AVX_X86_64 select CRYPTO_SKCIPHER select CRYPTO_CAST6 select CRYPTO_CAST_COMMON - select CRYPTO_GLUE_HELPER_X86 select CRYPTO_SIMD - select CRYPTO_XTS + imply CRYPTO_XTS + imply CRYPTO_CTR help The CAST6 encryption algorithm (synonymous with CAST-256) is described in RFC2612. @@ -1425,6 +1370,7 @@ config CRYPTO_DES3_EDE_X86_64 depends on X86 && 64BIT select CRYPTO_SKCIPHER select CRYPTO_LIB_DES + imply CRYPTO_CTR help Triple DES EDE (FIPS 46-3) algorithm. @@ -1454,18 +1400,6 @@ config CRYPTO_KHAZAD See also: -config CRYPTO_SALSA20 - tristate "Salsa20 stream cipher algorithm" - select CRYPTO_SKCIPHER - help - Salsa20 stream cipher algorithm. - - Salsa20 is a stream cipher submitted to eSTREAM, the ECRYPT - Stream Cipher Project. See - - The Salsa20 stream cipher algorithm is designed by Daniel J. - Bernstein . See - config CRYPTO_CHACHA20 tristate "ChaCha stream cipher algorithms" select CRYPTO_LIB_CHACHA_GENERIC @@ -1526,8 +1460,7 @@ config CRYPTO_SERPENT Serpent cipher algorithm, by Anderson, Biham & Knudsen. Keys are allowed to be from 0 to 256 bits in length, in steps - of 8 bits. Also includes the 'Tnepres' algorithm, a reversed - variant of Serpent for compatibility with old kerneli.org code. + of 8 bits. See also: @@ -1536,9 +1469,9 @@ config CRYPTO_SERPENT_SSE2_X86_64 tristate "Serpent cipher algorithm (x86_64/SSE2)" depends on X86 && 64BIT select CRYPTO_SKCIPHER - select CRYPTO_GLUE_HELPER_X86 select CRYPTO_SERPENT select CRYPTO_SIMD + imply CRYPTO_CTR help Serpent cipher algorithm, by Anderson, Biham & Knudsen. @@ -1555,9 +1488,9 @@ config CRYPTO_SERPENT_SSE2_586 tristate "Serpent cipher algorithm (i586/SSE2)" depends on X86 && !64BIT select CRYPTO_SKCIPHER - select CRYPTO_GLUE_HELPER_X86 select CRYPTO_SERPENT select CRYPTO_SIMD + imply CRYPTO_CTR help Serpent cipher algorithm, by Anderson, Biham & Knudsen. @@ -1574,10 +1507,10 @@ config CRYPTO_SERPENT_AVX_X86_64 tristate "Serpent cipher algorithm (x86_64/AVX)" depends on X86 && 64BIT select CRYPTO_SKCIPHER - select CRYPTO_GLUE_HELPER_X86 select CRYPTO_SERPENT select CRYPTO_SIMD - select CRYPTO_XTS + imply CRYPTO_XTS + imply CRYPTO_CTR help Serpent cipher algorithm, by Anderson, Biham & Knudsen. @@ -1675,6 +1608,7 @@ config CRYPTO_TWOFISH_586 depends on (X86 || UML_X86) && !64BIT select CRYPTO_ALGAPI select CRYPTO_TWOFISH_COMMON + imply CRYPTO_CTR help Twofish cipher algorithm. @@ -1691,6 +1625,7 @@ config CRYPTO_TWOFISH_X86_64 depends on (X86 || UML_X86) && 64BIT select CRYPTO_ALGAPI select CRYPTO_TWOFISH_COMMON + imply CRYPTO_CTR help Twofish cipher algorithm (x86_64). @@ -1708,7 +1643,6 @@ config CRYPTO_TWOFISH_X86_64_3WAY select CRYPTO_SKCIPHER select CRYPTO_TWOFISH_COMMON select CRYPTO_TWOFISH_X86_64 - select CRYPTO_GLUE_HELPER_X86 help Twofish cipher algorithm (x86_64, 3-way parallel). @@ -1727,11 +1661,11 @@ config CRYPTO_TWOFISH_AVX_X86_64 tristate "Twofish cipher algorithm (x86_64/AVX)" depends on X86 && 64BIT select CRYPTO_SKCIPHER - select CRYPTO_GLUE_HELPER_X86 select CRYPTO_SIMD select CRYPTO_TWOFISH_COMMON select CRYPTO_TWOFISH_X86_64 select CRYPTO_TWOFISH_X86_64_3WAY + imply CRYPTO_XTS help Twofish cipher algorithm (x86_64/AVX). diff --git a/crypto/Makefile b/crypto/Makefile index b279483fba50b07b530ac7e97cb841dc0e17b3b5..cf23affb16780686a83dea9684095bad03ac9539 100644 --- a/crypto/Makefile +++ b/crypto/Makefile @@ -67,9 +67,7 @@ obj-$(CONFIG_CRYPTO_XCBC) += xcbc.o obj-$(CONFIG_CRYPTO_NULL2) += crypto_null.o obj-$(CONFIG_CRYPTO_MD4) += md4.o obj-$(CONFIG_CRYPTO_MD5) += md5.o -obj-$(CONFIG_CRYPTO_RMD128) += rmd128.o obj-$(CONFIG_CRYPTO_RMD160) += rmd160.o -obj-$(CONFIG_CRYPTO_RMD256) += rmd256.o obj-$(CONFIG_CRYPTO_RMD320) += rmd320.o obj-$(CONFIG_CRYPTO_SHA1) += sha1_generic.o obj-$(CONFIG_CRYPTO_SHA256) += sha256_generic.o @@ -79,7 +77,6 @@ obj-$(CONFIG_CRYPTO_SM3) += sm3_generic.o obj-$(CONFIG_CRYPTO_STREEBOG) += streebog_generic.o obj-$(CONFIG_CRYPTO_WP512) += wp512.o CFLAGS_wp512.o := $(call cc-option,-fno-schedule-insns) # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=79149 -obj-$(CONFIG_CRYPTO_TGR192) += tgr192.o obj-$(CONFIG_CRYPTO_BLAKE2B) += blake2b_generic.o obj-$(CONFIG_CRYPTO_BLAKE2S) += blake2s_generic.o obj-$(CONFIG_CRYPTO_GF128MUL) += gf128mul.o @@ -141,7 +138,6 @@ obj-$(CONFIG_CRYPTO_TEA) += tea.o obj-$(CONFIG_CRYPTO_KHAZAD) += khazad.o obj-$(CONFIG_CRYPTO_ANUBIS) += anubis.o obj-$(CONFIG_CRYPTO_SEED) += seed.o -obj-$(CONFIG_CRYPTO_SALSA20) += salsa20_generic.o obj-$(CONFIG_CRYPTO_CHACHA20) += chacha_generic.o obj-$(CONFIG_CRYPTO_POLY1305) += poly1305_generic.o obj-$(CONFIG_CRYPTO_DEFLATE) += deflate.o diff --git a/crypto/adiantum.c b/crypto/adiantum.c index ce4d5725342cb31f4fb8e60d0ece879bab47c4ed..84450130cb6b382da9b42446004f49bbb0c15de1 100644 --- a/crypto/adiantum.c +++ b/crypto/adiantum.c @@ -32,6 +32,7 @@ #include #include +#include #include #include #include @@ -616,3 +617,4 @@ MODULE_DESCRIPTION("Adiantum length-preserving encryption mode"); MODULE_LICENSE("GPL v2"); MODULE_AUTHOR("Eric Biggers "); MODULE_ALIAS_CRYPTO("adiantum"); +MODULE_IMPORT_NS(CRYPTO_INTERNAL); diff --git a/crypto/ansi_cprng.c b/crypto/ansi_cprng.c index c475c1129ff2a67991ada8aa82d4f0093867e32b..3f512efaba3aa11209cf6a9e9a6db1ae87d2501f 100644 --- a/crypto/ansi_cprng.c +++ b/crypto/ansi_cprng.c @@ -7,6 +7,7 @@ * (C) Neil Horman */ +#include #include #include #include @@ -470,3 +471,4 @@ subsys_initcall(prng_mod_init); module_exit(prng_mod_fini); MODULE_ALIAS_CRYPTO("stdrng"); MODULE_ALIAS_CRYPTO("ansi_cprng"); +MODULE_IMPORT_NS(CRYPTO_INTERNAL); diff --git a/crypto/asymmetric_keys/asymmetric_type.c b/crypto/asymmetric_keys/asymmetric_type.c index 33e77d846caa8709ba9998606ce406524a4d21cf..ad8af3d70ac04d6f2b2ef6db6778aa0e6085cbd1 100644 --- a/crypto/asymmetric_keys/asymmetric_type.c +++ b/crypto/asymmetric_keys/asymmetric_type.c @@ -152,7 +152,8 @@ EXPORT_SYMBOL_GPL(asymmetric_key_generate_id); /** * asymmetric_key_id_same - Return true if two asymmetric keys IDs are the same. - * @kid_1, @kid_2: The key IDs to compare + * @kid1: The key ID to compare + * @kid2: The key ID to compare */ bool asymmetric_key_id_same(const struct asymmetric_key_id *kid1, const struct asymmetric_key_id *kid2) @@ -168,7 +169,8 @@ EXPORT_SYMBOL_GPL(asymmetric_key_id_same); /** * asymmetric_key_id_partial - Return true if two asymmetric keys IDs * partially match - * @kid_1, @kid_2: The key IDs to compare + * @kid1: The key ID to compare + * @kid2: The key ID to compare */ bool asymmetric_key_id_partial(const struct asymmetric_key_id *kid1, const struct asymmetric_key_id *kid2) diff --git a/crypto/asymmetric_keys/pkcs7_parser.h b/crypto/asymmetric_keys/pkcs7_parser.h index 6565fdc2d4cac2bc49bfa629edde67b875573cb7..e17f7ce4fb434832a3867ce2254de6ccc7f06ae7 100644 --- a/crypto/asymmetric_keys/pkcs7_parser.h +++ b/crypto/asymmetric_keys/pkcs7_parser.h @@ -41,10 +41,9 @@ struct pkcs7_signed_info { * * This contains the generated digest of _either_ the Content Data or * the Authenticated Attributes [RFC2315 9.3]. If the latter, one of - * the attributes contains the digest of the the Content Data within - * it. + * the attributes contains the digest of the Content Data within it. * - * THis also contains the issuing cert serial number and issuer's name + * This also contains the issuing cert serial number and issuer's name * [PKCS#7 or CMS ver 1] or issuing cert's SKID [CMS ver 3]. */ struct public_key_signature *sig; diff --git a/crypto/asymmetric_keys/pkcs7_trust.c b/crypto/asymmetric_keys/pkcs7_trust.c index 61af3c4d82ccf96b622d4cad6044ea985426d159..b531df2013c41d6c5d7ccdb1e00f296a94fc0c9a 100644 --- a/crypto/asymmetric_keys/pkcs7_trust.c +++ b/crypto/asymmetric_keys/pkcs7_trust.c @@ -16,7 +16,7 @@ #include #include "pkcs7_parser.h" -/** +/* * Check the trust on one PKCS#7 SignedInfo block. */ static int pkcs7_validate_trust_one(struct pkcs7_message *pkcs7, diff --git a/crypto/asymmetric_keys/pkcs7_verify.c b/crypto/asymmetric_keys/pkcs7_verify.c index ce49820caa97fb885766798060677361742f3645..0b4d07aa88111e332d16faeb2f97d8b594585741 100644 --- a/crypto/asymmetric_keys/pkcs7_verify.c +++ b/crypto/asymmetric_keys/pkcs7_verify.c @@ -141,11 +141,10 @@ int pkcs7_get_digest(struct pkcs7_message *pkcs7, const u8 **buf, u32 *len, *buf = sinfo->sig->digest; *len = sinfo->sig->digest_size; - for (i = 0; i < HASH_ALGO__LAST; i++) - if (!strcmp(hash_algo_name[i], sinfo->sig->hash_algo)) { - *hash_algo = i; - break; - } + i = match_string(hash_algo_name, HASH_ALGO__LAST, + sinfo->sig->hash_algo); + if (i >= 0) + *hash_algo = i; return 0; } diff --git a/crypto/asymmetric_keys/restrict.c b/crypto/asymmetric_keys/restrict.c index 77ebebada29c44095a0bd548c5606766bd3f7bf9..84cefe3b35856829ad1bcc117a14406828976a83 100644 --- a/crypto/asymmetric_keys/restrict.c +++ b/crypto/asymmetric_keys/restrict.c @@ -244,9 +244,10 @@ int restrict_link_by_key_or_keyring(struct key *dest_keyring, * @payload: The payload of the new key. * @trusted: A key or ring of keys that can be used to vouch for the new cert. * - * Check the new certificate only against the key or keys passed in the data - * parameter. If one of those is the signing key and validates the new - * certificate, then mark the new certificate as being ok to link. + * Check the new certificate against the key or keys passed in the data + * parameter and against the keys already linked to the destination keyring. If + * one of those is the signing key and validates the new certificate, then mark + * the new certificate as being ok to link. * * Returns 0 if the new certificate was accepted, -ENOKEY if we * couldn't find a matching parent certificate in the trusted list, diff --git a/crypto/blake2b_generic.c b/crypto/blake2b_generic.c index a2ffe60e06d34059c3eb0a53fb6f5e57ebc949aa..6704c03558896729cd40cf0c91b9c73d6182fe84 100644 --- a/crypto/blake2b_generic.c +++ b/crypto/blake2b_generic.c @@ -1,55 +1,27 @@ // SPDX-License-Identifier: (GPL-2.0-only OR Apache-2.0) /* - * BLAKE2b reference source code package - reference C implementations + * Generic implementation of the BLAKE2b digest algorithm. Based on the BLAKE2b + * reference implementation, but it has been heavily modified for use in the + * kernel. The reference implementation was: * - * Copyright 2012, Samuel Neves . You may use this under the - * terms of the CC0, the OpenSSL Licence, or the Apache Public License 2.0, at - * your option. The terms of these licenses can be found at: + * Copyright 2012, Samuel Neves . You may use this under + * the terms of the CC0, the OpenSSL Licence, or the Apache Public License + * 2.0, at your option. The terms of these licenses can be found at: * - * - CC0 1.0 Universal : http://creativecommons.org/publicdomain/zero/1.0 - * - OpenSSL license : https://www.openssl.org/source/license.html - * - Apache 2.0 : https://www.apache.org/licenses/LICENSE-2.0 + * - CC0 1.0 Universal : http://creativecommons.org/publicdomain/zero/1.0 + * - OpenSSL license : https://www.openssl.org/source/license.html + * - Apache 2.0 : https://www.apache.org/licenses/LICENSE-2.0 * - * More information about the BLAKE2 hash function can be found at - * https://blake2.net. - * - * Note: the original sources have been modified for inclusion in linux kernel - * in terms of coding style, using generic helpers and simplifications of error - * handling. + * More information about BLAKE2 can be found at https://blake2.net. */ #include #include -#include #include #include +#include #include -#define BLAKE2B_160_DIGEST_SIZE (160 / 8) -#define BLAKE2B_256_DIGEST_SIZE (256 / 8) -#define BLAKE2B_384_DIGEST_SIZE (384 / 8) -#define BLAKE2B_512_DIGEST_SIZE (512 / 8) - -enum blake2b_constant { - BLAKE2B_BLOCKBYTES = 128, - BLAKE2B_KEYBYTES = 64, -}; - -struct blake2b_state { - u64 h[8]; - u64 t[2]; - u64 f[2]; - u8 buf[BLAKE2B_BLOCKBYTES]; - size_t buflen; -}; - -static const u64 blake2b_IV[8] = { - 0x6a09e667f3bcc908ULL, 0xbb67ae8584caa73bULL, - 0x3c6ef372fe94f82bULL, 0xa54ff53a5f1d36f1ULL, - 0x510e527fade682d1ULL, 0x9b05688c2b3e6c1fULL, - 0x1f83d9abfb41bd6bULL, 0x5be0cd19137e2179ULL -}; - static const u8 blake2b_sigma[12][16] = { { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 }, { 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 }, @@ -95,8 +67,8 @@ static void blake2b_increment_counter(struct blake2b_state *S, const u64 inc) G(r,7,v[ 3],v[ 4],v[ 9],v[14]); \ } while (0) -static void blake2b_compress(struct blake2b_state *S, - const u8 block[BLAKE2B_BLOCKBYTES]) +static void blake2b_compress_one_generic(struct blake2b_state *S, + const u8 block[BLAKE2B_BLOCK_SIZE]) { u64 m[16]; u64 v[16]; @@ -108,14 +80,14 @@ static void blake2b_compress(struct blake2b_state *S, for (i = 0; i < 8; ++i) v[i] = S->h[i]; - v[ 8] = blake2b_IV[0]; - v[ 9] = blake2b_IV[1]; - v[10] = blake2b_IV[2]; - v[11] = blake2b_IV[3]; - v[12] = blake2b_IV[4] ^ S->t[0]; - v[13] = blake2b_IV[5] ^ S->t[1]; - v[14] = blake2b_IV[6] ^ S->f[0]; - v[15] = blake2b_IV[7] ^ S->f[1]; + v[ 8] = BLAKE2B_IV0; + v[ 9] = BLAKE2B_IV1; + v[10] = BLAKE2B_IV2; + v[11] = BLAKE2B_IV3; + v[12] = BLAKE2B_IV4 ^ S->t[0]; + v[13] = BLAKE2B_IV5 ^ S->t[1]; + v[14] = BLAKE2B_IV6 ^ S->f[0]; + v[15] = BLAKE2B_IV7 ^ S->f[1]; ROUND(0); ROUND(1); @@ -139,159 +111,54 @@ static void blake2b_compress(struct blake2b_state *S, #undef G #undef ROUND -struct blake2b_tfm_ctx { - u8 key[BLAKE2B_KEYBYTES]; - unsigned int keylen; -}; - -static int blake2b_setkey(struct crypto_shash *tfm, const u8 *key, - unsigned int keylen) +void blake2b_compress_generic(struct blake2b_state *state, + const u8 *block, size_t nblocks, u32 inc) { - struct blake2b_tfm_ctx *tctx = crypto_shash_ctx(tfm); - - if (keylen == 0 || keylen > BLAKE2B_KEYBYTES) - return -EINVAL; - - memcpy(tctx->key, key, keylen); - tctx->keylen = keylen; - - return 0; + do { + blake2b_increment_counter(state, inc); + blake2b_compress_one_generic(state, block); + block += BLAKE2B_BLOCK_SIZE; + } while (--nblocks); } +EXPORT_SYMBOL(blake2b_compress_generic); -static int blake2b_init(struct shash_desc *desc) +static int crypto_blake2b_update_generic(struct shash_desc *desc, + const u8 *in, unsigned int inlen) { - struct blake2b_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); - struct blake2b_state *state = shash_desc_ctx(desc); - const int digestsize = crypto_shash_digestsize(desc->tfm); - - memset(state, 0, sizeof(*state)); - memcpy(state->h, blake2b_IV, sizeof(state->h)); - - /* Parameter block is all zeros except index 0, no xor for 1..7 */ - state->h[0] ^= 0x01010000 | tctx->keylen << 8 | digestsize; - - if (tctx->keylen) { - /* - * Prefill the buffer with the key, next call to _update or - * _final will process it - */ - memcpy(state->buf, tctx->key, tctx->keylen); - state->buflen = BLAKE2B_BLOCKBYTES; - } - return 0; + return crypto_blake2b_update(desc, in, inlen, blake2b_compress_generic); } -static int blake2b_update(struct shash_desc *desc, const u8 *in, - unsigned int inlen) +static int crypto_blake2b_final_generic(struct shash_desc *desc, u8 *out) { - struct blake2b_state *state = shash_desc_ctx(desc); - const size_t left = state->buflen; - const size_t fill = BLAKE2B_BLOCKBYTES - left; - - if (!inlen) - return 0; - - if (inlen > fill) { - state->buflen = 0; - /* Fill buffer */ - memcpy(state->buf + left, in, fill); - blake2b_increment_counter(state, BLAKE2B_BLOCKBYTES); - /* Compress */ - blake2b_compress(state, state->buf); - in += fill; - inlen -= fill; - while (inlen > BLAKE2B_BLOCKBYTES) { - blake2b_increment_counter(state, BLAKE2B_BLOCKBYTES); - blake2b_compress(state, in); - in += BLAKE2B_BLOCKBYTES; - inlen -= BLAKE2B_BLOCKBYTES; - } - } - memcpy(state->buf + state->buflen, in, inlen); - state->buflen += inlen; - - return 0; + return crypto_blake2b_final(desc, out, blake2b_compress_generic); } -static int blake2b_final(struct shash_desc *desc, u8 *out) -{ - struct blake2b_state *state = shash_desc_ctx(desc); - const int digestsize = crypto_shash_digestsize(desc->tfm); - size_t i; - - blake2b_increment_counter(state, state->buflen); - /* Set last block */ - state->f[0] = (u64)-1; - /* Padding */ - memset(state->buf + state->buflen, 0, BLAKE2B_BLOCKBYTES - state->buflen); - blake2b_compress(state, state->buf); - - /* Avoid temporary buffer and switch the internal output to LE order */ - for (i = 0; i < ARRAY_SIZE(state->h); i++) - __cpu_to_le64s(&state->h[i]); - - memcpy(out, state->h, digestsize); - return 0; -} +#define BLAKE2B_ALG(name, driver_name, digest_size) \ + { \ + .base.cra_name = name, \ + .base.cra_driver_name = driver_name, \ + .base.cra_priority = 100, \ + .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY, \ + .base.cra_blocksize = BLAKE2B_BLOCK_SIZE, \ + .base.cra_ctxsize = sizeof(struct blake2b_tfm_ctx), \ + .base.cra_module = THIS_MODULE, \ + .digestsize = digest_size, \ + .setkey = crypto_blake2b_setkey, \ + .init = crypto_blake2b_init, \ + .update = crypto_blake2b_update_generic, \ + .final = crypto_blake2b_final_generic, \ + .descsize = sizeof(struct blake2b_state), \ + } static struct shash_alg blake2b_algs[] = { - { - .base.cra_name = "blake2b-160", - .base.cra_driver_name = "blake2b-160-generic", - .base.cra_priority = 100, - .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY, - .base.cra_blocksize = BLAKE2B_BLOCKBYTES, - .base.cra_ctxsize = sizeof(struct blake2b_tfm_ctx), - .base.cra_module = THIS_MODULE, - .digestsize = BLAKE2B_160_DIGEST_SIZE, - .setkey = blake2b_setkey, - .init = blake2b_init, - .update = blake2b_update, - .final = blake2b_final, - .descsize = sizeof(struct blake2b_state), - }, { - .base.cra_name = "blake2b-256", - .base.cra_driver_name = "blake2b-256-generic", - .base.cra_priority = 100, - .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY, - .base.cra_blocksize = BLAKE2B_BLOCKBYTES, - .base.cra_ctxsize = sizeof(struct blake2b_tfm_ctx), - .base.cra_module = THIS_MODULE, - .digestsize = BLAKE2B_256_DIGEST_SIZE, - .setkey = blake2b_setkey, - .init = blake2b_init, - .update = blake2b_update, - .final = blake2b_final, - .descsize = sizeof(struct blake2b_state), - }, { - .base.cra_name = "blake2b-384", - .base.cra_driver_name = "blake2b-384-generic", - .base.cra_priority = 100, - .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY, - .base.cra_blocksize = BLAKE2B_BLOCKBYTES, - .base.cra_ctxsize = sizeof(struct blake2b_tfm_ctx), - .base.cra_module = THIS_MODULE, - .digestsize = BLAKE2B_384_DIGEST_SIZE, - .setkey = blake2b_setkey, - .init = blake2b_init, - .update = blake2b_update, - .final = blake2b_final, - .descsize = sizeof(struct blake2b_state), - }, { - .base.cra_name = "blake2b-512", - .base.cra_driver_name = "blake2b-512-generic", - .base.cra_priority = 100, - .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY, - .base.cra_blocksize = BLAKE2B_BLOCKBYTES, - .base.cra_ctxsize = sizeof(struct blake2b_tfm_ctx), - .base.cra_module = THIS_MODULE, - .digestsize = BLAKE2B_512_DIGEST_SIZE, - .setkey = blake2b_setkey, - .init = blake2b_init, - .update = blake2b_update, - .final = blake2b_final, - .descsize = sizeof(struct blake2b_state), - } + BLAKE2B_ALG("blake2b-160", "blake2b-160-generic", + BLAKE2B_160_HASH_SIZE), + BLAKE2B_ALG("blake2b-256", "blake2b-256-generic", + BLAKE2B_256_HASH_SIZE), + BLAKE2B_ALG("blake2b-384", "blake2b-384-generic", + BLAKE2B_384_HASH_SIZE), + BLAKE2B_ALG("blake2b-512", "blake2b-512-generic", + BLAKE2B_512_HASH_SIZE), }; static int __init blake2b_mod_init(void) diff --git a/crypto/blake2s_generic.c b/crypto/blake2s_generic.c index 005783ff45ad059ec8fc9c415e10954508940bed..72fe480f9bd67de37747034a714379e884047e1c 100644 --- a/crypto/blake2s_generic.c +++ b/crypto/blake2s_generic.c @@ -1,149 +1,55 @@ // SPDX-License-Identifier: GPL-2.0 OR MIT /* + * shash interface to the generic implementation of BLAKE2s + * * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. */ #include -#include #include #include -#include #include #include -static int crypto_blake2s_setkey(struct crypto_shash *tfm, const u8 *key, - unsigned int keylen) +static int crypto_blake2s_update_generic(struct shash_desc *desc, + const u8 *in, unsigned int inlen) { - struct blake2s_tfm_ctx *tctx = crypto_shash_ctx(tfm); - - if (keylen == 0 || keylen > BLAKE2S_KEY_SIZE) - return -EINVAL; - - memcpy(tctx->key, key, keylen); - tctx->keylen = keylen; - - return 0; + return crypto_blake2s_update(desc, in, inlen, blake2s_compress_generic); } -static int crypto_blake2s_init(struct shash_desc *desc) +static int crypto_blake2s_final_generic(struct shash_desc *desc, u8 *out) { - struct blake2s_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); - struct blake2s_state *state = shash_desc_ctx(desc); - const int outlen = crypto_shash_digestsize(desc->tfm); - - if (tctx->keylen) - blake2s_init_key(state, outlen, tctx->key, tctx->keylen); - else - blake2s_init(state, outlen); - - return 0; + return crypto_blake2s_final(desc, out, blake2s_compress_generic); } -static int crypto_blake2s_update(struct shash_desc *desc, const u8 *in, - unsigned int inlen) -{ - struct blake2s_state *state = shash_desc_ctx(desc); - const size_t fill = BLAKE2S_BLOCK_SIZE - state->buflen; - - if (unlikely(!inlen)) - return 0; - if (inlen > fill) { - memcpy(state->buf + state->buflen, in, fill); - blake2s_compress_generic(state, state->buf, 1, BLAKE2S_BLOCK_SIZE); - state->buflen = 0; - in += fill; - inlen -= fill; - } - if (inlen > BLAKE2S_BLOCK_SIZE) { - const size_t nblocks = DIV_ROUND_UP(inlen, BLAKE2S_BLOCK_SIZE); - /* Hash one less (full) block than strictly possible */ - blake2s_compress_generic(state, in, nblocks - 1, BLAKE2S_BLOCK_SIZE); - in += BLAKE2S_BLOCK_SIZE * (nblocks - 1); - inlen -= BLAKE2S_BLOCK_SIZE * (nblocks - 1); +#define BLAKE2S_ALG(name, driver_name, digest_size) \ + { \ + .base.cra_name = name, \ + .base.cra_driver_name = driver_name, \ + .base.cra_priority = 100, \ + .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY, \ + .base.cra_blocksize = BLAKE2S_BLOCK_SIZE, \ + .base.cra_ctxsize = sizeof(struct blake2s_tfm_ctx), \ + .base.cra_module = THIS_MODULE, \ + .digestsize = digest_size, \ + .setkey = crypto_blake2s_setkey, \ + .init = crypto_blake2s_init, \ + .update = crypto_blake2s_update_generic, \ + .final = crypto_blake2s_final_generic, \ + .descsize = sizeof(struct blake2s_state), \ } - memcpy(state->buf + state->buflen, in, inlen); - state->buflen += inlen; - - return 0; -} - -static int crypto_blake2s_final(struct shash_desc *desc, u8 *out) -{ - struct blake2s_state *state = shash_desc_ctx(desc); - - blake2s_set_lastblock(state); - memset(state->buf + state->buflen, 0, - BLAKE2S_BLOCK_SIZE - state->buflen); /* Padding */ - blake2s_compress_generic(state, state->buf, 1, state->buflen); - cpu_to_le32_array(state->h, ARRAY_SIZE(state->h)); - memcpy(out, state->h, state->outlen); - memzero_explicit(state, sizeof(*state)); - - return 0; -} - -static struct shash_alg blake2s_algs[] = {{ - .base.cra_name = "blake2s-128", - .base.cra_driver_name = "blake2s-128-generic", - .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY, - .base.cra_ctxsize = sizeof(struct blake2s_tfm_ctx), - .base.cra_priority = 200, - .base.cra_blocksize = BLAKE2S_BLOCK_SIZE, - .base.cra_module = THIS_MODULE, - - .digestsize = BLAKE2S_128_HASH_SIZE, - .setkey = crypto_blake2s_setkey, - .init = crypto_blake2s_init, - .update = crypto_blake2s_update, - .final = crypto_blake2s_final, - .descsize = sizeof(struct blake2s_state), -}, { - .base.cra_name = "blake2s-160", - .base.cra_driver_name = "blake2s-160-generic", - .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY, - .base.cra_ctxsize = sizeof(struct blake2s_tfm_ctx), - .base.cra_priority = 200, - .base.cra_blocksize = BLAKE2S_BLOCK_SIZE, - .base.cra_module = THIS_MODULE, - - .digestsize = BLAKE2S_160_HASH_SIZE, - .setkey = crypto_blake2s_setkey, - .init = crypto_blake2s_init, - .update = crypto_blake2s_update, - .final = crypto_blake2s_final, - .descsize = sizeof(struct blake2s_state), -}, { - .base.cra_name = "blake2s-224", - .base.cra_driver_name = "blake2s-224-generic", - .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY, - .base.cra_ctxsize = sizeof(struct blake2s_tfm_ctx), - .base.cra_priority = 200, - .base.cra_blocksize = BLAKE2S_BLOCK_SIZE, - .base.cra_module = THIS_MODULE, - - .digestsize = BLAKE2S_224_HASH_SIZE, - .setkey = crypto_blake2s_setkey, - .init = crypto_blake2s_init, - .update = crypto_blake2s_update, - .final = crypto_blake2s_final, - .descsize = sizeof(struct blake2s_state), -}, { - .base.cra_name = "blake2s-256", - .base.cra_driver_name = "blake2s-256-generic", - .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY, - .base.cra_ctxsize = sizeof(struct blake2s_tfm_ctx), - .base.cra_priority = 200, - .base.cra_blocksize = BLAKE2S_BLOCK_SIZE, - .base.cra_module = THIS_MODULE, - .digestsize = BLAKE2S_256_HASH_SIZE, - .setkey = crypto_blake2s_setkey, - .init = crypto_blake2s_init, - .update = crypto_blake2s_update, - .final = crypto_blake2s_final, - .descsize = sizeof(struct blake2s_state), -}}; +static struct shash_alg blake2s_algs[] = { + BLAKE2S_ALG("blake2s-128", "blake2s-128-generic", + BLAKE2S_128_HASH_SIZE), + BLAKE2S_ALG("blake2s-160", "blake2s-160-generic", + BLAKE2S_160_HASH_SIZE), + BLAKE2S_ALG("blake2s-224", "blake2s-224-generic", + BLAKE2S_224_HASH_SIZE), + BLAKE2S_ALG("blake2s-256", "blake2s-256-generic", + BLAKE2S_256_HASH_SIZE), +}; static int __init blake2s_mod_init(void) { diff --git a/crypto/blowfish_generic.c b/crypto/blowfish_generic.c index c3c2041fe0c59ebfeac92c08a1e72445f4ba6fff..003b52c6880eabbc1af53bb2999452a420ae7962 100644 --- a/crypto/blowfish_generic.c +++ b/crypto/blowfish_generic.c @@ -14,7 +14,7 @@ #include #include #include -#include +#include #include #include #include @@ -36,12 +36,10 @@ static void bf_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) { struct bf_ctx *ctx = crypto_tfm_ctx(tfm); - const __be32 *in_blk = (const __be32 *)src; - __be32 *const out_blk = (__be32 *)dst; const u32 *P = ctx->p; const u32 *S = ctx->s; - u32 yl = be32_to_cpu(in_blk[0]); - u32 yr = be32_to_cpu(in_blk[1]); + u32 yl = get_unaligned_be32(src); + u32 yr = get_unaligned_be32(src + 4); ROUND(yr, yl, 0); ROUND(yl, yr, 1); @@ -63,19 +61,17 @@ static void bf_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) yl ^= P[16]; yr ^= P[17]; - out_blk[0] = cpu_to_be32(yr); - out_blk[1] = cpu_to_be32(yl); + put_unaligned_be32(yr, dst); + put_unaligned_be32(yl, dst + 4); } static void bf_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) { struct bf_ctx *ctx = crypto_tfm_ctx(tfm); - const __be32 *in_blk = (const __be32 *)src; - __be32 *const out_blk = (__be32 *)dst; const u32 *P = ctx->p; const u32 *S = ctx->s; - u32 yl = be32_to_cpu(in_blk[0]); - u32 yr = be32_to_cpu(in_blk[1]); + u32 yl = get_unaligned_be32(src); + u32 yr = get_unaligned_be32(src + 4); ROUND(yr, yl, 17); ROUND(yl, yr, 16); @@ -97,8 +93,8 @@ static void bf_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) yl ^= P[1]; yr ^= P[0]; - out_blk[0] = cpu_to_be32(yr); - out_blk[1] = cpu_to_be32(yl); + put_unaligned_be32(yr, dst); + put_unaligned_be32(yl, dst + 4); } static struct crypto_alg alg = { @@ -108,7 +104,6 @@ static struct crypto_alg alg = { .cra_flags = CRYPTO_ALG_TYPE_CIPHER, .cra_blocksize = BF_BLOCK_SIZE, .cra_ctxsize = sizeof(struct bf_ctx), - .cra_alignmask = 3, .cra_module = THIS_MODULE, .cra_u = { .cipher = { .cia_min_keysize = BF_MIN_KEY_SIZE, diff --git a/crypto/camellia_generic.c b/crypto/camellia_generic.c index 0b9f409f73705bedc097412cec6043475658639b..fd1a88af9e77fd091292e72b6c27753ca5d24331 100644 --- a/crypto/camellia_generic.c +++ b/crypto/camellia_generic.c @@ -9,14 +9,6 @@ * https://info.isl.ntt.co.jp/crypt/eng/camellia/specifications.html */ -/* - * - * NOTE --- NOTE --- NOTE --- NOTE - * This implementation assumes that all memory addresses passed - * as parameters are four-byte aligned. - * - */ - #include #include #include @@ -994,16 +986,14 @@ camellia_set_key(struct crypto_tfm *tfm, const u8 *in_key, static void camellia_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) { const struct camellia_ctx *cctx = crypto_tfm_ctx(tfm); - const __be32 *src = (const __be32 *)in; - __be32 *dst = (__be32 *)out; unsigned int max; u32 tmp[4]; - tmp[0] = be32_to_cpu(src[0]); - tmp[1] = be32_to_cpu(src[1]); - tmp[2] = be32_to_cpu(src[2]); - tmp[3] = be32_to_cpu(src[3]); + tmp[0] = get_unaligned_be32(in); + tmp[1] = get_unaligned_be32(in + 4); + tmp[2] = get_unaligned_be32(in + 8); + tmp[3] = get_unaligned_be32(in + 12); if (cctx->key_length == 16) max = 24; @@ -1013,25 +1003,23 @@ static void camellia_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) camellia_do_encrypt(cctx->key_table, tmp, max); /* do_encrypt returns 0,1 swapped with 2,3 */ - dst[0] = cpu_to_be32(tmp[2]); - dst[1] = cpu_to_be32(tmp[3]); - dst[2] = cpu_to_be32(tmp[0]); - dst[3] = cpu_to_be32(tmp[1]); + put_unaligned_be32(tmp[2], out); + put_unaligned_be32(tmp[3], out + 4); + put_unaligned_be32(tmp[0], out + 8); + put_unaligned_be32(tmp[1], out + 12); } static void camellia_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) { const struct camellia_ctx *cctx = crypto_tfm_ctx(tfm); - const __be32 *src = (const __be32 *)in; - __be32 *dst = (__be32 *)out; unsigned int max; u32 tmp[4]; - tmp[0] = be32_to_cpu(src[0]); - tmp[1] = be32_to_cpu(src[1]); - tmp[2] = be32_to_cpu(src[2]); - tmp[3] = be32_to_cpu(src[3]); + tmp[0] = get_unaligned_be32(in); + tmp[1] = get_unaligned_be32(in + 4); + tmp[2] = get_unaligned_be32(in + 8); + tmp[3] = get_unaligned_be32(in + 12); if (cctx->key_length == 16) max = 24; @@ -1041,10 +1029,10 @@ static void camellia_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) camellia_do_decrypt(cctx->key_table, tmp, max); /* do_decrypt returns 0,1 swapped with 2,3 */ - dst[0] = cpu_to_be32(tmp[2]); - dst[1] = cpu_to_be32(tmp[3]); - dst[2] = cpu_to_be32(tmp[0]); - dst[3] = cpu_to_be32(tmp[1]); + put_unaligned_be32(tmp[2], out); + put_unaligned_be32(tmp[3], out + 4); + put_unaligned_be32(tmp[0], out + 8); + put_unaligned_be32(tmp[1], out + 12); } static struct crypto_alg camellia_alg = { @@ -1054,7 +1042,6 @@ static struct crypto_alg camellia_alg = { .cra_flags = CRYPTO_ALG_TYPE_CIPHER, .cra_blocksize = CAMELLIA_BLOCK_SIZE, .cra_ctxsize = sizeof(struct camellia_ctx), - .cra_alignmask = 3, .cra_module = THIS_MODULE, .cra_u = { .cipher = { diff --git a/crypto/cast5_generic.c b/crypto/cast5_generic.c index 4095085d4e5178fa88ba4253da68fe1fc33cdc57..0257c14cefc25abe032e0d302aa8e40a05e6432c 100644 --- a/crypto/cast5_generic.c +++ b/crypto/cast5_generic.c @@ -13,7 +13,7 @@ */ -#include +#include #include #include #include @@ -302,8 +302,6 @@ static const u32 sb8[256] = { void __cast5_encrypt(struct cast5_ctx *c, u8 *outbuf, const u8 *inbuf) { - const __be32 *src = (const __be32 *)inbuf; - __be32 *dst = (__be32 *)outbuf; u32 l, r, t; u32 I; /* used by the Fx macros */ u32 *Km; @@ -315,8 +313,8 @@ void __cast5_encrypt(struct cast5_ctx *c, u8 *outbuf, const u8 *inbuf) /* (L0,R0) <-- (m1...m64). (Split the plaintext into left and * right 32-bit halves L0 = m1...m32 and R0 = m33...m64.) */ - l = be32_to_cpu(src[0]); - r = be32_to_cpu(src[1]); + l = get_unaligned_be32(inbuf); + r = get_unaligned_be32(inbuf + 4); /* (16 rounds) for i from 1 to 16, compute Li and Ri as follows: * Li = Ri-1; @@ -347,8 +345,8 @@ void __cast5_encrypt(struct cast5_ctx *c, u8 *outbuf, const u8 *inbuf) /* c1...c64 <-- (R16,L16). (Exchange final blocks L16, R16 and * concatenate to form the ciphertext.) */ - dst[0] = cpu_to_be32(r); - dst[1] = cpu_to_be32(l); + put_unaligned_be32(r, outbuf); + put_unaligned_be32(l, outbuf + 4); } EXPORT_SYMBOL_GPL(__cast5_encrypt); @@ -359,8 +357,6 @@ static void cast5_encrypt(struct crypto_tfm *tfm, u8 *outbuf, const u8 *inbuf) void __cast5_decrypt(struct cast5_ctx *c, u8 *outbuf, const u8 *inbuf) { - const __be32 *src = (const __be32 *)inbuf; - __be32 *dst = (__be32 *)outbuf; u32 l, r, t; u32 I; u32 *Km; @@ -369,8 +365,8 @@ void __cast5_decrypt(struct cast5_ctx *c, u8 *outbuf, const u8 *inbuf) Km = c->Km; Kr = c->Kr; - l = be32_to_cpu(src[0]); - r = be32_to_cpu(src[1]); + l = get_unaligned_be32(inbuf); + r = get_unaligned_be32(inbuf + 4); if (!(c->rr)) { t = l; l = r; r = t ^ F1(r, Km[15], Kr[15]); @@ -391,8 +387,8 @@ void __cast5_decrypt(struct cast5_ctx *c, u8 *outbuf, const u8 *inbuf) t = l; l = r; r = t ^ F2(r, Km[1], Kr[1]); t = l; l = r; r = t ^ F1(r, Km[0], Kr[0]); - dst[0] = cpu_to_be32(r); - dst[1] = cpu_to_be32(l); + put_unaligned_be32(r, outbuf); + put_unaligned_be32(l, outbuf + 4); } EXPORT_SYMBOL_GPL(__cast5_decrypt); @@ -513,7 +509,6 @@ static struct crypto_alg alg = { .cra_flags = CRYPTO_ALG_TYPE_CIPHER, .cra_blocksize = CAST5_BLOCK_SIZE, .cra_ctxsize = sizeof(struct cast5_ctx), - .cra_alignmask = 3, .cra_module = THIS_MODULE, .cra_u = { .cipher = { diff --git a/crypto/cast6_generic.c b/crypto/cast6_generic.c index c77ff6c8a2b21d3516b8116f91f6a2009a66afd8..75346380aa0bc7d8b5e910b913036f2954e81565 100644 --- a/crypto/cast6_generic.c +++ b/crypto/cast6_generic.c @@ -10,7 +10,7 @@ */ -#include +#include #include #include #include @@ -172,16 +172,14 @@ static inline void QBAR(u32 *block, const u8 *Kr, const u32 *Km) void __cast6_encrypt(const void *ctx, u8 *outbuf, const u8 *inbuf) { const struct cast6_ctx *c = ctx; - const __be32 *src = (const __be32 *)inbuf; - __be32 *dst = (__be32 *)outbuf; u32 block[4]; const u32 *Km; const u8 *Kr; - block[0] = be32_to_cpu(src[0]); - block[1] = be32_to_cpu(src[1]); - block[2] = be32_to_cpu(src[2]); - block[3] = be32_to_cpu(src[3]); + block[0] = get_unaligned_be32(inbuf); + block[1] = get_unaligned_be32(inbuf + 4); + block[2] = get_unaligned_be32(inbuf + 8); + block[3] = get_unaligned_be32(inbuf + 12); Km = c->Km[0]; Kr = c->Kr[0]; Q(block, Kr, Km); Km = c->Km[1]; Kr = c->Kr[1]; Q(block, Kr, Km); @@ -196,10 +194,10 @@ void __cast6_encrypt(const void *ctx, u8 *outbuf, const u8 *inbuf) Km = c->Km[10]; Kr = c->Kr[10]; QBAR(block, Kr, Km); Km = c->Km[11]; Kr = c->Kr[11]; QBAR(block, Kr, Km); - dst[0] = cpu_to_be32(block[0]); - dst[1] = cpu_to_be32(block[1]); - dst[2] = cpu_to_be32(block[2]); - dst[3] = cpu_to_be32(block[3]); + put_unaligned_be32(block[0], outbuf); + put_unaligned_be32(block[1], outbuf + 4); + put_unaligned_be32(block[2], outbuf + 8); + put_unaligned_be32(block[3], outbuf + 12); } EXPORT_SYMBOL_GPL(__cast6_encrypt); @@ -211,16 +209,14 @@ static void cast6_encrypt(struct crypto_tfm *tfm, u8 *outbuf, const u8 *inbuf) void __cast6_decrypt(const void *ctx, u8 *outbuf, const u8 *inbuf) { const struct cast6_ctx *c = ctx; - const __be32 *src = (const __be32 *)inbuf; - __be32 *dst = (__be32 *)outbuf; u32 block[4]; const u32 *Km; const u8 *Kr; - block[0] = be32_to_cpu(src[0]); - block[1] = be32_to_cpu(src[1]); - block[2] = be32_to_cpu(src[2]); - block[3] = be32_to_cpu(src[3]); + block[0] = get_unaligned_be32(inbuf); + block[1] = get_unaligned_be32(inbuf + 4); + block[2] = get_unaligned_be32(inbuf + 8); + block[3] = get_unaligned_be32(inbuf + 12); Km = c->Km[11]; Kr = c->Kr[11]; Q(block, Kr, Km); Km = c->Km[10]; Kr = c->Kr[10]; Q(block, Kr, Km); @@ -235,10 +231,10 @@ void __cast6_decrypt(const void *ctx, u8 *outbuf, const u8 *inbuf) Km = c->Km[1]; Kr = c->Kr[1]; QBAR(block, Kr, Km); Km = c->Km[0]; Kr = c->Kr[0]; QBAR(block, Kr, Km); - dst[0] = cpu_to_be32(block[0]); - dst[1] = cpu_to_be32(block[1]); - dst[2] = cpu_to_be32(block[2]); - dst[3] = cpu_to_be32(block[3]); + put_unaligned_be32(block[0], outbuf); + put_unaligned_be32(block[1], outbuf + 4); + put_unaligned_be32(block[2], outbuf + 8); + put_unaligned_be32(block[3], outbuf + 12); } EXPORT_SYMBOL_GPL(__cast6_decrypt); @@ -254,7 +250,6 @@ static struct crypto_alg alg = { .cra_flags = CRYPTO_ALG_TYPE_CIPHER, .cra_blocksize = CAST6_BLOCK_SIZE, .cra_ctxsize = sizeof(struct cast6_ctx), - .cra_alignmask = 3, .cra_module = THIS_MODULE, .cra_u = { .cipher = { diff --git a/crypto/cbc.c b/crypto/cbc.c index 0d9509dff891d54439152c7378b3ac9983098104..6c03e96b945f6672109cbdc1c59a90cacb7b4190 100644 --- a/crypto/cbc.c +++ b/crypto/cbc.c @@ -6,6 +6,7 @@ */ #include +#include #include #include #include diff --git a/crypto/ccm.c b/crypto/ccm.c index 494d7090118625054850442196ee484dbfa658e6..6b815ece51c6a930dc472fa3624e85591beb64d5 100644 --- a/crypto/ccm.c +++ b/crypto/ccm.c @@ -6,6 +6,7 @@ */ #include +#include #include #include #include @@ -954,3 +955,4 @@ MODULE_ALIAS_CRYPTO("ccm_base"); MODULE_ALIAS_CRYPTO("rfc4309"); MODULE_ALIAS_CRYPTO("ccm"); MODULE_ALIAS_CRYPTO("cbcmac"); +MODULE_IMPORT_NS(CRYPTO_INTERNAL); diff --git a/crypto/cfb.c b/crypto/cfb.c index 4e5219bbcd19de28495b50239ec27f1d54c7c9db..0d664dfb47bcd956b20e2bce80399235698614ea 100644 --- a/crypto/cfb.c +++ b/crypto/cfb.c @@ -20,6 +20,7 @@ */ #include +#include #include #include #include @@ -250,3 +251,4 @@ module_exit(crypto_cfb_module_exit); MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("CFB block cipher mode of operation"); MODULE_ALIAS_CRYPTO("cfb"); +MODULE_IMPORT_NS(CRYPTO_INTERNAL); diff --git a/crypto/cipher.c b/crypto/cipher.c index fd78150deb1c14c6830df06b4511df2e4802d293..b47141ed4a9f3da03237847d4cec4f08a50e3840 100644 --- a/crypto/cipher.c +++ b/crypto/cipher.c @@ -9,6 +9,7 @@ */ #include +#include #include #include #include @@ -53,7 +54,7 @@ int crypto_cipher_setkey(struct crypto_cipher *tfm, return cia->cia_setkey(crypto_cipher_tfm(tfm), key, keylen); } -EXPORT_SYMBOL_GPL(crypto_cipher_setkey); +EXPORT_SYMBOL_NS_GPL(crypto_cipher_setkey, CRYPTO_INTERNAL); static inline void cipher_crypt_one(struct crypto_cipher *tfm, u8 *dst, const u8 *src, bool enc) @@ -81,11 +82,11 @@ void crypto_cipher_encrypt_one(struct crypto_cipher *tfm, { cipher_crypt_one(tfm, dst, src, true); } -EXPORT_SYMBOL_GPL(crypto_cipher_encrypt_one); +EXPORT_SYMBOL_NS_GPL(crypto_cipher_encrypt_one, CRYPTO_INTERNAL); void crypto_cipher_decrypt_one(struct crypto_cipher *tfm, u8 *dst, const u8 *src) { cipher_crypt_one(tfm, dst, src, false); } -EXPORT_SYMBOL_GPL(crypto_cipher_decrypt_one); +EXPORT_SYMBOL_NS_GPL(crypto_cipher_decrypt_one, CRYPTO_INTERNAL); diff --git a/crypto/cmac.c b/crypto/cmac.c index df36be1efb81739d5618861533b85cd74acb508b..f4a5d3bfb37624e553b47030d1cee9de4a71e811 100644 --- a/crypto/cmac.c +++ b/crypto/cmac.c @@ -11,6 +11,7 @@ * Author: Kazunori Miyazawa */ +#include #include #include #include @@ -313,3 +314,4 @@ module_exit(crypto_cmac_module_exit); MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("CMAC keyed hash algorithm"); MODULE_ALIAS_CRYPTO("cmac"); +MODULE_IMPORT_NS(CRYPTO_INTERNAL); diff --git a/crypto/ctr.c b/crypto/ctr.c index c39fcffba27f521aefd1e6f40f6f6a74fd8c7ee7..23c698b220130423e7261298234b345ddc230310 100644 --- a/crypto/ctr.c +++ b/crypto/ctr.c @@ -7,6 +7,7 @@ #include #include +#include #include #include #include @@ -358,3 +359,4 @@ MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("CTR block cipher mode of operation"); MODULE_ALIAS_CRYPTO("rfc3686"); MODULE_ALIAS_CRYPTO("ctr"); +MODULE_IMPORT_NS(CRYPTO_INTERNAL); diff --git a/crypto/drbg.c b/crypto/drbg.c index 3132967a1749721e2ef8658a2e04fbfeebe315c0..1b4587e0ddad86e45f4f043aa0cc30f58d3b238d 100644 --- a/crypto/drbg.c +++ b/crypto/drbg.c @@ -98,6 +98,7 @@ */ #include +#include #include /*************************************************************** @@ -2161,3 +2162,4 @@ MODULE_DESCRIPTION("NIST SP800-90A Deterministic Random Bit Generator (DRBG) " CRYPTO_DRBG_HMAC_STRING CRYPTO_DRBG_CTR_STRING); MODULE_ALIAS_CRYPTO("stdrng"); +MODULE_IMPORT_NS(CRYPTO_INTERNAL); diff --git a/crypto/ecb.c b/crypto/ecb.c index 69a687cbdf216c0434cdf78199c5c3c604ea4d53..71fbb0543d64ea9aeef38ed84b4b5d16d3be9892 100644 --- a/crypto/ecb.c +++ b/crypto/ecb.c @@ -6,6 +6,7 @@ */ #include +#include #include #include #include diff --git a/crypto/ecdh_helper.c b/crypto/ecdh_helper.c index 66fcb2ea815445522ca9df6a94cfb8ddb7d919c4..fca63b559f655266fab1ff432dda85b21225c906 100644 --- a/crypto/ecdh_helper.c +++ b/crypto/ecdh_helper.c @@ -67,6 +67,9 @@ int crypto_ecdh_decode_key(const char *buf, unsigned int len, if (secret.type != CRYPTO_KPP_SECRET_TYPE_ECDH) return -EINVAL; + if (unlikely(len < secret.len)) + return -EINVAL; + ptr = ecdh_unpack_data(¶ms->curve_id, ptr, sizeof(params->curve_id)); ptr = ecdh_unpack_data(¶ms->key_size, ptr, sizeof(params->key_size)); if (secret.len != crypto_ecdh_key_len(params)) diff --git a/crypto/essiv.c b/crypto/essiv.c index d012be23d496d09f7157ee68d8541d35a89e3f7d..8bcc5bdcb2a95476227a4749f57d5a72c059bb27 100644 --- a/crypto/essiv.c +++ b/crypto/essiv.c @@ -30,6 +30,7 @@ #include #include +#include #include #include #include @@ -643,3 +644,4 @@ module_exit(essiv_module_exit); MODULE_DESCRIPTION("ESSIV skcipher/aead wrapper for block encryption"); MODULE_LICENSE("GPL v2"); MODULE_ALIAS_CRYPTO("essiv"); +MODULE_IMPORT_NS(CRYPTO_INTERNAL); diff --git a/crypto/fcrypt.c b/crypto/fcrypt.c index 58f935315cf8fe3a203adef960f4f5b2f62dc9bb..c36ea0c8be984bb1c305bb234aedaa9ea68080c9 100644 --- a/crypto/fcrypt.c +++ b/crypto/fcrypt.c @@ -396,7 +396,6 @@ static struct crypto_alg fcrypt_alg = { .cra_blocksize = 8, .cra_ctxsize = sizeof(struct fcrypt_ctx), .cra_module = THIS_MODULE, - .cra_alignmask = 3, .cra_u = { .cipher = { .cia_min_keysize = 8, .cia_max_keysize = 8, diff --git a/crypto/keywrap.c b/crypto/keywrap.c index 0355cce21b1e21711df1fadb26e67ecfd9b952de..3517773bc7f732b06f947c238744d75ae70c7e80 100644 --- a/crypto/keywrap.c +++ b/crypto/keywrap.c @@ -85,6 +85,7 @@ #include #include #include +#include #include struct crypto_kw_block { @@ -316,3 +317,4 @@ MODULE_LICENSE("Dual BSD/GPL"); MODULE_AUTHOR("Stephan Mueller "); MODULE_DESCRIPTION("Key Wrapping (RFC3394 / NIST SP800-38F)"); MODULE_ALIAS_CRYPTO("kw"); +MODULE_IMPORT_NS(CRYPTO_INTERNAL); diff --git a/crypto/michael_mic.c b/crypto/michael_mic.c index 63350c4ad461700d3bf9a64993925e09511f25c6..f4c31049601c9d91d389b627b16c6ba128ecef48 100644 --- a/crypto/michael_mic.c +++ b/crypto/michael_mic.c @@ -7,7 +7,7 @@ * Copyright (c) 2004 Jouni Malinen */ #include -#include +#include #include #include #include @@ -19,7 +19,7 @@ struct michael_mic_ctx { }; struct michael_mic_desc_ctx { - u8 pending[4]; + __le32 pending; size_t pending_len; u32 l, r; @@ -60,13 +60,12 @@ static int michael_update(struct shash_desc *desc, const u8 *data, unsigned int len) { struct michael_mic_desc_ctx *mctx = shash_desc_ctx(desc); - const __le32 *src; if (mctx->pending_len) { int flen = 4 - mctx->pending_len; if (flen > len) flen = len; - memcpy(&mctx->pending[mctx->pending_len], data, flen); + memcpy((u8 *)&mctx->pending + mctx->pending_len, data, flen); mctx->pending_len += flen; data += flen; len -= flen; @@ -74,23 +73,21 @@ static int michael_update(struct shash_desc *desc, const u8 *data, if (mctx->pending_len < 4) return 0; - src = (const __le32 *)mctx->pending; - mctx->l ^= le32_to_cpup(src); + mctx->l ^= le32_to_cpu(mctx->pending); michael_block(mctx->l, mctx->r); mctx->pending_len = 0; } - src = (const __le32 *)data; - while (len >= 4) { - mctx->l ^= le32_to_cpup(src++); + mctx->l ^= get_unaligned_le32(data); michael_block(mctx->l, mctx->r); + data += 4; len -= 4; } if (len > 0) { mctx->pending_len = len; - memcpy(mctx->pending, src, len); + memcpy(&mctx->pending, data, len); } return 0; @@ -100,8 +97,7 @@ static int michael_update(struct shash_desc *desc, const u8 *data, static int michael_final(struct shash_desc *desc, u8 *out) { struct michael_mic_desc_ctx *mctx = shash_desc_ctx(desc); - u8 *data = mctx->pending; - __le32 *dst = (__le32 *)out; + u8 *data = (u8 *)&mctx->pending; /* Last block and padding (0x5a, 4..7 x 0) */ switch (mctx->pending_len) { @@ -123,8 +119,8 @@ static int michael_final(struct shash_desc *desc, u8 *out) /* l ^= 0; */ michael_block(mctx->l, mctx->r); - dst[0] = cpu_to_le32(mctx->l); - dst[1] = cpu_to_le32(mctx->r); + put_unaligned_le32(mctx->l, out); + put_unaligned_le32(mctx->r, out + 4); return 0; } @@ -135,13 +131,11 @@ static int michael_setkey(struct crypto_shash *tfm, const u8 *key, { struct michael_mic_ctx *mctx = crypto_shash_ctx(tfm); - const __le32 *data = (const __le32 *)key; - if (keylen != 8) return -EINVAL; - mctx->l = le32_to_cpu(data[0]); - mctx->r = le32_to_cpu(data[1]); + mctx->l = get_unaligned_le32(key); + mctx->r = get_unaligned_le32(key + 4); return 0; } @@ -156,7 +150,6 @@ static struct shash_alg alg = { .cra_name = "michael_mic", .cra_driver_name = "michael_mic-generic", .cra_blocksize = 8, - .cra_alignmask = 3, .cra_ctxsize = sizeof(struct michael_mic_ctx), .cra_module = THIS_MODULE, } diff --git a/crypto/ofb.c b/crypto/ofb.c index 2ec68e3f2c552b3e925c41591042ebbade3be6dd..b630fdecceee33e8421875ad7cf45104fd1a1b12 100644 --- a/crypto/ofb.c +++ b/crypto/ofb.c @@ -8,6 +8,7 @@ */ #include +#include #include #include #include @@ -102,3 +103,4 @@ module_exit(crypto_ofb_module_exit); MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("OFB block cipher mode of operation"); MODULE_ALIAS_CRYPTO("ofb"); +MODULE_IMPORT_NS(CRYPTO_INTERNAL); diff --git a/crypto/pcbc.c b/crypto/pcbc.c index ae921fb74dc9bda81499e33d1281febbff6486ce..7030f59e46b6bc2760491176268b6df0cd8b7ea0 100644 --- a/crypto/pcbc.c +++ b/crypto/pcbc.c @@ -10,6 +10,7 @@ */ #include +#include #include #include #include @@ -191,3 +192,4 @@ module_exit(crypto_pcbc_module_exit); MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("PCBC block cipher mode of operation"); MODULE_ALIAS_CRYPTO("pcbc"); +MODULE_IMPORT_NS(CRYPTO_INTERNAL); diff --git a/crypto/ripemd.h b/crypto/ripemd.h index 93edbf52197d4ba94333a71fadfecc2034773e6f..b977785e2a62168ed90a9a272ddd2f4e7e1881b7 100644 --- a/crypto/ripemd.h +++ b/crypto/ripemd.h @@ -6,29 +6,15 @@ #ifndef _CRYPTO_RMD_H #define _CRYPTO_RMD_H -#define RMD128_DIGEST_SIZE 16 -#define RMD128_BLOCK_SIZE 64 - #define RMD160_DIGEST_SIZE 20 #define RMD160_BLOCK_SIZE 64 -#define RMD256_DIGEST_SIZE 32 -#define RMD256_BLOCK_SIZE 64 - -#define RMD320_DIGEST_SIZE 40 -#define RMD320_BLOCK_SIZE 64 - /* initial values */ #define RMD_H0 0x67452301UL #define RMD_H1 0xefcdab89UL #define RMD_H2 0x98badcfeUL #define RMD_H3 0x10325476UL #define RMD_H4 0xc3d2e1f0UL -#define RMD_H5 0x76543210UL -#define RMD_H6 0xfedcba98UL -#define RMD_H7 0x89abcdefUL -#define RMD_H8 0x01234567UL -#define RMD_H9 0x3c2d1e0fUL /* constants */ #define RMD_K1 0x00000000UL diff --git a/crypto/rmd128.c b/crypto/rmd128.c deleted file mode 100644 index 29308fb97e7e695c9e332b6ff2da1dd694f0413a..0000000000000000000000000000000000000000 --- a/crypto/rmd128.c +++ /dev/null @@ -1,323 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Cryptographic API. - * - * RIPEMD-128 - RACE Integrity Primitives Evaluation Message Digest. - * - * Based on the reference implementation by Antoon Bosselaers, ESAT-COSIC - * - * Copyright (c) 2008 Adrian-Ken Rueegsegger - */ -#include -#include -#include -#include -#include -#include - -#include "ripemd.h" - -struct rmd128_ctx { - u64 byte_count; - u32 state[4]; - __le32 buffer[16]; -}; - -#define K1 RMD_K1 -#define K2 RMD_K2 -#define K3 RMD_K3 -#define K4 RMD_K4 -#define KK1 RMD_K6 -#define KK2 RMD_K7 -#define KK3 RMD_K8 -#define KK4 RMD_K1 - -#define F1(x, y, z) (x ^ y ^ z) /* XOR */ -#define F2(x, y, z) (z ^ (x & (y ^ z))) /* x ? y : z */ -#define F3(x, y, z) ((x | ~y) ^ z) -#define F4(x, y, z) (y ^ (z & (x ^ y))) /* z ? x : y */ - -#define ROUND(a, b, c, d, f, k, x, s) { \ - (a) += f((b), (c), (d)) + le32_to_cpup(&(x)) + (k); \ - (a) = rol32((a), (s)); \ -} - -static void rmd128_transform(u32 *state, const __le32 *in) -{ - u32 aa, bb, cc, dd, aaa, bbb, ccc, ddd; - - /* Initialize left lane */ - aa = state[0]; - bb = state[1]; - cc = state[2]; - dd = state[3]; - - /* Initialize right lane */ - aaa = state[0]; - bbb = state[1]; - ccc = state[2]; - ddd = state[3]; - - /* round 1: left lane */ - ROUND(aa, bb, cc, dd, F1, K1, in[0], 11); - ROUND(dd, aa, bb, cc, F1, K1, in[1], 14); - ROUND(cc, dd, aa, bb, F1, K1, in[2], 15); - ROUND(bb, cc, dd, aa, F1, K1, in[3], 12); - ROUND(aa, bb, cc, dd, F1, K1, in[4], 5); - ROUND(dd, aa, bb, cc, F1, K1, in[5], 8); - ROUND(cc, dd, aa, bb, F1, K1, in[6], 7); - ROUND(bb, cc, dd, aa, F1, K1, in[7], 9); - ROUND(aa, bb, cc, dd, F1, K1, in[8], 11); - ROUND(dd, aa, bb, cc, F1, K1, in[9], 13); - ROUND(cc, dd, aa, bb, F1, K1, in[10], 14); - ROUND(bb, cc, dd, aa, F1, K1, in[11], 15); - ROUND(aa, bb, cc, dd, F1, K1, in[12], 6); - ROUND(dd, aa, bb, cc, F1, K1, in[13], 7); - ROUND(cc, dd, aa, bb, F1, K1, in[14], 9); - ROUND(bb, cc, dd, aa, F1, K1, in[15], 8); - - /* round 2: left lane */ - ROUND(aa, bb, cc, dd, F2, K2, in[7], 7); - ROUND(dd, aa, bb, cc, F2, K2, in[4], 6); - ROUND(cc, dd, aa, bb, F2, K2, in[13], 8); - ROUND(bb, cc, dd, aa, F2, K2, in[1], 13); - ROUND(aa, bb, cc, dd, F2, K2, in[10], 11); - ROUND(dd, aa, bb, cc, F2, K2, in[6], 9); - ROUND(cc, dd, aa, bb, F2, K2, in[15], 7); - ROUND(bb, cc, dd, aa, F2, K2, in[3], 15); - ROUND(aa, bb, cc, dd, F2, K2, in[12], 7); - ROUND(dd, aa, bb, cc, F2, K2, in[0], 12); - ROUND(cc, dd, aa, bb, F2, K2, in[9], 15); - ROUND(bb, cc, dd, aa, F2, K2, in[5], 9); - ROUND(aa, bb, cc, dd, F2, K2, in[2], 11); - ROUND(dd, aa, bb, cc, F2, K2, in[14], 7); - ROUND(cc, dd, aa, bb, F2, K2, in[11], 13); - ROUND(bb, cc, dd, aa, F2, K2, in[8], 12); - - /* round 3: left lane */ - ROUND(aa, bb, cc, dd, F3, K3, in[3], 11); - ROUND(dd, aa, bb, cc, F3, K3, in[10], 13); - ROUND(cc, dd, aa, bb, F3, K3, in[14], 6); - ROUND(bb, cc, dd, aa, F3, K3, in[4], 7); - ROUND(aa, bb, cc, dd, F3, K3, in[9], 14); - ROUND(dd, aa, bb, cc, F3, K3, in[15], 9); - ROUND(cc, dd, aa, bb, F3, K3, in[8], 13); - ROUND(bb, cc, dd, aa, F3, K3, in[1], 15); - ROUND(aa, bb, cc, dd, F3, K3, in[2], 14); - ROUND(dd, aa, bb, cc, F3, K3, in[7], 8); - ROUND(cc, dd, aa, bb, F3, K3, in[0], 13); - ROUND(bb, cc, dd, aa, F3, K3, in[6], 6); - ROUND(aa, bb, cc, dd, F3, K3, in[13], 5); - ROUND(dd, aa, bb, cc, F3, K3, in[11], 12); - ROUND(cc, dd, aa, bb, F3, K3, in[5], 7); - ROUND(bb, cc, dd, aa, F3, K3, in[12], 5); - - /* round 4: left lane */ - ROUND(aa, bb, cc, dd, F4, K4, in[1], 11); - ROUND(dd, aa, bb, cc, F4, K4, in[9], 12); - ROUND(cc, dd, aa, bb, F4, K4, in[11], 14); - ROUND(bb, cc, dd, aa, F4, K4, in[10], 15); - ROUND(aa, bb, cc, dd, F4, K4, in[0], 14); - ROUND(dd, aa, bb, cc, F4, K4, in[8], 15); - ROUND(cc, dd, aa, bb, F4, K4, in[12], 9); - ROUND(bb, cc, dd, aa, F4, K4, in[4], 8); - ROUND(aa, bb, cc, dd, F4, K4, in[13], 9); - ROUND(dd, aa, bb, cc, F4, K4, in[3], 14); - ROUND(cc, dd, aa, bb, F4, K4, in[7], 5); - ROUND(bb, cc, dd, aa, F4, K4, in[15], 6); - ROUND(aa, bb, cc, dd, F4, K4, in[14], 8); - ROUND(dd, aa, bb, cc, F4, K4, in[5], 6); - ROUND(cc, dd, aa, bb, F4, K4, in[6], 5); - ROUND(bb, cc, dd, aa, F4, K4, in[2], 12); - - /* round 1: right lane */ - ROUND(aaa, bbb, ccc, ddd, F4, KK1, in[5], 8); - ROUND(ddd, aaa, bbb, ccc, F4, KK1, in[14], 9); - ROUND(ccc, ddd, aaa, bbb, F4, KK1, in[7], 9); - ROUND(bbb, ccc, ddd, aaa, F4, KK1, in[0], 11); - ROUND(aaa, bbb, ccc, ddd, F4, KK1, in[9], 13); - ROUND(ddd, aaa, bbb, ccc, F4, KK1, in[2], 15); - ROUND(ccc, ddd, aaa, bbb, F4, KK1, in[11], 15); - ROUND(bbb, ccc, ddd, aaa, F4, KK1, in[4], 5); - ROUND(aaa, bbb, ccc, ddd, F4, KK1, in[13], 7); - ROUND(ddd, aaa, bbb, ccc, F4, KK1, in[6], 7); - ROUND(ccc, ddd, aaa, bbb, F4, KK1, in[15], 8); - ROUND(bbb, ccc, ddd, aaa, F4, KK1, in[8], 11); - ROUND(aaa, bbb, ccc, ddd, F4, KK1, in[1], 14); - ROUND(ddd, aaa, bbb, ccc, F4, KK1, in[10], 14); - ROUND(ccc, ddd, aaa, bbb, F4, KK1, in[3], 12); - ROUND(bbb, ccc, ddd, aaa, F4, KK1, in[12], 6); - - /* round 2: right lane */ - ROUND(aaa, bbb, ccc, ddd, F3, KK2, in[6], 9); - ROUND(ddd, aaa, bbb, ccc, F3, KK2, in[11], 13); - ROUND(ccc, ddd, aaa, bbb, F3, KK2, in[3], 15); - ROUND(bbb, ccc, ddd, aaa, F3, KK2, in[7], 7); - ROUND(aaa, bbb, ccc, ddd, F3, KK2, in[0], 12); - ROUND(ddd, aaa, bbb, ccc, F3, KK2, in[13], 8); - ROUND(ccc, ddd, aaa, bbb, F3, KK2, in[5], 9); - ROUND(bbb, ccc, ddd, aaa, F3, KK2, in[10], 11); - ROUND(aaa, bbb, ccc, ddd, F3, KK2, in[14], 7); - ROUND(ddd, aaa, bbb, ccc, F3, KK2, in[15], 7); - ROUND(ccc, ddd, aaa, bbb, F3, KK2, in[8], 12); - ROUND(bbb, ccc, ddd, aaa, F3, KK2, in[12], 7); - ROUND(aaa, bbb, ccc, ddd, F3, KK2, in[4], 6); - ROUND(ddd, aaa, bbb, ccc, F3, KK2, in[9], 15); - ROUND(ccc, ddd, aaa, bbb, F3, KK2, in[1], 13); - ROUND(bbb, ccc, ddd, aaa, F3, KK2, in[2], 11); - - /* round 3: right lane */ - ROUND(aaa, bbb, ccc, ddd, F2, KK3, in[15], 9); - ROUND(ddd, aaa, bbb, ccc, F2, KK3, in[5], 7); - ROUND(ccc, ddd, aaa, bbb, F2, KK3, in[1], 15); - ROUND(bbb, ccc, ddd, aaa, F2, KK3, in[3], 11); - ROUND(aaa, bbb, ccc, ddd, F2, KK3, in[7], 8); - ROUND(ddd, aaa, bbb, ccc, F2, KK3, in[14], 6); - ROUND(ccc, ddd, aaa, bbb, F2, KK3, in[6], 6); - ROUND(bbb, ccc, ddd, aaa, F2, KK3, in[9], 14); - ROUND(aaa, bbb, ccc, ddd, F2, KK3, in[11], 12); - ROUND(ddd, aaa, bbb, ccc, F2, KK3, in[8], 13); - ROUND(ccc, ddd, aaa, bbb, F2, KK3, in[12], 5); - ROUND(bbb, ccc, ddd, aaa, F2, KK3, in[2], 14); - ROUND(aaa, bbb, ccc, ddd, F2, KK3, in[10], 13); - ROUND(ddd, aaa, bbb, ccc, F2, KK3, in[0], 13); - ROUND(ccc, ddd, aaa, bbb, F2, KK3, in[4], 7); - ROUND(bbb, ccc, ddd, aaa, F2, KK3, in[13], 5); - - /* round 4: right lane */ - ROUND(aaa, bbb, ccc, ddd, F1, KK4, in[8], 15); - ROUND(ddd, aaa, bbb, ccc, F1, KK4, in[6], 5); - ROUND(ccc, ddd, aaa, bbb, F1, KK4, in[4], 8); - ROUND(bbb, ccc, ddd, aaa, F1, KK4, in[1], 11); - ROUND(aaa, bbb, ccc, ddd, F1, KK4, in[3], 14); - ROUND(ddd, aaa, bbb, ccc, F1, KK4, in[11], 14); - ROUND(ccc, ddd, aaa, bbb, F1, KK4, in[15], 6); - ROUND(bbb, ccc, ddd, aaa, F1, KK4, in[0], 14); - ROUND(aaa, bbb, ccc, ddd, F1, KK4, in[5], 6); - ROUND(ddd, aaa, bbb, ccc, F1, KK4, in[12], 9); - ROUND(ccc, ddd, aaa, bbb, F1, KK4, in[2], 12); - ROUND(bbb, ccc, ddd, aaa, F1, KK4, in[13], 9); - ROUND(aaa, bbb, ccc, ddd, F1, KK4, in[9], 12); - ROUND(ddd, aaa, bbb, ccc, F1, KK4, in[7], 5); - ROUND(ccc, ddd, aaa, bbb, F1, KK4, in[10], 15); - ROUND(bbb, ccc, ddd, aaa, F1, KK4, in[14], 8); - - /* combine results */ - ddd += cc + state[1]; /* final result for state[0] */ - state[1] = state[2] + dd + aaa; - state[2] = state[3] + aa + bbb; - state[3] = state[0] + bb + ccc; - state[0] = ddd; -} - -static int rmd128_init(struct shash_desc *desc) -{ - struct rmd128_ctx *rctx = shash_desc_ctx(desc); - - rctx->byte_count = 0; - - rctx->state[0] = RMD_H0; - rctx->state[1] = RMD_H1; - rctx->state[2] = RMD_H2; - rctx->state[3] = RMD_H3; - - memset(rctx->buffer, 0, sizeof(rctx->buffer)); - - return 0; -} - -static int rmd128_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - struct rmd128_ctx *rctx = shash_desc_ctx(desc); - const u32 avail = sizeof(rctx->buffer) - (rctx->byte_count & 0x3f); - - rctx->byte_count += len; - - /* Enough space in buffer? If so copy and we're done */ - if (avail > len) { - memcpy((char *)rctx->buffer + (sizeof(rctx->buffer) - avail), - data, len); - goto out; - } - - memcpy((char *)rctx->buffer + (sizeof(rctx->buffer) - avail), - data, avail); - - rmd128_transform(rctx->state, rctx->buffer); - data += avail; - len -= avail; - - while (len >= sizeof(rctx->buffer)) { - memcpy(rctx->buffer, data, sizeof(rctx->buffer)); - rmd128_transform(rctx->state, rctx->buffer); - data += sizeof(rctx->buffer); - len -= sizeof(rctx->buffer); - } - - memcpy(rctx->buffer, data, len); - -out: - return 0; -} - -/* Add padding and return the message digest. */ -static int rmd128_final(struct shash_desc *desc, u8 *out) -{ - struct rmd128_ctx *rctx = shash_desc_ctx(desc); - u32 i, index, padlen; - __le64 bits; - __le32 *dst = (__le32 *)out; - static const u8 padding[64] = { 0x80, }; - - bits = cpu_to_le64(rctx->byte_count << 3); - - /* Pad out to 56 mod 64 */ - index = rctx->byte_count & 0x3f; - padlen = (index < 56) ? (56 - index) : ((64+56) - index); - rmd128_update(desc, padding, padlen); - - /* Append length */ - rmd128_update(desc, (const u8 *)&bits, sizeof(bits)); - - /* Store state in digest */ - for (i = 0; i < 4; i++) - dst[i] = cpu_to_le32p(&rctx->state[i]); - - /* Wipe context */ - memset(rctx, 0, sizeof(*rctx)); - - return 0; -} - -static struct shash_alg alg = { - .digestsize = RMD128_DIGEST_SIZE, - .init = rmd128_init, - .update = rmd128_update, - .final = rmd128_final, - .descsize = sizeof(struct rmd128_ctx), - .base = { - .cra_name = "rmd128", - .cra_driver_name = "rmd128-generic", - .cra_blocksize = RMD128_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -}; - -static int __init rmd128_mod_init(void) -{ - return crypto_register_shash(&alg); -} - -static void __exit rmd128_mod_fini(void) -{ - crypto_unregister_shash(&alg); -} - -subsys_initcall(rmd128_mod_init); -module_exit(rmd128_mod_fini); - -MODULE_LICENSE("GPL"); -MODULE_AUTHOR("Adrian-Ken Rueegsegger "); -MODULE_DESCRIPTION("RIPEMD-128 Message Digest"); -MODULE_ALIAS_CRYPTO("rmd128"); diff --git a/crypto/rmd256.c b/crypto/rmd256.c deleted file mode 100644 index 3c730e9de5fdf114bca17ae1df4fc309b95729cc..0000000000000000000000000000000000000000 --- a/crypto/rmd256.c +++ /dev/null @@ -1,342 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Cryptographic API. - * - * RIPEMD-256 - RACE Integrity Primitives Evaluation Message Digest. - * - * Based on the reference implementation by Antoon Bosselaers, ESAT-COSIC - * - * Copyright (c) 2008 Adrian-Ken Rueegsegger - */ -#include -#include -#include -#include -#include -#include - -#include "ripemd.h" - -struct rmd256_ctx { - u64 byte_count; - u32 state[8]; - __le32 buffer[16]; -}; - -#define K1 RMD_K1 -#define K2 RMD_K2 -#define K3 RMD_K3 -#define K4 RMD_K4 -#define KK1 RMD_K6 -#define KK2 RMD_K7 -#define KK3 RMD_K8 -#define KK4 RMD_K1 - -#define F1(x, y, z) (x ^ y ^ z) /* XOR */ -#define F2(x, y, z) (z ^ (x & (y ^ z))) /* x ? y : z */ -#define F3(x, y, z) ((x | ~y) ^ z) -#define F4(x, y, z) (y ^ (z & (x ^ y))) /* z ? x : y */ - -#define ROUND(a, b, c, d, f, k, x, s) { \ - (a) += f((b), (c), (d)) + le32_to_cpup(&(x)) + (k); \ - (a) = rol32((a), (s)); \ -} - -static void rmd256_transform(u32 *state, const __le32 *in) -{ - u32 aa, bb, cc, dd, aaa, bbb, ccc, ddd; - - /* Initialize left lane */ - aa = state[0]; - bb = state[1]; - cc = state[2]; - dd = state[3]; - - /* Initialize right lane */ - aaa = state[4]; - bbb = state[5]; - ccc = state[6]; - ddd = state[7]; - - /* round 1: left lane */ - ROUND(aa, bb, cc, dd, F1, K1, in[0], 11); - ROUND(dd, aa, bb, cc, F1, K1, in[1], 14); - ROUND(cc, dd, aa, bb, F1, K1, in[2], 15); - ROUND(bb, cc, dd, aa, F1, K1, in[3], 12); - ROUND(aa, bb, cc, dd, F1, K1, in[4], 5); - ROUND(dd, aa, bb, cc, F1, K1, in[5], 8); - ROUND(cc, dd, aa, bb, F1, K1, in[6], 7); - ROUND(bb, cc, dd, aa, F1, K1, in[7], 9); - ROUND(aa, bb, cc, dd, F1, K1, in[8], 11); - ROUND(dd, aa, bb, cc, F1, K1, in[9], 13); - ROUND(cc, dd, aa, bb, F1, K1, in[10], 14); - ROUND(bb, cc, dd, aa, F1, K1, in[11], 15); - ROUND(aa, bb, cc, dd, F1, K1, in[12], 6); - ROUND(dd, aa, bb, cc, F1, K1, in[13], 7); - ROUND(cc, dd, aa, bb, F1, K1, in[14], 9); - ROUND(bb, cc, dd, aa, F1, K1, in[15], 8); - - /* round 1: right lane */ - ROUND(aaa, bbb, ccc, ddd, F4, KK1, in[5], 8); - ROUND(ddd, aaa, bbb, ccc, F4, KK1, in[14], 9); - ROUND(ccc, ddd, aaa, bbb, F4, KK1, in[7], 9); - ROUND(bbb, ccc, ddd, aaa, F4, KK1, in[0], 11); - ROUND(aaa, bbb, ccc, ddd, F4, KK1, in[9], 13); - ROUND(ddd, aaa, bbb, ccc, F4, KK1, in[2], 15); - ROUND(ccc, ddd, aaa, bbb, F4, KK1, in[11], 15); - ROUND(bbb, ccc, ddd, aaa, F4, KK1, in[4], 5); - ROUND(aaa, bbb, ccc, ddd, F4, KK1, in[13], 7); - ROUND(ddd, aaa, bbb, ccc, F4, KK1, in[6], 7); - ROUND(ccc, ddd, aaa, bbb, F4, KK1, in[15], 8); - ROUND(bbb, ccc, ddd, aaa, F4, KK1, in[8], 11); - ROUND(aaa, bbb, ccc, ddd, F4, KK1, in[1], 14); - ROUND(ddd, aaa, bbb, ccc, F4, KK1, in[10], 14); - ROUND(ccc, ddd, aaa, bbb, F4, KK1, in[3], 12); - ROUND(bbb, ccc, ddd, aaa, F4, KK1, in[12], 6); - - /* Swap contents of "a" registers */ - swap(aa, aaa); - - /* round 2: left lane */ - ROUND(aa, bb, cc, dd, F2, K2, in[7], 7); - ROUND(dd, aa, bb, cc, F2, K2, in[4], 6); - ROUND(cc, dd, aa, bb, F2, K2, in[13], 8); - ROUND(bb, cc, dd, aa, F2, K2, in[1], 13); - ROUND(aa, bb, cc, dd, F2, K2, in[10], 11); - ROUND(dd, aa, bb, cc, F2, K2, in[6], 9); - ROUND(cc, dd, aa, bb, F2, K2, in[15], 7); - ROUND(bb, cc, dd, aa, F2, K2, in[3], 15); - ROUND(aa, bb, cc, dd, F2, K2, in[12], 7); - ROUND(dd, aa, bb, cc, F2, K2, in[0], 12); - ROUND(cc, dd, aa, bb, F2, K2, in[9], 15); - ROUND(bb, cc, dd, aa, F2, K2, in[5], 9); - ROUND(aa, bb, cc, dd, F2, K2, in[2], 11); - ROUND(dd, aa, bb, cc, F2, K2, in[14], 7); - ROUND(cc, dd, aa, bb, F2, K2, in[11], 13); - ROUND(bb, cc, dd, aa, F2, K2, in[8], 12); - - /* round 2: right lane */ - ROUND(aaa, bbb, ccc, ddd, F3, KK2, in[6], 9); - ROUND(ddd, aaa, bbb, ccc, F3, KK2, in[11], 13); - ROUND(ccc, ddd, aaa, bbb, F3, KK2, in[3], 15); - ROUND(bbb, ccc, ddd, aaa, F3, KK2, in[7], 7); - ROUND(aaa, bbb, ccc, ddd, F3, KK2, in[0], 12); - ROUND(ddd, aaa, bbb, ccc, F3, KK2, in[13], 8); - ROUND(ccc, ddd, aaa, bbb, F3, KK2, in[5], 9); - ROUND(bbb, ccc, ddd, aaa, F3, KK2, in[10], 11); - ROUND(aaa, bbb, ccc, ddd, F3, KK2, in[14], 7); - ROUND(ddd, aaa, bbb, ccc, F3, KK2, in[15], 7); - ROUND(ccc, ddd, aaa, bbb, F3, KK2, in[8], 12); - ROUND(bbb, ccc, ddd, aaa, F3, KK2, in[12], 7); - ROUND(aaa, bbb, ccc, ddd, F3, KK2, in[4], 6); - ROUND(ddd, aaa, bbb, ccc, F3, KK2, in[9], 15); - ROUND(ccc, ddd, aaa, bbb, F3, KK2, in[1], 13); - ROUND(bbb, ccc, ddd, aaa, F3, KK2, in[2], 11); - - /* Swap contents of "b" registers */ - swap(bb, bbb); - - /* round 3: left lane */ - ROUND(aa, bb, cc, dd, F3, K3, in[3], 11); - ROUND(dd, aa, bb, cc, F3, K3, in[10], 13); - ROUND(cc, dd, aa, bb, F3, K3, in[14], 6); - ROUND(bb, cc, dd, aa, F3, K3, in[4], 7); - ROUND(aa, bb, cc, dd, F3, K3, in[9], 14); - ROUND(dd, aa, bb, cc, F3, K3, in[15], 9); - ROUND(cc, dd, aa, bb, F3, K3, in[8], 13); - ROUND(bb, cc, dd, aa, F3, K3, in[1], 15); - ROUND(aa, bb, cc, dd, F3, K3, in[2], 14); - ROUND(dd, aa, bb, cc, F3, K3, in[7], 8); - ROUND(cc, dd, aa, bb, F3, K3, in[0], 13); - ROUND(bb, cc, dd, aa, F3, K3, in[6], 6); - ROUND(aa, bb, cc, dd, F3, K3, in[13], 5); - ROUND(dd, aa, bb, cc, F3, K3, in[11], 12); - ROUND(cc, dd, aa, bb, F3, K3, in[5], 7); - ROUND(bb, cc, dd, aa, F3, K3, in[12], 5); - - /* round 3: right lane */ - ROUND(aaa, bbb, ccc, ddd, F2, KK3, in[15], 9); - ROUND(ddd, aaa, bbb, ccc, F2, KK3, in[5], 7); - ROUND(ccc, ddd, aaa, bbb, F2, KK3, in[1], 15); - ROUND(bbb, ccc, ddd, aaa, F2, KK3, in[3], 11); - ROUND(aaa, bbb, ccc, ddd, F2, KK3, in[7], 8); - ROUND(ddd, aaa, bbb, ccc, F2, KK3, in[14], 6); - ROUND(ccc, ddd, aaa, bbb, F2, KK3, in[6], 6); - ROUND(bbb, ccc, ddd, aaa, F2, KK3, in[9], 14); - ROUND(aaa, bbb, ccc, ddd, F2, KK3, in[11], 12); - ROUND(ddd, aaa, bbb, ccc, F2, KK3, in[8], 13); - ROUND(ccc, ddd, aaa, bbb, F2, KK3, in[12], 5); - ROUND(bbb, ccc, ddd, aaa, F2, KK3, in[2], 14); - ROUND(aaa, bbb, ccc, ddd, F2, KK3, in[10], 13); - ROUND(ddd, aaa, bbb, ccc, F2, KK3, in[0], 13); - ROUND(ccc, ddd, aaa, bbb, F2, KK3, in[4], 7); - ROUND(bbb, ccc, ddd, aaa, F2, KK3, in[13], 5); - - /* Swap contents of "c" registers */ - swap(cc, ccc); - - /* round 4: left lane */ - ROUND(aa, bb, cc, dd, F4, K4, in[1], 11); - ROUND(dd, aa, bb, cc, F4, K4, in[9], 12); - ROUND(cc, dd, aa, bb, F4, K4, in[11], 14); - ROUND(bb, cc, dd, aa, F4, K4, in[10], 15); - ROUND(aa, bb, cc, dd, F4, K4, in[0], 14); - ROUND(dd, aa, bb, cc, F4, K4, in[8], 15); - ROUND(cc, dd, aa, bb, F4, K4, in[12], 9); - ROUND(bb, cc, dd, aa, F4, K4, in[4], 8); - ROUND(aa, bb, cc, dd, F4, K4, in[13], 9); - ROUND(dd, aa, bb, cc, F4, K4, in[3], 14); - ROUND(cc, dd, aa, bb, F4, K4, in[7], 5); - ROUND(bb, cc, dd, aa, F4, K4, in[15], 6); - ROUND(aa, bb, cc, dd, F4, K4, in[14], 8); - ROUND(dd, aa, bb, cc, F4, K4, in[5], 6); - ROUND(cc, dd, aa, bb, F4, K4, in[6], 5); - ROUND(bb, cc, dd, aa, F4, K4, in[2], 12); - - /* round 4: right lane */ - ROUND(aaa, bbb, ccc, ddd, F1, KK4, in[8], 15); - ROUND(ddd, aaa, bbb, ccc, F1, KK4, in[6], 5); - ROUND(ccc, ddd, aaa, bbb, F1, KK4, in[4], 8); - ROUND(bbb, ccc, ddd, aaa, F1, KK4, in[1], 11); - ROUND(aaa, bbb, ccc, ddd, F1, KK4, in[3], 14); - ROUND(ddd, aaa, bbb, ccc, F1, KK4, in[11], 14); - ROUND(ccc, ddd, aaa, bbb, F1, KK4, in[15], 6); - ROUND(bbb, ccc, ddd, aaa, F1, KK4, in[0], 14); - ROUND(aaa, bbb, ccc, ddd, F1, KK4, in[5], 6); - ROUND(ddd, aaa, bbb, ccc, F1, KK4, in[12], 9); - ROUND(ccc, ddd, aaa, bbb, F1, KK4, in[2], 12); - ROUND(bbb, ccc, ddd, aaa, F1, KK4, in[13], 9); - ROUND(aaa, bbb, ccc, ddd, F1, KK4, in[9], 12); - ROUND(ddd, aaa, bbb, ccc, F1, KK4, in[7], 5); - ROUND(ccc, ddd, aaa, bbb, F1, KK4, in[10], 15); - ROUND(bbb, ccc, ddd, aaa, F1, KK4, in[14], 8); - - /* Swap contents of "d" registers */ - swap(dd, ddd); - - /* combine results */ - state[0] += aa; - state[1] += bb; - state[2] += cc; - state[3] += dd; - state[4] += aaa; - state[5] += bbb; - state[6] += ccc; - state[7] += ddd; -} - -static int rmd256_init(struct shash_desc *desc) -{ - struct rmd256_ctx *rctx = shash_desc_ctx(desc); - - rctx->byte_count = 0; - - rctx->state[0] = RMD_H0; - rctx->state[1] = RMD_H1; - rctx->state[2] = RMD_H2; - rctx->state[3] = RMD_H3; - rctx->state[4] = RMD_H5; - rctx->state[5] = RMD_H6; - rctx->state[6] = RMD_H7; - rctx->state[7] = RMD_H8; - - memset(rctx->buffer, 0, sizeof(rctx->buffer)); - - return 0; -} - -static int rmd256_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - struct rmd256_ctx *rctx = shash_desc_ctx(desc); - const u32 avail = sizeof(rctx->buffer) - (rctx->byte_count & 0x3f); - - rctx->byte_count += len; - - /* Enough space in buffer? If so copy and we're done */ - if (avail > len) { - memcpy((char *)rctx->buffer + (sizeof(rctx->buffer) - avail), - data, len); - goto out; - } - - memcpy((char *)rctx->buffer + (sizeof(rctx->buffer) - avail), - data, avail); - - rmd256_transform(rctx->state, rctx->buffer); - data += avail; - len -= avail; - - while (len >= sizeof(rctx->buffer)) { - memcpy(rctx->buffer, data, sizeof(rctx->buffer)); - rmd256_transform(rctx->state, rctx->buffer); - data += sizeof(rctx->buffer); - len -= sizeof(rctx->buffer); - } - - memcpy(rctx->buffer, data, len); - -out: - return 0; -} - -/* Add padding and return the message digest. */ -static int rmd256_final(struct shash_desc *desc, u8 *out) -{ - struct rmd256_ctx *rctx = shash_desc_ctx(desc); - u32 i, index, padlen; - __le64 bits; - __le32 *dst = (__le32 *)out; - static const u8 padding[64] = { 0x80, }; - - bits = cpu_to_le64(rctx->byte_count << 3); - - /* Pad out to 56 mod 64 */ - index = rctx->byte_count & 0x3f; - padlen = (index < 56) ? (56 - index) : ((64+56) - index); - rmd256_update(desc, padding, padlen); - - /* Append length */ - rmd256_update(desc, (const u8 *)&bits, sizeof(bits)); - - /* Store state in digest */ - for (i = 0; i < 8; i++) - dst[i] = cpu_to_le32p(&rctx->state[i]); - - /* Wipe context */ - memset(rctx, 0, sizeof(*rctx)); - - return 0; -} - -static struct shash_alg alg = { - .digestsize = RMD256_DIGEST_SIZE, - .init = rmd256_init, - .update = rmd256_update, - .final = rmd256_final, - .descsize = sizeof(struct rmd256_ctx), - .base = { - .cra_name = "rmd256", - .cra_driver_name = "rmd256-generic", - .cra_blocksize = RMD256_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -}; - -static int __init rmd256_mod_init(void) -{ - return crypto_register_shash(&alg); -} - -static void __exit rmd256_mod_fini(void) -{ - crypto_unregister_shash(&alg); -} - -subsys_initcall(rmd256_mod_init); -module_exit(rmd256_mod_fini); - -MODULE_LICENSE("GPL"); -MODULE_AUTHOR("Adrian-Ken Rueegsegger "); -MODULE_DESCRIPTION("RIPEMD-256 Message Digest"); -MODULE_ALIAS_CRYPTO("rmd256"); diff --git a/crypto/rmd320.c b/crypto/rmd320.c deleted file mode 100644 index c919ad6c4705a682486de0b3028c229869c65269..0000000000000000000000000000000000000000 --- a/crypto/rmd320.c +++ /dev/null @@ -1,391 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Cryptographic API. - * - * RIPEMD-320 - RACE Integrity Primitives Evaluation Message Digest. - * - * Based on the reference implementation by Antoon Bosselaers, ESAT-COSIC - * - * Copyright (c) 2008 Adrian-Ken Rueegsegger - */ -#include -#include -#include -#include -#include -#include - -#include "ripemd.h" - -struct rmd320_ctx { - u64 byte_count; - u32 state[10]; - __le32 buffer[16]; -}; - -#define K1 RMD_K1 -#define K2 RMD_K2 -#define K3 RMD_K3 -#define K4 RMD_K4 -#define K5 RMD_K5 -#define KK1 RMD_K6 -#define KK2 RMD_K7 -#define KK3 RMD_K8 -#define KK4 RMD_K9 -#define KK5 RMD_K1 - -#define F1(x, y, z) (x ^ y ^ z) /* XOR */ -#define F2(x, y, z) (z ^ (x & (y ^ z))) /* x ? y : z */ -#define F3(x, y, z) ((x | ~y) ^ z) -#define F4(x, y, z) (y ^ (z & (x ^ y))) /* z ? x : y */ -#define F5(x, y, z) (x ^ (y | ~z)) - -#define ROUND(a, b, c, d, e, f, k, x, s) { \ - (a) += f((b), (c), (d)) + le32_to_cpup(&(x)) + (k); \ - (a) = rol32((a), (s)) + (e); \ - (c) = rol32((c), 10); \ -} - -static void rmd320_transform(u32 *state, const __le32 *in) -{ - u32 aa, bb, cc, dd, ee, aaa, bbb, ccc, ddd, eee; - - /* Initialize left lane */ - aa = state[0]; - bb = state[1]; - cc = state[2]; - dd = state[3]; - ee = state[4]; - - /* Initialize right lane */ - aaa = state[5]; - bbb = state[6]; - ccc = state[7]; - ddd = state[8]; - eee = state[9]; - - /* round 1: left lane */ - ROUND(aa, bb, cc, dd, ee, F1, K1, in[0], 11); - ROUND(ee, aa, bb, cc, dd, F1, K1, in[1], 14); - ROUND(dd, ee, aa, bb, cc, F1, K1, in[2], 15); - ROUND(cc, dd, ee, aa, bb, F1, K1, in[3], 12); - ROUND(bb, cc, dd, ee, aa, F1, K1, in[4], 5); - ROUND(aa, bb, cc, dd, ee, F1, K1, in[5], 8); - ROUND(ee, aa, bb, cc, dd, F1, K1, in[6], 7); - ROUND(dd, ee, aa, bb, cc, F1, K1, in[7], 9); - ROUND(cc, dd, ee, aa, bb, F1, K1, in[8], 11); - ROUND(bb, cc, dd, ee, aa, F1, K1, in[9], 13); - ROUND(aa, bb, cc, dd, ee, F1, K1, in[10], 14); - ROUND(ee, aa, bb, cc, dd, F1, K1, in[11], 15); - ROUND(dd, ee, aa, bb, cc, F1, K1, in[12], 6); - ROUND(cc, dd, ee, aa, bb, F1, K1, in[13], 7); - ROUND(bb, cc, dd, ee, aa, F1, K1, in[14], 9); - ROUND(aa, bb, cc, dd, ee, F1, K1, in[15], 8); - - /* round 1: right lane */ - ROUND(aaa, bbb, ccc, ddd, eee, F5, KK1, in[5], 8); - ROUND(eee, aaa, bbb, ccc, ddd, F5, KK1, in[14], 9); - ROUND(ddd, eee, aaa, bbb, ccc, F5, KK1, in[7], 9); - ROUND(ccc, ddd, eee, aaa, bbb, F5, KK1, in[0], 11); - ROUND(bbb, ccc, ddd, eee, aaa, F5, KK1, in[9], 13); - ROUND(aaa, bbb, ccc, ddd, eee, F5, KK1, in[2], 15); - ROUND(eee, aaa, bbb, ccc, ddd, F5, KK1, in[11], 15); - ROUND(ddd, eee, aaa, bbb, ccc, F5, KK1, in[4], 5); - ROUND(ccc, ddd, eee, aaa, bbb, F5, KK1, in[13], 7); - ROUND(bbb, ccc, ddd, eee, aaa, F5, KK1, in[6], 7); - ROUND(aaa, bbb, ccc, ddd, eee, F5, KK1, in[15], 8); - ROUND(eee, aaa, bbb, ccc, ddd, F5, KK1, in[8], 11); - ROUND(ddd, eee, aaa, bbb, ccc, F5, KK1, in[1], 14); - ROUND(ccc, ddd, eee, aaa, bbb, F5, KK1, in[10], 14); - ROUND(bbb, ccc, ddd, eee, aaa, F5, KK1, in[3], 12); - ROUND(aaa, bbb, ccc, ddd, eee, F5, KK1, in[12], 6); - - /* Swap contents of "a" registers */ - swap(aa, aaa); - - /* round 2: left lane" */ - ROUND(ee, aa, bb, cc, dd, F2, K2, in[7], 7); - ROUND(dd, ee, aa, bb, cc, F2, K2, in[4], 6); - ROUND(cc, dd, ee, aa, bb, F2, K2, in[13], 8); - ROUND(bb, cc, dd, ee, aa, F2, K2, in[1], 13); - ROUND(aa, bb, cc, dd, ee, F2, K2, in[10], 11); - ROUND(ee, aa, bb, cc, dd, F2, K2, in[6], 9); - ROUND(dd, ee, aa, bb, cc, F2, K2, in[15], 7); - ROUND(cc, dd, ee, aa, bb, F2, K2, in[3], 15); - ROUND(bb, cc, dd, ee, aa, F2, K2, in[12], 7); - ROUND(aa, bb, cc, dd, ee, F2, K2, in[0], 12); - ROUND(ee, aa, bb, cc, dd, F2, K2, in[9], 15); - ROUND(dd, ee, aa, bb, cc, F2, K2, in[5], 9); - ROUND(cc, dd, ee, aa, bb, F2, K2, in[2], 11); - ROUND(bb, cc, dd, ee, aa, F2, K2, in[14], 7); - ROUND(aa, bb, cc, dd, ee, F2, K2, in[11], 13); - ROUND(ee, aa, bb, cc, dd, F2, K2, in[8], 12); - - /* round 2: right lane */ - ROUND(eee, aaa, bbb, ccc, ddd, F4, KK2, in[6], 9); - ROUND(ddd, eee, aaa, bbb, ccc, F4, KK2, in[11], 13); - ROUND(ccc, ddd, eee, aaa, bbb, F4, KK2, in[3], 15); - ROUND(bbb, ccc, ddd, eee, aaa, F4, KK2, in[7], 7); - ROUND(aaa, bbb, ccc, ddd, eee, F4, KK2, in[0], 12); - ROUND(eee, aaa, bbb, ccc, ddd, F4, KK2, in[13], 8); - ROUND(ddd, eee, aaa, bbb, ccc, F4, KK2, in[5], 9); - ROUND(ccc, ddd, eee, aaa, bbb, F4, KK2, in[10], 11); - ROUND(bbb, ccc, ddd, eee, aaa, F4, KK2, in[14], 7); - ROUND(aaa, bbb, ccc, ddd, eee, F4, KK2, in[15], 7); - ROUND(eee, aaa, bbb, ccc, ddd, F4, KK2, in[8], 12); - ROUND(ddd, eee, aaa, bbb, ccc, F4, KK2, in[12], 7); - ROUND(ccc, ddd, eee, aaa, bbb, F4, KK2, in[4], 6); - ROUND(bbb, ccc, ddd, eee, aaa, F4, KK2, in[9], 15); - ROUND(aaa, bbb, ccc, ddd, eee, F4, KK2, in[1], 13); - ROUND(eee, aaa, bbb, ccc, ddd, F4, KK2, in[2], 11); - - /* Swap contents of "b" registers */ - swap(bb, bbb); - - /* round 3: left lane" */ - ROUND(dd, ee, aa, bb, cc, F3, K3, in[3], 11); - ROUND(cc, dd, ee, aa, bb, F3, K3, in[10], 13); - ROUND(bb, cc, dd, ee, aa, F3, K3, in[14], 6); - ROUND(aa, bb, cc, dd, ee, F3, K3, in[4], 7); - ROUND(ee, aa, bb, cc, dd, F3, K3, in[9], 14); - ROUND(dd, ee, aa, bb, cc, F3, K3, in[15], 9); - ROUND(cc, dd, ee, aa, bb, F3, K3, in[8], 13); - ROUND(bb, cc, dd, ee, aa, F3, K3, in[1], 15); - ROUND(aa, bb, cc, dd, ee, F3, K3, in[2], 14); - ROUND(ee, aa, bb, cc, dd, F3, K3, in[7], 8); - ROUND(dd, ee, aa, bb, cc, F3, K3, in[0], 13); - ROUND(cc, dd, ee, aa, bb, F3, K3, in[6], 6); - ROUND(bb, cc, dd, ee, aa, F3, K3, in[13], 5); - ROUND(aa, bb, cc, dd, ee, F3, K3, in[11], 12); - ROUND(ee, aa, bb, cc, dd, F3, K3, in[5], 7); - ROUND(dd, ee, aa, bb, cc, F3, K3, in[12], 5); - - /* round 3: right lane */ - ROUND(ddd, eee, aaa, bbb, ccc, F3, KK3, in[15], 9); - ROUND(ccc, ddd, eee, aaa, bbb, F3, KK3, in[5], 7); - ROUND(bbb, ccc, ddd, eee, aaa, F3, KK3, in[1], 15); - ROUND(aaa, bbb, ccc, ddd, eee, F3, KK3, in[3], 11); - ROUND(eee, aaa, bbb, ccc, ddd, F3, KK3, in[7], 8); - ROUND(ddd, eee, aaa, bbb, ccc, F3, KK3, in[14], 6); - ROUND(ccc, ddd, eee, aaa, bbb, F3, KK3, in[6], 6); - ROUND(bbb, ccc, ddd, eee, aaa, F3, KK3, in[9], 14); - ROUND(aaa, bbb, ccc, ddd, eee, F3, KK3, in[11], 12); - ROUND(eee, aaa, bbb, ccc, ddd, F3, KK3, in[8], 13); - ROUND(ddd, eee, aaa, bbb, ccc, F3, KK3, in[12], 5); - ROUND(ccc, ddd, eee, aaa, bbb, F3, KK3, in[2], 14); - ROUND(bbb, ccc, ddd, eee, aaa, F3, KK3, in[10], 13); - ROUND(aaa, bbb, ccc, ddd, eee, F3, KK3, in[0], 13); - ROUND(eee, aaa, bbb, ccc, ddd, F3, KK3, in[4], 7); - ROUND(ddd, eee, aaa, bbb, ccc, F3, KK3, in[13], 5); - - /* Swap contents of "c" registers */ - swap(cc, ccc); - - /* round 4: left lane" */ - ROUND(cc, dd, ee, aa, bb, F4, K4, in[1], 11); - ROUND(bb, cc, dd, ee, aa, F4, K4, in[9], 12); - ROUND(aa, bb, cc, dd, ee, F4, K4, in[11], 14); - ROUND(ee, aa, bb, cc, dd, F4, K4, in[10], 15); - ROUND(dd, ee, aa, bb, cc, F4, K4, in[0], 14); - ROUND(cc, dd, ee, aa, bb, F4, K4, in[8], 15); - ROUND(bb, cc, dd, ee, aa, F4, K4, in[12], 9); - ROUND(aa, bb, cc, dd, ee, F4, K4, in[4], 8); - ROUND(ee, aa, bb, cc, dd, F4, K4, in[13], 9); - ROUND(dd, ee, aa, bb, cc, F4, K4, in[3], 14); - ROUND(cc, dd, ee, aa, bb, F4, K4, in[7], 5); - ROUND(bb, cc, dd, ee, aa, F4, K4, in[15], 6); - ROUND(aa, bb, cc, dd, ee, F4, K4, in[14], 8); - ROUND(ee, aa, bb, cc, dd, F4, K4, in[5], 6); - ROUND(dd, ee, aa, bb, cc, F4, K4, in[6], 5); - ROUND(cc, dd, ee, aa, bb, F4, K4, in[2], 12); - - /* round 4: right lane */ - ROUND(ccc, ddd, eee, aaa, bbb, F2, KK4, in[8], 15); - ROUND(bbb, ccc, ddd, eee, aaa, F2, KK4, in[6], 5); - ROUND(aaa, bbb, ccc, ddd, eee, F2, KK4, in[4], 8); - ROUND(eee, aaa, bbb, ccc, ddd, F2, KK4, in[1], 11); - ROUND(ddd, eee, aaa, bbb, ccc, F2, KK4, in[3], 14); - ROUND(ccc, ddd, eee, aaa, bbb, F2, KK4, in[11], 14); - ROUND(bbb, ccc, ddd, eee, aaa, F2, KK4, in[15], 6); - ROUND(aaa, bbb, ccc, ddd, eee, F2, KK4, in[0], 14); - ROUND(eee, aaa, bbb, ccc, ddd, F2, KK4, in[5], 6); - ROUND(ddd, eee, aaa, bbb, ccc, F2, KK4, in[12], 9); - ROUND(ccc, ddd, eee, aaa, bbb, F2, KK4, in[2], 12); - ROUND(bbb, ccc, ddd, eee, aaa, F2, KK4, in[13], 9); - ROUND(aaa, bbb, ccc, ddd, eee, F2, KK4, in[9], 12); - ROUND(eee, aaa, bbb, ccc, ddd, F2, KK4, in[7], 5); - ROUND(ddd, eee, aaa, bbb, ccc, F2, KK4, in[10], 15); - ROUND(ccc, ddd, eee, aaa, bbb, F2, KK4, in[14], 8); - - /* Swap contents of "d" registers */ - swap(dd, ddd); - - /* round 5: left lane" */ - ROUND(bb, cc, dd, ee, aa, F5, K5, in[4], 9); - ROUND(aa, bb, cc, dd, ee, F5, K5, in[0], 15); - ROUND(ee, aa, bb, cc, dd, F5, K5, in[5], 5); - ROUND(dd, ee, aa, bb, cc, F5, K5, in[9], 11); - ROUND(cc, dd, ee, aa, bb, F5, K5, in[7], 6); - ROUND(bb, cc, dd, ee, aa, F5, K5, in[12], 8); - ROUND(aa, bb, cc, dd, ee, F5, K5, in[2], 13); - ROUND(ee, aa, bb, cc, dd, F5, K5, in[10], 12); - ROUND(dd, ee, aa, bb, cc, F5, K5, in[14], 5); - ROUND(cc, dd, ee, aa, bb, F5, K5, in[1], 12); - ROUND(bb, cc, dd, ee, aa, F5, K5, in[3], 13); - ROUND(aa, bb, cc, dd, ee, F5, K5, in[8], 14); - ROUND(ee, aa, bb, cc, dd, F5, K5, in[11], 11); - ROUND(dd, ee, aa, bb, cc, F5, K5, in[6], 8); - ROUND(cc, dd, ee, aa, bb, F5, K5, in[15], 5); - ROUND(bb, cc, dd, ee, aa, F5, K5, in[13], 6); - - /* round 5: right lane */ - ROUND(bbb, ccc, ddd, eee, aaa, F1, KK5, in[12], 8); - ROUND(aaa, bbb, ccc, ddd, eee, F1, KK5, in[15], 5); - ROUND(eee, aaa, bbb, ccc, ddd, F1, KK5, in[10], 12); - ROUND(ddd, eee, aaa, bbb, ccc, F1, KK5, in[4], 9); - ROUND(ccc, ddd, eee, aaa, bbb, F1, KK5, in[1], 12); - ROUND(bbb, ccc, ddd, eee, aaa, F1, KK5, in[5], 5); - ROUND(aaa, bbb, ccc, ddd, eee, F1, KK5, in[8], 14); - ROUND(eee, aaa, bbb, ccc, ddd, F1, KK5, in[7], 6); - ROUND(ddd, eee, aaa, bbb, ccc, F1, KK5, in[6], 8); - ROUND(ccc, ddd, eee, aaa, bbb, F1, KK5, in[2], 13); - ROUND(bbb, ccc, ddd, eee, aaa, F1, KK5, in[13], 6); - ROUND(aaa, bbb, ccc, ddd, eee, F1, KK5, in[14], 5); - ROUND(eee, aaa, bbb, ccc, ddd, F1, KK5, in[0], 15); - ROUND(ddd, eee, aaa, bbb, ccc, F1, KK5, in[3], 13); - ROUND(ccc, ddd, eee, aaa, bbb, F1, KK5, in[9], 11); - ROUND(bbb, ccc, ddd, eee, aaa, F1, KK5, in[11], 11); - - /* Swap contents of "e" registers */ - swap(ee, eee); - - /* combine results */ - state[0] += aa; - state[1] += bb; - state[2] += cc; - state[3] += dd; - state[4] += ee; - state[5] += aaa; - state[6] += bbb; - state[7] += ccc; - state[8] += ddd; - state[9] += eee; -} - -static int rmd320_init(struct shash_desc *desc) -{ - struct rmd320_ctx *rctx = shash_desc_ctx(desc); - - rctx->byte_count = 0; - - rctx->state[0] = RMD_H0; - rctx->state[1] = RMD_H1; - rctx->state[2] = RMD_H2; - rctx->state[3] = RMD_H3; - rctx->state[4] = RMD_H4; - rctx->state[5] = RMD_H5; - rctx->state[6] = RMD_H6; - rctx->state[7] = RMD_H7; - rctx->state[8] = RMD_H8; - rctx->state[9] = RMD_H9; - - memset(rctx->buffer, 0, sizeof(rctx->buffer)); - - return 0; -} - -static int rmd320_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - struct rmd320_ctx *rctx = shash_desc_ctx(desc); - const u32 avail = sizeof(rctx->buffer) - (rctx->byte_count & 0x3f); - - rctx->byte_count += len; - - /* Enough space in buffer? If so copy and we're done */ - if (avail > len) { - memcpy((char *)rctx->buffer + (sizeof(rctx->buffer) - avail), - data, len); - goto out; - } - - memcpy((char *)rctx->buffer + (sizeof(rctx->buffer) - avail), - data, avail); - - rmd320_transform(rctx->state, rctx->buffer); - data += avail; - len -= avail; - - while (len >= sizeof(rctx->buffer)) { - memcpy(rctx->buffer, data, sizeof(rctx->buffer)); - rmd320_transform(rctx->state, rctx->buffer); - data += sizeof(rctx->buffer); - len -= sizeof(rctx->buffer); - } - - memcpy(rctx->buffer, data, len); - -out: - return 0; -} - -/* Add padding and return the message digest. */ -static int rmd320_final(struct shash_desc *desc, u8 *out) -{ - struct rmd320_ctx *rctx = shash_desc_ctx(desc); - u32 i, index, padlen; - __le64 bits; - __le32 *dst = (__le32 *)out; - static const u8 padding[64] = { 0x80, }; - - bits = cpu_to_le64(rctx->byte_count << 3); - - /* Pad out to 56 mod 64 */ - index = rctx->byte_count & 0x3f; - padlen = (index < 56) ? (56 - index) : ((64+56) - index); - rmd320_update(desc, padding, padlen); - - /* Append length */ - rmd320_update(desc, (const u8 *)&bits, sizeof(bits)); - - /* Store state in digest */ - for (i = 0; i < 10; i++) - dst[i] = cpu_to_le32p(&rctx->state[i]); - - /* Wipe context */ - memset(rctx, 0, sizeof(*rctx)); - - return 0; -} - -static struct shash_alg alg = { - .digestsize = RMD320_DIGEST_SIZE, - .init = rmd320_init, - .update = rmd320_update, - .final = rmd320_final, - .descsize = sizeof(struct rmd320_ctx), - .base = { - .cra_name = "rmd320", - .cra_driver_name = "rmd320-generic", - .cra_blocksize = RMD320_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -}; - -static int __init rmd320_mod_init(void) -{ - return crypto_register_shash(&alg); -} - -static void __exit rmd320_mod_fini(void) -{ - crypto_unregister_shash(&alg); -} - -subsys_initcall(rmd320_mod_init); -module_exit(rmd320_mod_fini); - -MODULE_LICENSE("GPL"); -MODULE_AUTHOR("Adrian-Ken Rueegsegger "); -MODULE_DESCRIPTION("RIPEMD-320 Message Digest"); -MODULE_ALIAS_CRYPTO("rmd320"); diff --git a/crypto/salsa20_generic.c b/crypto/salsa20_generic.c deleted file mode 100644 index 3418869dabefdef2abc4f2d5794d2509aef5d32f..0000000000000000000000000000000000000000 --- a/crypto/salsa20_generic.c +++ /dev/null @@ -1,212 +0,0 @@ -/* - * Salsa20: Salsa20 stream cipher algorithm - * - * Copyright (c) 2007 Tan Swee Heng - * - * Derived from: - * - salsa20.c: Public domain C code by Daniel J. Bernstein - * - * Salsa20 is a stream cipher candidate in eSTREAM, the ECRYPT Stream - * Cipher Project. It is designed by Daniel J. Bernstein . - * More information about eSTREAM and Salsa20 can be found here: - * https://www.ecrypt.eu.org/stream/ - * https://cr.yp.to/snuffle.html - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the Free - * Software Foundation; either version 2 of the License, or (at your option) - * any later version. - * - */ - -#include -#include -#include - -#define SALSA20_IV_SIZE 8 -#define SALSA20_MIN_KEY_SIZE 16 -#define SALSA20_MAX_KEY_SIZE 32 -#define SALSA20_BLOCK_SIZE 64 - -struct salsa20_ctx { - u32 initial_state[16]; -}; - -static void salsa20_block(u32 *state, __le32 *stream) -{ - u32 x[16]; - int i; - - memcpy(x, state, sizeof(x)); - - for (i = 0; i < 20; i += 2) { - x[ 4] ^= rol32((x[ 0] + x[12]), 7); - x[ 8] ^= rol32((x[ 4] + x[ 0]), 9); - x[12] ^= rol32((x[ 8] + x[ 4]), 13); - x[ 0] ^= rol32((x[12] + x[ 8]), 18); - x[ 9] ^= rol32((x[ 5] + x[ 1]), 7); - x[13] ^= rol32((x[ 9] + x[ 5]), 9); - x[ 1] ^= rol32((x[13] + x[ 9]), 13); - x[ 5] ^= rol32((x[ 1] + x[13]), 18); - x[14] ^= rol32((x[10] + x[ 6]), 7); - x[ 2] ^= rol32((x[14] + x[10]), 9); - x[ 6] ^= rol32((x[ 2] + x[14]), 13); - x[10] ^= rol32((x[ 6] + x[ 2]), 18); - x[ 3] ^= rol32((x[15] + x[11]), 7); - x[ 7] ^= rol32((x[ 3] + x[15]), 9); - x[11] ^= rol32((x[ 7] + x[ 3]), 13); - x[15] ^= rol32((x[11] + x[ 7]), 18); - x[ 1] ^= rol32((x[ 0] + x[ 3]), 7); - x[ 2] ^= rol32((x[ 1] + x[ 0]), 9); - x[ 3] ^= rol32((x[ 2] + x[ 1]), 13); - x[ 0] ^= rol32((x[ 3] + x[ 2]), 18); - x[ 6] ^= rol32((x[ 5] + x[ 4]), 7); - x[ 7] ^= rol32((x[ 6] + x[ 5]), 9); - x[ 4] ^= rol32((x[ 7] + x[ 6]), 13); - x[ 5] ^= rol32((x[ 4] + x[ 7]), 18); - x[11] ^= rol32((x[10] + x[ 9]), 7); - x[ 8] ^= rol32((x[11] + x[10]), 9); - x[ 9] ^= rol32((x[ 8] + x[11]), 13); - x[10] ^= rol32((x[ 9] + x[ 8]), 18); - x[12] ^= rol32((x[15] + x[14]), 7); - x[13] ^= rol32((x[12] + x[15]), 9); - x[14] ^= rol32((x[13] + x[12]), 13); - x[15] ^= rol32((x[14] + x[13]), 18); - } - - for (i = 0; i < 16; i++) - stream[i] = cpu_to_le32(x[i] + state[i]); - - if (++state[8] == 0) - state[9]++; -} - -static void salsa20_docrypt(u32 *state, u8 *dst, const u8 *src, - unsigned int bytes) -{ - __le32 stream[SALSA20_BLOCK_SIZE / sizeof(__le32)]; - - while (bytes >= SALSA20_BLOCK_SIZE) { - salsa20_block(state, stream); - crypto_xor_cpy(dst, src, (const u8 *)stream, - SALSA20_BLOCK_SIZE); - bytes -= SALSA20_BLOCK_SIZE; - dst += SALSA20_BLOCK_SIZE; - src += SALSA20_BLOCK_SIZE; - } - if (bytes) { - salsa20_block(state, stream); - crypto_xor_cpy(dst, src, (const u8 *)stream, bytes); - } -} - -static void salsa20_init(u32 *state, const struct salsa20_ctx *ctx, - const u8 *iv) -{ - memcpy(state, ctx->initial_state, sizeof(ctx->initial_state)); - state[6] = get_unaligned_le32(iv + 0); - state[7] = get_unaligned_le32(iv + 4); -} - -static int salsa20_setkey(struct crypto_skcipher *tfm, const u8 *key, - unsigned int keysize) -{ - static const char sigma[16] = "expand 32-byte k"; - static const char tau[16] = "expand 16-byte k"; - struct salsa20_ctx *ctx = crypto_skcipher_ctx(tfm); - const char *constants; - - if (keysize != SALSA20_MIN_KEY_SIZE && - keysize != SALSA20_MAX_KEY_SIZE) - return -EINVAL; - - ctx->initial_state[1] = get_unaligned_le32(key + 0); - ctx->initial_state[2] = get_unaligned_le32(key + 4); - ctx->initial_state[3] = get_unaligned_le32(key + 8); - ctx->initial_state[4] = get_unaligned_le32(key + 12); - if (keysize == 32) { /* recommended */ - key += 16; - constants = sigma; - } else { /* keysize == 16 */ - constants = tau; - } - ctx->initial_state[11] = get_unaligned_le32(key + 0); - ctx->initial_state[12] = get_unaligned_le32(key + 4); - ctx->initial_state[13] = get_unaligned_le32(key + 8); - ctx->initial_state[14] = get_unaligned_le32(key + 12); - ctx->initial_state[0] = get_unaligned_le32(constants + 0); - ctx->initial_state[5] = get_unaligned_le32(constants + 4); - ctx->initial_state[10] = get_unaligned_le32(constants + 8); - ctx->initial_state[15] = get_unaligned_le32(constants + 12); - - /* space for the nonce; it will be overridden for each request */ - ctx->initial_state[6] = 0; - ctx->initial_state[7] = 0; - - /* initial block number */ - ctx->initial_state[8] = 0; - ctx->initial_state[9] = 0; - - return 0; -} - -static int salsa20_crypt(struct skcipher_request *req) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - const struct salsa20_ctx *ctx = crypto_skcipher_ctx(tfm); - struct skcipher_walk walk; - u32 state[16]; - int err; - - err = skcipher_walk_virt(&walk, req, false); - - salsa20_init(state, ctx, req->iv); - - while (walk.nbytes > 0) { - unsigned int nbytes = walk.nbytes; - - if (nbytes < walk.total) - nbytes = round_down(nbytes, walk.stride); - - salsa20_docrypt(state, walk.dst.virt.addr, walk.src.virt.addr, - nbytes); - err = skcipher_walk_done(&walk, walk.nbytes - nbytes); - } - - return err; -} - -static struct skcipher_alg alg = { - .base.cra_name = "salsa20", - .base.cra_driver_name = "salsa20-generic", - .base.cra_priority = 100, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct salsa20_ctx), - .base.cra_module = THIS_MODULE, - - .min_keysize = SALSA20_MIN_KEY_SIZE, - .max_keysize = SALSA20_MAX_KEY_SIZE, - .ivsize = SALSA20_IV_SIZE, - .chunksize = SALSA20_BLOCK_SIZE, - .setkey = salsa20_setkey, - .encrypt = salsa20_crypt, - .decrypt = salsa20_crypt, -}; - -static int __init salsa20_generic_mod_init(void) -{ - return crypto_register_skcipher(&alg); -} - -static void __exit salsa20_generic_mod_fini(void) -{ - crypto_unregister_skcipher(&alg); -} - -subsys_initcall(salsa20_generic_mod_init); -module_exit(salsa20_generic_mod_fini); - -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION ("Salsa20 stream cipher algorithm"); -MODULE_ALIAS_CRYPTO("salsa20"); -MODULE_ALIAS_CRYPTO("salsa20-generic"); diff --git a/crypto/serpent_generic.c b/crypto/serpent_generic.c index 492c1d0bfe068e3226320a5663a75a087af58490..236c87547a174edda55c952f21d3ca6664d42349 100644 --- a/crypto/serpent_generic.c +++ b/crypto/serpent_generic.c @@ -5,17 +5,12 @@ * Serpent Cipher Algorithm. * * Copyright (C) 2002 Dag Arne Osvik - * 2003 Herbert Valerio Riedel - * - * Added tnepres support: - * Ruben Jesus Garcia Hernandez , 18.10.2004 - * Based on code by hvr */ #include #include #include -#include +#include #include #include #include @@ -453,19 +448,12 @@ void __serpent_encrypt(const void *c, u8 *dst, const u8 *src) { const struct serpent_ctx *ctx = c; const u32 *k = ctx->expkey; - const __le32 *s = (const __le32 *)src; - __le32 *d = (__le32 *)dst; u32 r0, r1, r2, r3, r4; -/* - * Note: The conversions between u8* and u32* might cause trouble - * on architectures with stricter alignment rules than x86 - */ - - r0 = le32_to_cpu(s[0]); - r1 = le32_to_cpu(s[1]); - r2 = le32_to_cpu(s[2]); - r3 = le32_to_cpu(s[3]); + r0 = get_unaligned_le32(src); + r1 = get_unaligned_le32(src + 4); + r2 = get_unaligned_le32(src + 8); + r3 = get_unaligned_le32(src + 12); K(r0, r1, r2, r3, 0); S0(r0, r1, r2, r3, r4); LK(r2, r1, r3, r0, r4, 1); @@ -501,10 +489,10 @@ void __serpent_encrypt(const void *c, u8 *dst, const u8 *src) S6(r0, r1, r3, r2, r4); LK(r3, r4, r1, r2, r0, 31); S7(r3, r4, r1, r2, r0); K(r0, r1, r2, r3, 32); - d[0] = cpu_to_le32(r0); - d[1] = cpu_to_le32(r1); - d[2] = cpu_to_le32(r2); - d[3] = cpu_to_le32(r3); + put_unaligned_le32(r0, dst); + put_unaligned_le32(r1, dst + 4); + put_unaligned_le32(r2, dst + 8); + put_unaligned_le32(r3, dst + 12); } EXPORT_SYMBOL_GPL(__serpent_encrypt); @@ -519,14 +507,12 @@ void __serpent_decrypt(const void *c, u8 *dst, const u8 *src) { const struct serpent_ctx *ctx = c; const u32 *k = ctx->expkey; - const __le32 *s = (const __le32 *)src; - __le32 *d = (__le32 *)dst; u32 r0, r1, r2, r3, r4; - r0 = le32_to_cpu(s[0]); - r1 = le32_to_cpu(s[1]); - r2 = le32_to_cpu(s[2]); - r3 = le32_to_cpu(s[3]); + r0 = get_unaligned_le32(src); + r1 = get_unaligned_le32(src + 4); + r2 = get_unaligned_le32(src + 8); + r3 = get_unaligned_le32(src + 12); K(r0, r1, r2, r3, 32); SI7(r0, r1, r2, r3, r4); KL(r1, r3, r0, r4, r2, 31); @@ -562,10 +548,10 @@ void __serpent_decrypt(const void *c, u8 *dst, const u8 *src) SI1(r3, r1, r2, r0, r4); KL(r4, r1, r2, r0, r3, 1); SI0(r4, r1, r2, r0, r3); K(r2, r3, r1, r4, 0); - d[0] = cpu_to_le32(r2); - d[1] = cpu_to_le32(r3); - d[2] = cpu_to_le32(r1); - d[3] = cpu_to_le32(r4); + put_unaligned_le32(r2, dst); + put_unaligned_le32(r3, dst + 4); + put_unaligned_le32(r1, dst + 8); + put_unaligned_le32(r4, dst + 12); } EXPORT_SYMBOL_GPL(__serpent_decrypt); @@ -576,66 +562,13 @@ static void serpent_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) __serpent_decrypt(ctx, dst, src); } -static int tnepres_setkey(struct crypto_tfm *tfm, const u8 *key, - unsigned int keylen) -{ - u8 rev_key[SERPENT_MAX_KEY_SIZE]; - int i; - - for (i = 0; i < keylen; ++i) - rev_key[keylen - i - 1] = key[i]; - - return serpent_setkey(tfm, rev_key, keylen); -} - -static void tnepres_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) -{ - const u32 * const s = (const u32 * const)src; - u32 * const d = (u32 * const)dst; - - u32 rs[4], rd[4]; - - rs[0] = swab32(s[3]); - rs[1] = swab32(s[2]); - rs[2] = swab32(s[1]); - rs[3] = swab32(s[0]); - - serpent_encrypt(tfm, (u8 *)rd, (u8 *)rs); - - d[0] = swab32(rd[3]); - d[1] = swab32(rd[2]); - d[2] = swab32(rd[1]); - d[3] = swab32(rd[0]); -} - -static void tnepres_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) -{ - const u32 * const s = (const u32 * const)src; - u32 * const d = (u32 * const)dst; - - u32 rs[4], rd[4]; - - rs[0] = swab32(s[3]); - rs[1] = swab32(s[2]); - rs[2] = swab32(s[1]); - rs[3] = swab32(s[0]); - - serpent_decrypt(tfm, (u8 *)rd, (u8 *)rs); - - d[0] = swab32(rd[3]); - d[1] = swab32(rd[2]); - d[2] = swab32(rd[1]); - d[3] = swab32(rd[0]); -} - -static struct crypto_alg srp_algs[2] = { { +static struct crypto_alg srp_alg = { .cra_name = "serpent", .cra_driver_name = "serpent-generic", .cra_priority = 100, .cra_flags = CRYPTO_ALG_TYPE_CIPHER, .cra_blocksize = SERPENT_BLOCK_SIZE, .cra_ctxsize = sizeof(struct serpent_ctx), - .cra_alignmask = 3, .cra_module = THIS_MODULE, .cra_u = { .cipher = { .cia_min_keysize = SERPENT_MIN_KEY_SIZE, @@ -643,38 +576,23 @@ static struct crypto_alg srp_algs[2] = { { .cia_setkey = serpent_setkey, .cia_encrypt = serpent_encrypt, .cia_decrypt = serpent_decrypt } } -}, { - .cra_name = "tnepres", - .cra_driver_name = "tnepres-generic", - .cra_flags = CRYPTO_ALG_TYPE_CIPHER, - .cra_blocksize = SERPENT_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct serpent_ctx), - .cra_alignmask = 3, - .cra_module = THIS_MODULE, - .cra_u = { .cipher = { - .cia_min_keysize = SERPENT_MIN_KEY_SIZE, - .cia_max_keysize = SERPENT_MAX_KEY_SIZE, - .cia_setkey = tnepres_setkey, - .cia_encrypt = tnepres_encrypt, - .cia_decrypt = tnepres_decrypt } } -} }; +}; static int __init serpent_mod_init(void) { - return crypto_register_algs(srp_algs, ARRAY_SIZE(srp_algs)); + return crypto_register_alg(&srp_alg); } static void __exit serpent_mod_fini(void) { - crypto_unregister_algs(srp_algs, ARRAY_SIZE(srp_algs)); + crypto_unregister_alg(&srp_alg); } subsys_initcall(serpent_mod_init); module_exit(serpent_mod_fini); MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("Serpent and tnepres (kerneli compatible serpent reversed) Cipher Algorithm"); +MODULE_DESCRIPTION("Serpent Cipher Algorithm"); MODULE_AUTHOR("Dag Arne Osvik "); -MODULE_ALIAS_CRYPTO("tnepres"); MODULE_ALIAS_CRYPTO("serpent"); MODULE_ALIAS_CRYPTO("serpent-generic"); diff --git a/crypto/skcipher.c b/crypto/skcipher.c index b4dae640de9f5b1e6923ae35ab83cabe61aa1274..a1537624541607c8095d627c505f0485d9b07155 100644 --- a/crypto/skcipher.c +++ b/crypto/skcipher.c @@ -10,6 +10,7 @@ */ #include +#include #include #include #include @@ -490,12 +491,6 @@ int skcipher_walk_virt(struct skcipher_walk *walk, } EXPORT_SYMBOL_GPL(skcipher_walk_virt); -void skcipher_walk_atomise(struct skcipher_walk *walk) -{ - walk->flags &= ~SKCIPHER_WALK_SLEEP; -} -EXPORT_SYMBOL_GPL(skcipher_walk_atomise); - int skcipher_walk_async(struct skcipher_walk *walk, struct skcipher_request *req) { @@ -986,3 +981,4 @@ EXPORT_SYMBOL_GPL(skcipher_alloc_instance_simple); MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("Symmetric key cipher type"); +MODULE_IMPORT_NS(CRYPTO_INTERNAL); diff --git a/crypto/tcrypt.c b/crypto/tcrypt.c index a647bb298fbce4bc47dc249f10dd75f2bdf88e58..6b7c158dc5087b6325896a0d5eac838296d99ab1 100644 --- a/crypto/tcrypt.c +++ b/crypto/tcrypt.c @@ -70,8 +70,8 @@ static const char *check[] = { "des", "md5", "des3_ede", "rot13", "sha1", "sha224", "sha256", "sm3", "blowfish", "twofish", "serpent", "sha384", "sha512", "md4", "aes", "cast6", "arc4", "michael_mic", "deflate", "crc32c", "tea", "xtea", - "khazad", "wp512", "wp384", "wp256", "tnepres", "xeta", "fcrypt", - "camellia", "seed", "salsa20", "rmd128", "rmd160", "rmd256", "rmd320", + "khazad", "wp512", "wp384", "wp256", "xeta", "fcrypt", + "camellia", "seed", "rmd160", "lzo", "lzo-rle", "cts", "sha3-224", "sha3-256", "sha3-384", "sha3-512", "streebog256", "streebog512", NULL @@ -199,8 +199,8 @@ static int test_mb_aead_jiffies(struct test_mb_aead_data *data, int enc, goto out; } - pr_cont("%d operations in %d seconds (%ld bytes)\n", - bcount * num_mb, secs, (long)bcount * blen * num_mb); + pr_cont("%d operations in %d seconds (%llu bytes)\n", + bcount * num_mb, secs, (u64)bcount * blen * num_mb); out: kfree(rc); @@ -471,8 +471,8 @@ static int test_aead_jiffies(struct aead_request *req, int enc, return ret; } - printk("%d operations in %d seconds (%ld bytes)\n", - bcount, secs, (long)bcount * blen); + pr_cont("%d operations in %d seconds (%llu bytes)\n", + bcount, secs, (u64)bcount * blen); return 0; } @@ -764,8 +764,8 @@ static int test_mb_ahash_jiffies(struct test_mb_ahash_data *data, int blen, goto out; } - pr_cont("%d operations in %d seconds (%ld bytes)\n", - bcount * num_mb, secs, (long)bcount * blen * num_mb); + pr_cont("%d operations in %d seconds (%llu bytes)\n", + bcount * num_mb, secs, (u64)bcount * blen * num_mb); out: kfree(rc); @@ -1201,8 +1201,8 @@ static int test_mb_acipher_jiffies(struct test_mb_skcipher_data *data, int enc, goto out; } - pr_cont("%d operations in %d seconds (%ld bytes)\n", - bcount * num_mb, secs, (long)bcount * blen * num_mb); + pr_cont("%d operations in %d seconds (%llu bytes)\n", + bcount * num_mb, secs, (u64)bcount * blen * num_mb); out: kfree(rc); @@ -1441,8 +1441,8 @@ static int test_acipher_jiffies(struct skcipher_request *req, int enc, return ret; } - pr_cont("%d operations in %d seconds (%ld bytes)\n", - bcount, secs, (long)bcount * blen); + pr_cont("%d operations in %d seconds (%llu bytes)\n", + bcount, secs, (u64)bcount * blen); return 0; } @@ -1806,27 +1806,11 @@ static int do_test(const char *alg, u32 type, u32 mask, int m, u32 num_mb) ret += tcrypt_test("wp256"); break; - case 25: - ret += tcrypt_test("ecb(tnepres)"); - break; - case 26: ret += tcrypt_test("ecb(anubis)"); ret += tcrypt_test("cbc(anubis)"); break; - case 27: - ret += tcrypt_test("tgr192"); - break; - - case 28: - ret += tcrypt_test("tgr160"); - break; - - case 29: - ret += tcrypt_test("tgr128"); - break; - case 30: ret += tcrypt_test("ecb(xeta)"); break; @@ -1847,10 +1831,6 @@ static int do_test(const char *alg, u32 type, u32 mask, int m, u32 num_mb) ret += tcrypt_test("sha224"); break; - case 34: - ret += tcrypt_test("salsa20"); - break; - case 35: ret += tcrypt_test("gcm(aes)"); break; @@ -1867,22 +1847,10 @@ static int do_test(const char *alg, u32 type, u32 mask, int m, u32 num_mb) ret += tcrypt_test("cts(cbc(aes))"); break; - case 39: - ret += tcrypt_test("rmd128"); - break; - case 40: ret += tcrypt_test("rmd160"); break; - case 41: - ret += tcrypt_test("rmd256"); - break; - - case 42: - ret += tcrypt_test("rmd320"); - break; - case 43: ret += tcrypt_test("ecb(seed)"); break; @@ -1955,10 +1923,6 @@ static int do_test(const char *alg, u32 type, u32 mask, int m, u32 num_mb) ret += tcrypt_test("xcbc(aes)"); break; - case 107: - ret += tcrypt_test("hmac(rmd128)"); - break; - case 108: ret += tcrypt_test("hmac(rmd160)"); break; @@ -2181,11 +2145,6 @@ static int do_test(const char *alg, u32 type, u32 mask, int m, u32 num_mb) speed_template_32_48_64); break; - case 206: - test_cipher_speed("salsa20", ENCRYPT, sec, NULL, 0, - speed_template_16_32); - break; - case 207: test_cipher_speed("ecb(serpent)", ENCRYPT, sec, NULL, 0, speed_template_16_32); @@ -2393,38 +2352,14 @@ static int do_test(const char *alg, u32 type, u32 mask, int m, u32 num_mb) test_hash_speed("wp512", sec, generic_hash_speed_template); if (mode > 300 && mode < 400) break; fallthrough; - case 310: - test_hash_speed("tgr128", sec, generic_hash_speed_template); - if (mode > 300 && mode < 400) break; - fallthrough; - case 311: - test_hash_speed("tgr160", sec, generic_hash_speed_template); - if (mode > 300 && mode < 400) break; - fallthrough; - case 312: - test_hash_speed("tgr192", sec, generic_hash_speed_template); - if (mode > 300 && mode < 400) break; - fallthrough; case 313: test_hash_speed("sha224", sec, generic_hash_speed_template); if (mode > 300 && mode < 400) break; fallthrough; - case 314: - test_hash_speed("rmd128", sec, generic_hash_speed_template); - if (mode > 300 && mode < 400) break; - fallthrough; case 315: test_hash_speed("rmd160", sec, generic_hash_speed_template); if (mode > 300 && mode < 400) break; fallthrough; - case 316: - test_hash_speed("rmd256", sec, generic_hash_speed_template); - if (mode > 300 && mode < 400) break; - fallthrough; - case 317: - test_hash_speed("rmd320", sec, generic_hash_speed_template); - if (mode > 300 && mode < 400) break; - fallthrough; case 318: klen = 16; test_hash_speed("ghash", sec, generic_hash_speed_template); @@ -2517,38 +2452,14 @@ static int do_test(const char *alg, u32 type, u32 mask, int m, u32 num_mb) test_ahash_speed("wp512", sec, generic_hash_speed_template); if (mode > 400 && mode < 500) break; fallthrough; - case 410: - test_ahash_speed("tgr128", sec, generic_hash_speed_template); - if (mode > 400 && mode < 500) break; - fallthrough; - case 411: - test_ahash_speed("tgr160", sec, generic_hash_speed_template); - if (mode > 400 && mode < 500) break; - fallthrough; - case 412: - test_ahash_speed("tgr192", sec, generic_hash_speed_template); - if (mode > 400 && mode < 500) break; - fallthrough; case 413: test_ahash_speed("sha224", sec, generic_hash_speed_template); if (mode > 400 && mode < 500) break; fallthrough; - case 414: - test_ahash_speed("rmd128", sec, generic_hash_speed_template); - if (mode > 400 && mode < 500) break; - fallthrough; case 415: test_ahash_speed("rmd160", sec, generic_hash_speed_template); if (mode > 400 && mode < 500) break; fallthrough; - case 416: - test_ahash_speed("rmd256", sec, generic_hash_speed_template); - if (mode > 400 && mode < 500) break; - fallthrough; - case 417: - test_ahash_speed("rmd320", sec, generic_hash_speed_template); - if (mode > 400 && mode < 500) break; - fallthrough; case 418: test_ahash_speed("sha3-224", sec, generic_hash_speed_template); if (mode > 400 && mode < 500) break; diff --git a/crypto/testmgr.c b/crypto/testmgr.c index 321e38eef51bc52296524b945b2b802d05d5443d..93359999c94bd48f198b5bc0bb1de1e9ac7d18e6 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -33,10 +33,13 @@ #include #include #include +#include #include #include "internal.h" +MODULE_IMPORT_NS(CRYPTO_INTERNAL); + static bool notests; module_param(notests, bool, 0644); MODULE_PARM_DESC(notests, "disable crypto self-tests"); @@ -4873,12 +4876,6 @@ static const struct alg_test_desc alg_test_descs[] = { .suite = { .cipher = __VECS(tea_tv_template) } - }, { - .alg = "ecb(tnepres)", - .test = alg_test_skcipher, - .suite = { - .cipher = __VECS(tnepres_tv_template) - } }, { .alg = "ecb(twofish)", .test = alg_test_skcipher, @@ -4954,12 +4951,6 @@ static const struct alg_test_desc alg_test_descs[] = { .suite = { .hash = __VECS(hmac_md5_tv_template) } - }, { - .alg = "hmac(rmd128)", - .test = alg_test_hash, - .suite = { - .hash = __VECS(hmac_rmd128_tv_template) - } }, { .alg = "hmac(rmd160)", .test = alg_test_hash, @@ -5272,30 +5263,12 @@ static const struct alg_test_desc alg_test_descs[] = { .aad_iv = 1, } } - }, { - .alg = "rmd128", - .test = alg_test_hash, - .suite = { - .hash = __VECS(rmd128_tv_template) - } }, { .alg = "rmd160", .test = alg_test_hash, .suite = { .hash = __VECS(rmd160_tv_template) } - }, { - .alg = "rmd256", - .test = alg_test_hash, - .suite = { - .hash = __VECS(rmd256_tv_template) - } - }, { - .alg = "rmd320", - .test = alg_test_hash, - .suite = { - .hash = __VECS(rmd320_tv_template) - } }, { .alg = "rsa", .test = alg_test_akcipher, @@ -5303,12 +5276,6 @@ static const struct alg_test_desc alg_test_descs[] = { .suite = { .akcipher = __VECS(rsa_tv_template) } - }, { - .alg = "salsa20", - .test = alg_test_skcipher, - .suite = { - .cipher = __VECS(salsa20_stream_tv_template) - } }, { .alg = "sha1", .test = alg_test_hash, @@ -5396,24 +5363,6 @@ static const struct alg_test_desc alg_test_descs[] = { .suite = { .hash = __VECS(streebog512_tv_template) } - }, { - .alg = "tgr128", - .test = alg_test_hash, - .suite = { - .hash = __VECS(tgr128_tv_template) - } - }, { - .alg = "tgr160", - .test = alg_test_hash, - .suite = { - .hash = __VECS(tgr160_tv_template) - } - }, { - .alg = "tgr192", - .test = alg_test_hash, - .suite = { - .hash = __VECS(tgr192_tv_template) - } }, { .alg = "vmac64(aes)", .test = alg_test_hash, diff --git a/crypto/testmgr.h b/crypto/testmgr.h index 8c83811c0e351f035d6a17f6860eb0d009d8b0d7..ced56ea0c9b43b2be118bfdee7759eb10c9b6631 100644 --- a/crypto/testmgr.h +++ b/crypto/testmgr.h @@ -3140,66 +3140,6 @@ static const struct hash_testvec md5_tv_template[] = { }; -/* - * RIPEMD-128 test vectors from ISO/IEC 10118-3:2004(E) - */ -static const struct hash_testvec rmd128_tv_template[] = { - { - .digest = "\xcd\xf2\x62\x13\xa1\x50\xdc\x3e" - "\xcb\x61\x0f\x18\xf6\xb3\x8b\x46", - }, { - .plaintext = "a", - .psize = 1, - .digest = "\x86\xbe\x7a\xfa\x33\x9d\x0f\xc7" - "\xcf\xc7\x85\xe7\x2f\x57\x8d\x33", - }, { - .plaintext = "abc", - .psize = 3, - .digest = "\xc1\x4a\x12\x19\x9c\x66\xe4\xba" - "\x84\x63\x6b\x0f\x69\x14\x4c\x77", - }, { - .plaintext = "message digest", - .psize = 14, - .digest = "\x9e\x32\x7b\x3d\x6e\x52\x30\x62" - "\xaf\xc1\x13\x2d\x7d\xf9\xd1\xb8", - }, { - .plaintext = "abcdefghijklmnopqrstuvwxyz", - .psize = 26, - .digest = "\xfd\x2a\xa6\x07\xf7\x1d\xc8\xf5" - "\x10\x71\x49\x22\xb3\x71\x83\x4e", - }, { - .plaintext = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcde" - "fghijklmnopqrstuvwxyz0123456789", - .psize = 62, - .digest = "\xd1\xe9\x59\xeb\x17\x9c\x91\x1f" - "\xae\xa4\x62\x4c\x60\xc5\xc7\x02", - }, { - .plaintext = "1234567890123456789012345678901234567890" - "1234567890123456789012345678901234567890", - .psize = 80, - .digest = "\x3f\x45\xef\x19\x47\x32\xc2\xdb" - "\xb2\xc4\xa2\xc7\x69\x79\x5f\xa3", - }, { - .plaintext = "abcdbcdecdefdefgefghfghighij" - "hijkijkljklmklmnlmnomnopnopq", - .psize = 56, - .digest = "\xa1\xaa\x06\x89\xd0\xfa\xfa\x2d" - "\xdc\x22\xe8\x8b\x49\x13\x3a\x06", - }, { - .plaintext = "abcdefghbcdefghicdefghijdefghijkefghijklfghi" - "jklmghijklmnhijklmnoijklmnopjklmnopqklmnopqr" - "lmnopqrsmnopqrstnopqrstu", - .psize = 112, - .digest = "\xd4\xec\xc9\x13\xe1\xdf\x77\x6b" - "\xf4\x8d\xe9\xd5\x5b\x1f\x25\x46", - }, { - .plaintext = "abcdbcdecdefdefgefghfghighijhijk", - .psize = 32, - .digest = "\x13\xfc\x13\xe8\xef\xff\x34\x7d" - "\xe1\x93\xff\x46\xdb\xac\xcf\xd4", - } -}; - /* * RIPEMD-160 test vectors from ISO/IEC 10118-3:2004(E) */ @@ -3260,134 +3200,6 @@ static const struct hash_testvec rmd160_tv_template[] = { } }; -/* - * RIPEMD-256 test vectors - */ -static const struct hash_testvec rmd256_tv_template[] = { - { - .digest = "\x02\xba\x4c\x4e\x5f\x8e\xcd\x18" - "\x77\xfc\x52\xd6\x4d\x30\xe3\x7a" - "\x2d\x97\x74\xfb\x1e\x5d\x02\x63" - "\x80\xae\x01\x68\xe3\xc5\x52\x2d", - }, { - .plaintext = "a", - .psize = 1, - .digest = "\xf9\x33\x3e\x45\xd8\x57\xf5\xd9" - "\x0a\x91\xba\xb7\x0a\x1e\xba\x0c" - "\xfb\x1b\xe4\xb0\x78\x3c\x9a\xcf" - "\xcd\x88\x3a\x91\x34\x69\x29\x25", - }, { - .plaintext = "abc", - .psize = 3, - .digest = "\xaf\xbd\x6e\x22\x8b\x9d\x8c\xbb" - "\xce\xf5\xca\x2d\x03\xe6\xdb\xa1" - "\x0a\xc0\xbc\x7d\xcb\xe4\x68\x0e" - "\x1e\x42\xd2\xe9\x75\x45\x9b\x65", - }, { - .plaintext = "message digest", - .psize = 14, - .digest = "\x87\xe9\x71\x75\x9a\x1c\xe4\x7a" - "\x51\x4d\x5c\x91\x4c\x39\x2c\x90" - "\x18\xc7\xc4\x6b\xc1\x44\x65\x55" - "\x4a\xfc\xdf\x54\xa5\x07\x0c\x0e", - }, { - .plaintext = "abcdefghijklmnopqrstuvwxyz", - .psize = 26, - .digest = "\x64\x9d\x30\x34\x75\x1e\xa2\x16" - "\x77\x6b\xf9\xa1\x8a\xcc\x81\xbc" - "\x78\x96\x11\x8a\x51\x97\x96\x87" - "\x82\xdd\x1f\xd9\x7d\x8d\x51\x33", - }, { - .plaintext = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcde" - "fghijklmnopqrstuvwxyz0123456789", - .psize = 62, - .digest = "\x57\x40\xa4\x08\xac\x16\xb7\x20" - "\xb8\x44\x24\xae\x93\x1c\xbb\x1f" - "\xe3\x63\xd1\xd0\xbf\x40\x17\xf1" - "\xa8\x9f\x7e\xa6\xde\x77\xa0\xb8", - }, { - .plaintext = "1234567890123456789012345678901234567890" - "1234567890123456789012345678901234567890", - .psize = 80, - .digest = "\x06\xfd\xcc\x7a\x40\x95\x48\xaa" - "\xf9\x13\x68\xc0\x6a\x62\x75\xb5" - "\x53\xe3\xf0\x99\xbf\x0e\xa4\xed" - "\xfd\x67\x78\xdf\x89\xa8\x90\xdd", - }, { - .plaintext = "abcdbcdecdefdefgefghfghighij" - "hijkijkljklmklmnlmnomnopnopq", - .psize = 56, - .digest = "\x38\x43\x04\x55\x83\xaa\xc6\xc8" - "\xc8\xd9\x12\x85\x73\xe7\xa9\x80" - "\x9a\xfb\x2a\x0f\x34\xcc\xc3\x6e" - "\xa9\xe7\x2f\x16\xf6\x36\x8e\x3f", - } -}; - -/* - * RIPEMD-320 test vectors - */ -static const struct hash_testvec rmd320_tv_template[] = { - { - .digest = "\x22\xd6\x5d\x56\x61\x53\x6c\xdc\x75\xc1" - "\xfd\xf5\xc6\xde\x7b\x41\xb9\xf2\x73\x25" - "\xeb\xc6\x1e\x85\x57\x17\x7d\x70\x5a\x0e" - "\xc8\x80\x15\x1c\x3a\x32\xa0\x08\x99\xb8", - }, { - .plaintext = "a", - .psize = 1, - .digest = "\xce\x78\x85\x06\x38\xf9\x26\x58\xa5\xa5" - "\x85\x09\x75\x79\x92\x6d\xda\x66\x7a\x57" - "\x16\x56\x2c\xfc\xf6\xfb\xe7\x7f\x63\x54" - "\x2f\x99\xb0\x47\x05\xd6\x97\x0d\xff\x5d", - }, { - .plaintext = "abc", - .psize = 3, - .digest = "\xde\x4c\x01\xb3\x05\x4f\x89\x30\xa7\x9d" - "\x09\xae\x73\x8e\x92\x30\x1e\x5a\x17\x08" - "\x5b\xef\xfd\xc1\xb8\xd1\x16\x71\x3e\x74" - "\xf8\x2f\xa9\x42\xd6\x4c\xdb\xc4\x68\x2d", - }, { - .plaintext = "message digest", - .psize = 14, - .digest = "\x3a\x8e\x28\x50\x2e\xd4\x5d\x42\x2f\x68" - "\x84\x4f\x9d\xd3\x16\xe7\xb9\x85\x33\xfa" - "\x3f\x2a\x91\xd2\x9f\x84\xd4\x25\xc8\x8d" - "\x6b\x4e\xff\x72\x7d\xf6\x6a\x7c\x01\x97", - }, { - .plaintext = "abcdefghijklmnopqrstuvwxyz", - .psize = 26, - .digest = "\xca\xbd\xb1\x81\x0b\x92\x47\x0a\x20\x93" - "\xaa\x6b\xce\x05\x95\x2c\x28\x34\x8c\xf4" - "\x3f\xf6\x08\x41\x97\x51\x66\xbb\x40\xed" - "\x23\x40\x04\xb8\x82\x44\x63\xe6\xb0\x09", - }, { - .plaintext = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcde" - "fghijklmnopqrstuvwxyz0123456789", - .psize = 62, - .digest = "\xed\x54\x49\x40\xc8\x6d\x67\xf2\x50\xd2" - "\x32\xc3\x0b\x7b\x3e\x57\x70\xe0\xc6\x0c" - "\x8c\xb9\xa4\xca\xfe\x3b\x11\x38\x8a\xf9" - "\x92\x0e\x1b\x99\x23\x0b\x84\x3c\x86\xa4", - }, { - .plaintext = "1234567890123456789012345678901234567890" - "1234567890123456789012345678901234567890", - .psize = 80, - .digest = "\x55\x78\x88\xaf\x5f\x6d\x8e\xd6\x2a\xb6" - "\x69\x45\xc6\xd2\xa0\xa4\x7e\xcd\x53\x41" - "\xe9\x15\xeb\x8f\xea\x1d\x05\x24\x95\x5f" - "\x82\x5d\xc7\x17\xe4\xa0\x08\xab\x2d\x42", - }, { - .plaintext = "abcdbcdecdefdefgefghfghighij" - "hijkijkljklmklmnlmnomnopnopq", - .psize = 56, - .digest = "\xd0\x34\xa7\x95\x0c\xf7\x22\x02\x1b\xa4" - "\xb8\x4d\xf7\x69\xa5\xde\x20\x60\xe2\x59" - "\xdf\x4c\x9b\xb4\xa4\x26\x8c\x0e\x93\x5b" - "\xbc\x74\x70\xa9\x69\xc9\xd0\x72\xa1\xac", - } -}; - static const struct hash_testvec crct10dif_tv_template[] = { { .plaintext = "abc", @@ -5138,132 +4950,6 @@ static const struct hash_testvec wp256_tv_template[] = { }, }; -/* - * TIGER test vectors from Tiger website - */ -static const struct hash_testvec tgr192_tv_template[] = { - { - .plaintext = "", - .psize = 0, - .digest = "\x24\xf0\x13\x0c\x63\xac\x93\x32" - "\x16\x16\x6e\x76\xb1\xbb\x92\x5f" - "\xf3\x73\xde\x2d\x49\x58\x4e\x7a", - }, { - .plaintext = "abc", - .psize = 3, - .digest = "\xf2\x58\xc1\xe8\x84\x14\xab\x2a" - "\x52\x7a\xb5\x41\xff\xc5\xb8\xbf" - "\x93\x5f\x7b\x95\x1c\x13\x29\x51", - }, { - .plaintext = "Tiger", - .psize = 5, - .digest = "\x9f\x00\xf5\x99\x07\x23\x00\xdd" - "\x27\x6a\xbb\x38\xc8\xeb\x6d\xec" - "\x37\x79\x0c\x11\x6f\x9d\x2b\xdf", - }, { - .plaintext = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+-", - .psize = 64, - .digest = "\x87\xfb\x2a\x90\x83\x85\x1c\xf7" - "\x47\x0d\x2c\xf8\x10\xe6\xdf\x9e" - "\xb5\x86\x44\x50\x34\xa5\xa3\x86", - }, { - .plaintext = "ABCDEFGHIJKLMNOPQRSTUVWXYZ=abcdefghijklmnopqrstuvwxyz+0123456789", - .psize = 64, - .digest = "\x46\x7d\xb8\x08\x63\xeb\xce\x48" - "\x8d\xf1\xcd\x12\x61\x65\x5d\xe9" - "\x57\x89\x65\x65\x97\x5f\x91\x97", - }, { - .plaintext = "Tiger - A Fast New Hash Function, " - "by Ross Anderson and Eli Biham, " - "proceedings of Fast Software Encryption 3, " - "Cambridge, 1996.", - .psize = 125, - .digest = "\x3d\x9a\xeb\x03\xd1\xbd\x1a\x63" - "\x57\xb2\x77\x4d\xfd\x6d\x5b\x24" - "\xdd\x68\x15\x1d\x50\x39\x74\xfc", - }, -}; - -static const struct hash_testvec tgr160_tv_template[] = { - { - .plaintext = "", - .psize = 0, - .digest = "\x24\xf0\x13\x0c\x63\xac\x93\x32" - "\x16\x16\x6e\x76\xb1\xbb\x92\x5f" - "\xf3\x73\xde\x2d", - }, { - .plaintext = "abc", - .psize = 3, - .digest = "\xf2\x58\xc1\xe8\x84\x14\xab\x2a" - "\x52\x7a\xb5\x41\xff\xc5\xb8\xbf" - "\x93\x5f\x7b\x95", - }, { - .plaintext = "Tiger", - .psize = 5, - .digest = "\x9f\x00\xf5\x99\x07\x23\x00\xdd" - "\x27\x6a\xbb\x38\xc8\xeb\x6d\xec" - "\x37\x79\x0c\x11", - }, { - .plaintext = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+-", - .psize = 64, - .digest = "\x87\xfb\x2a\x90\x83\x85\x1c\xf7" - "\x47\x0d\x2c\xf8\x10\xe6\xdf\x9e" - "\xb5\x86\x44\x50", - }, { - .plaintext = "ABCDEFGHIJKLMNOPQRSTUVWXYZ=abcdefghijklmnopqrstuvwxyz+0123456789", - .psize = 64, - .digest = "\x46\x7d\xb8\x08\x63\xeb\xce\x48" - "\x8d\xf1\xcd\x12\x61\x65\x5d\xe9" - "\x57\x89\x65\x65", - }, { - .plaintext = "Tiger - A Fast New Hash Function, " - "by Ross Anderson and Eli Biham, " - "proceedings of Fast Software Encryption 3, " - "Cambridge, 1996.", - .psize = 125, - .digest = "\x3d\x9a\xeb\x03\xd1\xbd\x1a\x63" - "\x57\xb2\x77\x4d\xfd\x6d\x5b\x24" - "\xdd\x68\x15\x1d", - }, -}; - -static const struct hash_testvec tgr128_tv_template[] = { - { - .plaintext = "", - .psize = 0, - .digest = "\x24\xf0\x13\x0c\x63\xac\x93\x32" - "\x16\x16\x6e\x76\xb1\xbb\x92\x5f", - }, { - .plaintext = "abc", - .psize = 3, - .digest = "\xf2\x58\xc1\xe8\x84\x14\xab\x2a" - "\x52\x7a\xb5\x41\xff\xc5\xb8\xbf", - }, { - .plaintext = "Tiger", - .psize = 5, - .digest = "\x9f\x00\xf5\x99\x07\x23\x00\xdd" - "\x27\x6a\xbb\x38\xc8\xeb\x6d\xec", - }, { - .plaintext = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+-", - .psize = 64, - .digest = "\x87\xfb\x2a\x90\x83\x85\x1c\xf7" - "\x47\x0d\x2c\xf8\x10\xe6\xdf\x9e", - }, { - .plaintext = "ABCDEFGHIJKLMNOPQRSTUVWXYZ=abcdefghijklmnopqrstuvwxyz+0123456789", - .psize = 64, - .digest = "\x46\x7d\xb8\x08\x63\xeb\xce\x48" - "\x8d\xf1\xcd\x12\x61\x65\x5d\xe9", - }, { - .plaintext = "Tiger - A Fast New Hash Function, " - "by Ross Anderson and Eli Biham, " - "proceedings of Fast Software Encryption 3, " - "Cambridge, 1996.", - .psize = 125, - .digest = "\x3d\x9a\xeb\x03\xd1\xbd\x1a\x63" - "\x57\xb2\x77\x4d\xfd\x6d\x5b\x24", - }, -}; - static const struct hash_testvec ghash_tv_template[] = { { @@ -5452,83 +5138,6 @@ static const struct hash_testvec hmac_md5_tv_template[] = }, }; -/* - * HMAC-RIPEMD128 test vectors from RFC2286 - */ -static const struct hash_testvec hmac_rmd128_tv_template[] = { - { - .key = "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b", - .ksize = 16, - .plaintext = "Hi There", - .psize = 8, - .digest = "\xfb\xf6\x1f\x94\x92\xaa\x4b\xbf" - "\x81\xc1\x72\xe8\x4e\x07\x34\xdb", - }, { - .key = "Jefe", - .ksize = 4, - .plaintext = "what do ya want for nothing?", - .psize = 28, - .digest = "\x87\x5f\x82\x88\x62\xb6\xb3\x34" - "\xb4\x27\xc5\x5f\x9f\x7f\xf0\x9b", - }, { - .key = "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa", - .ksize = 16, - .plaintext = "\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd" - "\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd" - "\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd" - "\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd", - .psize = 50, - .digest = "\x09\xf0\xb2\x84\x6d\x2f\x54\x3d" - "\xa3\x63\xcb\xec\x8d\x62\xa3\x8d", - }, { - .key = "\x01\x02\x03\x04\x05\x06\x07\x08" - "\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" - "\x11\x12\x13\x14\x15\x16\x17\x18\x19", - .ksize = 25, - .plaintext = "\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd" - "\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd" - "\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd" - "\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd", - .psize = 50, - .digest = "\xbd\xbb\xd7\xcf\x03\xe4\x4b\x5a" - "\xa6\x0a\xf8\x15\xbe\x4d\x22\x94", - }, { - .key = "\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c\x0c", - .ksize = 16, - .plaintext = "Test With Truncation", - .psize = 20, - .digest = "\xe7\x98\x08\xf2\x4b\x25\xfd\x03" - "\x1c\x15\x5f\x0d\x55\x1d\x9a\x3a", - }, { - .key = "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" - "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" - "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" - "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" - "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" - "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" - "\xaa\xaa", - .ksize = 80, - .plaintext = "Test Using Larger Than Block-Size Key - Hash Key First", - .psize = 54, - .digest = "\xdc\x73\x29\x28\xde\x98\x10\x4a" - "\x1f\x59\xd3\x73\xc1\x50\xac\xbb", - }, { - .key = "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" - "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" - "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" - "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" - "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" - "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" - "\xaa\xaa", - .ksize = 80, - .plaintext = "Test Using Larger Than Block-Size Key and Larger Than One " - "Block-Size Data", - .psize = 73, - .digest = "\x5c\x6b\xec\x96\x79\x3e\x16\xd4" - "\x06\x90\xc2\x37\x63\x5f\x30\xc5", - }, -}; - /* * HMAC-RIPEMD160 test vectors from RFC2286 */ @@ -11806,85 +11415,6 @@ static const struct cipher_testvec serpent_tv_template[] = { }, }; -static const struct cipher_testvec tnepres_tv_template[] = { - { /* KeySize=0 */ - .ptext = "\x00\x01\x02\x03\x04\x05\x06\x07" - "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", - .ctext = "\x41\xcc\x6b\x31\x59\x31\x45\x97" - "\x6d\x6f\xbb\x38\x4b\x37\x21\x28", - .len = 16, - }, - { /* KeySize=128, PT=0, I=1 */ - .ptext = "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00", - .key = "\x80\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00", - .klen = 16, - .ctext = "\x49\xaf\xbf\xad\x9d\x5a\x34\x05" - "\x2c\xd8\xff\xa5\x98\x6b\xd2\xdd", - .len = 16, - }, { /* KeySize=128 */ - .key = "\x00\x01\x02\x03\x04\x05\x06\x07" - "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", - .klen = 16, - .ptext = "\x00\x01\x02\x03\x04\x05\x06\x07" - "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", - .ctext = "\xea\xf4\xd7\xfc\xd8\x01\x34\x47" - "\x81\x45\x0b\xfa\x0c\xd6\xad\x6e", - .len = 16, - }, { /* KeySize=128, I=121 */ - .key = "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x80", - .klen = 16, - .ptext = zeroed_string, - .ctext = "\x3d\xda\xbf\xc0\x06\xda\xab\x06" - "\x46\x2a\xf4\xef\x81\x54\x4e\x26", - .len = 16, - }, { /* KeySize=192, PT=0, I=1 */ - .key = "\x80\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00", - .klen = 24, - .ptext = "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00", - .ctext = "\xe7\x8e\x54\x02\xc7\x19\x55\x68" - "\xac\x36\x78\xf7\xa3\xf6\x0c\x66", - .len = 16, - }, { /* KeySize=256, PT=0, I=1 */ - .key = "\x80\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00", - .klen = 32, - .ptext = "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00", - .ctext = "\xab\xed\x96\xe7\x66\xbf\x28\xcb" - "\xc0\xeb\xd2\x1a\x82\xef\x08\x19", - .len = 16, - }, { /* KeySize=256, I=257 */ - .key = "\x1f\x1e\x1d\x1c\x1b\x1a\x19\x18" - "\x17\x16\x15\x14\x13\x12\x11\x10" - "\x0f\x0e\x0d\x0c\x0b\x0a\x09\x08" - "\x07\x06\x05\x04\x03\x02\x01\x00", - .klen = 32, - .ptext = "\x0f\x0e\x0d\x0c\x0b\x0a\x09\x08" - "\x07\x06\x05\x04\x03\x02\x01\x00", - .ctext = "\x5c\xe7\x1c\x70\xd2\x88\x2e\x5b" - "\xb8\x32\xe4\x33\xf8\x9f\x26\xde", - .len = 16, - }, { /* KeySize=256 */ - .key = "\x00\x01\x02\x03\x04\x05\x06\x07" - "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" - "\x10\x11\x12\x13\x14\x15\x16\x17" - "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", - .klen = 32, - .ptext = "\x00\x01\x02\x03\x04\x05\x06\x07" - "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", - .ctext = "\x64\xa9\x1a\x37\xed\x9f\xe7\x49" - "\xa8\x4e\x76\xd6\xf5\x0d\x78\xee", - .len = 16, - } -}; - static const struct cipher_testvec serpent_cbc_tv_template[] = { { /* Generated with Crypto++ */ .key = "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9" @@ -24800,1168 +24330,6 @@ static const struct cipher_testvec seed_tv_template[] = { } }; -static const struct cipher_testvec salsa20_stream_tv_template[] = { - /* - * Testvectors from verified.test-vectors submitted to ECRYPT. - * They are truncated to size 39, 64, 111, 129 to test a variety - * of input length. - */ - { /* Set 3, vector 0 */ - .key = "\x00\x01\x02\x03\x04\x05\x06\x07" - "\x08\x09\x0A\x0B\x0C\x0D\x0E\x0F", - .klen = 16, - .iv = "\x00\x00\x00\x00\x00\x00\x00\x00", - .ptext = "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00", - .ctext = "\x2D\xD5\xC3\xF7\xBA\x2B\x20\xF7" - "\x68\x02\x41\x0C\x68\x86\x88\x89" - "\x5A\xD8\xC1\xBD\x4E\xA6\xC9\xB1" - "\x40\xFB\x9B\x90\xE2\x10\x49\xBF" - "\x58\x3F\x52\x79\x70\xEB\xC1", - .len = 39, - }, { /* Set 5, vector 0 */ - .key = "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00", - .klen = 16, - .iv = "\x80\x00\x00\x00\x00\x00\x00\x00", - .ptext = "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00", - .ctext = "\xB6\x6C\x1E\x44\x46\xDD\x95\x57" - "\xE5\x78\xE2\x23\xB0\xB7\x68\x01" - "\x7B\x23\xB2\x67\xBB\x02\x34\xAE" - "\x46\x26\xBF\x44\x3F\x21\x97\x76" - "\x43\x6F\xB1\x9F\xD0\xE8\x86\x6F" - "\xCD\x0D\xE9\xA9\x53\x8F\x4A\x09" - "\xCA\x9A\xC0\x73\x2E\x30\xBC\xF9" - "\x8E\x4F\x13\xE4\xB9\xE2\x01\xD9", - .len = 64, - }, { /* Set 3, vector 27 */ - .key = "\x1B\x1C\x1D\x1E\x1F\x20\x21\x22" - "\x23\x24\x25\x26\x27\x28\x29\x2A" - "\x2B\x2C\x2D\x2E\x2F\x30\x31\x32" - "\x33\x34\x35\x36\x37\x38\x39\x3A", - .klen = 32, - .iv = "\x00\x00\x00\x00\x00\x00\x00\x00", - .ptext = "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00", - .ctext = "\xAE\x39\x50\x8E\xAC\x9A\xEC\xE7" - "\xBF\x97\xBB\x20\xB9\xDE\xE4\x1F" - "\x87\xD9\x47\xF8\x28\x91\x35\x98" - "\xDB\x72\xCC\x23\x29\x48\x56\x5E" - "\x83\x7E\x0B\xF3\x7D\x5D\x38\x7B" - "\x2D\x71\x02\xB4\x3B\xB5\xD8\x23" - "\xB0\x4A\xDF\x3C\xEC\xB6\xD9\x3B" - "\x9B\xA7\x52\xBE\xC5\xD4\x50\x59" - "\x15\x14\xB4\x0E\x40\xE6\x53\xD1" - "\x83\x9C\x5B\xA0\x92\x29\x6B\x5E" - "\x96\x5B\x1E\x2F\xD3\xAC\xC1\x92" - "\xB1\x41\x3F\x19\x2F\xC4\x3B\xC6" - "\x95\x46\x45\x54\xE9\x75\x03\x08" - "\x44\xAF\xE5\x8A\x81\x12\x09", - .len = 111, - }, { /* Set 5, vector 27 */ - .key = "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00", - .klen = 32, - .iv = "\x00\x00\x00\x10\x00\x00\x00\x00", - .ptext = "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00", - .ctext = "\xD2\xDB\x1A\x5C\xF1\xC1\xAC\xDB" - "\xE8\x1A\x7A\x43\x40\xEF\x53\x43" - "\x5E\x7F\x4B\x1A\x50\x52\x3F\x8D" - "\x28\x3D\xCF\x85\x1D\x69\x6E\x60" - "\xF2\xDE\x74\x56\x18\x1B\x84\x10" - "\xD4\x62\xBA\x60\x50\xF0\x61\xF2" - "\x1C\x78\x7F\xC1\x24\x34\xAF\x58" - "\xBF\x2C\x59\xCA\x90\x77\xF3\xB0" - "\x5B\x4A\xDF\x89\xCE\x2C\x2F\xFC" - "\x67\xF0\xE3\x45\xE8\xB3\xB3\x75" - "\xA0\x95\x71\xA1\x29\x39\x94\xCA" - "\x45\x2F\xBD\xCB\x10\xB6\xBE\x9F" - "\x8E\xF9\xB2\x01\x0A\x5A\x0A\xB7" - "\x6B\x9D\x70\x8E\x4B\xD6\x2F\xCD" - "\x2E\x40\x48\x75\xE9\xE2\x21\x45" - "\x0B\xC9\xB6\xB5\x66\xBC\x9A\x59" - "\x5A", - .len = 129, - }, { /* large test vector generated using Crypto++ */ - .key = "\x00\x01\x02\x03\x04\x05\x06\x07" - "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" - "\x10\x11\x12\x13\x14\x15\x16\x17" - "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", - .klen = 32, - .iv = "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00", - .ptext = - "\x00\x01\x02\x03\x04\x05\x06\x07" - "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" - "\x10\x11\x12\x13\x14\x15\x16\x17" - "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f" - "\x20\x21\x22\x23\x24\x25\x26\x27" - "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f" - "\x30\x31\x32\x33\x34\x35\x36\x37" - "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f" - "\x40\x41\x42\x43\x44\x45\x46\x47" - "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f" - "\x50\x51\x52\x53\x54\x55\x56\x57" - "\x58\x59\x5a\x5b\x5c\x5d\x5e\x5f" - "\x60\x61\x62\x63\x64\x65\x66\x67" - "\x68\x69\x6a\x6b\x6c\x6d\x6e\x6f" - "\x70\x71\x72\x73\x74\x75\x76\x77" - "\x78\x79\x7a\x7b\x7c\x7d\x7e\x7f" - "\x80\x81\x82\x83\x84\x85\x86\x87" - "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f" - "\x90\x91\x92\x93\x94\x95\x96\x97" - "\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f" - "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7" - "\xa8\xa9\xaa\xab\xac\xad\xae\xaf" - "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7" - "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf" - "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7" - "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf" - "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7" - "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf" - "\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7" - "\xe8\xe9\xea\xeb\xec\xed\xee\xef" - "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7" - "\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff" - "\x00\x03\x06\x09\x0c\x0f\x12\x15" - "\x18\x1b\x1e\x21\x24\x27\x2a\x2d" - "\x30\x33\x36\x39\x3c\x3f\x42\x45" - "\x48\x4b\x4e\x51\x54\x57\x5a\x5d" - "\x60\x63\x66\x69\x6c\x6f\x72\x75" - "\x78\x7b\x7e\x81\x84\x87\x8a\x8d" - "\x90\x93\x96\x99\x9c\x9f\xa2\xa5" - "\xa8\xab\xae\xb1\xb4\xb7\xba\xbd" - "\xc0\xc3\xc6\xc9\xcc\xcf\xd2\xd5" - "\xd8\xdb\xde\xe1\xe4\xe7\xea\xed" - "\xf0\xf3\xf6\xf9\xfc\xff\x02\x05" - "\x08\x0b\x0e\x11\x14\x17\x1a\x1d" - "\x20\x23\x26\x29\x2c\x2f\x32\x35" - "\x38\x3b\x3e\x41\x44\x47\x4a\x4d" - "\x50\x53\x56\x59\x5c\x5f\x62\x65" - "\x68\x6b\x6e\x71\x74\x77\x7a\x7d" - "\x80\x83\x86\x89\x8c\x8f\x92\x95" - "\x98\x9b\x9e\xa1\xa4\xa7\xaa\xad" - "\xb0\xb3\xb6\xb9\xbc\xbf\xc2\xc5" - "\xc8\xcb\xce\xd1\xd4\xd7\xda\xdd" - "\xe0\xe3\xe6\xe9\xec\xef\xf2\xf5" - "\xf8\xfb\xfe\x01\x04\x07\x0a\x0d" - "\x10\x13\x16\x19\x1c\x1f\x22\x25" - "\x28\x2b\x2e\x31\x34\x37\x3a\x3d" - "\x40\x43\x46\x49\x4c\x4f\x52\x55" - "\x58\x5b\x5e\x61\x64\x67\x6a\x6d" - "\x70\x73\x76\x79\x7c\x7f\x82\x85" - "\x88\x8b\x8e\x91\x94\x97\x9a\x9d" - "\xa0\xa3\xa6\xa9\xac\xaf\xb2\xb5" - "\xb8\xbb\xbe\xc1\xc4\xc7\xca\xcd" - "\xd0\xd3\xd6\xd9\xdc\xdf\xe2\xe5" - "\xe8\xeb\xee\xf1\xf4\xf7\xfa\xfd" - "\x00\x05\x0a\x0f\x14\x19\x1e\x23" - "\x28\x2d\x32\x37\x3c\x41\x46\x4b" - "\x50\x55\x5a\x5f\x64\x69\x6e\x73" - "\x78\x7d\x82\x87\x8c\x91\x96\x9b" - "\xa0\xa5\xaa\xaf\xb4\xb9\xbe\xc3" - "\xc8\xcd\xd2\xd7\xdc\xe1\xe6\xeb" - "\xf0\xf5\xfa\xff\x04\x09\x0e\x13" - "\x18\x1d\x22\x27\x2c\x31\x36\x3b" - "\x40\x45\x4a\x4f\x54\x59\x5e\x63" - "\x68\x6d\x72\x77\x7c\x81\x86\x8b" - "\x90\x95\x9a\x9f\xa4\xa9\xae\xb3" - "\xb8\xbd\xc2\xc7\xcc\xd1\xd6\xdb" - "\xe0\xe5\xea\xef\xf4\xf9\xfe\x03" - "\x08\x0d\x12\x17\x1c\x21\x26\x2b" - "\x30\x35\x3a\x3f\x44\x49\x4e\x53" - "\x58\x5d\x62\x67\x6c\x71\x76\x7b" - "\x80\x85\x8a\x8f\x94\x99\x9e\xa3" - "\xa8\xad\xb2\xb7\xbc\xc1\xc6\xcb" - "\xd0\xd5\xda\xdf\xe4\xe9\xee\xf3" - "\xf8\xfd\x02\x07\x0c\x11\x16\x1b" - "\x20\x25\x2a\x2f\x34\x39\x3e\x43" - "\x48\x4d\x52\x57\x5c\x61\x66\x6b" - "\x70\x75\x7a\x7f\x84\x89\x8e\x93" - "\x98\x9d\xa2\xa7\xac\xb1\xb6\xbb" - "\xc0\xc5\xca\xcf\xd4\xd9\xde\xe3" - "\xe8\xed\xf2\xf7\xfc\x01\x06\x0b" - "\x10\x15\x1a\x1f\x24\x29\x2e\x33" - "\x38\x3d\x42\x47\x4c\x51\x56\x5b" - "\x60\x65\x6a\x6f\x74\x79\x7e\x83" - "\x88\x8d\x92\x97\x9c\xa1\xa6\xab" - "\xb0\xb5\xba\xbf\xc4\xc9\xce\xd3" - "\xd8\xdd\xe2\xe7\xec\xf1\xf6\xfb" - "\x00\x07\x0e\x15\x1c\x23\x2a\x31" - "\x38\x3f\x46\x4d\x54\x5b\x62\x69" - "\x70\x77\x7e\x85\x8c\x93\x9a\xa1" - "\xa8\xaf\xb6\xbd\xc4\xcb\xd2\xd9" - "\xe0\xe7\xee\xf5\xfc\x03\x0a\x11" - "\x18\x1f\x26\x2d\x34\x3b\x42\x49" - "\x50\x57\x5e\x65\x6c\x73\x7a\x81" - "\x88\x8f\x96\x9d\xa4\xab\xb2\xb9" - "\xc0\xc7\xce\xd5\xdc\xe3\xea\xf1" - "\xf8\xff\x06\x0d\x14\x1b\x22\x29" - "\x30\x37\x3e\x45\x4c\x53\x5a\x61" - "\x68\x6f\x76\x7d\x84\x8b\x92\x99" - "\xa0\xa7\xae\xb5\xbc\xc3\xca\xd1" - "\xd8\xdf\xe6\xed\xf4\xfb\x02\x09" - "\x10\x17\x1e\x25\x2c\x33\x3a\x41" - "\x48\x4f\x56\x5d\x64\x6b\x72\x79" - "\x80\x87\x8e\x95\x9c\xa3\xaa\xb1" - "\xb8\xbf\xc6\xcd\xd4\xdb\xe2\xe9" - "\xf0\xf7\xfe\x05\x0c\x13\x1a\x21" - "\x28\x2f\x36\x3d\x44\x4b\x52\x59" - "\x60\x67\x6e\x75\x7c\x83\x8a\x91" - "\x98\x9f\xa6\xad\xb4\xbb\xc2\xc9" - "\xd0\xd7\xde\xe5\xec\xf3\xfa\x01" - "\x08\x0f\x16\x1d\x24\x2b\x32\x39" - "\x40\x47\x4e\x55\x5c\x63\x6a\x71" - "\x78\x7f\x86\x8d\x94\x9b\xa2\xa9" - "\xb0\xb7\xbe\xc5\xcc\xd3\xda\xe1" - "\xe8\xef\xf6\xfd\x04\x0b\x12\x19" - "\x20\x27\x2e\x35\x3c\x43\x4a\x51" - "\x58\x5f\x66\x6d\x74\x7b\x82\x89" - "\x90\x97\x9e\xa5\xac\xb3\xba\xc1" - "\xc8\xcf\xd6\xdd\xe4\xeb\xf2\xf9" - "\x00\x09\x12\x1b\x24\x2d\x36\x3f" - "\x48\x51\x5a\x63\x6c\x75\x7e\x87" - "\x90\x99\xa2\xab\xb4\xbd\xc6\xcf" - "\xd8\xe1\xea\xf3\xfc\x05\x0e\x17" - "\x20\x29\x32\x3b\x44\x4d\x56\x5f" - "\x68\x71\x7a\x83\x8c\x95\x9e\xa7" - "\xb0\xb9\xc2\xcb\xd4\xdd\xe6\xef" - "\xf8\x01\x0a\x13\x1c\x25\x2e\x37" - "\x40\x49\x52\x5b\x64\x6d\x76\x7f" - "\x88\x91\x9a\xa3\xac\xb5\xbe\xc7" - "\xd0\xd9\xe2\xeb\xf4\xfd\x06\x0f" - "\x18\x21\x2a\x33\x3c\x45\x4e\x57" - "\x60\x69\x72\x7b\x84\x8d\x96\x9f" - "\xa8\xb1\xba\xc3\xcc\xd5\xde\xe7" - "\xf0\xf9\x02\x0b\x14\x1d\x26\x2f" - "\x38\x41\x4a\x53\x5c\x65\x6e\x77" - "\x80\x89\x92\x9b\xa4\xad\xb6\xbf" - "\xc8\xd1\xda\xe3\xec\xf5\xfe\x07" - "\x10\x19\x22\x2b\x34\x3d\x46\x4f" - "\x58\x61\x6a\x73\x7c\x85\x8e\x97" - "\xa0\xa9\xb2\xbb\xc4\xcd\xd6\xdf" - "\xe8\xf1\xfa\x03\x0c\x15\x1e\x27" - "\x30\x39\x42\x4b\x54\x5d\x66\x6f" - "\x78\x81\x8a\x93\x9c\xa5\xae\xb7" - "\xc0\xc9\xd2\xdb\xe4\xed\xf6\xff" - "\x08\x11\x1a\x23\x2c\x35\x3e\x47" - "\x50\x59\x62\x6b\x74\x7d\x86\x8f" - "\x98\xa1\xaa\xb3\xbc\xc5\xce\xd7" - "\xe0\xe9\xf2\xfb\x04\x0d\x16\x1f" - "\x28\x31\x3a\x43\x4c\x55\x5e\x67" - "\x70\x79\x82\x8b\x94\x9d\xa6\xaf" - "\xb8\xc1\xca\xd3\xdc\xe5\xee\xf7" - "\x00\x0b\x16\x21\x2c\x37\x42\x4d" - "\x58\x63\x6e\x79\x84\x8f\x9a\xa5" - "\xb0\xbb\xc6\xd1\xdc\xe7\xf2\xfd" - "\x08\x13\x1e\x29\x34\x3f\x4a\x55" - "\x60\x6b\x76\x81\x8c\x97\xa2\xad" - "\xb8\xc3\xce\xd9\xe4\xef\xfa\x05" - "\x10\x1b\x26\x31\x3c\x47\x52\x5d" - "\x68\x73\x7e\x89\x94\x9f\xaa\xb5" - "\xc0\xcb\xd6\xe1\xec\xf7\x02\x0d" - "\x18\x23\x2e\x39\x44\x4f\x5a\x65" - "\x70\x7b\x86\x91\x9c\xa7\xb2\xbd" - "\xc8\xd3\xde\xe9\xf4\xff\x0a\x15" - "\x20\x2b\x36\x41\x4c\x57\x62\x6d" - "\x78\x83\x8e\x99\xa4\xaf\xba\xc5" - "\xd0\xdb\xe6\xf1\xfc\x07\x12\x1d" - "\x28\x33\x3e\x49\x54\x5f\x6a\x75" - "\x80\x8b\x96\xa1\xac\xb7\xc2\xcd" - "\xd8\xe3\xee\xf9\x04\x0f\x1a\x25" - "\x30\x3b\x46\x51\x5c\x67\x72\x7d" - "\x88\x93\x9e\xa9\xb4\xbf\xca\xd5" - "\xe0\xeb\xf6\x01\x0c\x17\x22\x2d" - "\x38\x43\x4e\x59\x64\x6f\x7a\x85" - "\x90\x9b\xa6\xb1\xbc\xc7\xd2\xdd" - "\xe8\xf3\xfe\x09\x14\x1f\x2a\x35" - "\x40\x4b\x56\x61\x6c\x77\x82\x8d" - "\x98\xa3\xae\xb9\xc4\xcf\xda\xe5" - "\xf0\xfb\x06\x11\x1c\x27\x32\x3d" - "\x48\x53\x5e\x69\x74\x7f\x8a\x95" - "\xa0\xab\xb6\xc1\xcc\xd7\xe2\xed" - "\xf8\x03\x0e\x19\x24\x2f\x3a\x45" - "\x50\x5b\x66\x71\x7c\x87\x92\x9d" - "\xa8\xb3\xbe\xc9\xd4\xdf\xea\xf5" - "\x00\x0d\x1a\x27\x34\x41\x4e\x5b" - "\x68\x75\x82\x8f\x9c\xa9\xb6\xc3" - "\xd0\xdd\xea\xf7\x04\x11\x1e\x2b" - "\x38\x45\x52\x5f\x6c\x79\x86\x93" - "\xa0\xad\xba\xc7\xd4\xe1\xee\xfb" - "\x08\x15\x22\x2f\x3c\x49\x56\x63" - "\x70\x7d\x8a\x97\xa4\xb1\xbe\xcb" - "\xd8\xe5\xf2\xff\x0c\x19\x26\x33" - "\x40\x4d\x5a\x67\x74\x81\x8e\x9b" - "\xa8\xb5\xc2\xcf\xdc\xe9\xf6\x03" - "\x10\x1d\x2a\x37\x44\x51\x5e\x6b" - "\x78\x85\x92\x9f\xac\xb9\xc6\xd3" - "\xe0\xed\xfa\x07\x14\x21\x2e\x3b" - "\x48\x55\x62\x6f\x7c\x89\x96\xa3" - "\xb0\xbd\xca\xd7\xe4\xf1\xfe\x0b" - "\x18\x25\x32\x3f\x4c\x59\x66\x73" - "\x80\x8d\x9a\xa7\xb4\xc1\xce\xdb" - "\xe8\xf5\x02\x0f\x1c\x29\x36\x43" - "\x50\x5d\x6a\x77\x84\x91\x9e\xab" - "\xb8\xc5\xd2\xdf\xec\xf9\x06\x13" - "\x20\x2d\x3a\x47\x54\x61\x6e\x7b" - "\x88\x95\xa2\xaf\xbc\xc9\xd6\xe3" - "\xf0\xfd\x0a\x17\x24\x31\x3e\x4b" - "\x58\x65\x72\x7f\x8c\x99\xa6\xb3" - "\xc0\xcd\xda\xe7\xf4\x01\x0e\x1b" - "\x28\x35\x42\x4f\x5c\x69\x76\x83" - "\x90\x9d\xaa\xb7\xc4\xd1\xde\xeb" - "\xf8\x05\x12\x1f\x2c\x39\x46\x53" - "\x60\x6d\x7a\x87\x94\xa1\xae\xbb" - "\xc8\xd5\xe2\xef\xfc\x09\x16\x23" - "\x30\x3d\x4a\x57\x64\x71\x7e\x8b" - "\x98\xa5\xb2\xbf\xcc\xd9\xe6\xf3" - "\x00\x0f\x1e\x2d\x3c\x4b\x5a\x69" - "\x78\x87\x96\xa5\xb4\xc3\xd2\xe1" - "\xf0\xff\x0e\x1d\x2c\x3b\x4a\x59" - "\x68\x77\x86\x95\xa4\xb3\xc2\xd1" - "\xe0\xef\xfe\x0d\x1c\x2b\x3a\x49" - "\x58\x67\x76\x85\x94\xa3\xb2\xc1" - "\xd0\xdf\xee\xfd\x0c\x1b\x2a\x39" - "\x48\x57\x66\x75\x84\x93\xa2\xb1" - "\xc0\xcf\xde\xed\xfc\x0b\x1a\x29" - "\x38\x47\x56\x65\x74\x83\x92\xa1" - "\xb0\xbf\xce\xdd\xec\xfb\x0a\x19" - "\x28\x37\x46\x55\x64\x73\x82\x91" - "\xa0\xaf\xbe\xcd\xdc\xeb\xfa\x09" - "\x18\x27\x36\x45\x54\x63\x72\x81" - "\x90\x9f\xae\xbd\xcc\xdb\xea\xf9" - "\x08\x17\x26\x35\x44\x53\x62\x71" - "\x80\x8f\x9e\xad\xbc\xcb\xda\xe9" - "\xf8\x07\x16\x25\x34\x43\x52\x61" - "\x70\x7f\x8e\x9d\xac\xbb\xca\xd9" - "\xe8\xf7\x06\x15\x24\x33\x42\x51" - "\x60\x6f\x7e\x8d\x9c\xab\xba\xc9" - "\xd8\xe7\xf6\x05\x14\x23\x32\x41" - "\x50\x5f\x6e\x7d\x8c\x9b\xaa\xb9" - "\xc8\xd7\xe6\xf5\x04\x13\x22\x31" - "\x40\x4f\x5e\x6d\x7c\x8b\x9a\xa9" - "\xb8\xc7\xd6\xe5\xf4\x03\x12\x21" - "\x30\x3f\x4e\x5d\x6c\x7b\x8a\x99" - "\xa8\xb7\xc6\xd5\xe4\xf3\x02\x11" - "\x20\x2f\x3e\x4d\x5c\x6b\x7a\x89" - "\x98\xa7\xb6\xc5\xd4\xe3\xf2\x01" - "\x10\x1f\x2e\x3d\x4c\x5b\x6a\x79" - "\x88\x97\xa6\xb5\xc4\xd3\xe2\xf1" - "\x00\x11\x22\x33\x44\x55\x66\x77" - "\x88\x99\xaa\xbb\xcc\xdd\xee\xff" - "\x10\x21\x32\x43\x54\x65\x76\x87" - "\x98\xa9\xba\xcb\xdc\xed\xfe\x0f" - "\x20\x31\x42\x53\x64\x75\x86\x97" - "\xa8\xb9\xca\xdb\xec\xfd\x0e\x1f" - "\x30\x41\x52\x63\x74\x85\x96\xa7" - "\xb8\xc9\xda\xeb\xfc\x0d\x1e\x2f" - "\x40\x51\x62\x73\x84\x95\xa6\xb7" - "\xc8\xd9\xea\xfb\x0c\x1d\x2e\x3f" - "\x50\x61\x72\x83\x94\xa5\xb6\xc7" - "\xd8\xe9\xfa\x0b\x1c\x2d\x3e\x4f" - "\x60\x71\x82\x93\xa4\xb5\xc6\xd7" - "\xe8\xf9\x0a\x1b\x2c\x3d\x4e\x5f" - "\x70\x81\x92\xa3\xb4\xc5\xd6\xe7" - "\xf8\x09\x1a\x2b\x3c\x4d\x5e\x6f" - "\x80\x91\xa2\xb3\xc4\xd5\xe6\xf7" - "\x08\x19\x2a\x3b\x4c\x5d\x6e\x7f" - "\x90\xa1\xb2\xc3\xd4\xe5\xf6\x07" - "\x18\x29\x3a\x4b\x5c\x6d\x7e\x8f" - "\xa0\xb1\xc2\xd3\xe4\xf5\x06\x17" - "\x28\x39\x4a\x5b\x6c\x7d\x8e\x9f" - "\xb0\xc1\xd2\xe3\xf4\x05\x16\x27" - "\x38\x49\x5a\x6b\x7c\x8d\x9e\xaf" - "\xc0\xd1\xe2\xf3\x04\x15\x26\x37" - "\x48\x59\x6a\x7b\x8c\x9d\xae\xbf" - "\xd0\xe1\xf2\x03\x14\x25\x36\x47" - "\x58\x69\x7a\x8b\x9c\xad\xbe\xcf" - "\xe0\xf1\x02\x13\x24\x35\x46\x57" - "\x68\x79\x8a\x9b\xac\xbd\xce\xdf" - "\xf0\x01\x12\x23\x34\x45\x56\x67" - "\x78\x89\x9a\xab\xbc\xcd\xde\xef" - "\x00\x13\x26\x39\x4c\x5f\x72\x85" - "\x98\xab\xbe\xd1\xe4\xf7\x0a\x1d" - "\x30\x43\x56\x69\x7c\x8f\xa2\xb5" - "\xc8\xdb\xee\x01\x14\x27\x3a\x4d" - "\x60\x73\x86\x99\xac\xbf\xd2\xe5" - "\xf8\x0b\x1e\x31\x44\x57\x6a\x7d" - "\x90\xa3\xb6\xc9\xdc\xef\x02\x15" - "\x28\x3b\x4e\x61\x74\x87\x9a\xad" - "\xc0\xd3\xe6\xf9\x0c\x1f\x32\x45" - "\x58\x6b\x7e\x91\xa4\xb7\xca\xdd" - "\xf0\x03\x16\x29\x3c\x4f\x62\x75" - "\x88\x9b\xae\xc1\xd4\xe7\xfa\x0d" - "\x20\x33\x46\x59\x6c\x7f\x92\xa5" - "\xb8\xcb\xde\xf1\x04\x17\x2a\x3d" - "\x50\x63\x76\x89\x9c\xaf\xc2\xd5" - "\xe8\xfb\x0e\x21\x34\x47\x5a\x6d" - "\x80\x93\xa6\xb9\xcc\xdf\xf2\x05" - "\x18\x2b\x3e\x51\x64\x77\x8a\x9d" - "\xb0\xc3\xd6\xe9\xfc\x0f\x22\x35" - "\x48\x5b\x6e\x81\x94\xa7\xba\xcd" - "\xe0\xf3\x06\x19\x2c\x3f\x52\x65" - "\x78\x8b\x9e\xb1\xc4\xd7\xea\xfd" - "\x10\x23\x36\x49\x5c\x6f\x82\x95" - "\xa8\xbb\xce\xe1\xf4\x07\x1a\x2d" - "\x40\x53\x66\x79\x8c\x9f\xb2\xc5" - "\xd8\xeb\xfe\x11\x24\x37\x4a\x5d" - "\x70\x83\x96\xa9\xbc\xcf\xe2\xf5" - "\x08\x1b\x2e\x41\x54\x67\x7a\x8d" - "\xa0\xb3\xc6\xd9\xec\xff\x12\x25" - "\x38\x4b\x5e\x71\x84\x97\xaa\xbd" - "\xd0\xe3\xf6\x09\x1c\x2f\x42\x55" - "\x68\x7b\x8e\xa1\xb4\xc7\xda\xed" - "\x00\x15\x2a\x3f\x54\x69\x7e\x93" - "\xa8\xbd\xd2\xe7\xfc\x11\x26\x3b" - "\x50\x65\x7a\x8f\xa4\xb9\xce\xe3" - "\xf8\x0d\x22\x37\x4c\x61\x76\x8b" - "\xa0\xb5\xca\xdf\xf4\x09\x1e\x33" - "\x48\x5d\x72\x87\x9c\xb1\xc6\xdb" - "\xf0\x05\x1a\x2f\x44\x59\x6e\x83" - "\x98\xad\xc2\xd7\xec\x01\x16\x2b" - "\x40\x55\x6a\x7f\x94\xa9\xbe\xd3" - "\xe8\xfd\x12\x27\x3c\x51\x66\x7b" - "\x90\xa5\xba\xcf\xe4\xf9\x0e\x23" - "\x38\x4d\x62\x77\x8c\xa1\xb6\xcb" - "\xe0\xf5\x0a\x1f\x34\x49\x5e\x73" - "\x88\x9d\xb2\xc7\xdc\xf1\x06\x1b" - "\x30\x45\x5a\x6f\x84\x99\xae\xc3" - "\xd8\xed\x02\x17\x2c\x41\x56\x6b" - "\x80\x95\xaa\xbf\xd4\xe9\xfe\x13" - "\x28\x3d\x52\x67\x7c\x91\xa6\xbb" - "\xd0\xe5\xfa\x0f\x24\x39\x4e\x63" - "\x78\x8d\xa2\xb7\xcc\xe1\xf6\x0b" - "\x20\x35\x4a\x5f\x74\x89\x9e\xb3" - "\xc8\xdd\xf2\x07\x1c\x31\x46\x5b" - "\x70\x85\x9a\xaf\xc4\xd9\xee\x03" - "\x18\x2d\x42\x57\x6c\x81\x96\xab" - "\xc0\xd5\xea\xff\x14\x29\x3e\x53" - "\x68\x7d\x92\xa7\xbc\xd1\xe6\xfb" - "\x10\x25\x3a\x4f\x64\x79\x8e\xa3" - "\xb8\xcd\xe2\xf7\x0c\x21\x36\x4b" - "\x60\x75\x8a\x9f\xb4\xc9\xde\xf3" - "\x08\x1d\x32\x47\x5c\x71\x86\x9b" - "\xb0\xc5\xda\xef\x04\x19\x2e\x43" - "\x58\x6d\x82\x97\xac\xc1\xd6\xeb" - "\x00\x17\x2e\x45\x5c\x73\x8a\xa1" - "\xb8\xcf\xe6\xfd\x14\x2b\x42\x59" - "\x70\x87\x9e\xb5\xcc\xe3\xfa\x11" - "\x28\x3f\x56\x6d\x84\x9b\xb2\xc9" - "\xe0\xf7\x0e\x25\x3c\x53\x6a\x81" - "\x98\xaf\xc6\xdd\xf4\x0b\x22\x39" - "\x50\x67\x7e\x95\xac\xc3\xda\xf1" - "\x08\x1f\x36\x4d\x64\x7b\x92\xa9" - "\xc0\xd7\xee\x05\x1c\x33\x4a\x61" - "\x78\x8f\xa6\xbd\xd4\xeb\x02\x19" - "\x30\x47\x5e\x75\x8c\xa3\xba\xd1" - "\xe8\xff\x16\x2d\x44\x5b\x72\x89" - "\xa0\xb7\xce\xe5\xfc\x13\x2a\x41" - "\x58\x6f\x86\x9d\xb4\xcb\xe2\xf9" - "\x10\x27\x3e\x55\x6c\x83\x9a\xb1" - "\xc8\xdf\xf6\x0d\x24\x3b\x52\x69" - "\x80\x97\xae\xc5\xdc\xf3\x0a\x21" - "\x38\x4f\x66\x7d\x94\xab\xc2\xd9" - "\xf0\x07\x1e\x35\x4c\x63\x7a\x91" - "\xa8\xbf\xd6\xed\x04\x1b\x32\x49" - "\x60\x77\x8e\xa5\xbc\xd3\xea\x01" - "\x18\x2f\x46\x5d\x74\x8b\xa2\xb9" - "\xd0\xe7\xfe\x15\x2c\x43\x5a\x71" - "\x88\x9f\xb6\xcd\xe4\xfb\x12\x29" - "\x40\x57\x6e\x85\x9c\xb3\xca\xe1" - "\xf8\x0f\x26\x3d\x54\x6b\x82\x99" - "\xb0\xc7\xde\xf5\x0c\x23\x3a\x51" - "\x68\x7f\x96\xad\xc4\xdb\xf2\x09" - "\x20\x37\x4e\x65\x7c\x93\xaa\xc1" - "\xd8\xef\x06\x1d\x34\x4b\x62\x79" - "\x90\xa7\xbe\xd5\xec\x03\x1a\x31" - "\x48\x5f\x76\x8d\xa4\xbb\xd2\xe9" - "\x00\x19\x32\x4b\x64\x7d\x96\xaf" - "\xc8\xe1\xfa\x13\x2c\x45\x5e\x77" - "\x90\xa9\xc2\xdb\xf4\x0d\x26\x3f" - "\x58\x71\x8a\xa3\xbc\xd5\xee\x07" - "\x20\x39\x52\x6b\x84\x9d\xb6\xcf" - "\xe8\x01\x1a\x33\x4c\x65\x7e\x97" - "\xb0\xc9\xe2\xfb\x14\x2d\x46\x5f" - "\x78\x91\xaa\xc3\xdc\xf5\x0e\x27" - "\x40\x59\x72\x8b\xa4\xbd\xd6\xef" - "\x08\x21\x3a\x53\x6c\x85\x9e\xb7" - "\xd0\xe9\x02\x1b\x34\x4d\x66\x7f" - "\x98\xb1\xca\xe3\xfc\x15\x2e\x47" - "\x60\x79\x92\xab\xc4\xdd\xf6\x0f" - "\x28\x41\x5a\x73\x8c\xa5\xbe\xd7" - "\xf0\x09\x22\x3b\x54\x6d\x86\x9f" - "\xb8\xd1\xea\x03\x1c\x35\x4e\x67" - "\x80\x99\xb2\xcb\xe4\xfd\x16\x2f" - "\x48\x61\x7a\x93\xac\xc5\xde\xf7" - "\x10\x29\x42\x5b\x74\x8d\xa6\xbf" - "\xd8\xf1\x0a\x23\x3c\x55\x6e\x87" - "\xa0\xb9\xd2\xeb\x04\x1d\x36\x4f" - "\x68\x81\x9a\xb3\xcc\xe5\xfe\x17" - "\x30\x49\x62\x7b\x94\xad\xc6\xdf" - "\xf8\x11\x2a\x43\x5c\x75\x8e\xa7" - "\xc0\xd9\xf2\x0b\x24\x3d\x56\x6f" - "\x88\xa1\xba\xd3\xec\x05\x1e\x37" - "\x50\x69\x82\x9b\xb4\xcd\xe6\xff" - "\x18\x31\x4a\x63\x7c\x95\xae\xc7" - "\xe0\xf9\x12\x2b\x44\x5d\x76\x8f" - "\xa8\xc1\xda\xf3\x0c\x25\x3e\x57" - "\x70\x89\xa2\xbb\xd4\xed\x06\x1f" - "\x38\x51\x6a\x83\x9c\xb5\xce\xe7" - "\x00\x1b\x36\x51\x6c\x87\xa2\xbd" - "\xd8\xf3\x0e\x29\x44\x5f\x7a\x95" - "\xb0\xcb\xe6\x01\x1c\x37\x52\x6d" - "\x88\xa3\xbe\xd9\xf4\x0f\x2a\x45" - "\x60\x7b\x96\xb1\xcc\xe7\x02\x1d" - "\x38\x53\x6e\x89\xa4\xbf\xda\xf5" - "\x10\x2b\x46\x61\x7c\x97\xb2\xcd" - "\xe8\x03\x1e\x39\x54\x6f\x8a\xa5" - "\xc0\xdb\xf6\x11\x2c\x47\x62\x7d" - "\x98\xb3\xce\xe9\x04\x1f\x3a\x55" - "\x70\x8b\xa6\xc1\xdc\xf7\x12\x2d" - "\x48\x63\x7e\x99\xb4\xcf\xea\x05" - "\x20\x3b\x56\x71\x8c\xa7\xc2\xdd" - "\xf8\x13\x2e\x49\x64\x7f\x9a\xb5" - "\xd0\xeb\x06\x21\x3c\x57\x72\x8d" - "\xa8\xc3\xde\xf9\x14\x2f\x4a\x65" - "\x80\x9b\xb6\xd1\xec\x07\x22\x3d" - "\x58\x73\x8e\xa9\xc4\xdf\xfa\x15" - "\x30\x4b\x66\x81\x9c\xb7\xd2\xed" - "\x08\x23\x3e\x59\x74\x8f\xaa\xc5" - "\xe0\xfb\x16\x31\x4c\x67\x82\x9d" - "\xb8\xd3\xee\x09\x24\x3f\x5a\x75" - "\x90\xab\xc6\xe1\xfc\x17\x32\x4d" - "\x68\x83\x9e\xb9\xd4\xef\x0a\x25" - "\x40\x5b\x76\x91\xac\xc7\xe2\xfd" - "\x18\x33\x4e\x69\x84\x9f\xba\xd5" - "\xf0\x0b\x26\x41\x5c\x77\x92\xad" - "\xc8\xe3\xfe\x19\x34\x4f\x6a\x85" - "\xa0\xbb\xd6\xf1\x0c\x27\x42\x5d" - "\x78\x93\xae\xc9\xe4\xff\x1a\x35" - "\x50\x6b\x86\xa1\xbc\xd7\xf2\x0d" - "\x28\x43\x5e\x79\x94\xaf\xca\xe5" - "\x00\x1d\x3a\x57\x74\x91\xae\xcb" - "\xe8\x05\x22\x3f\x5c\x79\x96\xb3" - "\xd0\xed\x0a\x27\x44\x61\x7e\x9b" - "\xb8\xd5\xf2\x0f\x2c\x49\x66\x83" - "\xa0\xbd\xda\xf7\x14\x31\x4e\x6b" - "\x88\xa5\xc2\xdf\xfc\x19\x36\x53" - "\x70\x8d\xaa\xc7\xe4\x01\x1e\x3b" - "\x58\x75\x92\xaf\xcc\xe9\x06\x23" - "\x40\x5d\x7a\x97\xb4\xd1\xee\x0b" - "\x28\x45\x62\x7f\x9c\xb9\xd6\xf3" - "\x10\x2d\x4a\x67\x84\xa1\xbe\xdb" - "\xf8\x15\x32\x4f\x6c\x89\xa6\xc3" - "\xe0\xfd\x1a\x37\x54\x71\x8e\xab" - "\xc8\xe5\x02\x1f\x3c\x59\x76\x93" - "\xb0\xcd\xea\x07\x24\x41\x5e\x7b" - "\x98\xb5\xd2\xef\x0c\x29\x46\x63" - "\x80\x9d\xba\xd7\xf4\x11\x2e\x4b" - "\x68\x85\xa2\xbf\xdc\xf9\x16\x33" - "\x50\x6d\x8a\xa7\xc4\xe1\xfe\x1b" - "\x38\x55\x72\x8f\xac\xc9\xe6\x03" - "\x20\x3d\x5a\x77\x94\xb1\xce\xeb" - "\x08\x25\x42\x5f\x7c\x99\xb6\xd3" - "\xf0\x0d\x2a\x47\x64\x81\x9e\xbb" - "\xd8\xf5\x12\x2f\x4c\x69\x86\xa3" - "\xc0\xdd\xfa\x17\x34\x51\x6e\x8b" - "\xa8\xc5\xe2\xff\x1c\x39\x56\x73" - "\x90\xad\xca\xe7\x04\x21\x3e\x5b" - "\x78\x95\xb2\xcf\xec\x09\x26\x43" - "\x60\x7d\x9a\xb7\xd4\xf1\x0e\x2b" - "\x48\x65\x82\x9f\xbc\xd9\xf6\x13" - "\x30\x4d\x6a\x87\xa4\xc1\xde\xfb" - "\x18\x35\x52\x6f\x8c\xa9\xc6\xe3" - "\x00\x1f\x3e\x5d\x7c\x9b\xba\xd9" - "\xf8\x17\x36\x55\x74\x93\xb2\xd1" - "\xf0\x0f\x2e\x4d\x6c\x8b\xaa\xc9" - "\xe8\x07\x26\x45\x64\x83\xa2\xc1" - "\xe0\xff\x1e\x3d\x5c\x7b\x9a\xb9" - "\xd8\xf7\x16\x35\x54\x73\x92\xb1" - "\xd0\xef\x0e\x2d\x4c\x6b\x8a\xa9" - "\xc8\xe7\x06\x25\x44\x63\x82\xa1" - "\xc0\xdf\xfe\x1d\x3c\x5b\x7a\x99" - "\xb8\xd7\xf6\x15\x34\x53\x72\x91" - "\xb0\xcf\xee\x0d\x2c\x4b\x6a\x89" - "\xa8\xc7\xe6\x05\x24\x43\x62\x81" - "\xa0\xbf\xde\xfd\x1c\x3b\x5a\x79" - "\x98\xb7\xd6\xf5\x14\x33\x52\x71" - "\x90\xaf\xce\xed\x0c\x2b\x4a\x69" - "\x88\xa7\xc6\xe5\x04\x23\x42\x61" - "\x80\x9f\xbe\xdd\xfc\x1b\x3a\x59" - "\x78\x97\xb6\xd5\xf4\x13\x32\x51" - "\x70\x8f\xae\xcd\xec\x0b\x2a\x49" - "\x68\x87\xa6\xc5\xe4\x03\x22\x41" - "\x60\x7f\x9e\xbd\xdc\xfb\x1a\x39" - "\x58\x77\x96\xb5\xd4\xf3\x12\x31" - "\x50\x6f\x8e\xad\xcc\xeb\x0a\x29" - "\x48\x67\x86\xa5\xc4\xe3\x02\x21" - "\x40\x5f\x7e\x9d\xbc\xdb\xfa\x19" - "\x38\x57\x76\x95\xb4\xd3\xf2\x11" - "\x30\x4f\x6e\x8d\xac\xcb\xea\x09" - "\x28\x47\x66\x85\xa4\xc3\xe2\x01" - "\x20\x3f\x5e\x7d\x9c\xbb\xda\xf9" - "\x18\x37\x56\x75\x94\xb3\xd2\xf1" - "\x10\x2f\x4e\x6d\x8c\xab\xca\xe9" - "\x08\x27\x46\x65\x84\xa3\xc2\xe1" - "\x00\x21\x42\x63", - .ctext = - "\xb5\x81\xf5\x64\x18\x73\xe3\xf0" - "\x4c\x13\xf2\x77\x18\x60\x65\x5e" - "\x29\x01\xce\x98\x55\x53\xf9\x0c" - "\x2a\x08\xd5\x09\xb3\x57\x55\x56" - "\xc5\xe9\x56\x90\xcb\x6a\xa3\xc0" - "\xff\xc4\x79\xb4\xd2\x97\x5d\xc4" - "\x43\xd1\xfe\x94\x7b\x88\x06\x5a" - "\xb2\x9e\x2c\xfc\x44\x03\xb7\x90" - "\xa0\xc1\xba\x6a\x33\xb8\xc7\xb2" - "\x9d\xe1\x12\x4f\xc0\x64\xd4\x01" - "\xfe\x8c\x7a\x66\xf7\xe6\x5a\x91" - "\xbb\xde\x56\x86\xab\x65\x21\x30" - "\x00\x84\x65\x24\xa5\x7d\x85\xb4" - "\xe3\x17\xed\x3a\xb7\x6f\xb4\x0b" - "\x0b\xaf\x15\xae\x5a\x8f\xf2\x0c" - "\x2f\x27\xf4\x09\xd8\xd2\x96\xb7" - "\x71\xf2\xc5\x99\x4d\x7e\x7f\x75" - "\x77\x89\x30\x8b\x59\xdb\xa2\xb2" - "\xa0\xf3\x19\x39\x2b\xc5\x7e\x3f" - "\x4f\xd9\xd3\x56\x28\x97\x44\xdc" - "\xc0\x8b\x77\x24\xd9\x52\xe7\xc5" - "\xaf\xf6\x7d\x59\xb2\x44\x05\x1d" - "\xb1\xb0\x11\xa5\x0f\xec\x33\xe1" - "\x6d\x1b\x4e\x1f\xff\x57\x91\xb4" - "\x5b\x9a\x96\xc5\x53\xbc\xae\x20" - "\x3c\xbb\x14\xe2\xe8\x22\x33\xc1" - "\x5e\x76\x9e\x46\x99\xf6\x2a\x15" - "\xc6\x97\x02\xa0\x66\x43\xd1\xa6" - "\x31\xa6\x9f\xfb\xf4\xd3\x69\xe5" - "\xcd\x76\x95\xb8\x7a\x82\x7f\x21" - "\x45\xff\x3f\xce\x55\xf6\x95\x10" - "\x08\x77\x10\x43\xc6\xf3\x09\xe5" - "\x68\xe7\x3c\xad\x00\x52\x45\x0d" - "\xfe\x2d\xc6\xc2\x94\x8c\x12\x1d" - "\xe6\x25\xae\x98\x12\x8e\x19\x9c" - "\x81\x68\xb1\x11\xf6\x69\xda\xe3" - "\x62\x08\x18\x7a\x25\x49\x28\xac" - "\xba\x71\x12\x0b\xe4\xa2\xe5\xc7" - "\x5d\x8e\xec\x49\x40\x21\xbf\x5a" - "\x98\xf3\x02\x68\x55\x03\x7f\x8a" - "\xe5\x94\x0c\x32\x5c\x07\x82\x63" - "\xaf\x6f\x91\x40\x84\x8e\x52\x25" - "\xd0\xb0\x29\x53\x05\xe2\x50\x7a" - "\x34\xeb\xc9\x46\x20\xa8\x3d\xde" - "\x7f\x16\x5f\x36\xc5\x2e\xdc\xd1" - "\x15\x47\xc7\x50\x40\x6d\x91\xc5" - "\xe7\x93\x95\x1a\xd3\x57\xbc\x52" - "\x33\xee\x14\x19\x22\x52\x89\xa7" - "\x4a\x25\x56\x77\x4b\xca\xcf\x0a" - "\xe1\xf5\x35\x85\x30\x7e\x59\x4a" - "\xbd\x14\x5b\xdf\xe3\x46\xcb\xac" - "\x1f\x6c\x96\x0e\xf4\x81\xd1\x99" - "\xca\x88\x63\x3d\x02\x58\x6b\xa9" - "\xe5\x9f\xb3\x00\xb2\x54\xc6\x74" - "\x1c\xbf\x46\xab\x97\xcc\xf8\x54" - "\x04\x07\x08\x52\xe6\xc0\xda\x93" - "\x74\x7d\x93\x99\x5d\x78\x68\xa6" - "\x2e\x6b\xd3\x6a\x69\xcc\x12\x6b" - "\xd4\xc7\xa5\xc6\xe7\xf6\x03\x04" - "\x5d\xcd\x61\x5e\x17\x40\xdc\xd1" - "\x5c\xf5\x08\xdf\x5c\x90\x85\xa4" - "\xaf\xf6\x78\xbb\x0d\xf1\xf4\xa4" - "\x54\x26\x72\x9e\x61\xfa\x86\xcf" - "\xe8\x9e\xa1\xe0\xc7\x48\x23\xae" - "\x5a\x90\xae\x75\x0a\x74\x18\x89" - "\x05\xb1\x92\xb2\x7f\xd0\x1b\xa6" - "\x62\x07\x25\x01\xc7\xc2\x4f\xf9" - "\xe8\xfe\x63\x95\x80\x07\xb4\x26" - "\xcc\xd1\x26\xb6\xc4\x3f\x9e\xcb" - "\x8e\x3b\x2e\x44\x16\xd3\x10\x9a" - "\x95\x08\xeb\xc8\xcb\xeb\xbf\x6f" - "\x0b\xcd\x1f\xc8\xca\x86\xaa\xec" - "\x33\xe6\x69\xf4\x45\x25\x86\x3a" - "\x22\x94\x4f\x00\x23\x6a\x44\xc2" - "\x49\x97\x33\xab\x36\x14\x0a\x70" - "\x24\xc3\xbe\x04\x3b\x79\xa0\xf9" - "\xb8\xe7\x76\x29\x22\x83\xd7\xf2" - "\x94\xf4\x41\x49\xba\x5f\x7b\x07" - "\xb5\xfb\xdb\x03\x1a\x9f\xb6\x4c" - "\xc2\x2e\x37\x40\x49\xc3\x38\x16" - "\xe2\x4f\x77\x82\xb0\x68\x4c\x71" - "\x1d\x57\x61\x9c\xd9\x4e\x54\x99" - "\x47\x13\x28\x73\x3c\xbb\x00\x90" - "\xf3\x4d\xc9\x0e\xfd\xe7\xb1\x71" - "\xd3\x15\x79\xbf\xcc\x26\x2f\xbd" - "\xad\x6c\x50\x69\x6c\x3e\x6d\x80" - "\x9a\xea\x78\xaf\x19\xb2\x0d\x4d" - "\xad\x04\x07\xae\x22\x90\x4a\x93" - "\x32\x0e\x36\x9b\x1b\x46\xba\x3b" - "\xb4\xac\xc6\xd1\xa2\x31\x53\x3b" - "\x2a\x3d\x45\xfe\x03\x61\x10\x85" - "\x17\x69\xa6\x78\xcc\x6c\x87\x49" - "\x53\xf9\x80\x10\xde\x80\xa2\x41" - "\x6a\xc3\x32\x02\xad\x6d\x3c\x56" - "\x00\x71\x51\x06\xa7\xbd\xfb\xef" - "\x3c\xb5\x9f\xfc\x48\x7d\x53\x7c" - "\x66\xb0\x49\x23\xc4\x47\x10\x0e" - "\xe5\x6c\x74\x13\xe6\xc5\x3f\xaa" - "\xde\xff\x07\x44\xdd\x56\x1b\xad" - "\x09\x77\xfb\x5b\x12\xb8\x0d\x38" - "\x17\x37\x35\x7b\x9b\xbc\xfe\xd4" - "\x7e\x8b\xda\x7e\x5b\x04\xa7\x22" - "\xa7\x31\xa1\x20\x86\xc7\x1b\x99" - "\xdb\xd1\x89\xf4\x94\xa3\x53\x69" - "\x8d\xe7\xe8\x74\x11\x8d\x74\xd6" - "\x07\x37\x91\x9f\xfd\x67\x50\x3a" - "\xc9\xe1\xf4\x36\xd5\xa0\x47\xd1" - "\xf9\xe5\x39\xa3\x31\xac\x07\x36" - "\x23\xf8\x66\x18\x14\x28\x34\x0f" - "\xb8\xd0\xe7\x29\xb3\x04\x4b\x55" - "\x01\x41\xb2\x75\x8d\xcb\x96\x85" - "\x3a\xfb\xab\x2b\x9e\xfa\x58\x20" - "\x44\x1f\xc0\x14\x22\x75\x61\xe8" - "\xaa\x19\xcf\xf1\x82\x56\xf4\xd7" - "\x78\x7b\x3d\x5f\xb3\x9e\x0b\x8a" - "\x57\x50\xdb\x17\x41\x65\x4d\xa3" - "\x02\xc9\x9c\x9c\x53\xfb\x39\x39" - "\x9b\x1d\x72\x24\xda\xb7\x39\xbe" - "\x13\x3b\xfa\x29\xda\x9e\x54\x64" - "\x6e\xba\xd8\xa1\xcb\xb3\x36\xfa" - "\xcb\x47\x85\xe9\x61\x38\xbc\xbe" - "\xc5\x00\x38\x2a\x54\xf7\xc4\xb9" - "\xb3\xd3\x7b\xa0\xa0\xf8\x72\x7f" - "\x8c\x8e\x82\x0e\xc6\x1c\x75\x9d" - "\xca\x8e\x61\x87\xde\xad\x80\xd2" - "\xf5\xf9\x80\xef\x15\x75\xaf\xf5" - "\x80\xfb\xff\x6d\x1e\x25\xb7\x40" - "\x61\x6a\x39\x5a\x6a\xb5\x31\xab" - "\x97\x8a\x19\x89\x44\x40\xc0\xa6" - "\xb4\x4e\x30\x32\x7b\x13\xe7\x67" - "\xa9\x8b\x57\x04\xc2\x01\xa6\xf4" - "\x28\x99\xad\x2c\x76\xa3\x78\xc2" - "\x4a\xe6\xca\x5c\x50\x6a\xc1\xb0" - "\x62\x4b\x10\x8e\x7c\x17\x43\xb3" - "\x17\x66\x1c\x3e\x8d\x69\xf0\x5a" - "\x71\xf5\x97\xdc\xd1\x45\xdd\x28" - "\xf3\x5d\xdf\x53\x7b\x11\xe5\xbc" - "\x4c\xdb\x1b\x51\x6b\xe9\xfb\x3d" - "\xc1\xc3\x2c\xb9\x71\xf5\xb6\xb2" - "\x13\x36\x79\x80\x53\xe8\xd3\xa6" - "\x0a\xaf\xfd\x56\x97\xf7\x40\x8e" - "\x45\xce\xf8\xb0\x9e\x5c\x33\x82" - "\xb0\x44\x56\xfc\x05\x09\xe9\x2a" - "\xac\x26\x80\x14\x1d\xc8\x3a\x35" - "\x4c\x82\x97\xfd\x76\xb7\xa9\x0a" - "\x35\x58\x79\x8e\x0f\x66\xea\xaf" - "\x51\x6c\x09\xa9\x6e\x9b\xcb\x9a" - "\x31\x47\xa0\x2f\x7c\x71\xb4\x4a" - "\x11\xaa\x8c\x66\xc5\x64\xe6\x3a" - "\x54\xda\x24\x6a\xc4\x41\x65\x46" - "\x82\xa0\x0a\x0f\x5f\xfb\x25\xd0" - "\x2c\x91\xa7\xee\xc4\x81\x07\x86" - "\x75\x5e\x33\x69\x97\xe4\x2c\xa8" - "\x9d\x9f\x0b\x6a\xbe\xad\x98\xda" - "\x6d\x94\x41\xda\x2c\x1e\x89\xc4" - "\xc2\xaf\x1e\x00\x05\x0b\x83\x60" - "\xbd\x43\xea\x15\x23\x7f\xb9\xac" - "\xee\x4f\x2c\xaf\x2a\xf3\xdf\xd0" - "\xf3\x19\x31\xbb\x4a\x74\x84\x17" - "\x52\x32\x2c\x7d\x61\xe4\xcb\xeb" - "\x80\x38\x15\x52\xcb\x6f\xea\xe5" - "\x73\x9c\xd9\x24\x69\xc6\x95\x32" - "\x21\xc8\x11\xe4\xdc\x36\xd7\x93" - "\x38\x66\xfb\xb2\x7f\x3a\xb9\xaf" - "\x31\xdd\x93\x75\x78\x8a\x2c\x94" - "\x87\x1a\x58\xec\x9e\x7d\x4d\xba" - "\xe1\xe5\x4d\xfc\xbc\xa4\x2a\x14" - "\xef\xcc\xa7\xec\xab\x43\x09\x18" - "\xd3\xab\x68\xd1\x07\x99\x44\x47" - "\xd6\x83\x85\x3b\x30\xea\xa9\x6b" - "\x63\xea\xc4\x07\xfb\x43\x2f\xa4" - "\xaa\xb0\xab\x03\x89\xce\x3f\x8c" - "\x02\x7c\x86\x54\xbc\x88\xaf\x75" - "\xd2\xdc\x63\x17\xd3\x26\xf6\x96" - "\xa9\x3c\xf1\x61\x8c\x11\x18\xcc" - "\xd6\xea\x5b\xe2\xcd\xf0\xf1\xb2" - "\xe5\x35\x90\x1f\x85\x4c\x76\x5b" - "\x66\xce\x44\xa4\x32\x9f\xe6\x7b" - "\x71\x6e\x9f\x58\x15\x67\x72\x87" - "\x64\x8e\x3a\x44\x45\xd4\x76\xfa" - "\xc2\xf6\xef\x85\x05\x18\x7a\x9b" - "\xba\x41\x54\xac\xf0\xfc\x59\x12" - "\x3f\xdf\xa0\xe5\x8a\x65\xfd\x3a" - "\x62\x8d\x83\x2c\x03\xbe\x05\x76" - "\x2e\x53\x49\x97\x94\x33\xae\x40" - "\x81\x15\xdb\x6e\xad\xaa\xf5\x4b" - "\xe3\x98\x70\xdf\xe0\x7c\xcd\xdb" - "\x02\xd4\x7d\x2f\xc1\xe6\xb4\xf3" - "\xd7\x0d\x7a\xd9\x23\x9e\x87\x2d" - "\xce\x87\xad\xcc\x72\x05\x00\x29" - "\xdc\x73\x7f\x64\xc1\x15\x0e\xc2" - "\xdf\xa7\x5f\xeb\x41\xa1\xcd\xef" - "\x5c\x50\x79\x2a\x56\x56\x71\x8c" - "\xac\xc0\x79\x50\x69\xca\x59\x32" - "\x65\xf2\x54\xe4\x52\x38\x76\xd1" - "\x5e\xde\x26\x9e\xfb\x75\x2e\x11" - "\xb5\x10\xf4\x17\x73\xf5\x89\xc7" - "\x4f\x43\x5c\x8e\x7c\xb9\x05\x52" - "\x24\x40\x99\xfe\x9b\x85\x0b\x6c" - "\x22\x3e\x8b\xae\x86\xa1\xd2\x79" - "\x05\x68\x6b\xab\xe3\x41\x49\xed" - "\x15\xa1\x8d\x40\x2d\x61\xdf\x1a" - "\x59\xc9\x26\x8b\xef\x30\x4c\x88" - "\x4b\x10\xf8\x8d\xa6\x92\x9f\x4b" - "\xf3\xc4\x53\x0b\x89\x5d\x28\x92" - "\xcf\x78\xb2\xc0\x5d\xed\x7e\xfc" - "\xc0\x12\x23\x5f\x5a\x78\x86\x43" - "\x6e\x27\xf7\x5a\xa7\x6a\xed\x19" - "\x04\xf0\xb3\x12\xd1\xbd\x0e\x89" - "\x6e\xbc\x96\xa8\xd8\x49\x39\x9f" - "\x7e\x67\xf0\x2e\x3e\x01\xa9\xba" - "\xec\x8b\x62\x8e\xcb\x4a\x70\x43" - "\xc7\xc2\xc4\xca\x82\x03\x73\xe9" - "\x11\xdf\xcf\x54\xea\xc9\xb0\x95" - "\x51\xc0\x13\x3d\x92\x05\xfa\xf4" - "\xa9\x34\xc8\xce\x6c\x3d\x54\xcc" - "\xc4\xaf\xf1\xdc\x11\x44\x26\xa2" - "\xaf\xf1\x85\x75\x7d\x03\x61\x68" - "\x4e\x78\xc6\x92\x7d\x86\x7d\x77" - "\xdc\x71\x72\xdb\xc6\xae\xa1\xcb" - "\x70\x9a\x0b\x19\xbe\x4a\x6c\x2a" - "\xe2\xba\x6c\x64\x9a\x13\x28\xdf" - "\x85\x75\xe6\x43\xf6\x87\x08\x68" - "\x6e\xba\x6e\x79\x9f\x04\xbc\x23" - "\x50\xf6\x33\x5c\x1f\x24\x25\xbe" - "\x33\x47\x80\x45\x56\xa3\xa7\xd7" - "\x7a\xb1\x34\x0b\x90\x3c\x9c\xad" - "\x44\x5f\x9e\x0e\x9d\xd4\xbd\x93" - "\x5e\xfa\x3c\xe0\xb0\xd9\xed\xf3" - "\xd6\x2e\xff\x24\xd8\x71\x6c\xed" - "\xaf\x55\xeb\x22\xac\x93\x68\x32" - "\x05\x5b\x47\xdd\xc6\x4a\xcb\xc7" - "\x10\xe1\x3c\x92\x1a\xf3\x23\x78" - "\x2b\xa1\xd2\x80\xf4\x12\xb1\x20" - "\x8f\xff\x26\x35\xdd\xfb\xc7\x4e" - "\x78\xf1\x2d\x50\x12\x77\xa8\x60" - "\x7c\x0f\xf5\x16\x2f\x63\x70\x2a" - "\xc0\x96\x80\x4e\x0a\xb4\x93\x35" - "\x5d\x1d\x3f\x56\xf7\x2f\xbb\x90" - "\x11\x16\x8f\xa2\xec\x47\xbe\xac" - "\x56\x01\x26\x56\xb1\x8c\xb2\x10" - "\xf9\x1a\xca\xf5\xd1\xb7\x39\x20" - "\x63\xf1\x69\x20\x4f\x13\x12\x1f" - "\x5b\x65\xfc\x98\xf7\xc4\x7a\xbe" - "\xf7\x26\x4d\x2b\x84\x7b\x42\xad" - "\xd8\x7a\x0a\xb4\xd8\x74\xbf\xc1" - "\xf0\x6e\xb4\x29\xa3\xbb\xca\x46" - "\x67\x70\x6a\x2d\xce\x0e\xa2\x8a" - "\xa9\x87\xbf\x05\xc4\xc1\x04\xa3" - "\xab\xd4\x45\x43\x8c\xb6\x02\xb0" - "\x41\xc8\xfc\x44\x3d\x59\xaa\x2e" - "\x44\x21\x2a\x8d\x88\x9d\x57\xf4" - "\xa0\x02\x77\xb8\xa6\xa0\xe6\x75" - "\x5c\x82\x65\x3e\x03\x5c\x29\x8f" - "\x38\x55\xab\x33\x26\xef\x9f\x43" - "\x52\xfd\x68\xaf\x36\xb4\xbb\x9a" - "\x58\x09\x09\x1b\xc3\x65\x46\x46" - "\x1d\xa7\x94\x18\x23\x50\x2c\xca" - "\x2c\x55\x19\x97\x01\x9d\x93\x3b" - "\x63\x86\xf2\x03\x67\x45\xd2\x72" - "\x28\x52\x6c\xf4\xe3\x1c\xb5\x11" - "\x13\xf1\xeb\x21\xc7\xd9\x56\x82" - "\x2b\x82\x39\xbd\x69\x54\xed\x62" - "\xc3\xe2\xde\x73\xd4\x6a\x12\xae" - "\x13\x21\x7f\x4b\x5b\xfc\xbf\xe8" - "\x2b\xbe\x56\xba\x68\x8b\x9a\xb1" - "\x6e\xfa\xbf\x7e\x5a\x4b\xf1\xac" - "\x98\x65\x85\xd1\x93\x53\xd3\x7b" - "\x09\xdd\x4b\x10\x6d\x84\xb0\x13" - "\x65\xbd\xcf\x52\x09\xc4\x85\xe2" - "\x84\x74\x15\x65\xb7\xf7\x51\xaf" - "\x55\xad\xa4\xd1\x22\x54\x70\x94" - "\xa0\x1c\x90\x41\xfd\x99\xd7\x5a" - "\x31\xef\xaa\x25\xd0\x7f\x4f\xea" - "\x1d\x55\x42\xe5\x49\xb0\xd0\x46" - "\x62\x36\x43\xb2\x82\x15\x75\x50" - "\xa4\x72\xeb\x54\x27\x1f\x8a\xe4" - "\x7d\xe9\x66\xc5\xf1\x53\xa4\xd1" - "\x0c\xeb\xb8\xf8\xbc\xd4\xe2\xe7" - "\xe1\xf8\x4b\xcb\xa9\xa1\xaf\x15" - "\x83\xcb\x72\xd0\x33\x79\x00\x2d" - "\x9f\xd7\xf1\x2e\x1e\x10\xe4\x45" - "\xc0\x75\x3a\x39\xea\x68\xf7\x5d" - "\x1b\x73\x8f\xe9\x8e\x0f\x72\x47" - "\xae\x35\x0a\x31\x7a\x14\x4d\x4a" - "\x6f\x47\xf7\x7e\x91\x6e\x74\x8b" - "\x26\x47\xf9\xc3\xf9\xde\x70\xf5" - "\x61\xab\xa9\x27\x9f\x82\xe4\x9c" - "\x89\x91\x3f\x2e\x6a\xfd\xb5\x49" - "\xe9\xfd\x59\x14\x36\x49\x40\x6d" - "\x32\xd8\x85\x42\xf3\xa5\xdf\x0c" - "\xa8\x27\xd7\x54\xe2\x63\x2f\xf2" - "\x7e\x8b\x8b\xe7\xf1\x9a\x95\x35" - "\x43\xdc\x3a\xe4\xb6\xf4\xd0\xdf" - "\x9c\xcb\x94\xf3\x21\xa0\x77\x50" - "\xe2\xc6\xc4\xc6\x5f\x09\x64\x5b" - "\x92\x90\xd8\xe1\xd1\xed\x4b\x42" - "\xd7\x37\xaf\x65\x3d\x11\x39\xb6" - "\x24\x8a\x60\xae\xd6\x1e\xbf\x0e" - "\x0d\xd7\xdc\x96\x0e\x65\x75\x4e" - "\x29\x06\x9d\xa4\x51\x3a\x10\x63" - "\x8f\x17\x07\xd5\x8e\x3c\xf4\x28" - "\x00\x5a\x5b\x05\x19\xd8\xc0\x6c" - "\xe5\x15\xe4\x9c\x9d\x71\x9d\x5e" - "\x94\x29\x1a\xa7\x80\xfa\x0e\x33" - "\x03\xdd\xb7\x3e\x9a\xa9\x26\x18" - "\x37\xa9\x64\x08\x4d\x94\x5a\x88" - "\xca\x35\xce\x81\x02\xe3\x1f\x1b" - "\x89\x1a\x77\x85\xe3\x41\x6d\x32" - "\x42\x19\x23\x7d\xc8\x73\xee\x25" - "\x85\x0d\xf8\x31\x25\x79\x1b\x6f" - "\x79\x25\xd2\xd8\xd4\x23\xfd\xf7" - "\x82\x36\x6a\x0c\x46\x22\x15\xe9" - "\xff\x72\x41\x91\x91\x7d\x3a\xb7" - "\xdd\x65\x99\x70\xf6\x8d\x84\xf8" - "\x67\x15\x20\x11\xd6\xb2\x55\x7b" - "\xdb\x87\xee\xef\x55\x89\x2a\x59" - "\x2b\x07\x8f\x43\x8a\x59\x3c\x01" - "\x8b\x65\x54\xa1\x66\xd5\x38\xbd" - "\xc6\x30\xa9\xcc\x49\xb6\xa8\x1b" - "\xb8\xc0\x0e\xe3\x45\x28\xe2\xff" - "\x41\x9f\x7e\x7c\xd1\xae\x9e\x25" - "\x3f\x4c\x7c\x7c\xf4\xa8\x26\x4d" - "\x5c\xfd\x4b\x27\x18\xf9\x61\x76" - "\x48\xba\x0c\x6b\xa9\x4d\xfc\xf5" - "\x3b\x35\x7e\x2f\x4a\xa9\xc2\x9a" - "\xae\xab\x86\x09\x89\xc9\xc2\x40" - "\x39\x2c\x81\xb3\xb8\x17\x67\xc2" - "\x0d\x32\x4a\x3a\x67\x81\xd7\x1a" - "\x34\x52\xc5\xdb\x0a\xf5\x63\x39" - "\xea\x1f\xe1\x7c\xa1\x9e\xc1\x35" - "\xe3\xb1\x18\x45\x67\xf9\x22\x38" - "\x95\xd9\x34\x34\x86\xc6\x41\x94" - "\x15\xf9\x5b\x41\xa6\x87\x8b\xf8" - "\xd5\xe1\x1b\xe2\x5b\xf3\x86\x10" - "\xff\xe6\xae\x69\x76\xbc\x0d\xb4" - "\x09\x90\x0c\xa2\x65\x0c\xad\x74" - "\xf5\xd7\xff\xda\xc1\xce\x85\xbe" - "\x00\xa7\xff\x4d\x2f\x65\xd3\x8c" - "\x86\x2d\x05\xe8\xed\x3e\x6b\x8b" - "\x0f\x3d\x83\x8c\xf1\x1d\x5b\x96" - "\x2e\xb1\x9c\xc2\x98\xe1\x70\xb9" - "\xba\x5c\x8a\x43\xd6\x34\xa7\x2d" - "\xc9\x92\xae\xf2\xa5\x7b\x05\x49" - "\xa7\x33\x34\x86\xca\xe4\x96\x23" - "\x76\x5b\xf2\xc6\xf1\x51\x28\x42" - "\x7b\xcc\x76\x8f\xfa\xa2\xad\x31" - "\xd4\xd6\x7a\x6d\x25\x25\x54\xe4" - "\x3f\x50\x59\xe1\x5c\x05\xb7\x27" - "\x48\xbf\x07\xec\x1b\x13\xbe\x2b" - "\xa1\x57\x2b\xd5\xab\xd7\xd0\x4c" - "\x1e\xcb\x71\x9b\xc5\x90\x85\xd3" - "\xde\x59\xec\x71\xeb\x89\xbb\xd0" - "\x09\x50\xe1\x16\x3f\xfd\x1c\x34" - "\xc3\x1c\xa1\x10\x77\x53\x98\xef" - "\xf2\xfd\xa5\x01\x59\xc2\x9b\x26" - "\xc7\x42\xd9\x49\xda\x58\x2b\x6e" - "\x9f\x53\x19\x76\x7e\xd9\xc9\x0e" - "\x68\xc8\x7f\x51\x22\x42\xef\x49" - "\xa4\x55\xb6\x36\xac\x09\xc7\x31" - "\x88\x15\x4b\x2e\x8f\x3a\x08\xf7" - "\xd8\xf7\xa8\xc5\xa9\x33\xa6\x45" - "\xe4\xc4\x94\x76\xf3\x0d\x8f\x7e" - "\xc8\xf6\xbc\x23\x0a\xb6\x4c\xd3" - "\x6a\xcd\x36\xc2\x90\x5c\x5c\x3c" - "\x65\x7b\xc2\xd6\xcc\xe6\x0d\x87" - "\x73\x2e\x71\x79\x16\x06\x63\x28" - "\x09\x15\xd8\x89\x38\x38\x3d\xb5" - "\x42\x1c\x08\x24\xf7\x2a\xd2\x9d" - "\xc8\xca\xef\xf9\x27\xd8\x07\x86" - "\xf7\x43\x0b\x55\x15\x3f\x9f\x83" - "\xef\xdc\x49\x9d\x2a\xc1\x54\x62" - "\xbd\x9b\x66\x55\x9f\xb7\x12\xf3" - "\x1b\x4d\x9d\x2a\x5c\xed\x87\x75" - "\x87\x26\xec\x61\x2c\xb4\x0f\x89" - "\xb0\xfb\x2e\x68\x5d\x15\xc7\x8d" - "\x2e\xc0\xd9\xec\xaf\x4f\xd2\x25" - "\x29\xe8\xd2\x26\x2b\x67\xe9\xfc" - "\x2b\xa8\x67\x96\x12\x1f\x5b\x96" - "\xc6\x14\x53\xaf\x44\xea\xd6\xe2" - "\x94\x98\xe4\x12\x93\x4c\x92\xe0" - "\x18\xa5\x8d\x2d\xe4\x71\x3c\x47" - "\x4c\xf7\xe6\x47\x9e\xc0\x68\xdf" - "\xd4\xf5\x5a\x74\xb1\x2b\x29\x03" - "\x19\x07\xaf\x90\x62\x5c\x68\x98" - "\x48\x16\x11\x02\x9d\xee\xb4\x9b" - "\xe5\x42\x7f\x08\xfd\x16\x32\x0b" - "\xd0\xb3\xfa\x2b\xb7\x99\xf9\x29" - "\xcd\x20\x45\x9f\xb3\x1a\x5d\xa2" - "\xaf\x4d\xe0\xbd\x42\x0d\xbc\x74" - "\x99\x9c\x8e\x53\x1a\xb4\x3e\xbd" - "\xa2\x9a\x2d\xf7\xf8\x39\x0f\x67" - "\x63\xfc\x6b\xc0\xaf\xb3\x4b\x4f" - "\x55\xc4\xcf\xa7\xc8\x04\x11\x3e" - "\x14\x32\xbb\x1b\x38\x77\xd6\x7f" - "\x54\x4c\xdf\x75\xf3\x07\x2d\x33" - "\x9b\xa8\x20\xe1\x7b\x12\xb5\xf3" - "\xef\x2f\xce\x72\xe5\x24\x60\xc1" - "\x30\xe2\xab\xa1\x8e\x11\x09\xa8" - "\x21\x33\x44\xfe\x7f\x35\x32\x93" - "\x39\xa7\xad\x8b\x79\x06\xb2\xcb" - "\x4e\xa9\x5f\xc7\xba\x74\x29\xec" - "\x93\xa0\x4e\x54\x93\xc0\xbc\x55" - "\x64\xf0\x48\xe5\x57\x99\xee\x75" - "\xd6\x79\x0f\x66\xb7\xc6\x57\x76" - "\xf7\xb7\xf3\x9c\xc5\x60\xe8\x7f" - "\x83\x76\xd6\x0e\xaa\xe6\x90\x39" - "\x1d\xa6\x32\x6a\x34\xe3\x55\xf8" - "\x58\xa0\x58\x7d\x33\xe0\x22\x39" - "\x44\x64\x87\x86\x5a\x2f\xa7\x7e" - "\x0f\x38\xea\xb0\x30\xcc\x61\xa5" - "\x6a\x32\xae\x1e\xf7\xe9\xd0\xa9" - "\x0c\x32\x4b\xb5\x49\x28\xab\x85" - "\x2f\x8e\x01\x36\x38\x52\xd0\xba" - "\xd6\x02\x78\xf8\x0e\x3e\x9c\x8b" - "\x6b\x45\x99\x3f\x5c\xfe\x58\xf1" - "\x5c\x94\x04\xe1\xf5\x18\x6d\x51" - "\xb2\x5d\x18\x20\xb6\xc2\x9a\x42" - "\x1d\xb3\xab\x3c\xb6\x3a\x13\x03" - "\xb2\x46\x82\x4f\xfc\x64\xbc\x4f" - "\xca\xfa\x9c\xc0\xd5\xa7\xbd\x11" - "\xb7\xe4\x5a\xf6\x6f\x4d\x4d\x54" - "\xea\xa4\x98\x66\xd4\x22\x3b\xd3" - "\x8f\x34\x47\xd9\x7c\xf4\x72\x3b" - "\x4d\x02\x77\xf6\xd6\xdd\x08\x0a" - "\x81\xe1\x86\x89\x3e\x56\x10\x3c" - "\xba\xd7\x81\x8c\x08\xbc\x8b\xe2" - "\x53\xec\xa7\x89\xee\xc8\x56\xb5" - "\x36\x2c\xb2\x03\xba\x99\xdd\x7c" - "\x48\xa0\xb0\xbc\x91\x33\xe9\xa8" - "\xcb\xcd\xcf\x59\x5f\x1f\x15\xe2" - "\x56\xf5\x4e\x01\x35\x27\x45\x77" - "\x47\xc8\xbc\xcb\x7e\x39\xc1\x97" - "\x28\xd3\x84\xfc\x2c\x3e\xc8\xad" - "\x9c\xf8\x8a\x61\x9c\x28\xaa\xc5" - "\x99\x20\x43\x85\x9d\xa5\xe2\x8b" - "\xb8\xae\xeb\xd0\x32\x0d\x52\x78" - "\x09\x56\x3f\xc7\xd8\x7e\x26\xfc" - "\x37\xfb\x6f\x04\xfc\xfa\x92\x10" - "\xac\xf8\x3e\x21\xdc\x8c\x21\x16" - "\x7d\x67\x6e\xf6\xcd\xda\xb6\x98" - "\x23\xab\x23\x3c\xb2\x10\xa0\x53" - "\x5a\x56\x9f\xc5\xd0\xff\xbb\xe4" - "\x98\x3c\x69\x1e\xdb\x38\x8f\x7e" - "\x0f\xd2\x98\x88\x81\x8b\x45\x67" - "\xea\x33\xf1\xeb\xe9\x97\x55\x2e" - "\xd9\xaa\xeb\x5a\xec\xda\xe1\x68" - "\xa8\x9d\x3c\x84\x7c\x05\x3d\x62" - "\x87\x8f\x03\x21\x28\x95\x0c\x89" - "\x25\x22\x4a\xb0\x93\xa9\x50\xa2" - "\x2f\x57\x6e\x18\x42\x19\x54\x0c" - "\x55\x67\xc6\x11\x49\xf4\x5c\xd2" - "\xe9\x3d\xdd\x8b\x48\x71\x21\x00" - "\xc3\x9a\x6c\x85\x74\x28\x83\x4a" - "\x1b\x31\x05\xe1\x06\x92\xe7\xda" - "\x85\x73\x78\x45\x20\x7f\xae\x13" - "\x7c\x33\x06\x22\xf4\x83\xf9\x35" - "\x3f\x6c\x71\xa8\x4e\x48\xbe\x9b" - "\xce\x8a\xba\xda\xbe\x28\x08\xf7" - "\xe2\x14\x8c\x71\xea\x72\xf9\x33" - "\xf2\x88\x3f\xd7\xbb\x69\x6c\x29" - "\x19\xdc\x84\xce\x1f\x12\x4f\xc8" - "\xaf\xa5\x04\xba\x5a\xab\xb0\xd9" - "\x14\x1f\x6c\x68\x98\x39\x89\x7a" - "\xd9\xd8\x2f\xdf\xa8\x47\x4a\x25" - "\xe2\xfb\x33\xf4\x59\x78\xe1\x68" - "\x85\xcf\xfe\x59\x20\xd4\x05\x1d" - "\x80\x99\xae\xbc\xca\xae\x0f\x2f" - "\x65\x43\x34\x8e\x7e\xac\xd3\x93" - "\x2f\xac\x6d\x14\x3d\x02\x07\x70" - "\x9d\xa4\xf3\x1b\x5c\x36\xfc\x01" - "\x73\x34\x85\x0c\x6c\xd6\xf1\xbd" - "\x3f\xdf\xee\xf5\xd9\xba\x56\xef" - "\xf4\x9b\x6b\xee\x9f\x5a\x78\x6d" - "\x32\x19\xf4\xf7\xf8\x4c\x69\x0b" - "\x4b\xbc\xbb\xb7\xf2\x85\xaf\x70" - "\x75\x24\x6c\x54\xa7\x0e\x4d\x1d" - "\x01\xbf\x08\xac\xcf\x7f\x2c\xe3" - "\x14\x89\x5e\x70\x5a\x99\x92\xcd" - "\x01\x84\xc8\xd2\xab\xe5\x4f\x58" - "\xe7\x0f\x2f\x0e\xff\x68\xea\xfd" - "\x15\xb3\x17\xe6\xb0\xe7\x85\xd8" - "\x23\x2e\x05\xc7\xc9\xc4\x46\x1f" - "\xe1\x9e\x49\x20\x23\x24\x4d\x7e" - "\x29\x65\xff\xf4\xb6\xfd\x1a\x85" - "\xc4\x16\xec\xfc\xea\x7b\xd6\x2c" - "\x43\xf8\xb7\xbf\x79\xc0\x85\xcd" - "\xef\xe1\x98\xd3\xa5\xf7\x90\x8c" - "\xe9\x7f\x80\x6b\xd2\xac\x4c\x30" - "\xa7\xc6\x61\x6c\xd2\xf9\x2c\xff" - "\x30\xbc\x22\x81\x7d\x93\x12\xe4" - "\x0a\xcd\xaf\xdd\xe8\xab\x0a\x1e" - "\x13\xa4\x27\xc3\x5f\xf7\x4b\xbb" - "\x37\x09\x4b\x91\x6f\x92\x4f\xaf" - "\x52\xee\xdf\xef\x09\x6f\xf7\x5c" - "\x6e\x12\x17\x72\x63\x57\xc7\xba" - "\x3b\x6b\x38\x32\x73\x1b\x9c\x80" - "\xc1\x7a\xc6\xcf\xcd\x35\xc0\x6b" - "\x31\x1a\x6b\xe9\xd8\x2c\x29\x3f" - "\x96\xfb\xb6\xcd\x13\x91\x3b\xc2" - "\xd2\xa3\x31\x8d\xa4\xcd\x57\xcd" - "\x13\x3d\x64\xfd\x06\xce\xe6\xdc" - "\x0c\x24\x43\x31\x40\x57\xf1\x72" - "\x17\xe3\x3a\x63\x6d\x35\xcf\x5d" - "\x97\x40\x59\xdd\xf7\x3c\x02\xf7" - "\x1c\x7e\x05\xbb\xa9\x0d\x01\xb1" - "\x8e\xc0\x30\xa9\x53\x24\xc9\x89" - "\x84\x6d\xaa\xd0\xcd\x91\xc2\x4d" - "\x91\xb0\x89\xe2\xbf\x83\x44\xaa" - "\x28\x72\x23\xa0\xc2\xad\xad\x1c" - "\xfc\x3f\x09\x7a\x0b\xdc\xc5\x1b" - "\x87\x13\xc6\x5b\x59\x8d\xf2\xc8" - "\xaf\xdf\x11\x95", - .len = 4100, - }, -}; - static const struct cipher_testvec chacha20_tv_template[] = { { /* RFC7539 A.2. Test Vector #1 */ .key = "\x00\x00\x00\x00\x00\x00\x00\x00" diff --git a/crypto/tgr192.c b/crypto/tgr192.c deleted file mode 100644 index aa29c529b44e61bd876e0542e83d8e1900cdf292..0000000000000000000000000000000000000000 --- a/crypto/tgr192.c +++ /dev/null @@ -1,682 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Cryptographic API. - * - * Tiger hashing Algorithm - * - * Copyright (C) 1998 Free Software Foundation, Inc. - * - * The Tiger algorithm was developed by Ross Anderson and Eli Biham. - * It was optimized for 64-bit processors while still delievering - * decent performance on 32 and 16-bit processors. - * - * This version is derived from the GnuPG implementation and the - * Tiger-Perl interface written by Rafael Sevilla - * - * Adapted for Linux Kernel Crypto by Aaron Grothe - * ajgrothe@yahoo.com, February 22, 2005 - */ -#include -#include -#include -#include -#include -#include -#include - -#define TGR192_DIGEST_SIZE 24 -#define TGR160_DIGEST_SIZE 20 -#define TGR128_DIGEST_SIZE 16 - -#define TGR192_BLOCK_SIZE 64 - -struct tgr192_ctx { - u64 a, b, c; - u8 hash[64]; - int count; - u32 nblocks; -}; - -static const u64 sbox1[256] = { - 0x02aab17cf7e90c5eULL, 0xac424b03e243a8ecULL, 0x72cd5be30dd5fcd3ULL, - 0x6d019b93f6f97f3aULL, 0xcd9978ffd21f9193ULL, 0x7573a1c9708029e2ULL, - 0xb164326b922a83c3ULL, 0x46883eee04915870ULL, 0xeaace3057103ece6ULL, - 0xc54169b808a3535cULL, 0x4ce754918ddec47cULL, 0x0aa2f4dfdc0df40cULL, - 0x10b76f18a74dbefaULL, 0xc6ccb6235ad1ab6aULL, 0x13726121572fe2ffULL, - 0x1a488c6f199d921eULL, 0x4bc9f9f4da0007caULL, 0x26f5e6f6e85241c7ULL, - 0x859079dbea5947b6ULL, 0x4f1885c5c99e8c92ULL, 0xd78e761ea96f864bULL, - 0x8e36428c52b5c17dULL, 0x69cf6827373063c1ULL, 0xb607c93d9bb4c56eULL, - 0x7d820e760e76b5eaULL, 0x645c9cc6f07fdc42ULL, 0xbf38a078243342e0ULL, - 0x5f6b343c9d2e7d04ULL, 0xf2c28aeb600b0ec6ULL, 0x6c0ed85f7254bcacULL, - 0x71592281a4db4fe5ULL, 0x1967fa69ce0fed9fULL, 0xfd5293f8b96545dbULL, - 0xc879e9d7f2a7600bULL, 0x860248920193194eULL, 0xa4f9533b2d9cc0b3ULL, - 0x9053836c15957613ULL, 0xdb6dcf8afc357bf1ULL, 0x18beea7a7a370f57ULL, - 0x037117ca50b99066ULL, 0x6ab30a9774424a35ULL, 0xf4e92f02e325249bULL, - 0x7739db07061ccae1ULL, 0xd8f3b49ceca42a05ULL, 0xbd56be3f51382f73ULL, - 0x45faed5843b0bb28ULL, 0x1c813d5c11bf1f83ULL, 0x8af0e4b6d75fa169ULL, - 0x33ee18a487ad9999ULL, 0x3c26e8eab1c94410ULL, 0xb510102bc0a822f9ULL, - 0x141eef310ce6123bULL, 0xfc65b90059ddb154ULL, 0xe0158640c5e0e607ULL, - 0x884e079826c3a3cfULL, 0x930d0d9523c535fdULL, 0x35638d754e9a2b00ULL, - 0x4085fccf40469dd5ULL, 0xc4b17ad28be23a4cULL, 0xcab2f0fc6a3e6a2eULL, - 0x2860971a6b943fcdULL, 0x3dde6ee212e30446ULL, 0x6222f32ae01765aeULL, - 0x5d550bb5478308feULL, 0xa9efa98da0eda22aULL, 0xc351a71686c40da7ULL, - 0x1105586d9c867c84ULL, 0xdcffee85fda22853ULL, 0xccfbd0262c5eef76ULL, - 0xbaf294cb8990d201ULL, 0xe69464f52afad975ULL, 0x94b013afdf133e14ULL, - 0x06a7d1a32823c958ULL, 0x6f95fe5130f61119ULL, 0xd92ab34e462c06c0ULL, - 0xed7bde33887c71d2ULL, 0x79746d6e6518393eULL, 0x5ba419385d713329ULL, - 0x7c1ba6b948a97564ULL, 0x31987c197bfdac67ULL, 0xde6c23c44b053d02ULL, - 0x581c49fed002d64dULL, 0xdd474d6338261571ULL, 0xaa4546c3e473d062ULL, - 0x928fce349455f860ULL, 0x48161bbacaab94d9ULL, 0x63912430770e6f68ULL, - 0x6ec8a5e602c6641cULL, 0x87282515337ddd2bULL, 0x2cda6b42034b701bULL, - 0xb03d37c181cb096dULL, 0xe108438266c71c6fULL, 0x2b3180c7eb51b255ULL, - 0xdf92b82f96c08bbcULL, 0x5c68c8c0a632f3baULL, 0x5504cc861c3d0556ULL, - 0xabbfa4e55fb26b8fULL, 0x41848b0ab3baceb4ULL, 0xb334a273aa445d32ULL, - 0xbca696f0a85ad881ULL, 0x24f6ec65b528d56cULL, 0x0ce1512e90f4524aULL, - 0x4e9dd79d5506d35aULL, 0x258905fac6ce9779ULL, 0x2019295b3e109b33ULL, - 0xf8a9478b73a054ccULL, 0x2924f2f934417eb0ULL, 0x3993357d536d1bc4ULL, - 0x38a81ac21db6ff8bULL, 0x47c4fbf17d6016bfULL, 0x1e0faadd7667e3f5ULL, - 0x7abcff62938beb96ULL, 0xa78dad948fc179c9ULL, 0x8f1f98b72911e50dULL, - 0x61e48eae27121a91ULL, 0x4d62f7ad31859808ULL, 0xeceba345ef5ceaebULL, - 0xf5ceb25ebc9684ceULL, 0xf633e20cb7f76221ULL, 0xa32cdf06ab8293e4ULL, - 0x985a202ca5ee2ca4ULL, 0xcf0b8447cc8a8fb1ULL, 0x9f765244979859a3ULL, - 0xa8d516b1a1240017ULL, 0x0bd7ba3ebb5dc726ULL, 0xe54bca55b86adb39ULL, - 0x1d7a3afd6c478063ULL, 0x519ec608e7669eddULL, 0x0e5715a2d149aa23ULL, - 0x177d4571848ff194ULL, 0xeeb55f3241014c22ULL, 0x0f5e5ca13a6e2ec2ULL, - 0x8029927b75f5c361ULL, 0xad139fabc3d6e436ULL, 0x0d5df1a94ccf402fULL, - 0x3e8bd948bea5dfc8ULL, 0xa5a0d357bd3ff77eULL, 0xa2d12e251f74f645ULL, - 0x66fd9e525e81a082ULL, 0x2e0c90ce7f687a49ULL, 0xc2e8bcbeba973bc5ULL, - 0x000001bce509745fULL, 0x423777bbe6dab3d6ULL, 0xd1661c7eaef06eb5ULL, - 0xa1781f354daacfd8ULL, 0x2d11284a2b16affcULL, 0xf1fc4f67fa891d1fULL, - 0x73ecc25dcb920adaULL, 0xae610c22c2a12651ULL, 0x96e0a810d356b78aULL, - 0x5a9a381f2fe7870fULL, 0xd5ad62ede94e5530ULL, 0xd225e5e8368d1427ULL, - 0x65977b70c7af4631ULL, 0x99f889b2de39d74fULL, 0x233f30bf54e1d143ULL, - 0x9a9675d3d9a63c97ULL, 0x5470554ff334f9a8ULL, 0x166acb744a4f5688ULL, - 0x70c74caab2e4aeadULL, 0xf0d091646f294d12ULL, 0x57b82a89684031d1ULL, - 0xefd95a5a61be0b6bULL, 0x2fbd12e969f2f29aULL, 0x9bd37013feff9fe8ULL, - 0x3f9b0404d6085a06ULL, 0x4940c1f3166cfe15ULL, 0x09542c4dcdf3defbULL, - 0xb4c5218385cd5ce3ULL, 0xc935b7dc4462a641ULL, 0x3417f8a68ed3b63fULL, - 0xb80959295b215b40ULL, 0xf99cdaef3b8c8572ULL, 0x018c0614f8fcb95dULL, - 0x1b14accd1a3acdf3ULL, 0x84d471f200bb732dULL, 0xc1a3110e95e8da16ULL, - 0x430a7220bf1a82b8ULL, 0xb77e090d39df210eULL, 0x5ef4bd9f3cd05e9dULL, - 0x9d4ff6da7e57a444ULL, 0xda1d60e183d4a5f8ULL, 0xb287c38417998e47ULL, - 0xfe3edc121bb31886ULL, 0xc7fe3ccc980ccbefULL, 0xe46fb590189bfd03ULL, - 0x3732fd469a4c57dcULL, 0x7ef700a07cf1ad65ULL, 0x59c64468a31d8859ULL, - 0x762fb0b4d45b61f6ULL, 0x155baed099047718ULL, 0x68755e4c3d50baa6ULL, - 0xe9214e7f22d8b4dfULL, 0x2addbf532eac95f4ULL, 0x32ae3909b4bd0109ULL, - 0x834df537b08e3450ULL, 0xfa209da84220728dULL, 0x9e691d9b9efe23f7ULL, - 0x0446d288c4ae8d7fULL, 0x7b4cc524e169785bULL, 0x21d87f0135ca1385ULL, - 0xcebb400f137b8aa5ULL, 0x272e2b66580796beULL, 0x3612264125c2b0deULL, - 0x057702bdad1efbb2ULL, 0xd4babb8eacf84be9ULL, 0x91583139641bc67bULL, - 0x8bdc2de08036e024ULL, 0x603c8156f49f68edULL, 0xf7d236f7dbef5111ULL, - 0x9727c4598ad21e80ULL, 0xa08a0896670a5fd7ULL, 0xcb4a8f4309eba9cbULL, - 0x81af564b0f7036a1ULL, 0xc0b99aa778199abdULL, 0x959f1ec83fc8e952ULL, - 0x8c505077794a81b9ULL, 0x3acaaf8f056338f0ULL, 0x07b43f50627a6778ULL, - 0x4a44ab49f5eccc77ULL, 0x3bc3d6e4b679ee98ULL, 0x9cc0d4d1cf14108cULL, - 0x4406c00b206bc8a0ULL, 0x82a18854c8d72d89ULL, 0x67e366b35c3c432cULL, - 0xb923dd61102b37f2ULL, 0x56ab2779d884271dULL, 0xbe83e1b0ff1525afULL, - 0xfb7c65d4217e49a9ULL, 0x6bdbe0e76d48e7d4ULL, 0x08df828745d9179eULL, - 0x22ea6a9add53bd34ULL, 0xe36e141c5622200aULL, 0x7f805d1b8cb750eeULL, - 0xafe5c7a59f58e837ULL, 0xe27f996a4fb1c23cULL, 0xd3867dfb0775f0d0ULL, - 0xd0e673de6e88891aULL, 0x123aeb9eafb86c25ULL, 0x30f1d5d5c145b895ULL, - 0xbb434a2dee7269e7ULL, 0x78cb67ecf931fa38ULL, 0xf33b0372323bbf9cULL, - 0x52d66336fb279c74ULL, 0x505f33ac0afb4eaaULL, 0xe8a5cd99a2cce187ULL, - 0x534974801e2d30bbULL, 0x8d2d5711d5876d90ULL, 0x1f1a412891bc038eULL, - 0xd6e2e71d82e56648ULL, 0x74036c3a497732b7ULL, 0x89b67ed96361f5abULL, - 0xffed95d8f1ea02a2ULL, 0xe72b3bd61464d43dULL, 0xa6300f170bdc4820ULL, - 0xebc18760ed78a77aULL -}; - -static const u64 sbox2[256] = { - 0xe6a6be5a05a12138ULL, 0xb5a122a5b4f87c98ULL, 0x563c6089140b6990ULL, - 0x4c46cb2e391f5dd5ULL, 0xd932addbc9b79434ULL, 0x08ea70e42015aff5ULL, - 0xd765a6673e478cf1ULL, 0xc4fb757eab278d99ULL, 0xdf11c6862d6e0692ULL, - 0xddeb84f10d7f3b16ULL, 0x6f2ef604a665ea04ULL, 0x4a8e0f0ff0e0dfb3ULL, - 0xa5edeef83dbcba51ULL, 0xfc4f0a2a0ea4371eULL, 0xe83e1da85cb38429ULL, - 0xdc8ff882ba1b1ce2ULL, 0xcd45505e8353e80dULL, 0x18d19a00d4db0717ULL, - 0x34a0cfeda5f38101ULL, 0x0be77e518887caf2ULL, 0x1e341438b3c45136ULL, - 0xe05797f49089ccf9ULL, 0xffd23f9df2591d14ULL, 0x543dda228595c5cdULL, - 0x661f81fd99052a33ULL, 0x8736e641db0f7b76ULL, 0x15227725418e5307ULL, - 0xe25f7f46162eb2faULL, 0x48a8b2126c13d9feULL, 0xafdc541792e76eeaULL, - 0x03d912bfc6d1898fULL, 0x31b1aafa1b83f51bULL, 0xf1ac2796e42ab7d9ULL, - 0x40a3a7d7fcd2ebacULL, 0x1056136d0afbbcc5ULL, 0x7889e1dd9a6d0c85ULL, - 0xd33525782a7974aaULL, 0xa7e25d09078ac09bULL, 0xbd4138b3eac6edd0ULL, - 0x920abfbe71eb9e70ULL, 0xa2a5d0f54fc2625cULL, 0xc054e36b0b1290a3ULL, - 0xf6dd59ff62fe932bULL, 0x3537354511a8ac7dULL, 0xca845e9172fadcd4ULL, - 0x84f82b60329d20dcULL, 0x79c62ce1cd672f18ULL, 0x8b09a2add124642cULL, - 0xd0c1e96a19d9e726ULL, 0x5a786a9b4ba9500cULL, 0x0e020336634c43f3ULL, - 0xc17b474aeb66d822ULL, 0x6a731ae3ec9baac2ULL, 0x8226667ae0840258ULL, - 0x67d4567691caeca5ULL, 0x1d94155c4875adb5ULL, 0x6d00fd985b813fdfULL, - 0x51286efcb774cd06ULL, 0x5e8834471fa744afULL, 0xf72ca0aee761ae2eULL, - 0xbe40e4cdaee8e09aULL, 0xe9970bbb5118f665ULL, 0x726e4beb33df1964ULL, - 0x703b000729199762ULL, 0x4631d816f5ef30a7ULL, 0xb880b5b51504a6beULL, - 0x641793c37ed84b6cULL, 0x7b21ed77f6e97d96ULL, 0x776306312ef96b73ULL, - 0xae528948e86ff3f4ULL, 0x53dbd7f286a3f8f8ULL, 0x16cadce74cfc1063ULL, - 0x005c19bdfa52c6ddULL, 0x68868f5d64d46ad3ULL, 0x3a9d512ccf1e186aULL, - 0x367e62c2385660aeULL, 0xe359e7ea77dcb1d7ULL, 0x526c0773749abe6eULL, - 0x735ae5f9d09f734bULL, 0x493fc7cc8a558ba8ULL, 0xb0b9c1533041ab45ULL, - 0x321958ba470a59bdULL, 0x852db00b5f46c393ULL, 0x91209b2bd336b0e5ULL, - 0x6e604f7d659ef19fULL, 0xb99a8ae2782ccb24ULL, 0xccf52ab6c814c4c7ULL, - 0x4727d9afbe11727bULL, 0x7e950d0c0121b34dULL, 0x756f435670ad471fULL, - 0xf5add442615a6849ULL, 0x4e87e09980b9957aULL, 0x2acfa1df50aee355ULL, - 0xd898263afd2fd556ULL, 0xc8f4924dd80c8fd6ULL, 0xcf99ca3d754a173aULL, - 0xfe477bacaf91bf3cULL, 0xed5371f6d690c12dULL, 0x831a5c285e687094ULL, - 0xc5d3c90a3708a0a4ULL, 0x0f7f903717d06580ULL, 0x19f9bb13b8fdf27fULL, - 0xb1bd6f1b4d502843ULL, 0x1c761ba38fff4012ULL, 0x0d1530c4e2e21f3bULL, - 0x8943ce69a7372c8aULL, 0xe5184e11feb5ce66ULL, 0x618bdb80bd736621ULL, - 0x7d29bad68b574d0bULL, 0x81bb613e25e6fe5bULL, 0x071c9c10bc07913fULL, - 0xc7beeb7909ac2d97ULL, 0xc3e58d353bc5d757ULL, 0xeb017892f38f61e8ULL, - 0xd4effb9c9b1cc21aULL, 0x99727d26f494f7abULL, 0xa3e063a2956b3e03ULL, - 0x9d4a8b9a4aa09c30ULL, 0x3f6ab7d500090fb4ULL, 0x9cc0f2a057268ac0ULL, - 0x3dee9d2dedbf42d1ULL, 0x330f49c87960a972ULL, 0xc6b2720287421b41ULL, - 0x0ac59ec07c00369cULL, 0xef4eac49cb353425ULL, 0xf450244eef0129d8ULL, - 0x8acc46e5caf4deb6ULL, 0x2ffeab63989263f7ULL, 0x8f7cb9fe5d7a4578ULL, - 0x5bd8f7644e634635ULL, 0x427a7315bf2dc900ULL, 0x17d0c4aa2125261cULL, - 0x3992486c93518e50ULL, 0xb4cbfee0a2d7d4c3ULL, 0x7c75d6202c5ddd8dULL, - 0xdbc295d8e35b6c61ULL, 0x60b369d302032b19ULL, 0xce42685fdce44132ULL, - 0x06f3ddb9ddf65610ULL, 0x8ea4d21db5e148f0ULL, 0x20b0fce62fcd496fULL, - 0x2c1b912358b0ee31ULL, 0xb28317b818f5a308ULL, 0xa89c1e189ca6d2cfULL, - 0x0c6b18576aaadbc8ULL, 0xb65deaa91299fae3ULL, 0xfb2b794b7f1027e7ULL, - 0x04e4317f443b5bebULL, 0x4b852d325939d0a6ULL, 0xd5ae6beefb207ffcULL, - 0x309682b281c7d374ULL, 0xbae309a194c3b475ULL, 0x8cc3f97b13b49f05ULL, - 0x98a9422ff8293967ULL, 0x244b16b01076ff7cULL, 0xf8bf571c663d67eeULL, - 0x1f0d6758eee30da1ULL, 0xc9b611d97adeb9b7ULL, 0xb7afd5887b6c57a2ULL, - 0x6290ae846b984fe1ULL, 0x94df4cdeacc1a5fdULL, 0x058a5bd1c5483affULL, - 0x63166cc142ba3c37ULL, 0x8db8526eb2f76f40ULL, 0xe10880036f0d6d4eULL, - 0x9e0523c9971d311dULL, 0x45ec2824cc7cd691ULL, 0x575b8359e62382c9ULL, - 0xfa9e400dc4889995ULL, 0xd1823ecb45721568ULL, 0xdafd983b8206082fULL, - 0xaa7d29082386a8cbULL, 0x269fcd4403b87588ULL, 0x1b91f5f728bdd1e0ULL, - 0xe4669f39040201f6ULL, 0x7a1d7c218cf04adeULL, 0x65623c29d79ce5ceULL, - 0x2368449096c00bb1ULL, 0xab9bf1879da503baULL, 0xbc23ecb1a458058eULL, - 0x9a58df01bb401eccULL, 0xa070e868a85f143dULL, 0x4ff188307df2239eULL, - 0x14d565b41a641183ULL, 0xee13337452701602ULL, 0x950e3dcf3f285e09ULL, - 0x59930254b9c80953ULL, 0x3bf299408930da6dULL, 0xa955943f53691387ULL, - 0xa15edecaa9cb8784ULL, 0x29142127352be9a0ULL, 0x76f0371fff4e7afbULL, - 0x0239f450274f2228ULL, 0xbb073af01d5e868bULL, 0xbfc80571c10e96c1ULL, - 0xd267088568222e23ULL, 0x9671a3d48e80b5b0ULL, 0x55b5d38ae193bb81ULL, - 0x693ae2d0a18b04b8ULL, 0x5c48b4ecadd5335fULL, 0xfd743b194916a1caULL, - 0x2577018134be98c4ULL, 0xe77987e83c54a4adULL, 0x28e11014da33e1b9ULL, - 0x270cc59e226aa213ULL, 0x71495f756d1a5f60ULL, 0x9be853fb60afef77ULL, - 0xadc786a7f7443dbfULL, 0x0904456173b29a82ULL, 0x58bc7a66c232bd5eULL, - 0xf306558c673ac8b2ULL, 0x41f639c6b6c9772aULL, 0x216defe99fda35daULL, - 0x11640cc71c7be615ULL, 0x93c43694565c5527ULL, 0xea038e6246777839ULL, - 0xf9abf3ce5a3e2469ULL, 0x741e768d0fd312d2ULL, 0x0144b883ced652c6ULL, - 0xc20b5a5ba33f8552ULL, 0x1ae69633c3435a9dULL, 0x97a28ca4088cfdecULL, - 0x8824a43c1e96f420ULL, 0x37612fa66eeea746ULL, 0x6b4cb165f9cf0e5aULL, - 0x43aa1c06a0abfb4aULL, 0x7f4dc26ff162796bULL, 0x6cbacc8e54ed9b0fULL, - 0xa6b7ffefd2bb253eULL, 0x2e25bc95b0a29d4fULL, 0x86d6a58bdef1388cULL, - 0xded74ac576b6f054ULL, 0x8030bdbc2b45805dULL, 0x3c81af70e94d9289ULL, - 0x3eff6dda9e3100dbULL, 0xb38dc39fdfcc8847ULL, 0x123885528d17b87eULL, - 0xf2da0ed240b1b642ULL, 0x44cefadcd54bf9a9ULL, 0x1312200e433c7ee6ULL, - 0x9ffcc84f3a78c748ULL, 0xf0cd1f72248576bbULL, 0xec6974053638cfe4ULL, - 0x2ba7b67c0cec4e4cULL, 0xac2f4df3e5ce32edULL, 0xcb33d14326ea4c11ULL, - 0xa4e9044cc77e58bcULL, 0x5f513293d934fcefULL, 0x5dc9645506e55444ULL, - 0x50de418f317de40aULL, 0x388cb31a69dde259ULL, 0x2db4a83455820a86ULL, - 0x9010a91e84711ae9ULL, 0x4df7f0b7b1498371ULL, 0xd62a2eabc0977179ULL, - 0x22fac097aa8d5c0eULL -}; - -static const u64 sbox3[256] = { - 0xf49fcc2ff1daf39bULL, 0x487fd5c66ff29281ULL, 0xe8a30667fcdca83fULL, - 0x2c9b4be3d2fcce63ULL, 0xda3ff74b93fbbbc2ULL, 0x2fa165d2fe70ba66ULL, - 0xa103e279970e93d4ULL, 0xbecdec77b0e45e71ULL, 0xcfb41e723985e497ULL, - 0xb70aaa025ef75017ULL, 0xd42309f03840b8e0ULL, 0x8efc1ad035898579ULL, - 0x96c6920be2b2abc5ULL, 0x66af4163375a9172ULL, 0x2174abdcca7127fbULL, - 0xb33ccea64a72ff41ULL, 0xf04a4933083066a5ULL, 0x8d970acdd7289af5ULL, - 0x8f96e8e031c8c25eULL, 0xf3fec02276875d47ULL, 0xec7bf310056190ddULL, - 0xf5adb0aebb0f1491ULL, 0x9b50f8850fd58892ULL, 0x4975488358b74de8ULL, - 0xa3354ff691531c61ULL, 0x0702bbe481d2c6eeULL, 0x89fb24057deded98ULL, - 0xac3075138596e902ULL, 0x1d2d3580172772edULL, 0xeb738fc28e6bc30dULL, - 0x5854ef8f63044326ULL, 0x9e5c52325add3bbeULL, 0x90aa53cf325c4623ULL, - 0xc1d24d51349dd067ULL, 0x2051cfeea69ea624ULL, 0x13220f0a862e7e4fULL, - 0xce39399404e04864ULL, 0xd9c42ca47086fcb7ULL, 0x685ad2238a03e7ccULL, - 0x066484b2ab2ff1dbULL, 0xfe9d5d70efbf79ecULL, 0x5b13b9dd9c481854ULL, - 0x15f0d475ed1509adULL, 0x0bebcd060ec79851ULL, 0xd58c6791183ab7f8ULL, - 0xd1187c5052f3eee4ULL, 0xc95d1192e54e82ffULL, 0x86eea14cb9ac6ca2ULL, - 0x3485beb153677d5dULL, 0xdd191d781f8c492aULL, 0xf60866baa784ebf9ULL, - 0x518f643ba2d08c74ULL, 0x8852e956e1087c22ULL, 0xa768cb8dc410ae8dULL, - 0x38047726bfec8e1aULL, 0xa67738b4cd3b45aaULL, 0xad16691cec0dde19ULL, - 0xc6d4319380462e07ULL, 0xc5a5876d0ba61938ULL, 0x16b9fa1fa58fd840ULL, - 0x188ab1173ca74f18ULL, 0xabda2f98c99c021fULL, 0x3e0580ab134ae816ULL, - 0x5f3b05b773645abbULL, 0x2501a2be5575f2f6ULL, 0x1b2f74004e7e8ba9ULL, - 0x1cd7580371e8d953ULL, 0x7f6ed89562764e30ULL, 0xb15926ff596f003dULL, - 0x9f65293da8c5d6b9ULL, 0x6ecef04dd690f84cULL, 0x4782275fff33af88ULL, - 0xe41433083f820801ULL, 0xfd0dfe409a1af9b5ULL, 0x4325a3342cdb396bULL, - 0x8ae77e62b301b252ULL, 0xc36f9e9f6655615aULL, 0x85455a2d92d32c09ULL, - 0xf2c7dea949477485ULL, 0x63cfb4c133a39ebaULL, 0x83b040cc6ebc5462ULL, - 0x3b9454c8fdb326b0ULL, 0x56f56a9e87ffd78cULL, 0x2dc2940d99f42bc6ULL, - 0x98f7df096b096e2dULL, 0x19a6e01e3ad852bfULL, 0x42a99ccbdbd4b40bULL, - 0xa59998af45e9c559ULL, 0x366295e807d93186ULL, 0x6b48181bfaa1f773ULL, - 0x1fec57e2157a0a1dULL, 0x4667446af6201ad5ULL, 0xe615ebcacfb0f075ULL, - 0xb8f31f4f68290778ULL, 0x22713ed6ce22d11eULL, 0x3057c1a72ec3c93bULL, - 0xcb46acc37c3f1f2fULL, 0xdbb893fd02aaf50eULL, 0x331fd92e600b9fcfULL, - 0xa498f96148ea3ad6ULL, 0xa8d8426e8b6a83eaULL, 0xa089b274b7735cdcULL, - 0x87f6b3731e524a11ULL, 0x118808e5cbc96749ULL, 0x9906e4c7b19bd394ULL, - 0xafed7f7e9b24a20cULL, 0x6509eadeeb3644a7ULL, 0x6c1ef1d3e8ef0edeULL, - 0xb9c97d43e9798fb4ULL, 0xa2f2d784740c28a3ULL, 0x7b8496476197566fULL, - 0x7a5be3e6b65f069dULL, 0xf96330ed78be6f10ULL, 0xeee60de77a076a15ULL, - 0x2b4bee4aa08b9bd0ULL, 0x6a56a63ec7b8894eULL, 0x02121359ba34fef4ULL, - 0x4cbf99f8283703fcULL, 0x398071350caf30c8ULL, 0xd0a77a89f017687aULL, - 0xf1c1a9eb9e423569ULL, 0x8c7976282dee8199ULL, 0x5d1737a5dd1f7abdULL, - 0x4f53433c09a9fa80ULL, 0xfa8b0c53df7ca1d9ULL, 0x3fd9dcbc886ccb77ULL, - 0xc040917ca91b4720ULL, 0x7dd00142f9d1dcdfULL, 0x8476fc1d4f387b58ULL, - 0x23f8e7c5f3316503ULL, 0x032a2244e7e37339ULL, 0x5c87a5d750f5a74bULL, - 0x082b4cc43698992eULL, 0xdf917becb858f63cULL, 0x3270b8fc5bf86ddaULL, - 0x10ae72bb29b5dd76ULL, 0x576ac94e7700362bULL, 0x1ad112dac61efb8fULL, - 0x691bc30ec5faa427ULL, 0xff246311cc327143ULL, 0x3142368e30e53206ULL, - 0x71380e31e02ca396ULL, 0x958d5c960aad76f1ULL, 0xf8d6f430c16da536ULL, - 0xc8ffd13f1be7e1d2ULL, 0x7578ae66004ddbe1ULL, 0x05833f01067be646ULL, - 0xbb34b5ad3bfe586dULL, 0x095f34c9a12b97f0ULL, 0x247ab64525d60ca8ULL, - 0xdcdbc6f3017477d1ULL, 0x4a2e14d4decad24dULL, 0xbdb5e6d9be0a1eebULL, - 0x2a7e70f7794301abULL, 0xdef42d8a270540fdULL, 0x01078ec0a34c22c1ULL, - 0xe5de511af4c16387ULL, 0x7ebb3a52bd9a330aULL, 0x77697857aa7d6435ULL, - 0x004e831603ae4c32ULL, 0xe7a21020ad78e312ULL, 0x9d41a70c6ab420f2ULL, - 0x28e06c18ea1141e6ULL, 0xd2b28cbd984f6b28ULL, 0x26b75f6c446e9d83ULL, - 0xba47568c4d418d7fULL, 0xd80badbfe6183d8eULL, 0x0e206d7f5f166044ULL, - 0xe258a43911cbca3eULL, 0x723a1746b21dc0bcULL, 0xc7caa854f5d7cdd3ULL, - 0x7cac32883d261d9cULL, 0x7690c26423ba942cULL, 0x17e55524478042b8ULL, - 0xe0be477656a2389fULL, 0x4d289b5e67ab2da0ULL, 0x44862b9c8fbbfd31ULL, - 0xb47cc8049d141365ULL, 0x822c1b362b91c793ULL, 0x4eb14655fb13dfd8ULL, - 0x1ecbba0714e2a97bULL, 0x6143459d5cde5f14ULL, 0x53a8fbf1d5f0ac89ULL, - 0x97ea04d81c5e5b00ULL, 0x622181a8d4fdb3f3ULL, 0xe9bcd341572a1208ULL, - 0x1411258643cce58aULL, 0x9144c5fea4c6e0a4ULL, 0x0d33d06565cf620fULL, - 0x54a48d489f219ca1ULL, 0xc43e5eac6d63c821ULL, 0xa9728b3a72770dafULL, - 0xd7934e7b20df87efULL, 0xe35503b61a3e86e5ULL, 0xcae321fbc819d504ULL, - 0x129a50b3ac60bfa6ULL, 0xcd5e68ea7e9fb6c3ULL, 0xb01c90199483b1c7ULL, - 0x3de93cd5c295376cULL, 0xaed52edf2ab9ad13ULL, 0x2e60f512c0a07884ULL, - 0xbc3d86a3e36210c9ULL, 0x35269d9b163951ceULL, 0x0c7d6e2ad0cdb5faULL, - 0x59e86297d87f5733ULL, 0x298ef221898db0e7ULL, 0x55000029d1a5aa7eULL, - 0x8bc08ae1b5061b45ULL, 0xc2c31c2b6c92703aULL, 0x94cc596baf25ef42ULL, - 0x0a1d73db22540456ULL, 0x04b6a0f9d9c4179aULL, 0xeffdafa2ae3d3c60ULL, - 0xf7c8075bb49496c4ULL, 0x9cc5c7141d1cd4e3ULL, 0x78bd1638218e5534ULL, - 0xb2f11568f850246aULL, 0xedfabcfa9502bc29ULL, 0x796ce5f2da23051bULL, - 0xaae128b0dc93537cULL, 0x3a493da0ee4b29aeULL, 0xb5df6b2c416895d7ULL, - 0xfcabbd25122d7f37ULL, 0x70810b58105dc4b1ULL, 0xe10fdd37f7882a90ULL, - 0x524dcab5518a3f5cULL, 0x3c9e85878451255bULL, 0x4029828119bd34e2ULL, - 0x74a05b6f5d3ceccbULL, 0xb610021542e13ecaULL, 0x0ff979d12f59e2acULL, - 0x6037da27e4f9cc50ULL, 0x5e92975a0df1847dULL, 0xd66de190d3e623feULL, - 0x5032d6b87b568048ULL, 0x9a36b7ce8235216eULL, 0x80272a7a24f64b4aULL, - 0x93efed8b8c6916f7ULL, 0x37ddbff44cce1555ULL, 0x4b95db5d4b99bd25ULL, - 0x92d3fda169812fc0ULL, 0xfb1a4a9a90660bb6ULL, 0x730c196946a4b9b2ULL, - 0x81e289aa7f49da68ULL, 0x64669a0f83b1a05fULL, 0x27b3ff7d9644f48bULL, - 0xcc6b615c8db675b3ULL, 0x674f20b9bcebbe95ULL, 0x6f31238275655982ULL, - 0x5ae488713e45cf05ULL, 0xbf619f9954c21157ULL, 0xeabac46040a8eae9ULL, - 0x454c6fe9f2c0c1cdULL, 0x419cf6496412691cULL, 0xd3dc3bef265b0f70ULL, - 0x6d0e60f5c3578a9eULL -}; - -static const u64 sbox4[256] = { - 0x5b0e608526323c55ULL, 0x1a46c1a9fa1b59f5ULL, 0xa9e245a17c4c8ffaULL, - 0x65ca5159db2955d7ULL, 0x05db0a76ce35afc2ULL, 0x81eac77ea9113d45ULL, - 0x528ef88ab6ac0a0dULL, 0xa09ea253597be3ffULL, 0x430ddfb3ac48cd56ULL, - 0xc4b3a67af45ce46fULL, 0x4ececfd8fbe2d05eULL, 0x3ef56f10b39935f0ULL, - 0x0b22d6829cd619c6ULL, 0x17fd460a74df2069ULL, 0x6cf8cc8e8510ed40ULL, - 0xd6c824bf3a6ecaa7ULL, 0x61243d581a817049ULL, 0x048bacb6bbc163a2ULL, - 0xd9a38ac27d44cc32ULL, 0x7fddff5baaf410abULL, 0xad6d495aa804824bULL, - 0xe1a6a74f2d8c9f94ULL, 0xd4f7851235dee8e3ULL, 0xfd4b7f886540d893ULL, - 0x247c20042aa4bfdaULL, 0x096ea1c517d1327cULL, 0xd56966b4361a6685ULL, - 0x277da5c31221057dULL, 0x94d59893a43acff7ULL, 0x64f0c51ccdc02281ULL, - 0x3d33bcc4ff6189dbULL, 0xe005cb184ce66af1ULL, 0xff5ccd1d1db99beaULL, - 0xb0b854a7fe42980fULL, 0x7bd46a6a718d4b9fULL, 0xd10fa8cc22a5fd8cULL, - 0xd31484952be4bd31ULL, 0xc7fa975fcb243847ULL, 0x4886ed1e5846c407ULL, - 0x28cddb791eb70b04ULL, 0xc2b00be2f573417fULL, 0x5c9590452180f877ULL, - 0x7a6bddfff370eb00ULL, 0xce509e38d6d9d6a4ULL, 0xebeb0f00647fa702ULL, - 0x1dcc06cf76606f06ULL, 0xe4d9f28ba286ff0aULL, 0xd85a305dc918c262ULL, - 0x475b1d8732225f54ULL, 0x2d4fb51668ccb5feULL, 0xa679b9d9d72bba20ULL, - 0x53841c0d912d43a5ULL, 0x3b7eaa48bf12a4e8ULL, 0x781e0e47f22f1ddfULL, - 0xeff20ce60ab50973ULL, 0x20d261d19dffb742ULL, 0x16a12b03062a2e39ULL, - 0x1960eb2239650495ULL, 0x251c16fed50eb8b8ULL, 0x9ac0c330f826016eULL, - 0xed152665953e7671ULL, 0x02d63194a6369570ULL, 0x5074f08394b1c987ULL, - 0x70ba598c90b25ce1ULL, 0x794a15810b9742f6ULL, 0x0d5925e9fcaf8c6cULL, - 0x3067716cd868744eULL, 0x910ab077e8d7731bULL, 0x6a61bbdb5ac42f61ULL, - 0x93513efbf0851567ULL, 0xf494724b9e83e9d5ULL, 0xe887e1985c09648dULL, - 0x34b1d3c675370cfdULL, 0xdc35e433bc0d255dULL, 0xd0aab84234131be0ULL, - 0x08042a50b48b7eafULL, 0x9997c4ee44a3ab35ULL, 0x829a7b49201799d0ULL, - 0x263b8307b7c54441ULL, 0x752f95f4fd6a6ca6ULL, 0x927217402c08c6e5ULL, - 0x2a8ab754a795d9eeULL, 0xa442f7552f72943dULL, 0x2c31334e19781208ULL, - 0x4fa98d7ceaee6291ULL, 0x55c3862f665db309ULL, 0xbd0610175d53b1f3ULL, - 0x46fe6cb840413f27ULL, 0x3fe03792df0cfa59ULL, 0xcfe700372eb85e8fULL, - 0xa7be29e7adbce118ULL, 0xe544ee5cde8431ddULL, 0x8a781b1b41f1873eULL, - 0xa5c94c78a0d2f0e7ULL, 0x39412e2877b60728ULL, 0xa1265ef3afc9a62cULL, - 0xbcc2770c6a2506c5ULL, 0x3ab66dd5dce1ce12ULL, 0xe65499d04a675b37ULL, - 0x7d8f523481bfd216ULL, 0x0f6f64fcec15f389ULL, 0x74efbe618b5b13c8ULL, - 0xacdc82b714273e1dULL, 0xdd40bfe003199d17ULL, 0x37e99257e7e061f8ULL, - 0xfa52626904775aaaULL, 0x8bbbf63a463d56f9ULL, 0xf0013f1543a26e64ULL, - 0xa8307e9f879ec898ULL, 0xcc4c27a4150177ccULL, 0x1b432f2cca1d3348ULL, - 0xde1d1f8f9f6fa013ULL, 0x606602a047a7ddd6ULL, 0xd237ab64cc1cb2c7ULL, - 0x9b938e7225fcd1d3ULL, 0xec4e03708e0ff476ULL, 0xfeb2fbda3d03c12dULL, - 0xae0bced2ee43889aULL, 0x22cb8923ebfb4f43ULL, 0x69360d013cf7396dULL, - 0x855e3602d2d4e022ULL, 0x073805bad01f784cULL, 0x33e17a133852f546ULL, - 0xdf4874058ac7b638ULL, 0xba92b29c678aa14aULL, 0x0ce89fc76cfaadcdULL, - 0x5f9d4e0908339e34ULL, 0xf1afe9291f5923b9ULL, 0x6e3480f60f4a265fULL, - 0xeebf3a2ab29b841cULL, 0xe21938a88f91b4adULL, 0x57dfeff845c6d3c3ULL, - 0x2f006b0bf62caaf2ULL, 0x62f479ef6f75ee78ULL, 0x11a55ad41c8916a9ULL, - 0xf229d29084fed453ULL, 0x42f1c27b16b000e6ULL, 0x2b1f76749823c074ULL, - 0x4b76eca3c2745360ULL, 0x8c98f463b91691bdULL, 0x14bcc93cf1ade66aULL, - 0x8885213e6d458397ULL, 0x8e177df0274d4711ULL, 0xb49b73b5503f2951ULL, - 0x10168168c3f96b6bULL, 0x0e3d963b63cab0aeULL, 0x8dfc4b5655a1db14ULL, - 0xf789f1356e14de5cULL, 0x683e68af4e51dac1ULL, 0xc9a84f9d8d4b0fd9ULL, - 0x3691e03f52a0f9d1ULL, 0x5ed86e46e1878e80ULL, 0x3c711a0e99d07150ULL, - 0x5a0865b20c4e9310ULL, 0x56fbfc1fe4f0682eULL, 0xea8d5de3105edf9bULL, - 0x71abfdb12379187aULL, 0x2eb99de1bee77b9cULL, 0x21ecc0ea33cf4523ULL, - 0x59a4d7521805c7a1ULL, 0x3896f5eb56ae7c72ULL, 0xaa638f3db18f75dcULL, - 0x9f39358dabe9808eULL, 0xb7defa91c00b72acULL, 0x6b5541fd62492d92ULL, - 0x6dc6dee8f92e4d5bULL, 0x353f57abc4beea7eULL, 0x735769d6da5690ceULL, - 0x0a234aa642391484ULL, 0xf6f9508028f80d9dULL, 0xb8e319a27ab3f215ULL, - 0x31ad9c1151341a4dULL, 0x773c22a57bef5805ULL, 0x45c7561a07968633ULL, - 0xf913da9e249dbe36ULL, 0xda652d9b78a64c68ULL, 0x4c27a97f3bc334efULL, - 0x76621220e66b17f4ULL, 0x967743899acd7d0bULL, 0xf3ee5bcae0ed6782ULL, - 0x409f753600c879fcULL, 0x06d09a39b5926db6ULL, 0x6f83aeb0317ac588ULL, - 0x01e6ca4a86381f21ULL, 0x66ff3462d19f3025ULL, 0x72207c24ddfd3bfbULL, - 0x4af6b6d3e2ece2ebULL, 0x9c994dbec7ea08deULL, 0x49ace597b09a8bc4ULL, - 0xb38c4766cf0797baULL, 0x131b9373c57c2a75ULL, 0xb1822cce61931e58ULL, - 0x9d7555b909ba1c0cULL, 0x127fafdd937d11d2ULL, 0x29da3badc66d92e4ULL, - 0xa2c1d57154c2ecbcULL, 0x58c5134d82f6fe24ULL, 0x1c3ae3515b62274fULL, - 0xe907c82e01cb8126ULL, 0xf8ed091913e37fcbULL, 0x3249d8f9c80046c9ULL, - 0x80cf9bede388fb63ULL, 0x1881539a116cf19eULL, 0x5103f3f76bd52457ULL, - 0x15b7e6f5ae47f7a8ULL, 0xdbd7c6ded47e9ccfULL, 0x44e55c410228bb1aULL, - 0xb647d4255edb4e99ULL, 0x5d11882bb8aafc30ULL, 0xf5098bbb29d3212aULL, - 0x8fb5ea14e90296b3ULL, 0x677b942157dd025aULL, 0xfb58e7c0a390acb5ULL, - 0x89d3674c83bd4a01ULL, 0x9e2da4df4bf3b93bULL, 0xfcc41e328cab4829ULL, - 0x03f38c96ba582c52ULL, 0xcad1bdbd7fd85db2ULL, 0xbbb442c16082ae83ULL, - 0xb95fe86ba5da9ab0ULL, 0xb22e04673771a93fULL, 0x845358c9493152d8ULL, - 0xbe2a488697b4541eULL, 0x95a2dc2dd38e6966ULL, 0xc02c11ac923c852bULL, - 0x2388b1990df2a87bULL, 0x7c8008fa1b4f37beULL, 0x1f70d0c84d54e503ULL, - 0x5490adec7ece57d4ULL, 0x002b3c27d9063a3aULL, 0x7eaea3848030a2bfULL, - 0xc602326ded2003c0ULL, 0x83a7287d69a94086ULL, 0xc57a5fcb30f57a8aULL, - 0xb56844e479ebe779ULL, 0xa373b40f05dcbce9ULL, 0xd71a786e88570ee2ULL, - 0x879cbacdbde8f6a0ULL, 0x976ad1bcc164a32fULL, 0xab21e25e9666d78bULL, - 0x901063aae5e5c33cULL, 0x9818b34448698d90ULL, 0xe36487ae3e1e8abbULL, - 0xafbdf931893bdcb4ULL, 0x6345a0dc5fbbd519ULL, 0x8628fe269b9465caULL, - 0x1e5d01603f9c51ecULL, 0x4de44006a15049b7ULL, 0xbf6c70e5f776cbb1ULL, - 0x411218f2ef552bedULL, 0xcb0c0708705a36a3ULL, 0xe74d14754f986044ULL, - 0xcd56d9430ea8280eULL, 0xc12591d7535f5065ULL, 0xc83223f1720aef96ULL, - 0xc3a0396f7363a51fULL -}; - - -static void tgr192_round(u64 * ra, u64 * rb, u64 * rc, u64 x, int mul) -{ - u64 a = *ra; - u64 b = *rb; - u64 c = *rc; - - c ^= x; - a -= sbox1[c & 0xff] ^ sbox2[(c >> 16) & 0xff] - ^ sbox3[(c >> 32) & 0xff] ^ sbox4[(c >> 48) & 0xff]; - b += sbox4[(c >> 8) & 0xff] ^ sbox3[(c >> 24) & 0xff] - ^ sbox2[(c >> 40) & 0xff] ^ sbox1[(c >> 56) & 0xff]; - b *= mul; - - *ra = a; - *rb = b; - *rc = c; -} - - -static void tgr192_pass(u64 * ra, u64 * rb, u64 * rc, u64 * x, int mul) -{ - u64 a = *ra; - u64 b = *rb; - u64 c = *rc; - - tgr192_round(&a, &b, &c, x[0], mul); - tgr192_round(&b, &c, &a, x[1], mul); - tgr192_round(&c, &a, &b, x[2], mul); - tgr192_round(&a, &b, &c, x[3], mul); - tgr192_round(&b, &c, &a, x[4], mul); - tgr192_round(&c, &a, &b, x[5], mul); - tgr192_round(&a, &b, &c, x[6], mul); - tgr192_round(&b, &c, &a, x[7], mul); - - *ra = a; - *rb = b; - *rc = c; -} - - -static void tgr192_key_schedule(u64 * x) -{ - x[0] -= x[7] ^ 0xa5a5a5a5a5a5a5a5ULL; - x[1] ^= x[0]; - x[2] += x[1]; - x[3] -= x[2] ^ ((~x[1]) << 19); - x[4] ^= x[3]; - x[5] += x[4]; - x[6] -= x[5] ^ ((~x[4]) >> 23); - x[7] ^= x[6]; - x[0] += x[7]; - x[1] -= x[0] ^ ((~x[7]) << 19); - x[2] ^= x[1]; - x[3] += x[2]; - x[4] -= x[3] ^ ((~x[2]) >> 23); - x[5] ^= x[4]; - x[6] += x[5]; - x[7] -= x[6] ^ 0x0123456789abcdefULL; -} - - -/**************** - * Transform the message DATA which consists of 512 bytes (8 words) - */ - -static void tgr192_transform(struct tgr192_ctx *tctx, const u8 * data) -{ - u64 a, b, c, aa, bb, cc; - u64 x[8]; - int i; - - for (i = 0; i < 8; i++) - x[i] = get_unaligned_le64(data + i * sizeof(__le64)); - - /* save */ - a = aa = tctx->a; - b = bb = tctx->b; - c = cc = tctx->c; - - tgr192_pass(&a, &b, &c, x, 5); - tgr192_key_schedule(x); - tgr192_pass(&c, &a, &b, x, 7); - tgr192_key_schedule(x); - tgr192_pass(&b, &c, &a, x, 9); - - - /* feedforward */ - a ^= aa; - b -= bb; - c += cc; - /* store */ - tctx->a = a; - tctx->b = b; - tctx->c = c; -} - -static int tgr192_init(struct shash_desc *desc) -{ - struct tgr192_ctx *tctx = shash_desc_ctx(desc); - - tctx->a = 0x0123456789abcdefULL; - tctx->b = 0xfedcba9876543210ULL; - tctx->c = 0xf096a5b4c3b2e187ULL; - tctx->nblocks = 0; - tctx->count = 0; - - return 0; -} - - -/* Update the message digest with the contents - * of INBUF with length INLEN. */ -static int tgr192_update(struct shash_desc *desc, const u8 *inbuf, - unsigned int len) -{ - struct tgr192_ctx *tctx = shash_desc_ctx(desc); - - if (tctx->count == 64) { /* flush the buffer */ - tgr192_transform(tctx, tctx->hash); - tctx->count = 0; - tctx->nblocks++; - } - if (!inbuf) { - return 0; - } - if (tctx->count) { - for (; len && tctx->count < 64; len--) { - tctx->hash[tctx->count++] = *inbuf++; - } - tgr192_update(desc, NULL, 0); - if (!len) { - return 0; - } - - } - - while (len >= 64) { - tgr192_transform(tctx, inbuf); - tctx->count = 0; - tctx->nblocks++; - len -= 64; - inbuf += 64; - } - for (; len && tctx->count < 64; len--) { - tctx->hash[tctx->count++] = *inbuf++; - } - - return 0; -} - - - -/* The routine terminates the computation */ -static int tgr192_final(struct shash_desc *desc, u8 * out) -{ - struct tgr192_ctx *tctx = shash_desc_ctx(desc); - __be64 *dst = (__be64 *)out; - __be64 *be64p; - __le32 *le32p; - u32 t, msb, lsb; - - tgr192_update(desc, NULL, 0); /* flush */ - - msb = 0; - t = tctx->nblocks; - if ((lsb = t << 6) < t) { /* multiply by 64 to make a byte count */ - msb++; - } - msb += t >> 26; - t = lsb; - if ((lsb = t + tctx->count) < t) { /* add the count */ - msb++; - } - t = lsb; - if ((lsb = t << 3) < t) { /* multiply by 8 to make a bit count */ - msb++; - } - msb += t >> 29; - - if (tctx->count < 56) { /* enough room */ - tctx->hash[tctx->count++] = 0x01; /* pad */ - while (tctx->count < 56) { - tctx->hash[tctx->count++] = 0; /* pad */ - } - } else { /* need one extra block */ - tctx->hash[tctx->count++] = 0x01; /* pad character */ - while (tctx->count < 64) { - tctx->hash[tctx->count++] = 0; - } - tgr192_update(desc, NULL, 0); /* flush */ - memset(tctx->hash, 0, 56); /* fill next block with zeroes */ - } - /* append the 64 bit count */ - le32p = (__le32 *)&tctx->hash[56]; - le32p[0] = cpu_to_le32(lsb); - le32p[1] = cpu_to_le32(msb); - - tgr192_transform(tctx, tctx->hash); - - be64p = (__be64 *)tctx->hash; - dst[0] = be64p[0] = cpu_to_be64(tctx->a); - dst[1] = be64p[1] = cpu_to_be64(tctx->b); - dst[2] = be64p[2] = cpu_to_be64(tctx->c); - - return 0; -} - -static int tgr160_final(struct shash_desc *desc, u8 * out) -{ - u8 D[64]; - - tgr192_final(desc, D); - memcpy(out, D, TGR160_DIGEST_SIZE); - memzero_explicit(D, TGR192_DIGEST_SIZE); - - return 0; -} - -static int tgr128_final(struct shash_desc *desc, u8 * out) -{ - u8 D[64]; - - tgr192_final(desc, D); - memcpy(out, D, TGR128_DIGEST_SIZE); - memzero_explicit(D, TGR192_DIGEST_SIZE); - - return 0; -} - -static struct shash_alg tgr_algs[3] = { { - .digestsize = TGR192_DIGEST_SIZE, - .init = tgr192_init, - .update = tgr192_update, - .final = tgr192_final, - .descsize = sizeof(struct tgr192_ctx), - .base = { - .cra_name = "tgr192", - .cra_driver_name = "tgr192-generic", - .cra_blocksize = TGR192_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -}, { - .digestsize = TGR160_DIGEST_SIZE, - .init = tgr192_init, - .update = tgr192_update, - .final = tgr160_final, - .descsize = sizeof(struct tgr192_ctx), - .base = { - .cra_name = "tgr160", - .cra_driver_name = "tgr160-generic", - .cra_blocksize = TGR192_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -}, { - .digestsize = TGR128_DIGEST_SIZE, - .init = tgr192_init, - .update = tgr192_update, - .final = tgr128_final, - .descsize = sizeof(struct tgr192_ctx), - .base = { - .cra_name = "tgr128", - .cra_driver_name = "tgr128-generic", - .cra_blocksize = TGR192_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -} }; - -static int __init tgr192_mod_init(void) -{ - return crypto_register_shashes(tgr_algs, ARRAY_SIZE(tgr_algs)); -} - -static void __exit tgr192_mod_fini(void) -{ - crypto_unregister_shashes(tgr_algs, ARRAY_SIZE(tgr_algs)); -} - -MODULE_ALIAS_CRYPTO("tgr192"); -MODULE_ALIAS_CRYPTO("tgr160"); -MODULE_ALIAS_CRYPTO("tgr128"); - -subsys_initcall(tgr192_mod_init); -module_exit(tgr192_mod_fini); - -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("Tiger Message Digest Algorithm"); diff --git a/crypto/twofish_generic.c b/crypto/twofish_generic.c index 4f7c033224f92c2130c9523be7280a4da5d04694..86b2f067a4162acad710c0c9fe9125e8c7aa8ef0 100644 --- a/crypto/twofish_generic.c +++ b/crypto/twofish_generic.c @@ -24,7 +24,7 @@ * Third Edition. */ -#include +#include #include #include #include @@ -83,11 +83,11 @@ * whitening subkey number m. */ #define INPACK(n, x, m) \ - x = le32_to_cpu(src[n]) ^ ctx->w[m] + x = get_unaligned_le32(in + (n) * 4) ^ ctx->w[m] #define OUTUNPACK(n, x, m) \ x ^= ctx->w[m]; \ - dst[n] = cpu_to_le32(x) + put_unaligned_le32(x, out + (n) * 4) @@ -95,8 +95,6 @@ static void twofish_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) { struct twofish_ctx *ctx = crypto_tfm_ctx(tfm); - const __le32 *src = (const __le32 *)in; - __le32 *dst = (__le32 *)out; /* The four 32-bit chunks of the text. */ u32 a, b, c, d; @@ -132,8 +130,6 @@ static void twofish_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) static void twofish_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) { struct twofish_ctx *ctx = crypto_tfm_ctx(tfm); - const __le32 *src = (const __le32 *)in; - __le32 *dst = (__le32 *)out; /* The four 32-bit chunks of the text. */ u32 a, b, c, d; @@ -172,7 +168,6 @@ static struct crypto_alg alg = { .cra_flags = CRYPTO_ALG_TYPE_CIPHER, .cra_blocksize = TF_BLOCK_SIZE, .cra_ctxsize = sizeof(struct twofish_ctx), - .cra_alignmask = 3, .cra_module = THIS_MODULE, .cra_u = { .cipher = { .cia_min_keysize = TF_MIN_KEY_SIZE, diff --git a/crypto/vmac.c b/crypto/vmac.c index 9b565d1040d65bb8d4b5277f3bf94a3a33df2282..4633b2dda1e0a5a4fe7f9f07e4b53468d500c44d 100644 --- a/crypto/vmac.c +++ b/crypto/vmac.c @@ -36,6 +36,7 @@ #include #include #include +#include #include /* @@ -693,3 +694,4 @@ module_exit(vmac_module_exit); MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("VMAC hash algorithm"); MODULE_ALIAS_CRYPTO("vmac64"); +MODULE_IMPORT_NS(CRYPTO_INTERNAL); diff --git a/crypto/xcbc.c b/crypto/xcbc.c index af3b7eb5d7c7a2fd3ae5f87348fb514efe4218b2..6074c5c1da492e0f9045f5ac7b105abf9fa2748c 100644 --- a/crypto/xcbc.c +++ b/crypto/xcbc.c @@ -6,6 +6,7 @@ * Kazunori Miyazawa */ +#include #include #include #include @@ -272,3 +273,4 @@ module_exit(crypto_xcbc_module_exit); MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("XCBC keyed hash algorithm"); MODULE_ALIAS_CRYPTO("xcbc"); +MODULE_IMPORT_NS(CRYPTO_INTERNAL); diff --git a/crypto/xor.c b/crypto/xor.c index 8f899f898ec9f903734693cd6f4ba8a593bbcc14..8e72e5d5db0dede501ecbdb2bee28ab94aa1a834 100644 --- a/crypto/xor.c +++ b/crypto/xor.c @@ -95,7 +95,7 @@ do_xor_speed(struct xor_block_template *tmpl, void *b1, void *b2) for (i = 0; i < 3; i++) { start = ktime_get(); for (j = 0; j < REPS; j++) { - mb(); /* prevent loop optimzation */ + mb(); /* prevent loop optimization */ tmpl->do_2(BENCH_SIZE, b1, b2); mb(); } diff --git a/crypto/xts.c b/crypto/xts.c index ad45b009774b1172eb9e7e34c3f2a14c1f6b65c4..6c12f30dbdd6d19f3efc16a5da5c2f27d9a2cac6 100644 --- a/crypto/xts.c +++ b/crypto/xts.c @@ -7,6 +7,7 @@ * Based on ecb.c * Copyright (c) 2006 Herbert Xu */ +#include #include #include #include @@ -464,3 +465,4 @@ module_exit(xts_module_exit); MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("XTS block cipher mode"); MODULE_ALIAS_CRYPTO("xts"); +MODULE_IMPORT_NS(CRYPTO_INTERNAL); diff --git a/drivers/Kconfig b/drivers/Kconfig index dcecc9f6e33f78de53cf18655624b27d2abe76c4..62c753a73651ffba3ebe876a4049132c22590011 100644 --- a/drivers/Kconfig +++ b/drivers/Kconfig @@ -6,6 +6,7 @@ menu "Device Drivers" source "drivers/amba/Kconfig" source "drivers/eisa/Kconfig" source "drivers/pci/Kconfig" +source "drivers/cxl/Kconfig" source "drivers/pcmcia/Kconfig" source "drivers/rapidio/Kconfig" diff --git a/drivers/Makefile b/drivers/Makefile index fd11b9ac4cc369a1d68ffacb9791f67d1a43791e..6fba7daba59194dcc785d05f64e3c0296db9a2b7 100644 --- a/drivers/Makefile +++ b/drivers/Makefile @@ -27,7 +27,7 @@ obj-y += idle/ obj-y += char/ipmi/ obj-$(CONFIG_ACPI) += acpi/ -obj-$(CONFIG_SFI) += sfi/ + # PnP must come after ACPI since it will eventually need to check if acpi # was used and do nothing if so obj-$(CONFIG_PNP) += pnp/ @@ -73,6 +73,7 @@ obj-$(CONFIG_NVM) += lightnvm/ obj-y += base/ block/ misc/ mfd/ nfc/ obj-$(CONFIG_LIBNVDIMM) += nvdimm/ obj-$(CONFIG_DAX) += dax/ +obj-$(CONFIG_CXL_BUS) += cxl/ obj-$(CONFIG_DMA_SHARED_BUFFER) += dma-buf/ obj-$(CONFIG_NUBUS) += nubus/ obj-y += macintosh/ diff --git a/drivers/accessibility/speakup/serialio.c b/drivers/accessibility/speakup/serialio.c index 403b01d66367e69651afeffa60dfc155a25f1aac..53580bdc5baa1be549edeea82b577f878a16ad92 100644 --- a/drivers/accessibility/speakup/serialio.c +++ b/drivers/accessibility/speakup/serialio.c @@ -27,11 +27,11 @@ static const struct old_serial_port *serstate; static int timeouts; static int spk_serial_out(struct spk_synth *in_synth, const char ch); -static void spk_serial_send_xchar(char ch); -static void spk_serial_tiocmset(unsigned int set, unsigned int clear); -static unsigned char spk_serial_in(void); -static unsigned char spk_serial_in_nowait(void); -static void spk_serial_flush_buffer(void); +static void spk_serial_send_xchar(struct spk_synth *in_synth, char ch); +static void spk_serial_tiocmset(struct spk_synth *in_synth, unsigned int set, unsigned int clear); +static unsigned char spk_serial_in(struct spk_synth *in_synth); +static unsigned char spk_serial_in_nowait(struct spk_synth *in_synth); +static void spk_serial_flush_buffer(struct spk_synth *in_synth); static int spk_serial_wait_for_xmitr(struct spk_synth *in_synth); struct spk_io_ops spk_serial_io_ops = { @@ -150,7 +150,7 @@ static void start_serial_interrupt(int irq) outb(1, speakup_info.port_tts + UART_FCR); /* Turn FIFO On */ } -static void spk_serial_send_xchar(char ch) +static void spk_serial_send_xchar(struct spk_synth *synth, char ch) { int timeout = SPK_XMITR_TIMEOUT; @@ -162,7 +162,7 @@ static void spk_serial_send_xchar(char ch) outb(ch, speakup_info.port_tts); } -static void spk_serial_tiocmset(unsigned int set, unsigned int clear) +static void spk_serial_tiocmset(struct spk_synth *in_synth, unsigned int set, unsigned int clear) { int old = inb(speakup_info.port_tts + UART_MCR); @@ -251,7 +251,7 @@ static int spk_serial_wait_for_xmitr(struct spk_synth *in_synth) return 1; } -static unsigned char spk_serial_in(void) +static unsigned char spk_serial_in(struct spk_synth *in_synth) { int tmout = SPK_SERIAL_TIMEOUT; @@ -265,7 +265,7 @@ static unsigned char spk_serial_in(void) return inb_p(speakup_info.port_tts + UART_RX); } -static unsigned char spk_serial_in_nowait(void) +static unsigned char spk_serial_in_nowait(struct spk_synth *in_synth) { unsigned char lsr; @@ -275,7 +275,7 @@ static unsigned char spk_serial_in_nowait(void) return inb_p(speakup_info.port_tts + UART_RX); } -static void spk_serial_flush_buffer(void) +static void spk_serial_flush_buffer(struct spk_synth *in_synth) { /* TODO: flush the UART 16550 buffer */ } @@ -307,7 +307,7 @@ const char *spk_serial_synth_immediate(struct spk_synth *synth, } EXPORT_SYMBOL_GPL(spk_serial_synth_immediate); -void spk_serial_release(void) +void spk_serial_release(struct spk_synth *synth) { spk_stop_serial_interrupt(); if (speakup_info.port_tts == 0) diff --git a/drivers/accessibility/speakup/speakup_acntpc.c b/drivers/accessibility/speakup/speakup_acntpc.c index c94328a5bd4ac928d21b8204b1719db3a5d56ddd..c1ec087dca1307f8934942ac4a8f13103ab746f3 100644 --- a/drivers/accessibility/speakup/speakup_acntpc.c +++ b/drivers/accessibility/speakup/speakup_acntpc.c @@ -25,7 +25,7 @@ #define PROCSPEECH '\r' static int synth_probe(struct spk_synth *synth); -static void accent_release(void); +static void accent_release(struct spk_synth *synth); static const char *synth_immediate(struct spk_synth *synth, const char *buf); static void do_catch_up(struct spk_synth *synth); static void synth_flush(struct spk_synth *synth); @@ -294,7 +294,7 @@ static int synth_probe(struct spk_synth *synth) return 0; } -static void accent_release(void) +static void accent_release(struct spk_synth *synth) { spk_stop_serial_interrupt(); if (speakup_info.port_tts) diff --git a/drivers/accessibility/speakup/speakup_apollo.c b/drivers/accessibility/speakup/speakup_apollo.c index 0877b4044c28d079ceb56cb84dd09e852c73ce6b..cd63581b2e99edec7b7020e578bc8f3c3164cfb6 100644 --- a/drivers/accessibility/speakup/speakup_apollo.c +++ b/drivers/accessibility/speakup/speakup_apollo.c @@ -163,8 +163,8 @@ static void do_catch_up(struct spk_synth *synth) full_time_val = full_time->u.n.value; spin_unlock_irqrestore(&speakup_info.spinlock, flags); if (!synth->io_ops->synth_out(synth, ch)) { - synth->io_ops->tiocmset(0, UART_MCR_RTS); - synth->io_ops->tiocmset(UART_MCR_RTS, 0); + synth->io_ops->tiocmset(synth, 0, UART_MCR_RTS); + synth->io_ops->tiocmset(synth, UART_MCR_RTS, 0); schedule_timeout(msecs_to_jiffies(full_time_val)); continue; } diff --git a/drivers/accessibility/speakup/speakup_audptr.c b/drivers/accessibility/speakup/speakup_audptr.c index e6a6a9665d8fc8ae79e70151bb8d48415a56a05f..e89fd72579e609937821e0f43399a4034e5d49c0 100644 --- a/drivers/accessibility/speakup/speakup_audptr.c +++ b/drivers/accessibility/speakup/speakup_audptr.c @@ -119,8 +119,8 @@ static struct spk_synth synth_audptr = { static void synth_flush(struct spk_synth *synth) { - synth->io_ops->flush_buffer(); - synth->io_ops->send_xchar(SYNTH_CLEAR); + synth->io_ops->flush_buffer(synth); + synth->io_ops->send_xchar(synth, SYNTH_CLEAR); synth->io_ops->synth_out(synth, PROCSPEECH); } @@ -130,11 +130,11 @@ static void synth_version(struct spk_synth *synth) char synth_id[40] = ""; synth->synth_immediate(synth, "\x05[Q]"); - synth_id[test] = synth->io_ops->synth_in(); + synth_id[test] = synth->io_ops->synth_in(synth); if (synth_id[test] == 'A') { do { /* read version string from synth */ - synth_id[++test] = synth->io_ops->synth_in(); + synth_id[++test] = synth->io_ops->synth_in(synth); } while (synth_id[test] != '\n' && test < 32); synth_id[++test] = 0x00; } diff --git a/drivers/accessibility/speakup/speakup_decext.c b/drivers/accessibility/speakup/speakup_decext.c index 7408eb29cf3893fc64f0e07bced6103f38229d6c..092cfd08a9e18d8a274d7e4d14ea5d0a47b1afec 100644 --- a/drivers/accessibility/speakup/speakup_decext.c +++ b/drivers/accessibility/speakup/speakup_decext.c @@ -218,7 +218,7 @@ static void do_catch_up(struct spk_synth *synth) static void synth_flush(struct spk_synth *synth) { in_escape = 0; - synth->io_ops->flush_buffer(); + synth->io_ops->flush_buffer(synth); synth->synth_immediate(synth, "\033P;10z\033\\"); } diff --git a/drivers/accessibility/speakup/speakup_decpc.c b/drivers/accessibility/speakup/speakup_decpc.c index 96f24c848cc56329c68e897613ebc9bc8319f0d0..dec314dee214f0ccdd63f0a6f94f615f892fcbdc 100644 --- a/drivers/accessibility/speakup/speakup_decpc.c +++ b/drivers/accessibility/speakup/speakup_decpc.c @@ -125,7 +125,7 @@ enum { PRIMARY_DIC = 0, USER_DIC, COMMAND_DIC, ABBREV_DIC }; #define SYNTH_IO_EXTENT 8 static int synth_probe(struct spk_synth *synth); -static void dtpc_release(void); +static void dtpc_release(struct spk_synth *synth); static const char *synth_immediate(struct spk_synth *synth, const char *buf); static void do_catch_up(struct spk_synth *synth); static void synth_flush(struct spk_synth *synth); @@ -474,7 +474,7 @@ static int synth_probe(struct spk_synth *synth) return 0; } -static void dtpc_release(void) +static void dtpc_release(struct spk_synth *synth) { spk_stop_serial_interrupt(); if (speakup_info.port_tts) diff --git a/drivers/accessibility/speakup/speakup_dectlk.c b/drivers/accessibility/speakup/speakup_dectlk.c index ab6d61e80b1cbac8d66951190a89d5f6c99ee7f7..580ec796816bcf0eefdc66e23329ee10f173f19f 100644 --- a/drivers/accessibility/speakup/speakup_dectlk.c +++ b/drivers/accessibility/speakup/speakup_dectlk.c @@ -78,6 +78,8 @@ static struct kobj_attribute direct_attribute = __ATTR(direct, 0644, spk_var_show, spk_var_store); static struct kobj_attribute full_time_attribute = __ATTR(full_time, 0644, spk_var_show, spk_var_store); +static struct kobj_attribute flush_time_attribute = + __ATTR(flush_time, 0644, spk_var_show, spk_var_store); static struct kobj_attribute jiffy_delta_attribute = __ATTR(jiffy_delta, 0644, spk_var_show, spk_var_store); static struct kobj_attribute trigger_time_attribute = @@ -99,6 +101,7 @@ static struct attribute *synth_attrs[] = { &delay_time_attribute.attr, &direct_attribute.attr, &full_time_attribute.attr, + &flush_time_attribute.attr, &jiffy_delta_attribute.attr, &trigger_time_attribute.attr, NULL, /* need to NULL terminate the list of attributes */ @@ -118,6 +121,7 @@ static struct spk_synth synth_dectlk = { .trigger = 50, .jiffies = 50, .full = 40000, + .flush_time = 4000, .dev_name = SYNTH_DEFAULT_DEV, .startup = SYNTH_START, .checkval = SYNTH_CHECK, @@ -200,18 +204,23 @@ static void do_catch_up(struct spk_synth *synth) static u_char last = '\0'; unsigned long flags; unsigned long jiff_max; - unsigned long timeout = msecs_to_jiffies(4000); + unsigned long timeout; DEFINE_WAIT(wait); struct var_t *jiffy_delta; struct var_t *delay_time; + struct var_t *flush_time; int jiffy_delta_val; int delay_time_val; + int timeout_val; jiffy_delta = spk_get_var(JIFFY); delay_time = spk_get_var(DELAY); + flush_time = spk_get_var(FLUSH); spin_lock_irqsave(&speakup_info.spinlock, flags); jiffy_delta_val = jiffy_delta->u.n.value; + timeout_val = flush_time->u.n.value; spin_unlock_irqrestore(&speakup_info.spinlock, flags); + timeout = msecs_to_jiffies(timeout_val); jiff_max = jiffies + jiffy_delta_val; while (!kthread_should_stop()) { @@ -289,7 +298,7 @@ static void synth_flush(struct spk_synth *synth) synth->io_ops->synth_out(synth, ']'); in_escape = 0; is_flushing = 1; - synth->io_ops->flush_buffer(); + synth->io_ops->flush_buffer(synth); synth->io_ops->synth_out(synth, SYNTH_CLEAR); } diff --git a/drivers/accessibility/speakup/speakup_dtlk.c b/drivers/accessibility/speakup/speakup_dtlk.c index dbebed0eeeec519cf53bf7e942aa89aa37796994..92838d3ae9eb8d3462f8f33ee0525b14fd7d7e63 100644 --- a/drivers/accessibility/speakup/speakup_dtlk.c +++ b/drivers/accessibility/speakup/speakup_dtlk.c @@ -24,7 +24,7 @@ #define PROCSPEECH 0x00 static int synth_probe(struct spk_synth *synth); -static void dtlk_release(void); +static void dtlk_release(struct spk_synth *synth); static const char *synth_immediate(struct spk_synth *synth, const char *buf); static void do_catch_up(struct spk_synth *synth); static void synth_flush(struct spk_synth *synth); @@ -365,7 +365,7 @@ static int synth_probe(struct spk_synth *synth) return 0; } -static void dtlk_release(void) +static void dtlk_release(struct spk_synth *synth) { spk_stop_serial_interrupt(); if (speakup_info.port_tts) diff --git a/drivers/accessibility/speakup/speakup_keypc.c b/drivers/accessibility/speakup/speakup_keypc.c index 414827e888fcb9fdf8d8bca90ad8f2817d950ec8..311f4aa0be22d6bc62f8e53e36580025ad6ef66a 100644 --- a/drivers/accessibility/speakup/speakup_keypc.c +++ b/drivers/accessibility/speakup/speakup_keypc.c @@ -24,7 +24,7 @@ #define SYNTH_CLEAR 0x03 static int synth_probe(struct spk_synth *synth); -static void keynote_release(void); +static void keynote_release(struct spk_synth *synth); static const char *synth_immediate(struct spk_synth *synth, const char *buf); static void do_catch_up(struct spk_synth *synth); static void synth_flush(struct spk_synth *synth); @@ -295,7 +295,7 @@ static int synth_probe(struct spk_synth *synth) return 0; } -static void keynote_release(void) +static void keynote_release(struct spk_synth *synth) { spk_stop_serial_interrupt(); if (synth_port) diff --git a/drivers/accessibility/speakup/speakup_ltlk.c b/drivers/accessibility/speakup/speakup_ltlk.c index 3c59519a871f76248ae400e9f18adc6c720529ba..3e59b387d0c4f92e9c14be441bc1235b350f5b62 100644 --- a/drivers/accessibility/speakup/speakup_ltlk.c +++ b/drivers/accessibility/speakup/speakup_ltlk.c @@ -132,7 +132,7 @@ static void synth_interrogate(struct spk_synth *synth) synth->synth_immediate(synth, "\x18\x01?"); for (i = 0; i < 50; i++) { - buf[i] = synth->io_ops->synth_in(); + buf[i] = synth->io_ops->synth_in(synth); if (i > 2 && buf[i] == 0x7f) break; } diff --git a/drivers/accessibility/speakup/speakup_soft.c b/drivers/accessibility/speakup/speakup_soft.c index 9a7029539f35ddfa30d3535c447f9315c2198f9c..c3f97c572fb64f790e90fb011e3c54347786bfde 100644 --- a/drivers/accessibility/speakup/speakup_soft.c +++ b/drivers/accessibility/speakup/speakup_soft.c @@ -24,7 +24,7 @@ #define CLEAR_SYNTH 0x18 static int softsynth_probe(struct spk_synth *synth); -static void softsynth_release(void); +static void softsynth_release(struct spk_synth *synth); static int softsynth_is_alive(struct spk_synth *synth); static unsigned char get_index(struct spk_synth *synth); @@ -402,7 +402,7 @@ static int softsynth_probe(struct spk_synth *synth) return 0; } -static void softsynth_release(void) +static void softsynth_release(struct spk_synth *synth) { misc_deregister(&synth_device); misc_deregister(&synthu_device); diff --git a/drivers/accessibility/speakup/speakup_spkout.c b/drivers/accessibility/speakup/speakup_spkout.c index 6e933bf1de2e2ecb8396c8a33e02297b27826b8d..bd3d8dc300ff331b4128784a4a66914e3fa6ca7b 100644 --- a/drivers/accessibility/speakup/speakup_spkout.c +++ b/drivers/accessibility/speakup/speakup_spkout.c @@ -117,8 +117,8 @@ static struct spk_synth synth_spkout = { static void synth_flush(struct spk_synth *synth) { - synth->io_ops->flush_buffer(); - synth->io_ops->send_xchar(SYNTH_CLEAR); + synth->io_ops->flush_buffer(synth); + synth->io_ops->send_xchar(synth, SYNTH_CLEAR); } module_param_named(ser, synth_spkout.ser, int, 0444); diff --git a/drivers/accessibility/speakup/spk_priv.h b/drivers/accessibility/speakup/spk_priv.h index 0f4bcbe5ddb93974fcfbe7979d8cff60f8b0c403..9da57ead17cbe309890640e76899acd54e7e804d 100644 --- a/drivers/accessibility/speakup/spk_priv.h +++ b/drivers/accessibility/speakup/spk_priv.h @@ -34,8 +34,8 @@ const struct old_serial_port *spk_serial_init(int index); void spk_stop_serial_interrupt(void); -void spk_serial_release(void); -void spk_ttyio_release(void); +void spk_serial_release(struct spk_synth *synth); +void spk_ttyio_release(struct spk_synth *synth); void spk_ttyio_register_ldisc(void); void spk_ttyio_unregister_ldisc(void); diff --git a/drivers/accessibility/speakup/spk_ttyio.c b/drivers/accessibility/speakup/spk_ttyio.c index 835d17455fcdb1a57e77f861c80a2033b6874122..9af1d4c124d378d312e943c260c43afd426c8156 100644 --- a/drivers/accessibility/speakup/spk_ttyio.c +++ b/drivers/accessibility/speakup/spk_ttyio.c @@ -12,14 +12,15 @@ struct spk_ldisc_data { char buf; struct completion completion; bool buf_free; + struct spk_synth *synth; }; -static struct spk_synth *spk_ttyio_synth; -static struct tty_struct *speakup_tty; -/* mutex to protect against speakup_tty disappearing from underneath us while - * we are using it. this can happen when the device physically unplugged, - * while in use. it also serialises access to speakup_tty. +/* + * This allows to catch within spk_ttyio_ldisc_open whether it is getting set + * on for a speakup-driven device. */ +static struct tty_struct *speakup_tty; +/* This mutex serializes the use of such global speakup_tty variable */ static DEFINE_MUTEX(speakup_tty_mutex); static int ser_to_dev(int ser, dev_t *dev_no) @@ -67,22 +68,20 @@ static int spk_ttyio_ldisc_open(struct tty_struct *tty) static void spk_ttyio_ldisc_close(struct tty_struct *tty) { - mutex_lock(&speakup_tty_mutex); - kfree(speakup_tty->disc_data); - speakup_tty = NULL; - mutex_unlock(&speakup_tty_mutex); + kfree(tty->disc_data); } static int spk_ttyio_receive_buf2(struct tty_struct *tty, const unsigned char *cp, char *fp, int count) { struct spk_ldisc_data *ldisc_data = tty->disc_data; + struct spk_synth *synth = ldisc_data->synth; - if (spk_ttyio_synth->read_buff_add) { + if (synth->read_buff_add) { int i; for (i = 0; i < count; i++) - spk_ttyio_synth->read_buff_add(cp[i]); + synth->read_buff_add(cp[i]); return count; } @@ -114,11 +113,11 @@ static struct tty_ldisc_ops spk_ttyio_ldisc_ops = { static int spk_ttyio_out(struct spk_synth *in_synth, const char ch); static int spk_ttyio_out_unicode(struct spk_synth *in_synth, u16 ch); -static void spk_ttyio_send_xchar(char ch); -static void spk_ttyio_tiocmset(unsigned int set, unsigned int clear); -static unsigned char spk_ttyio_in(void); -static unsigned char spk_ttyio_in_nowait(void); -static void spk_ttyio_flush_buffer(void); +static void spk_ttyio_send_xchar(struct spk_synth *in_synth, char ch); +static void spk_ttyio_tiocmset(struct spk_synth *in_synth, unsigned int set, unsigned int clear); +static unsigned char spk_ttyio_in(struct spk_synth *in_synth); +static unsigned char spk_ttyio_in_nowait(struct spk_synth *in_synth); +static void spk_ttyio_flush_buffer(struct spk_synth *in_synth); static int spk_ttyio_wait_for_xmitr(struct spk_synth *in_synth); struct spk_io_ops spk_ttyio_ops = { @@ -187,13 +186,17 @@ static int spk_ttyio_initialise_ldisc(struct spk_synth *synth) mutex_lock(&speakup_tty_mutex); speakup_tty = tty; ret = tty_set_ldisc(tty, N_SPEAKUP); - if (ret) - speakup_tty = NULL; + speakup_tty = NULL; mutex_unlock(&speakup_tty_mutex); - if (!ret) + if (!ret) { /* Success */ + struct spk_ldisc_data *ldisc_data = tty->disc_data; + + ldisc_data->synth = synth; + synth->dev = tty; return 0; + } pr_err("speakup: Failed to set N_SPEAKUP on tty\n"); @@ -221,29 +224,30 @@ void spk_ttyio_unregister_ldisc(void) static int spk_ttyio_out(struct spk_synth *in_synth, const char ch) { - mutex_lock(&speakup_tty_mutex); - if (in_synth->alive && speakup_tty && speakup_tty->ops->write) { - int ret = speakup_tty->ops->write(speakup_tty, &ch, 1); - - mutex_unlock(&speakup_tty_mutex); - if (ret == 0) - /* No room */ - return 0; - if (ret < 0) { - pr_warn("%s: I/O error, deactivating speakup\n", - in_synth->long_name); - /* No synth any more, so nobody will restart TTYs, - * and we thus need to do it ourselves. Now that there - * is no synth we can let application flood anyway - */ - in_synth->alive = 0; - speakup_start_ttys(); - return 0; - } + struct tty_struct *tty = in_synth->dev; + int ret; + + if (!in_synth->alive || !tty->ops->write) + return 0; + + ret = tty->ops->write(tty, &ch, 1); + + if (ret == 0) + /* No room */ + return 0; + + if (ret > 0) + /* Success */ return 1; - } - mutex_unlock(&speakup_tty_mutex); + pr_warn("%s: I/O error, deactivating speakup\n", + in_synth->long_name); + /* No synth any more, so nobody will restart TTYs, + * and we thus need to do it ourselves. Now that there + * is no synth we can let application flood anyway + */ + in_synth->alive = 0; + speakup_start_ttys(); return 0; } @@ -264,47 +268,20 @@ static int spk_ttyio_out_unicode(struct spk_synth *in_synth, u16 ch) return ret; } -static int check_tty(struct tty_struct *tty) -{ - if (!tty) { - pr_warn("%s: I/O error, deactivating speakup\n", - spk_ttyio_synth->long_name); - /* No synth any more, so nobody will restart TTYs, and we thus - * need to do it ourselves. Now that there is no synth we can - * let application flood anyway - */ - spk_ttyio_synth->alive = 0; - speakup_start_ttys(); - return 1; - } - - return 0; -} - -static void spk_ttyio_send_xchar(char ch) +static void spk_ttyio_send_xchar(struct spk_synth *in_synth, char ch) { - mutex_lock(&speakup_tty_mutex); - if (check_tty(speakup_tty)) { - mutex_unlock(&speakup_tty_mutex); - return; - } + struct tty_struct *tty = in_synth->dev; - if (speakup_tty->ops->send_xchar) - speakup_tty->ops->send_xchar(speakup_tty, ch); - mutex_unlock(&speakup_tty_mutex); + if (tty->ops->send_xchar) + tty->ops->send_xchar(tty, ch); } -static void spk_ttyio_tiocmset(unsigned int set, unsigned int clear) +static void spk_ttyio_tiocmset(struct spk_synth *in_synth, unsigned int set, unsigned int clear) { - mutex_lock(&speakup_tty_mutex); - if (check_tty(speakup_tty)) { - mutex_unlock(&speakup_tty_mutex); - return; - } + struct tty_struct *tty = in_synth->dev; - if (speakup_tty->ops->tiocmset) - speakup_tty->ops->tiocmset(speakup_tty, set, clear); - mutex_unlock(&speakup_tty_mutex); + if (tty->ops->tiocmset) + tty->ops->tiocmset(tty, set, clear); } static int spk_ttyio_wait_for_xmitr(struct spk_synth *in_synth) @@ -312,9 +289,10 @@ static int spk_ttyio_wait_for_xmitr(struct spk_synth *in_synth) return 1; } -static unsigned char ttyio_in(int timeout) +static unsigned char ttyio_in(struct spk_synth *in_synth, int timeout) { - struct spk_ldisc_data *ldisc_data = speakup_tty->disc_data; + struct tty_struct *tty = in_synth->dev; + struct spk_ldisc_data *ldisc_data = tty->disc_data; char rv; if (!timeout) { @@ -334,35 +312,29 @@ static unsigned char ttyio_in(int timeout) mb(); ldisc_data->buf_free = true; /* Let TTY push more characters */ - tty_schedule_flip(speakup_tty->port); + tty_schedule_flip(tty->port); return rv; } -static unsigned char spk_ttyio_in(void) +static unsigned char spk_ttyio_in(struct spk_synth *in_synth) { - return ttyio_in(SPK_SYNTH_TIMEOUT); + return ttyio_in(in_synth, SPK_SYNTH_TIMEOUT); } -static unsigned char spk_ttyio_in_nowait(void) +static unsigned char spk_ttyio_in_nowait(struct spk_synth *in_synth) { - u8 rv = ttyio_in(0); + u8 rv = ttyio_in(in_synth, 0); return (rv == 0xff) ? 0 : rv; } -static void spk_ttyio_flush_buffer(void) +static void spk_ttyio_flush_buffer(struct spk_synth *in_synth) { - mutex_lock(&speakup_tty_mutex); - if (check_tty(speakup_tty)) { - mutex_unlock(&speakup_tty_mutex); - return; - } + struct tty_struct *tty = in_synth->dev; - if (speakup_tty->ops->flush_buffer) - speakup_tty->ops->flush_buffer(speakup_tty); - - mutex_unlock(&speakup_tty_mutex); + if (tty->ops->flush_buffer) + tty->ops->flush_buffer(tty); } int spk_ttyio_synth_probe(struct spk_synth *synth) @@ -373,37 +345,38 @@ int spk_ttyio_synth_probe(struct spk_synth *synth) return rv; synth->alive = 1; - spk_ttyio_synth = synth; return 0; } EXPORT_SYMBOL_GPL(spk_ttyio_synth_probe); -void spk_ttyio_release(void) +void spk_ttyio_release(struct spk_synth *in_synth) { - if (!speakup_tty) - return; + struct tty_struct *tty = in_synth->dev; - tty_lock(speakup_tty); + tty_lock(tty); - if (speakup_tty->ops->close) - speakup_tty->ops->close(speakup_tty, NULL); + if (tty->ops->close) + tty->ops->close(tty, NULL); + + tty_ldisc_flush(tty); + tty_unlock(tty); + tty_kclose(tty); - tty_ldisc_flush(speakup_tty); - tty_unlock(speakup_tty); - tty_kclose(speakup_tty); + in_synth->dev = NULL; } EXPORT_SYMBOL_GPL(spk_ttyio_release); -const char *spk_ttyio_synth_immediate(struct spk_synth *synth, const char *buff) +const char *spk_ttyio_synth_immediate(struct spk_synth *in_synth, const char *buff) { + struct tty_struct *tty = in_synth->dev; u_char ch; while ((ch = *buff)) { if (ch == '\n') - ch = synth->procspeech; - if (tty_write_room(speakup_tty) < 1 || - !synth->io_ops->synth_out(synth, ch)) + ch = in_synth->procspeech; + if (tty_write_room(tty) < 1 || + !in_synth->io_ops->synth_out(in_synth, ch)) return buff; buff++; } diff --git a/drivers/accessibility/speakup/spk_types.h b/drivers/accessibility/speakup/spk_types.h index 91fca3033a45a6a987c282b3d035aa084d0a45e2..6a96ad94bc3f84255c6b14fb96db35ee1f017688 100644 --- a/drivers/accessibility/speakup/spk_types.h +++ b/drivers/accessibility/speakup/spk_types.h @@ -48,7 +48,7 @@ enum var_id_t { ATTRIB_BLEEP, BLEEPS, RATE, PITCH, VOL, TONE, PUNCT, VOICE, FREQUENCY, LANG, DIRECT, PAUSE, - CAPS_START, CAPS_STOP, CHARTAB, INFLECTION, + CAPS_START, CAPS_STOP, CHARTAB, INFLECTION, FLUSH, MAXVARS }; @@ -157,11 +157,11 @@ struct spk_synth; struct spk_io_ops { int (*synth_out)(struct spk_synth *synth, const char ch); int (*synth_out_unicode)(struct spk_synth *synth, u16 ch); - void (*send_xchar)(char ch); - void (*tiocmset)(unsigned int set, unsigned int clear); - unsigned char (*synth_in)(void); - unsigned char (*synth_in_nowait)(void); - void (*flush_buffer)(void); + void (*send_xchar)(struct spk_synth *synth, char ch); + void (*tiocmset)(struct spk_synth *synth, unsigned int set, unsigned int clear); + unsigned char (*synth_in)(struct spk_synth *synth); + unsigned char (*synth_in_nowait)(struct spk_synth *synth); + void (*flush_buffer)(struct spk_synth *synth); int (*wait_for_xmitr)(struct spk_synth *synth); }; @@ -178,6 +178,7 @@ struct spk_synth { int trigger; int jiffies; int full; + int flush_time; int ser; char *dev_name; short flags; @@ -188,7 +189,7 @@ struct spk_synth { int *default_vol; struct spk_io_ops *io_ops; int (*probe)(struct spk_synth *synth); - void (*release)(void); + void (*release)(struct spk_synth *synth); const char *(*synth_immediate)(struct spk_synth *synth, const char *buff); void (*catch_up)(struct spk_synth *synth); @@ -200,6 +201,8 @@ struct spk_synth { struct synth_indexing indexing; int alive; struct attribute_group attributes; + + void *dev; }; /* diff --git a/drivers/accessibility/speakup/synth.c b/drivers/accessibility/speakup/synth.c index ac47dbac72075d03f7a67c51c8c8ce66c9351f6f..2b8699673bace5571ea623d55409a6d8bdf03225 100644 --- a/drivers/accessibility/speakup/synth.c +++ b/drivers/accessibility/speakup/synth.c @@ -137,14 +137,14 @@ EXPORT_SYMBOL_GPL(spk_do_catch_up_unicode); void spk_synth_flush(struct spk_synth *synth) { - synth->io_ops->flush_buffer(); + synth->io_ops->flush_buffer(synth); synth->io_ops->synth_out(synth, synth->clear); } EXPORT_SYMBOL_GPL(spk_synth_flush); unsigned char spk_synth_get_index(struct spk_synth *synth) { - return synth->io_ops->synth_in_nowait(); + return synth->io_ops->synth_in_nowait(synth); } EXPORT_SYMBOL_GPL(spk_synth_get_index); @@ -348,6 +348,7 @@ struct var_t synth_time_vars[] = { { TRIGGER, .u.n = {NULL, 20, 10, 2000, 0, 0, NULL } }, { JIFFY, .u.n = {NULL, 50, 20, 200, 0, 0, NULL } }, { FULL, .u.n = {NULL, 400, 200, 60000, 0, 0, NULL } }, + { FLUSH, .u.n = {NULL, 4000, 100, 4000, 0, 0, NULL } }, V_LAST_VAR }; @@ -408,6 +409,8 @@ static int do_synth_init(struct spk_synth *in_synth) synth_time_vars[2].u.n.default_val = synth->jiffies; synth_time_vars[3].u.n.value = synth_time_vars[3].u.n.default_val = synth->full; + synth_time_vars[4].u.n.value = + synth_time_vars[4].u.n.default_val = synth->flush_time; synth_printf("%s", synth->init); for (var = synth->vars; (var->var_id >= 0) && (var->var_id < MAXVARS); var++) @@ -440,7 +443,7 @@ void synth_release(void) sysfs_remove_group(speakup_kobj, &synth->attributes); for (var = synth->vars; var->var_id != MAXVARS; var++) speakup_unregister_var(var->var_id); - synth->release(); + synth->release(synth); synth = NULL; } diff --git a/drivers/accessibility/speakup/varhandlers.c b/drivers/accessibility/speakup/varhandlers.c index d7f6bec7ff06f6439e12b2d818a12cc91143bae9..067c0da97dcbf16cea24f32cdd069ffb7f27af65 100644 --- a/drivers/accessibility/speakup/varhandlers.c +++ b/drivers/accessibility/speakup/varhandlers.c @@ -23,6 +23,7 @@ static struct st_var_header var_headers[] = { { "trigger_time", TRIGGER, VAR_TIME, NULL, NULL }, { "jiffy_delta", JIFFY, VAR_TIME, NULL, NULL }, { "full_time", FULL, VAR_TIME, NULL, NULL }, + { "flush_time", FLUSH, VAR_TIME, NULL, NULL }, { "spell_delay", SPELL_DELAY, VAR_NUM, &spk_spell_delay, NULL }, { "bleeps", BLEEPS, VAR_NUM, &spk_bleeps, NULL }, { "attrib_bleep", ATTRIB_BLEEP, VAR_NUM, &spk_attrib_bleep, NULL }, diff --git a/drivers/acpi/Kconfig b/drivers/acpi/Kconfig index fc06945d3f99d24c9e42a5483d00fc99f2040f1c..eedec61e3476ebec2d85e16efcd15bc80d3f5a05 100644 --- a/drivers/acpi/Kconfig +++ b/drivers/acpi/Kconfig @@ -87,6 +87,14 @@ config ACPI_SPCR_TABLE This table provides information about the configuration of the earlycon console. +config ACPI_FPDT + bool "ACPI Firmware Performance Data Table (FPDT) support" + depends on X86_64 + help + Enable support for the Firmware Performance Data Table (FPDT). + This table provides information on the timing of the system + boot, S3 suspend and S3 resume firmware code paths. + config ACPI_LPIT bool depends on X86_64 @@ -327,21 +335,7 @@ config ACPI_THERMAL the module will be called thermal. config ACPI_PLATFORM_PROFILE - tristate "ACPI Platform Profile Driver" - default m - help - This driver adds support for platform-profiles on platforms that - support it. - - Platform-profiles can be used to control the platform behaviour. For - example whether to operate in a lower power mode, in a higher - power performance mode or between the two. - - This driver provides the sysfs interface and is used as the registration - point for platform specific drivers. - - Which profiles are supported is determined on a per-platform basis and - should be obtained from the platform specific driver. + tristate config ACPI_CUSTOM_DSDT_FILE string "Custom DSDT Table file to include" diff --git a/drivers/acpi/Makefile b/drivers/acpi/Makefile index 52b627c7f9772ccf46286b152732bcbb2d6dcfb1..700b41adf2db6dd82f272b5a0983d89a9ff7b00e 100644 --- a/drivers/acpi/Makefile +++ b/drivers/acpi/Makefile @@ -57,6 +57,7 @@ acpi-$(CONFIG_X86) += x86/utils.o acpi-$(CONFIG_X86) += x86/s2idle.o acpi-$(CONFIG_DEBUG_FS) += debugfs.o acpi-y += acpi_lpat.o +acpi-$(CONFIG_ACPI_FPDT) += acpi_fpdt.o acpi-$(CONFIG_ACPI_LPIT) += acpi_lpit.o acpi-$(CONFIG_ACPI_GENERIC_GSI) += irq.o acpi-$(CONFIG_ACPI_WATCHDOG) += acpi_watchdog.o diff --git a/drivers/acpi/acpi_fpdt.c b/drivers/acpi/acpi_fpdt.c new file mode 100644 index 0000000000000000000000000000000000000000..a89a806a7a2a943ba4a0cb5a2c5d691270509a2f --- /dev/null +++ b/drivers/acpi/acpi_fpdt.c @@ -0,0 +1,264 @@ +// SPDX-License-Identifier: GPL-2.0-only + +/* + * FPDT support for exporting boot and suspend/resume performance data + * + * Copyright (C) 2021 Intel Corporation. All rights reserved. + */ + +#define pr_fmt(fmt) "ACPI FPDT: " fmt + +#include + +/* + * FPDT contains ACPI table header and a number of fpdt_subtable_entries. + * Each fpdt_subtable_entry points to a subtable: FBPT or S3PT. + * Each FPDT subtable (FBPT/S3PT) is composed of a fpdt_subtable_header + * and a number of fpdt performance records. + * Each FPDT performance record is composed of a fpdt_record_header and + * performance data fields, for boot or suspend or resume phase. + */ +enum fpdt_subtable_type { + SUBTABLE_FBPT, + SUBTABLE_S3PT, +}; + +struct fpdt_subtable_entry { + u16 type; /* refer to enum fpdt_subtable_type */ + u8 length; + u8 revision; + u32 reserved; + u64 address; /* physical address of the S3PT/FBPT table */ +}; + +struct fpdt_subtable_header { + u32 signature; + u32 length; +}; + +enum fpdt_record_type { + RECORD_S3_RESUME, + RECORD_S3_SUSPEND, + RECORD_BOOT, +}; + +struct fpdt_record_header { + u16 type; /* refer to enum fpdt_record_type */ + u8 length; + u8 revision; +}; + +struct resume_performance_record { + struct fpdt_record_header header; + u32 resume_count; + u64 resume_prev; + u64 resume_avg; +} __attribute__((packed)); + +struct boot_performance_record { + struct fpdt_record_header header; + u32 reserved; + u64 firmware_start; + u64 bootloader_load; + u64 bootloader_launch; + u64 exitbootservice_start; + u64 exitbootservice_end; +} __attribute__((packed)); + +struct suspend_performance_record { + struct fpdt_record_header header; + u64 suspend_start; + u64 suspend_end; +} __attribute__((packed)); + + +static struct resume_performance_record *record_resume; +static struct suspend_performance_record *record_suspend; +static struct boot_performance_record *record_boot; + +#define FPDT_ATTR(phase, name) \ +static ssize_t name##_show(struct kobject *kobj, \ + struct kobj_attribute *attr, char *buf) \ +{ \ + return sprintf(buf, "%llu\n", record_##phase->name); \ +} \ +static struct kobj_attribute name##_attr = \ +__ATTR(name##_ns, 0444, name##_show, NULL) + +FPDT_ATTR(resume, resume_prev); +FPDT_ATTR(resume, resume_avg); +FPDT_ATTR(suspend, suspend_start); +FPDT_ATTR(suspend, suspend_end); +FPDT_ATTR(boot, firmware_start); +FPDT_ATTR(boot, bootloader_load); +FPDT_ATTR(boot, bootloader_launch); +FPDT_ATTR(boot, exitbootservice_start); +FPDT_ATTR(boot, exitbootservice_end); + +static ssize_t resume_count_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + return sprintf(buf, "%u\n", record_resume->resume_count); +} + +static struct kobj_attribute resume_count_attr = +__ATTR_RO(resume_count); + +static struct attribute *resume_attrs[] = { + &resume_count_attr.attr, + &resume_prev_attr.attr, + &resume_avg_attr.attr, + NULL +}; + +static const struct attribute_group resume_attr_group = { + .attrs = resume_attrs, + .name = "resume", +}; + +static struct attribute *suspend_attrs[] = { + &suspend_start_attr.attr, + &suspend_end_attr.attr, + NULL +}; + +static const struct attribute_group suspend_attr_group = { + .attrs = suspend_attrs, + .name = "suspend", +}; + +static struct attribute *boot_attrs[] = { + &firmware_start_attr.attr, + &bootloader_load_attr.attr, + &bootloader_launch_attr.attr, + &exitbootservice_start_attr.attr, + &exitbootservice_end_attr.attr, + NULL +}; + +static const struct attribute_group boot_attr_group = { + .attrs = boot_attrs, + .name = "boot", +}; + +static struct kobject *fpdt_kobj; + +static int fpdt_process_subtable(u64 address, u32 subtable_type) +{ + struct fpdt_subtable_header *subtable_header; + struct fpdt_record_header *record_header; + char *signature = (subtable_type == SUBTABLE_FBPT ? "FBPT" : "S3PT"); + u32 length, offset; + int result; + + subtable_header = acpi_os_map_memory(address, sizeof(*subtable_header)); + if (!subtable_header) + return -ENOMEM; + + if (strncmp((char *)&subtable_header->signature, signature, 4)) { + pr_info(FW_BUG "subtable signature and type mismatch!\n"); + return -EINVAL; + } + + length = subtable_header->length; + acpi_os_unmap_memory(subtable_header, sizeof(*subtable_header)); + + subtable_header = acpi_os_map_memory(address, length); + if (!subtable_header) + return -ENOMEM; + + offset = sizeof(*subtable_header); + while (offset < length) { + record_header = (void *)subtable_header + offset; + offset += record_header->length; + + switch (record_header->type) { + case RECORD_S3_RESUME: + if (subtable_type != SUBTABLE_S3PT) { + pr_err(FW_BUG "Invalid record %d for subtable %s\n", + record_header->type, signature); + return -EINVAL; + } + if (record_resume) { + pr_err("Duplicate resume performance record found.\n"); + continue; + } + record_resume = (struct resume_performance_record *)record_header; + result = sysfs_create_group(fpdt_kobj, &resume_attr_group); + if (result) + return result; + break; + case RECORD_S3_SUSPEND: + if (subtable_type != SUBTABLE_S3PT) { + pr_err(FW_BUG "Invalid %d for subtable %s\n", + record_header->type, signature); + continue; + } + if (record_suspend) { + pr_err("Duplicate suspend performance record found.\n"); + continue; + } + record_suspend = (struct suspend_performance_record *)record_header; + result = sysfs_create_group(fpdt_kobj, &suspend_attr_group); + if (result) + return result; + break; + case RECORD_BOOT: + if (subtable_type != SUBTABLE_FBPT) { + pr_err(FW_BUG "Invalid %d for subtable %s\n", + record_header->type, signature); + return -EINVAL; + } + if (record_boot) { + pr_err("Duplicate boot performance record found.\n"); + continue; + } + record_boot = (struct boot_performance_record *)record_header; + result = sysfs_create_group(fpdt_kobj, &boot_attr_group); + if (result) + return result; + break; + + default: + pr_err(FW_BUG "Invalid record %d found.\n", record_header->type); + return -EINVAL; + } + } + return 0; +} + +static int __init acpi_init_fpdt(void) +{ + acpi_status status; + struct acpi_table_header *header; + struct fpdt_subtable_entry *subtable; + u32 offset = sizeof(*header); + + status = acpi_get_table(ACPI_SIG_FPDT, 0, &header); + + if (ACPI_FAILURE(status)) + return 0; + + fpdt_kobj = kobject_create_and_add("fpdt", acpi_kobj); + if (!fpdt_kobj) + return -ENOMEM; + + while (offset < header->length) { + subtable = (void *)header + offset; + switch (subtable->type) { + case SUBTABLE_FBPT: + case SUBTABLE_S3PT: + fpdt_process_subtable(subtable->address, + subtable->type); + break; + default: + pr_info(FW_BUG "Invalid subtable type %d found.\n", + subtable->type); + break; + } + offset += sizeof(*subtable); + } + return 0; +} + +fs_initcall(acpi_init_fpdt); diff --git a/drivers/acpi/acpica/acobject.h b/drivers/acpi/acpica/acobject.h index af47a3ffd2a42474ae2815338e18db8b2a11c746..9db5ae0f79ea04b93dbd006c334a5370cbe7f97f 100644 --- a/drivers/acpi/acpica/acobject.h +++ b/drivers/acpi/acpica/acobject.h @@ -284,6 +284,7 @@ struct acpi_object_addr_handler { acpi_adr_space_handler handler; struct acpi_namespace_node *node; /* Parent device */ void *context; + acpi_mutex context_mutex; acpi_adr_space_setup setup; union acpi_operand_object *region_list; /* Regions using this handler */ union acpi_operand_object *next; diff --git a/drivers/acpi/acpica/evhandler.c b/drivers/acpi/acpica/evhandler.c index ea9485e6a475488ba47d74c2a072b22bd7aa3c83..c0cd7147a5a394cf2bb94f8c70c9ce23066a264a 100644 --- a/drivers/acpi/acpica/evhandler.c +++ b/drivers/acpi/acpica/evhandler.c @@ -489,6 +489,13 @@ acpi_ev_install_space_handler(struct acpi_namespace_node *node, /* Init handler obj */ + status = + acpi_os_create_mutex(&handler_obj->address_space.context_mutex); + if (ACPI_FAILURE(status)) { + acpi_ut_remove_reference(handler_obj); + goto unlock_and_exit; + } + handler_obj->address_space.space_id = (u8)space_id; handler_obj->address_space.handler_flags = flags; handler_obj->address_space.region_list = NULL; diff --git a/drivers/acpi/acpica/evregion.c b/drivers/acpi/acpica/evregion.c index 3ed7d9ae95cf428b3ded46f8978b84d852b936c7..4ef43c8ef5e78a0cc1fb8ba16745dba86dbbd9c0 100644 --- a/drivers/acpi/acpica/evregion.c +++ b/drivers/acpi/acpica/evregion.c @@ -112,6 +112,8 @@ acpi_ev_address_space_dispatch(union acpi_operand_object *region_obj, union acpi_operand_object *region_obj2; void *region_context = NULL; struct acpi_connection_info *context; + acpi_mutex context_mutex; + u8 context_locked; acpi_physical_address address; ACPI_FUNCTION_TRACE(ev_address_space_dispatch); @@ -136,6 +138,8 @@ acpi_ev_address_space_dispatch(union acpi_operand_object *region_obj, } context = handler_desc->address_space.context; + context_mutex = handler_desc->address_space.context_mutex; + context_locked = FALSE; /* * It may be the case that the region has never been initialized. @@ -204,6 +208,23 @@ acpi_ev_address_space_dispatch(union acpi_operand_object *region_obj, handler = handler_desc->address_space.handler; address = (region_obj->region.address + region_offset); + ACPI_DEBUG_PRINT((ACPI_DB_OPREGION, + "Handler %p (@%p) Address %8.8X%8.8X [%s]\n", + ®ion_obj->region.handler->address_space, handler, + ACPI_FORMAT_UINT64(address), + acpi_ut_get_region_name(region_obj->region. + space_id))); + + if (!(handler_desc->address_space.handler_flags & + ACPI_ADDR_HANDLER_DEFAULT_INSTALLED)) { + /* + * For handlers other than the default (supplied) handlers, we must + * exit the interpreter because the handler *might* block -- we don't + * know what it will do, so we can't hold the lock on the interpreter. + */ + acpi_ex_exit_interpreter(); + } + /* * Special handling for generic_serial_bus and general_purpose_io: * There are three extra parameters that must be passed to the @@ -212,48 +233,39 @@ acpi_ev_address_space_dispatch(union acpi_operand_object *region_obj, * 2) Length of the above buffer * 3) Actual access length from the access_as() op * + * Since we pass these extra parameters via the context, which is + * shared between threads, we must lock the context to avoid these + * parameters being changed from another thread before the handler + * has completed running. + * * In addition, for general_purpose_io, the Address and bit_width fields * are defined as follows: * 1) Address is the pin number index of the field (bit offset from * the previous Connection) * 2) bit_width is the actual bit length of the field (number of pins) */ - if ((region_obj->region.space_id == ACPI_ADR_SPACE_GSBUS) && + if ((region_obj->region.space_id == ACPI_ADR_SPACE_GSBUS || + region_obj->region.space_id == ACPI_ADR_SPACE_GPIO) && context && field_obj) { - /* Get the Connection (resource_template) buffer */ + status = + acpi_os_acquire_mutex(context_mutex, ACPI_WAIT_FOREVER); + if (ACPI_FAILURE(status)) { + goto re_enter_interpreter; + } - context->connection = field_obj->field.resource_buffer; - context->length = field_obj->field.resource_length; - context->access_length = field_obj->field.access_length; - } - if ((region_obj->region.space_id == ACPI_ADR_SPACE_GPIO) && - context && field_obj) { + context_locked = TRUE; /* Get the Connection (resource_template) buffer */ context->connection = field_obj->field.resource_buffer; context->length = field_obj->field.resource_length; context->access_length = field_obj->field.access_length; - address = field_obj->field.pin_number_index; - bit_width = field_obj->field.bit_length; - } - - ACPI_DEBUG_PRINT((ACPI_DB_OPREGION, - "Handler %p (@%p) Address %8.8X%8.8X [%s]\n", - ®ion_obj->region.handler->address_space, handler, - ACPI_FORMAT_UINT64(address), - acpi_ut_get_region_name(region_obj->region. - space_id))); - if (!(handler_desc->address_space.handler_flags & - ACPI_ADDR_HANDLER_DEFAULT_INSTALLED)) { - /* - * For handlers other than the default (supplied) handlers, we must - * exit the interpreter because the handler *might* block -- we don't - * know what it will do, so we can't hold the lock on the interpreter. - */ - acpi_ex_exit_interpreter(); + if (region_obj->region.space_id == ACPI_ADR_SPACE_GPIO) { + address = field_obj->field.pin_number_index; + bit_width = field_obj->field.bit_length; + } } /* Call the handler */ @@ -261,6 +273,10 @@ acpi_ev_address_space_dispatch(union acpi_operand_object *region_obj, status = handler(function, address, bit_width, value, context, region_obj2->extra.region_context); + if (context_locked) { + acpi_os_release_mutex(context_mutex); + } + if (ACPI_FAILURE(status)) { ACPI_EXCEPTION((AE_INFO, status, "Returned by Handler for [%s]", acpi_ut_get_region_name(region_obj->region. @@ -277,6 +293,7 @@ acpi_ev_address_space_dispatch(union acpi_operand_object *region_obj, } } +re_enter_interpreter: if (!(handler_desc->address_space.handler_flags & ACPI_ADDR_HANDLER_DEFAULT_INSTALLED)) { /* diff --git a/drivers/acpi/acpica/evxfregn.c b/drivers/acpi/acpica/evxfregn.c index 7672d70da850dd388a261cdbb48479ad9a0705ef..b1ff0a8f9c14fe3672c84893d20c948b86b0b695 100644 --- a/drivers/acpi/acpica/evxfregn.c +++ b/drivers/acpi/acpica/evxfregn.c @@ -201,6 +201,8 @@ acpi_remove_address_space_handler(acpi_handle device, /* Now we can delete the handler object */ + acpi_os_release_mutex(handler_obj->address_space. + context_mutex); acpi_ut_remove_reference(handler_obj); goto unlock_and_exit; } diff --git a/drivers/acpi/pci_root.c b/drivers/acpi/pci_root.c index 0bf072cef6cfd32057edece53d01b634357d95fe..dcd593766a643319d78d36f3759840397a3203c1 100644 --- a/drivers/acpi/pci_root.c +++ b/drivers/acpi/pci_root.c @@ -56,8 +56,6 @@ static struct acpi_scan_handler pci_root_handler = { }, }; -static DEFINE_MUTEX(osc_lock); - /** * acpi_is_root_bridge - determine whether an ACPI CA node is a PCI root bridge * @handle: the ACPI CA node in question. @@ -223,12 +221,7 @@ static acpi_status acpi_pci_query_osc(struct acpi_pci_root *root, static acpi_status acpi_pci_osc_support(struct acpi_pci_root *root, u32 flags) { - acpi_status status; - - mutex_lock(&osc_lock); - status = acpi_pci_query_osc(root, flags, NULL); - mutex_unlock(&osc_lock); - return status; + return acpi_pci_query_osc(root, flags, NULL); } struct acpi_pci_root *acpi_pci_find_root(acpi_handle handle) @@ -353,10 +346,10 @@ EXPORT_SYMBOL_GPL(acpi_get_pci_dev); * _OSC bits the BIOS has granted control of, but its contents are meaningless * on failure. **/ -acpi_status acpi_pci_osc_control_set(acpi_handle handle, u32 *mask, u32 req) +static acpi_status acpi_pci_osc_control_set(acpi_handle handle, u32 *mask, u32 req) { struct acpi_pci_root *root; - acpi_status status = AE_OK; + acpi_status status; u32 ctrl, capbuf[3]; if (!mask) @@ -370,18 +363,16 @@ acpi_status acpi_pci_osc_control_set(acpi_handle handle, u32 *mask, u32 req) if (!root) return AE_NOT_EXIST; - mutex_lock(&osc_lock); - *mask = ctrl | root->osc_control_set; /* No need to evaluate _OSC if the control was already granted. */ if ((root->osc_control_set & ctrl) == ctrl) - goto out; + return AE_OK; /* Need to check the available controls bits before requesting them. */ while (*mask) { status = acpi_pci_query_osc(root, root->osc_support_set, mask); if (ACPI_FAILURE(status)) - goto out; + return status; if (ctrl == *mask) break; decode_osc_control(root, "platform does not support", @@ -392,21 +383,19 @@ acpi_status acpi_pci_osc_control_set(acpi_handle handle, u32 *mask, u32 req) if ((ctrl & req) != req) { decode_osc_control(root, "not requesting control; platform does not support", req & ~(ctrl)); - status = AE_SUPPORT; - goto out; + return AE_SUPPORT; } capbuf[OSC_QUERY_DWORD] = 0; capbuf[OSC_SUPPORT_DWORD] = root->osc_support_set; capbuf[OSC_CONTROL_DWORD] = ctrl; status = acpi_pci_run_osc(handle, capbuf, mask); - if (ACPI_SUCCESS(status)) - root->osc_control_set = *mask; -out: - mutex_unlock(&osc_lock); - return status; + if (ACPI_FAILURE(status)) + return status; + + root->osc_control_set = *mask; + return AE_OK; } -EXPORT_SYMBOL(acpi_pci_osc_control_set); static void negotiate_os_control(struct acpi_pci_root *root, int *no_aspm, bool is_pcie) @@ -452,9 +441,8 @@ static void negotiate_os_control(struct acpi_pci_root *root, int *no_aspm, if ((status == AE_NOT_FOUND) && !is_pcie) return; - dev_info(&device->dev, "_OSC failed (%s)%s\n", - acpi_format_exception(status), - pcie_aspm_support_enabled() ? "; disabling ASPM" : ""); + dev_info(&device->dev, "_OSC: platform retains control of PCIe features (%s)\n", + acpi_format_exception(status)); return; } @@ -510,7 +498,7 @@ static void negotiate_os_control(struct acpi_pci_root *root, int *no_aspm, } else { decode_osc_control(root, "OS requested", requested); decode_osc_control(root, "platform willing to grant", control); - dev_info(&device->dev, "_OSC failed (%s); disabling ASPM\n", + dev_info(&device->dev, "_OSC: platform retains control of PCIe features (%s)\n", acpi_format_exception(status)); /* diff --git a/drivers/acpi/platform_profile.c b/drivers/acpi/platform_profile.c index 4a59c5993bde432b482f85d6bf7c345b6467b3dd..dd2fbf38e414889589b126315650d0410b9caba7 100644 --- a/drivers/acpi/platform_profile.c +++ b/drivers/acpi/platform_profile.c @@ -17,6 +17,7 @@ static const char * const profile_names[] = { [PLATFORM_PROFILE_COOL] = "cool", [PLATFORM_PROFILE_QUIET] = "quiet", [PLATFORM_PROFILE_BALANCED] = "balanced", + [PLATFORM_PROFILE_BALANCED_PERFORMANCE] = "balanced-performance", [PLATFORM_PROFILE_PERFORMANCE] = "performance", }; static_assert(ARRAY_SIZE(profile_names) == PLATFORM_PROFILE_LAST); diff --git a/drivers/acpi/thermal.c b/drivers/acpi/thermal.c index 4f906380b03122590cf42a383ce3b434eba37c57..95105db642b9823dbdcab603c516d7f6edf1df6e 100644 --- a/drivers/acpi/thermal.c +++ b/drivers/acpi/thermal.c @@ -670,27 +670,24 @@ static int thermal_get_trend(struct thermal_zone_device *thermal, return 0; } - -static int thermal_notify(struct thermal_zone_device *thermal, int trip, - enum thermal_trip_type trip_type) +static void acpi_thermal_zone_device_hot(struct thermal_zone_device *thermal) { - u8 type = 0; struct acpi_thermal *tz = thermal->devdata; - if (trip_type == THERMAL_TRIP_CRITICAL) - type = ACPI_THERMAL_NOTIFY_CRITICAL; - else if (trip_type == THERMAL_TRIP_HOT) - type = ACPI_THERMAL_NOTIFY_HOT; - else - return 0; - acpi_bus_generate_netlink_event(tz->device->pnp.device_class, - dev_name(&tz->device->dev), type, 1); + dev_name(&tz->device->dev), + ACPI_THERMAL_NOTIFY_HOT, 1); +} - if (trip_type == THERMAL_TRIP_CRITICAL && nocrt) - return 1; +static void acpi_thermal_zone_device_critical(struct thermal_zone_device *thermal) +{ + struct acpi_thermal *tz = thermal->devdata; - return 0; + acpi_bus_generate_netlink_event(tz->device->pnp.device_class, + dev_name(&tz->device->dev), + ACPI_THERMAL_NOTIFY_CRITICAL, 1); + + thermal_zone_device_critical(thermal); } static int acpi_thermal_cooling_device_cb(struct thermal_zone_device *thermal, @@ -760,25 +757,6 @@ static int acpi_thermal_cooling_device_cb(struct thermal_zone_device *thermal, } } - for (i = 0; i < tz->devices.count; i++) { - handle = tz->devices.handles[i]; - status = acpi_bus_get_device(handle, &dev); - if (ACPI_SUCCESS(status) && (dev == device)) { - if (bind) - result = thermal_zone_bind_cooling_device - (thermal, THERMAL_TRIPS_NONE, - cdev, THERMAL_NO_LIMIT, - THERMAL_NO_LIMIT, - THERMAL_WEIGHT_DEFAULT); - else - result = thermal_zone_unbind_cooling_device - (thermal, THERMAL_TRIPS_NONE, - cdev); - if (result) - goto failed; - } - } - failed: return result; } @@ -805,7 +783,8 @@ static struct thermal_zone_device_ops acpi_thermal_zone_ops = { .get_trip_temp = thermal_get_trip_temp, .get_crit_temp = thermal_get_crit_temp, .get_trend = thermal_get_trend, - .notify = thermal_notify, + .hot = acpi_thermal_zone_device_hot, + .critical = acpi_thermal_zone_device_critical, }; static int acpi_thermal_register_thermal_zone(struct acpi_thermal *tz) diff --git a/drivers/amba/bus.c b/drivers/amba/bus.c index ecc304149067cafb06b9258df7333430dde7a0df..939ca220bf78d7cd931e1e31592b351b05407fe6 100644 --- a/drivers/amba/bus.c +++ b/drivers/amba/bus.c @@ -56,31 +56,28 @@ amba_lookup(const struct amba_id *table, struct amba_device *dev) return NULL; } -static int amba_match(struct device *dev, struct device_driver *drv) +static int amba_get_enable_pclk(struct amba_device *pcdev) { - struct amba_device *pcdev = to_amba_device(dev); - struct amba_driver *pcdrv = to_amba_driver(drv); + int ret; - /* When driver_override is set, only bind to the matching driver */ - if (pcdev->driver_override) - return !strcmp(pcdev->driver_override, drv->name); + pcdev->pclk = clk_get(&pcdev->dev, "apb_pclk"); + if (IS_ERR(pcdev->pclk)) + return PTR_ERR(pcdev->pclk); - return amba_lookup(pcdrv->id_table, pcdev) != NULL; + ret = clk_prepare_enable(pcdev->pclk); + if (ret) + clk_put(pcdev->pclk); + + return ret; } -static int amba_uevent(struct device *dev, struct kobj_uevent_env *env) +static void amba_put_disable_pclk(struct amba_device *pcdev) { - struct amba_device *pcdev = to_amba_device(dev); - int retval = 0; - - retval = add_uevent_var(env, "AMBA_ID=%08x", pcdev->periphid); - if (retval) - return retval; - - retval = add_uevent_var(env, "MODALIAS=amba:d%08X", pcdev->periphid); - return retval; + clk_disable_unprepare(pcdev->pclk); + clk_put(pcdev->pclk); } + static ssize_t driver_override_show(struct device *_dev, struct device_attribute *attr, char *buf) { @@ -152,102 +149,29 @@ static struct attribute *amba_dev_attrs[] = { }; ATTRIBUTE_GROUPS(amba_dev); -#ifdef CONFIG_PM -/* - * Hooks to provide runtime PM of the pclk (bus clock). It is safe to - * enable/disable the bus clock at runtime PM suspend/resume as this - * does not result in loss of context. - */ -static int amba_pm_runtime_suspend(struct device *dev) +static int amba_match(struct device *dev, struct device_driver *drv) { struct amba_device *pcdev = to_amba_device(dev); - int ret = pm_generic_runtime_suspend(dev); + struct amba_driver *pcdrv = to_amba_driver(drv); - if (ret == 0 && dev->driver) { - if (pm_runtime_is_irq_safe(dev)) - clk_disable(pcdev->pclk); - else - clk_disable_unprepare(pcdev->pclk); - } + /* When driver_override is set, only bind to the matching driver */ + if (pcdev->driver_override) + return !strcmp(pcdev->driver_override, drv->name); - return ret; + return amba_lookup(pcdrv->id_table, pcdev) != NULL; } -static int amba_pm_runtime_resume(struct device *dev) +static int amba_uevent(struct device *dev, struct kobj_uevent_env *env) { struct amba_device *pcdev = to_amba_device(dev); - int ret; - - if (dev->driver) { - if (pm_runtime_is_irq_safe(dev)) - ret = clk_enable(pcdev->pclk); - else - ret = clk_prepare_enable(pcdev->pclk); - /* Failure is probably fatal to the system, but... */ - if (ret) - return ret; - } - - return pm_generic_runtime_resume(dev); -} -#endif /* CONFIG_PM */ - -static const struct dev_pm_ops amba_pm = { - .suspend = pm_generic_suspend, - .resume = pm_generic_resume, - .freeze = pm_generic_freeze, - .thaw = pm_generic_thaw, - .poweroff = pm_generic_poweroff, - .restore = pm_generic_restore, - SET_RUNTIME_PM_OPS( - amba_pm_runtime_suspend, - amba_pm_runtime_resume, - NULL - ) -}; - -/* - * Primecells are part of the Advanced Microcontroller Bus Architecture, - * so we call the bus "amba". - * DMA configuration for platform and AMBA bus is same. So here we reuse - * platform's DMA config routine. - */ -struct bus_type amba_bustype = { - .name = "amba", - .dev_groups = amba_dev_groups, - .match = amba_match, - .uevent = amba_uevent, - .dma_configure = platform_dma_configure, - .pm = &amba_pm, -}; -EXPORT_SYMBOL_GPL(amba_bustype); - -static int __init amba_init(void) -{ - return bus_register(&amba_bustype); -} - -postcore_initcall(amba_init); - -static int amba_get_enable_pclk(struct amba_device *pcdev) -{ - int ret; - - pcdev->pclk = clk_get(&pcdev->dev, "apb_pclk"); - if (IS_ERR(pcdev->pclk)) - return PTR_ERR(pcdev->pclk); - - ret = clk_prepare_enable(pcdev->pclk); - if (ret) - clk_put(pcdev->pclk); + int retval = 0; - return ret; -} + retval = add_uevent_var(env, "AMBA_ID=%08x", pcdev->periphid); + if (retval) + return retval; -static void amba_put_disable_pclk(struct amba_device *pcdev) -{ - clk_disable_unprepare(pcdev->pclk); - clk_put(pcdev->pclk); + retval = add_uevent_var(env, "MODALIAS=amba:d%08X", pcdev->periphid); + return retval; } /* @@ -299,10 +223,10 @@ static int amba_remove(struct device *dev) { struct amba_device *pcdev = to_amba_device(dev); struct amba_driver *drv = to_amba_driver(dev->driver); - int ret; pm_runtime_get_sync(dev); - ret = drv->remove(pcdev); + if (drv->remove) + drv->remove(pcdev); pm_runtime_put_noidle(dev); /* Undo the runtime PM settings in amba_probe() */ @@ -313,15 +237,101 @@ static int amba_remove(struct device *dev) amba_put_disable_pclk(pcdev); dev_pm_domain_detach(dev, true); - return ret; + return 0; } static void amba_shutdown(struct device *dev) { - struct amba_driver *drv = to_amba_driver(dev->driver); - drv->shutdown(to_amba_device(dev)); + struct amba_driver *drv; + + if (!dev->driver) + return; + + drv = to_amba_driver(dev->driver); + if (drv->shutdown) + drv->shutdown(to_amba_device(dev)); +} + +#ifdef CONFIG_PM +/* + * Hooks to provide runtime PM of the pclk (bus clock). It is safe to + * enable/disable the bus clock at runtime PM suspend/resume as this + * does not result in loss of context. + */ +static int amba_pm_runtime_suspend(struct device *dev) +{ + struct amba_device *pcdev = to_amba_device(dev); + int ret = pm_generic_runtime_suspend(dev); + + if (ret == 0 && dev->driver) { + if (pm_runtime_is_irq_safe(dev)) + clk_disable(pcdev->pclk); + else + clk_disable_unprepare(pcdev->pclk); + } + + return ret; +} + +static int amba_pm_runtime_resume(struct device *dev) +{ + struct amba_device *pcdev = to_amba_device(dev); + int ret; + + if (dev->driver) { + if (pm_runtime_is_irq_safe(dev)) + ret = clk_enable(pcdev->pclk); + else + ret = clk_prepare_enable(pcdev->pclk); + /* Failure is probably fatal to the system, but... */ + if (ret) + return ret; + } + + return pm_generic_runtime_resume(dev); +} +#endif /* CONFIG_PM */ + +static const struct dev_pm_ops amba_pm = { + .suspend = pm_generic_suspend, + .resume = pm_generic_resume, + .freeze = pm_generic_freeze, + .thaw = pm_generic_thaw, + .poweroff = pm_generic_poweroff, + .restore = pm_generic_restore, + SET_RUNTIME_PM_OPS( + amba_pm_runtime_suspend, + amba_pm_runtime_resume, + NULL + ) +}; + +/* + * Primecells are part of the Advanced Microcontroller Bus Architecture, + * so we call the bus "amba". + * DMA configuration for platform and AMBA bus is same. So here we reuse + * platform's DMA config routine. + */ +struct bus_type amba_bustype = { + .name = "amba", + .dev_groups = amba_dev_groups, + .match = amba_match, + .uevent = amba_uevent, + .probe = amba_probe, + .remove = amba_remove, + .shutdown = amba_shutdown, + .dma_configure = platform_dma_configure, + .pm = &amba_pm, +}; +EXPORT_SYMBOL_GPL(amba_bustype); + +static int __init amba_init(void) +{ + return bus_register(&amba_bustype); } +postcore_initcall(amba_init); + /** * amba_driver_register - register an AMBA device driver * @drv: amba device driver structure @@ -332,12 +342,10 @@ static void amba_shutdown(struct device *dev) */ int amba_driver_register(struct amba_driver *drv) { - drv->drv.bus = &amba_bustype; + if (!drv->probe) + return -EINVAL; -#define SETFN(fn) if (drv->fn) drv->drv.fn = amba_##fn - SETFN(probe); - SETFN(remove); - SETFN(shutdown); + drv->drv.bus = &amba_bustype; return driver_register(&drv->drv); } diff --git a/drivers/android/binderfs.c b/drivers/android/binderfs.c index 7b4f154f07e6c96eb8f5e6416f7a6127976be2ee..e80ba93c62a90975230dc5e8d79a72e4a26cce2d 100644 --- a/drivers/android/binderfs.c +++ b/drivers/android/binderfs.c @@ -355,7 +355,8 @@ static inline bool is_binderfs_control_device(const struct dentry *dentry) return info->control_dentry == dentry; } -static int binderfs_rename(struct inode *old_dir, struct dentry *old_dentry, +static int binderfs_rename(struct user_namespace *mnt_userns, + struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry, unsigned int flags) { @@ -363,7 +364,8 @@ static int binderfs_rename(struct inode *old_dir, struct dentry *old_dentry, is_binderfs_control_device(new_dentry)) return -EPERM; - return simple_rename(old_dir, old_dentry, new_dir, new_dentry, flags); + return simple_rename(&init_user_ns, old_dir, old_dentry, new_dir, + new_dentry, flags); } static int binderfs_unlink(struct inode *dir, struct dentry *dentry) diff --git a/drivers/auxdisplay/Kconfig b/drivers/auxdisplay/Kconfig index a2b59b84bb881c82b06dcd9f1aef6f25304234b4..1509cb74705a30ade4f89e8c2f6af9cb9bacb594 100644 --- a/drivers/auxdisplay/Kconfig +++ b/drivers/auxdisplay/Kconfig @@ -507,6 +507,3 @@ config PANEL depends on PARPORT select AUXDISPLAY select PARPORT_PANEL - -config CHARLCD - tristate "Character LCD core support" if COMPILE_TEST diff --git a/drivers/auxdisplay/cfag12864b.c b/drivers/auxdisplay/cfag12864b.c index 7eebae7e322c605e97d2592223beb92ab82054c3..fd430e6866a117d4cd30cd48317f7504efbdeac6 100644 --- a/drivers/auxdisplay/cfag12864b.c +++ b/drivers/auxdisplay/cfag12864b.c @@ -5,7 +5,7 @@ * Description: cfag12864b LCD driver * Depends: ks0108 * - * Author: Copyright (C) Miguel Ojeda Sandonis + * Author: Copyright (C) Miguel Ojeda * Date: 2006-10-31 */ @@ -376,5 +376,5 @@ module_init(cfag12864b_init); module_exit(cfag12864b_exit); MODULE_LICENSE("GPL v2"); -MODULE_AUTHOR("Miguel Ojeda Sandonis "); +MODULE_AUTHOR("Miguel Ojeda "); MODULE_DESCRIPTION("cfag12864b LCD driver"); diff --git a/drivers/auxdisplay/cfag12864bfb.c b/drivers/auxdisplay/cfag12864bfb.c index 2002291ab338829ceafe909b3c72cd46bb37ce18..d66821adf45324908cbf87c17022e8e83d764754 100644 --- a/drivers/auxdisplay/cfag12864bfb.c +++ b/drivers/auxdisplay/cfag12864bfb.c @@ -5,7 +5,7 @@ * Description: cfag12864b LCD framebuffer driver * Depends: cfag12864b * - * Author: Copyright (C) Miguel Ojeda Sandonis + * Author: Copyright (C) Miguel Ojeda * Date: 2006-10-31 */ @@ -171,5 +171,5 @@ module_init(cfag12864bfb_init); module_exit(cfag12864bfb_exit); MODULE_LICENSE("GPL v2"); -MODULE_AUTHOR("Miguel Ojeda Sandonis "); +MODULE_AUTHOR("Miguel Ojeda "); MODULE_DESCRIPTION("cfag12864b LCD framebuffer driver"); diff --git a/drivers/auxdisplay/ht16k33.c b/drivers/auxdisplay/ht16k33.c index d951d54b26f5239b47d36f2ec6b95a71e54310d7..1e69cc6d21a0dca208267f58476a1dde739203fd 100644 --- a/drivers/auxdisplay/ht16k33.c +++ b/drivers/auxdisplay/ht16k33.c @@ -117,8 +117,7 @@ static void ht16k33_fb_queue(struct ht16k33_priv *priv) { struct ht16k33_fbdev *fbdev = &priv->fbdev; - schedule_delayed_work(&fbdev->work, - msecs_to_jiffies(HZ / fbdev->refresh_rate)); + schedule_delayed_work(&fbdev->work, HZ / fbdev->refresh_rate); } /* @@ -402,11 +401,6 @@ static int ht16k33_probe(struct i2c_client *client, return -EIO; } - if (client->irq <= 0) { - dev_err(&client->dev, "No IRQ specified\n"); - return -EINVAL; - } - priv = devm_kzalloc(&client->dev, sizeof(*priv), GFP_KERNEL); if (!priv) return -ENOMEM; @@ -459,9 +453,12 @@ static int ht16k33_probe(struct i2c_client *client, if (err) goto err_fbdev_info; - err = ht16k33_keypad_probe(client, &priv->keypad); - if (err) - goto err_fbdev_unregister; + /* Keypad */ + if (client->irq > 0) { + err = ht16k33_keypad_probe(client, &priv->keypad); + if (err) + goto err_fbdev_unregister; + } /* Backlight */ memset(&bl_props, 0, sizeof(struct backlight_properties)); diff --git a/drivers/auxdisplay/ks0108.c b/drivers/auxdisplay/ks0108.c index abfe3fa9e6f459bf33e026babdfdf297f4222bbe..03c95ad4216c5a764716ff5bf773c736838c112b 100644 --- a/drivers/auxdisplay/ks0108.c +++ b/drivers/auxdisplay/ks0108.c @@ -5,7 +5,7 @@ * Description: ks0108 LCD Controller driver * Depends: parport * - * Author: Copyright (C) Miguel Ojeda Sandonis + * Author: Copyright (C) Miguel Ojeda * Date: 2006-10-31 */ @@ -182,6 +182,6 @@ module_init(ks0108_init); module_exit(ks0108_exit); MODULE_LICENSE("GPL v2"); -MODULE_AUTHOR("Miguel Ojeda Sandonis "); +MODULE_AUTHOR("Miguel Ojeda "); MODULE_DESCRIPTION("ks0108 LCD Controller driver"); diff --git a/drivers/base/Kconfig b/drivers/base/Kconfig index 040be48ce046643672fc846899ee78d55a6dde31..ffcbe2bc460eb5effddb4ce546ba0d375e313e08 100644 --- a/drivers/base/Kconfig +++ b/drivers/base/Kconfig @@ -161,7 +161,7 @@ config HMEM_REPORTING default n depends on NUMA help - Enable reporting for heterogenous memory access attributes under + Enable reporting for heterogeneous memory access attributes under their non-uniform memory nodes. source "drivers/base/test/Kconfig" @@ -213,4 +213,10 @@ config GENERIC_ARCH_TOPOLOGY appropriate scaling, sysfs interface for reading capacity values at runtime. +config GENERIC_ARCH_NUMA + bool + help + Enable support for generic NUMA implementation. Currently, RISC-V + and ARM64 use it. + endmenu diff --git a/drivers/base/Makefile b/drivers/base/Makefile index 5e7bf9669a81f8c21d8347ddfb182f0171e4e3f9..8b93a7f291ecd736d3decb878bcc90c34b543484 100644 --- a/drivers/base/Makefile +++ b/drivers/base/Makefile @@ -24,6 +24,7 @@ obj-$(CONFIG_PINCTRL) += pinctrl.o obj-$(CONFIG_DEV_COREDUMP) += devcoredump.o obj-$(CONFIG_GENERIC_MSI_IRQ_DOMAIN) += platform-msi.o obj-$(CONFIG_GENERIC_ARCH_TOPOLOGY) += arch_topology.o +obj-$(CONFIG_GENERIC_ARCH_NUMA) += arch_numa.o obj-y += test/ diff --git a/arch/arm64/mm/numa.c b/drivers/base/arch_numa.c similarity index 93% rename from arch/arm64/mm/numa.c rename to drivers/base/arch_numa.c index a8303bc6b62a41eda320767c55d5b88f7a62bff7..4cc4e117727d8ac28787da15a2825bb0d852fdc3 100644 --- a/arch/arm64/mm/numa.c +++ b/drivers/base/arch_numa.c @@ -13,7 +13,6 @@ #include #include -#include #include struct pglist_data *node_data[MAX_NUMNODES] __read_mostly; @@ -356,11 +355,12 @@ static int __init numa_register_nodes(void) /* Check that valid nid is set to memblks */ for_each_mem_region(mblk) { int mblk_nid = memblock_get_region_node(mblk); + phys_addr_t start = mblk->base; + phys_addr_t end = mblk->base + mblk->size - 1; if (mblk_nid == NUMA_NO_NODE || mblk_nid >= MAX_NUMNODES) { - pr_warn("Warning: invalid memblk node %d [mem %#010Lx-%#010Lx]\n", - mblk_nid, mblk->base, - mblk->base + mblk->size - 1); + pr_warn("Warning: invalid memblk node %d [mem %pap-%pap]\n", + mblk_nid, &start, &end); return -EINVAL; } } @@ -428,14 +428,14 @@ static int __init numa_init(int (*init_func)(void)) static int __init dummy_numa_init(void) { phys_addr_t start = memblock_start_of_DRAM(); - phys_addr_t end = memblock_end_of_DRAM(); + phys_addr_t end = memblock_end_of_DRAM() - 1; int ret; if (numa_off) pr_info("NUMA disabled\n"); /* Forced off on command line. */ - pr_info("Faking a node at [mem %#018Lx-%#018Lx]\n", start, end - 1); + pr_info("Faking a node at [mem %pap-%pap]\n", &start, &end); - ret = numa_add_memblk(0, start, end); + ret = numa_add_memblk(0, start, end + 1); if (ret) { pr_err("NUMA init failed\n"); return ret; @@ -445,16 +445,36 @@ static int __init dummy_numa_init(void) return 0; } +#ifdef CONFIG_ACPI_NUMA +static int __init arch_acpi_numa_init(void) +{ + int ret; + + ret = acpi_numa_init(); + if (ret) { + pr_info("Failed to initialise from firmware\n"); + return ret; + } + + return srat_disabled() ? -EINVAL : 0; +} +#else +static int __init arch_acpi_numa_init(void) +{ + return -EOPNOTSUPP; +} +#endif + /** - * arm64_numa_init() - Initialize NUMA + * arch_numa_init() - Initialize NUMA * * Try each configured NUMA initialization method until one succeeds. The * last fallback is dummy single node config encompassing whole memory. */ -void __init arm64_numa_init(void) +void __init arch_numa_init(void) { if (!numa_off) { - if (!acpi_disabled && !numa_init(arm64_acpi_numa_init)) + if (!acpi_disabled && !numa_init(arch_acpi_numa_init)) return; if (acpi_disabled && !numa_init(of_numa_init)) return; diff --git a/drivers/base/auxiliary.c b/drivers/base/auxiliary.c index 8336535f1e110219992b2c765a6b81cfdfdbb53e..d8b314e7d0fdc88cfa33f37f2ccefdd329cb286e 100644 --- a/drivers/base/auxiliary.c +++ b/drivers/base/auxiliary.c @@ -15,6 +15,7 @@ #include #include #include +#include "base.h" static const struct auxiliary_device_id *auxiliary_match_id(const struct auxiliary_device_id *id, const struct auxiliary_device *auxdev) @@ -260,19 +261,11 @@ void auxiliary_driver_unregister(struct auxiliary_driver *auxdrv) } EXPORT_SYMBOL_GPL(auxiliary_driver_unregister); -static int __init auxiliary_bus_init(void) +void __init auxiliary_bus_init(void) { - return bus_register(&auxiliary_bus_type); + WARN_ON(bus_register(&auxiliary_bus_type)); } -static void __exit auxiliary_bus_exit(void) -{ - bus_unregister(&auxiliary_bus_type); -} - -module_init(auxiliary_bus_init); -module_exit(auxiliary_bus_exit); - MODULE_LICENSE("GPL v2"); MODULE_DESCRIPTION("Auxiliary Bus"); MODULE_AUTHOR("David Ertman "); diff --git a/drivers/base/base.h b/drivers/base/base.h index f5600a83124fafa376826260e8bd04d57ae13531..52b3d7b75c2750e85c0b5f1b238964dd42cb5fa9 100644 --- a/drivers/base/base.h +++ b/drivers/base/base.h @@ -119,6 +119,11 @@ static inline int hypervisor_init(void) { return 0; } extern int platform_bus_init(void); extern void cpu_dev_init(void); extern void container_dev_init(void); +#ifdef CONFIG_AUXILIARY_BUS +extern void auxiliary_bus_init(void); +#else +static inline void auxiliary_bus_init(void) { } +#endif struct kobject *virtual_device_parent(struct device *dev); diff --git a/drivers/base/bus.c b/drivers/base/bus.c index a9c23ecebc7c8de3e1060ed4243cf75d431f8099..36d0c654ea6124c94f41851261b43c4f933c9e01 100644 --- a/drivers/base/bus.c +++ b/drivers/base/bus.c @@ -633,7 +633,7 @@ int bus_add_driver(struct device_driver *drv) error = driver_add_groups(drv, bus->drv_groups); if (error) { /* How the hell do we get out of this pickle? Give up */ - printk(KERN_ERR "%s: driver_create_groups(%s) failed\n", + printk(KERN_ERR "%s: driver_add_groups(%s) failed\n", __func__, drv->name); } @@ -729,23 +729,6 @@ int device_reprobe(struct device *dev) } EXPORT_SYMBOL_GPL(device_reprobe); -/** - * find_bus - locate bus by name. - * @name: name of bus. - * - * Call kset_find_obj() to iterate over list of buses to - * find a bus by name. Return bus if found. - * - * Note that kset_find_obj increments bus' reference count. - */ -#if 0 -struct bus_type *find_bus(char *name) -{ - struct kobject *k = kset_find_obj(bus_kset, name); - return k ? to_bus(k) : NULL; -} -#endif /* 0 */ - static int bus_add_groups(struct bus_type *bus, const struct attribute_group **groups) { diff --git a/drivers/base/core.c b/drivers/base/core.c index 7c0406e675e98fcd71c9f62ea66d03dfd71b8932..f29839382f8166b4106e96eb9a0cc41f1e2d7b34 100644 --- a/drivers/base/core.c +++ b/drivers/base/core.c @@ -149,6 +149,21 @@ void fwnode_links_purge(struct fwnode_handle *fwnode) fwnode_links_purge_consumers(fwnode); } +static void fw_devlink_purge_absent_suppliers(struct fwnode_handle *fwnode) +{ + struct fwnode_handle *child; + + /* Don't purge consumer links of an added child */ + if (fwnode->dev) + return; + + fwnode->flags |= FWNODE_FLAG_NOT_DEVICE; + fwnode_links_purge_consumers(fwnode); + + fwnode_for_each_available_child_node(fwnode, child) + fw_devlink_purge_absent_suppliers(child); +} + #ifdef CONFIG_SRCU static DEFINE_MUTEX(device_links_lock); DEFINE_STATIC_SRCU(device_links_srcu); @@ -245,7 +260,8 @@ int device_is_dependent(struct device *dev, void *target) return ret; list_for_each_entry(link, &dev->links.consumers, s_node) { - if (link->flags == (DL_FLAG_SYNC_STATE_ONLY | DL_FLAG_MANAGED)) + if ((link->flags & ~DL_FLAG_INFERRED) == + (DL_FLAG_SYNC_STATE_ONLY | DL_FLAG_MANAGED)) continue; if (link->consumer == target) @@ -318,7 +334,8 @@ static int device_reorder_to_tail(struct device *dev, void *not_used) device_for_each_child(dev, NULL, device_reorder_to_tail); list_for_each_entry(link, &dev->links.consumers, s_node) { - if (link->flags == (DL_FLAG_SYNC_STATE_ONLY | DL_FLAG_MANAGED)) + if ((link->flags & ~DL_FLAG_INFERRED) == + (DL_FLAG_SYNC_STATE_ONLY | DL_FLAG_MANAGED)) continue; device_reorder_to_tail(link->consumer, NULL); } @@ -566,7 +583,8 @@ postcore_initcall(devlink_class_init); #define DL_MANAGED_LINK_FLAGS (DL_FLAG_AUTOREMOVE_CONSUMER | \ DL_FLAG_AUTOREMOVE_SUPPLIER | \ DL_FLAG_AUTOPROBE_CONSUMER | \ - DL_FLAG_SYNC_STATE_ONLY) + DL_FLAG_SYNC_STATE_ONLY | \ + DL_FLAG_INFERRED) #define DL_ADD_VALID_FLAGS (DL_MANAGED_LINK_FLAGS | DL_FLAG_STATELESS | \ DL_FLAG_PM_RUNTIME | DL_FLAG_RPM_ACTIVE) @@ -635,7 +653,7 @@ struct device_link *device_link_add(struct device *consumer, if (!consumer || !supplier || flags & ~DL_ADD_VALID_FLAGS || (flags & DL_FLAG_STATELESS && flags & DL_MANAGED_LINK_FLAGS) || (flags & DL_FLAG_SYNC_STATE_ONLY && - flags != DL_FLAG_SYNC_STATE_ONLY) || + (flags & ~DL_FLAG_INFERRED) != DL_FLAG_SYNC_STATE_ONLY) || (flags & DL_FLAG_AUTOPROBE_CONSUMER && flags & (DL_FLAG_AUTOREMOVE_CONSUMER | DL_FLAG_AUTOREMOVE_SUPPLIER))) @@ -691,6 +709,10 @@ struct device_link *device_link_add(struct device *consumer, if (link->consumer != consumer) continue; + if (link->flags & DL_FLAG_INFERRED && + !(flags & DL_FLAG_INFERRED)) + link->flags &= ~DL_FLAG_INFERRED; + if (flags & DL_FLAG_PM_RUNTIME) { if (!(link->flags & DL_FLAG_PM_RUNTIME)) { pm_runtime_new_link(consumer); @@ -950,6 +972,10 @@ int device_links_check_suppliers(struct device *dev) mutex_lock(&fwnode_link_lock); if (dev->fwnode && !list_empty(&dev->fwnode->suppliers) && !fw_devlink_is_permissive()) { + dev_dbg(dev, "probe deferral - wait for supplier %pfwP\n", + list_first_entry(&dev->fwnode->suppliers, + struct fwnode_link, + c_hook)->supplier); mutex_unlock(&fwnode_link_lock); return -EPROBE_DEFER; } @@ -964,6 +990,8 @@ int device_links_check_suppliers(struct device *dev) if (link->status != DL_STATE_AVAILABLE && !(link->flags & DL_FLAG_SYNC_STATE_ONLY)) { device_links_missing_supplier(dev); + dev_dbg(dev, "probe deferral - supplier %s not ready\n", + dev_name(link->supplier)); ret = -EPROBE_DEFER; break; } @@ -1142,12 +1170,22 @@ void device_links_driver_bound(struct device *dev) LIST_HEAD(sync_list); /* - * If a device probes successfully, it's expected to have created all + * If a device binds successfully, it's expected to have created all * the device links it needs to or make new device links as it needs - * them. So, it no longer needs to wait on any suppliers. + * them. So, fw_devlink no longer needs to create device links to any + * of the device's suppliers. + * + * Also, if a child firmware node of this bound device is not added as + * a device by now, assume it is never going to be added and make sure + * other devices don't defer probe indefinitely by waiting for such a + * child device. */ - if (dev->fwnode && dev->fwnode->dev == dev) + if (dev->fwnode && dev->fwnode->dev == dev) { + struct fwnode_handle *child; fwnode_links_purge_suppliers(dev->fwnode); + fwnode_for_each_available_child_node(dev->fwnode, child) + fw_devlink_purge_absent_suppliers(child); + } device_remove_file(dev, &dev_attr_waiting_for_supplier); device_links_write_lock(); @@ -1458,7 +1496,14 @@ static void device_links_purge(struct device *dev) device_links_write_unlock(); } -static u32 fw_devlink_flags = DL_FLAG_SYNC_STATE_ONLY; +#define FW_DEVLINK_FLAGS_PERMISSIVE (DL_FLAG_INFERRED | \ + DL_FLAG_SYNC_STATE_ONLY) +#define FW_DEVLINK_FLAGS_ON (DL_FLAG_INFERRED | \ + DL_FLAG_AUTOPROBE_CONSUMER) +#define FW_DEVLINK_FLAGS_RPM (FW_DEVLINK_FLAGS_ON | \ + DL_FLAG_PM_RUNTIME) + +static u32 fw_devlink_flags = FW_DEVLINK_FLAGS_PERMISSIVE; static int __init fw_devlink_setup(char *arg) { if (!arg) @@ -1467,17 +1512,23 @@ static int __init fw_devlink_setup(char *arg) if (strcmp(arg, "off") == 0) { fw_devlink_flags = 0; } else if (strcmp(arg, "permissive") == 0) { - fw_devlink_flags = DL_FLAG_SYNC_STATE_ONLY; + fw_devlink_flags = FW_DEVLINK_FLAGS_PERMISSIVE; } else if (strcmp(arg, "on") == 0) { - fw_devlink_flags = DL_FLAG_AUTOPROBE_CONSUMER; + fw_devlink_flags = FW_DEVLINK_FLAGS_ON; } else if (strcmp(arg, "rpm") == 0) { - fw_devlink_flags = DL_FLAG_AUTOPROBE_CONSUMER | - DL_FLAG_PM_RUNTIME; + fw_devlink_flags = FW_DEVLINK_FLAGS_RPM; } return 0; } early_param("fw_devlink", fw_devlink_setup); +static bool fw_devlink_strict; +static int __init fw_devlink_strict_setup(char *arg) +{ + return strtobool(arg, &fw_devlink_strict); +} +early_param("fw_devlink.strict", fw_devlink_strict_setup); + u32 fw_devlink_get_flags(void) { return fw_devlink_flags; @@ -1485,7 +1536,12 @@ u32 fw_devlink_get_flags(void) static bool fw_devlink_is_permissive(void) { - return fw_devlink_flags == DL_FLAG_SYNC_STATE_ONLY; + return fw_devlink_flags == FW_DEVLINK_FLAGS_PERMISSIVE; +} + +bool fw_devlink_is_strict(void) +{ + return fw_devlink_strict && !fw_devlink_is_permissive(); } static void fw_devlink_parse_fwnode(struct fwnode_handle *fwnode) @@ -1507,6 +1563,53 @@ static void fw_devlink_parse_fwtree(struct fwnode_handle *fwnode) fw_devlink_parse_fwtree(child); } +/** + * fw_devlink_relax_cycle - Convert cyclic links to SYNC_STATE_ONLY links + * @con: Device to check dependencies for. + * @sup: Device to check against. + * + * Check if @sup depends on @con or any device dependent on it (its child or + * its consumer etc). When such a cyclic dependency is found, convert all + * device links created solely by fw_devlink into SYNC_STATE_ONLY device links. + * This is the equivalent of doing fw_devlink=permissive just between the + * devices in the cycle. We need to do this because, at this point, fw_devlink + * can't tell which of these dependencies is not a real dependency. + * + * Return 1 if a cycle is found. Otherwise, return 0. + */ +static int fw_devlink_relax_cycle(struct device *con, void *sup) +{ + struct device_link *link; + int ret; + + if (con == sup) + return 1; + + ret = device_for_each_child(con, sup, fw_devlink_relax_cycle); + if (ret) + return ret; + + list_for_each_entry(link, &con->links.consumers, s_node) { + if ((link->flags & ~DL_FLAG_INFERRED) == + (DL_FLAG_SYNC_STATE_ONLY | DL_FLAG_MANAGED)) + continue; + + if (!fw_devlink_relax_cycle(link->consumer, sup)) + continue; + + ret = 1; + + if (!(link->flags & DL_FLAG_INFERRED)) + continue; + + pm_runtime_drop_link(link); + link->flags = DL_FLAG_MANAGED | FW_DEVLINK_FLAGS_PERMISSIVE; + dev_dbg(link->consumer, "Relaxing link with %s\n", + dev_name(link->supplier)); + } + return ret; +} + /** * fw_devlink_create_devlink - Create a device link from a consumer to fwnode * @con - Consumer device for the device link @@ -1534,16 +1637,40 @@ static int fw_devlink_create_devlink(struct device *con, sup_dev = get_dev_from_fwnode(sup_handle); if (sup_dev) { + /* + * If it's one of those drivers that don't actually bind to + * their device using driver core, then don't wait on this + * supplier device indefinitely. + */ + if (sup_dev->links.status == DL_DEV_NO_DRIVER && + sup_handle->flags & FWNODE_FLAG_INITIALIZED) { + ret = -EINVAL; + goto out; + } + /* * If this fails, it is due to cycles in device links. Just * give up on this link and treat it as invalid. */ - if (!device_link_add(con, sup_dev, flags)) + if (!device_link_add(con, sup_dev, flags) && + !(flags & DL_FLAG_SYNC_STATE_ONLY)) { + dev_info(con, "Fixing up cyclic dependency with %s\n", + dev_name(sup_dev)); + device_links_write_lock(); + fw_devlink_relax_cycle(con, sup_dev); + device_links_write_unlock(); + device_link_add(con, sup_dev, + FW_DEVLINK_FLAGS_PERMISSIVE); ret = -EINVAL; + } goto out; } + /* Supplier that's already initialized without a struct device. */ + if (sup_handle->flags & FWNODE_FLAG_INITIALIZED) + return -EINVAL; + /* * DL_FLAG_SYNC_STATE_ONLY doesn't block probing and supports * cycles. So cycle detection isn't necessary and shouldn't be @@ -1632,7 +1759,7 @@ static void __fw_devlink_link_to_consumers(struct device *dev) con_dev = NULL; } else { own_link = false; - dl_flags = DL_FLAG_SYNC_STATE_ONLY; + dl_flags = FW_DEVLINK_FLAGS_PERMISSIVE; } } @@ -1687,7 +1814,7 @@ static void __fw_devlink_link_to_suppliers(struct device *dev, if (own_link) dl_flags = fw_devlink_get_flags(); else - dl_flags = DL_FLAG_SYNC_STATE_ONLY; + dl_flags = FW_DEVLINK_FLAGS_PERMISSIVE; list_for_each_entry_safe(link, tmp, &fwnode->suppliers, c_hook) { int ret; diff --git a/drivers/base/devtmpfs.c b/drivers/base/devtmpfs.c index eac184e6d65774da9a39cf70e6f71848a1a373d5..653c8c6ac7a73a2137598065b79a078fdaaf1841 100644 --- a/drivers/base/devtmpfs.c +++ b/drivers/base/devtmpfs.c @@ -162,7 +162,7 @@ static int dev_mkdir(const char *name, umode_t mode) if (IS_ERR(dentry)) return PTR_ERR(dentry); - err = vfs_mkdir(d_inode(path.dentry), dentry, mode); + err = vfs_mkdir(&init_user_ns, d_inode(path.dentry), dentry, mode); if (!err) /* mark as kernel-created inode */ d_inode(dentry)->i_private = &thread; @@ -212,7 +212,8 @@ static int handle_create(const char *nodename, umode_t mode, kuid_t uid, if (IS_ERR(dentry)) return PTR_ERR(dentry); - err = vfs_mknod(d_inode(path.dentry), dentry, mode, dev->devt); + err = vfs_mknod(&init_user_ns, d_inode(path.dentry), dentry, mode, + dev->devt); if (!err) { struct iattr newattrs; @@ -221,7 +222,7 @@ static int handle_create(const char *nodename, umode_t mode, kuid_t uid, newattrs.ia_gid = gid; newattrs.ia_valid = ATTR_MODE|ATTR_UID|ATTR_GID; inode_lock(d_inode(dentry)); - notify_change(dentry, &newattrs, NULL); + notify_change(&init_user_ns, dentry, &newattrs, NULL); inode_unlock(d_inode(dentry)); /* mark as kernel-created inode */ @@ -242,7 +243,8 @@ static int dev_rmdir(const char *name) return PTR_ERR(dentry); if (d_really_is_positive(dentry)) { if (d_inode(dentry)->i_private == &thread) - err = vfs_rmdir(d_inode(parent.dentry), dentry); + err = vfs_rmdir(&init_user_ns, d_inode(parent.dentry), + dentry); else err = -EPERM; } else { @@ -328,9 +330,10 @@ static int handle_remove(const char *nodename, struct device *dev) newattrs.ia_valid = ATTR_UID|ATTR_GID|ATTR_MODE; inode_lock(d_inode(dentry)); - notify_change(dentry, &newattrs, NULL); + notify_change(&init_user_ns, dentry, &newattrs, NULL); inode_unlock(d_inode(dentry)); - err = vfs_unlink(d_inode(parent.dentry), dentry, NULL); + err = vfs_unlink(&init_user_ns, d_inode(parent.dentry), + dentry, NULL); if (!err || err == -ENOENT) deleted = 1; } diff --git a/drivers/base/init.c b/drivers/base/init.c index 908e6520e804b2967ab029ca6a617cc0d1902f16..a9f57c22fb9e2f128e9d8708483ca1f631a8fa7e 100644 --- a/drivers/base/init.c +++ b/drivers/base/init.c @@ -32,6 +32,7 @@ void __init driver_init(void) */ of_core_init(); platform_bus_init(); + auxiliary_bus_init(); cpu_dev_init(); memory_dev_init(); container_dev_init(); diff --git a/drivers/base/memory.c b/drivers/base/memory.c index eef4ffb6122c9f3f9b6bd7904089c30f1968de57..f35298425575ef8775d50fd2f714e2cf417f9e81 100644 --- a/drivers/base/memory.c +++ b/drivers/base/memory.c @@ -35,7 +35,7 @@ static const char *const online_type_to_str[] = { [MMOP_ONLINE_MOVABLE] = "online_movable", }; -int memhp_online_type_from_str(const char *str) +int mhp_online_type_from_str(const char *str) { int i; @@ -253,7 +253,7 @@ static int memory_subsys_offline(struct device *dev) static ssize_t state_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { - const int online_type = memhp_online_type_from_str(buf); + const int online_type = mhp_online_type_from_str(buf); struct memory_block *mem = to_memory_block(dev); int ret; @@ -290,20 +290,20 @@ static ssize_t state_store(struct device *dev, struct device_attribute *attr, } /* - * phys_device is a bad name for this. What I really want - * is a way to differentiate between memory ranges that - * are part of physical devices that constitute - * a complete removable unit or fru. - * i.e. do these ranges belong to the same physical device, - * s.t. if I offline all of these sections I can then - * remove the physical device? + * Legacy interface that we cannot remove: s390x exposes the storage increment + * covered by a memory block, allowing for identifying which memory blocks + * comprise a storage increment. Since a memory block spans complete + * storage increments nowadays, this interface is basically unused. Other + * archs never exposed != 0. */ static ssize_t phys_device_show(struct device *dev, struct device_attribute *attr, char *buf) { struct memory_block *mem = to_memory_block(dev); + unsigned long start_pfn = section_nr_to_pfn(mem->start_section_nr); - return sysfs_emit(buf, "%d\n", mem->phys_device); + return sysfs_emit(buf, "%d\n", + arch_get_memory_phys_device(start_pfn)); } #ifdef CONFIG_MEMORY_HOTREMOVE @@ -387,19 +387,19 @@ static ssize_t auto_online_blocks_show(struct device *dev, struct device_attribute *attr, char *buf) { return sysfs_emit(buf, "%s\n", - online_type_to_str[memhp_default_online_type]); + online_type_to_str[mhp_default_online_type]); } static ssize_t auto_online_blocks_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { - const int online_type = memhp_online_type_from_str(buf); + const int online_type = mhp_online_type_from_str(buf); if (online_type < 0) return -EINVAL; - memhp_default_online_type = online_type; + mhp_default_online_type = online_type; return count; } @@ -488,11 +488,7 @@ static DEVICE_ATTR_WO(soft_offline_page); static DEVICE_ATTR_WO(hard_offline_page); #endif -/* - * Note that phys_device is optional. It is here to allow for - * differentiation between which *physical* devices each - * section belongs to... - */ +/* See phys_device_show(). */ int __weak arch_get_memory_phys_device(unsigned long start_pfn) { return 0; @@ -574,7 +570,6 @@ int register_memory(struct memory_block *memory) static int init_memory_block(unsigned long block_id, unsigned long state) { struct memory_block *mem; - unsigned long start_pfn; int ret = 0; mem = find_memory_block_by_id(block_id); @@ -588,8 +583,6 @@ static int init_memory_block(unsigned long block_id, unsigned long state) mem->start_section_nr = block_id * sections_per_block; mem->state = state; - start_pfn = section_nr_to_pfn(mem->start_section_nr); - mem->phys_device = arch_get_memory_phys_device(start_pfn); mem->nid = NUMA_NO_NODE; ret = register_memory(mem); diff --git a/drivers/base/node.c b/drivers/base/node.c index 04f71c7bc3f83867cefd7fa9ba0cda4d236c2209..f449dbb2c74666a83caabb55b39e8c9de31a622c 100644 --- a/drivers/base/node.c +++ b/drivers/base/node.c @@ -372,14 +372,19 @@ static ssize_t node_read_meminfo(struct device *dev, struct pglist_data *pgdat = NODE_DATA(nid); struct sysinfo i; unsigned long sreclaimable, sunreclaimable; + unsigned long swapcached = 0; si_meminfo_node(&i, nid); sreclaimable = node_page_state_pages(pgdat, NR_SLAB_RECLAIMABLE_B); sunreclaimable = node_page_state_pages(pgdat, NR_SLAB_UNRECLAIMABLE_B); +#ifdef CONFIG_SWAP + swapcached = node_page_state_pages(pgdat, NR_SWAPCACHE); +#endif len = sysfs_emit_at(buf, len, "Node %d MemTotal: %8lu kB\n" "Node %d MemFree: %8lu kB\n" "Node %d MemUsed: %8lu kB\n" + "Node %d SwapCached: %8lu kB\n" "Node %d Active: %8lu kB\n" "Node %d Inactive: %8lu kB\n" "Node %d Active(anon): %8lu kB\n" @@ -391,6 +396,7 @@ static ssize_t node_read_meminfo(struct device *dev, nid, K(i.totalram), nid, K(i.freeram), nid, K(i.totalram - i.freeram), + nid, K(swapcached), nid, K(node_page_state(pgdat, NR_ACTIVE_ANON) + node_page_state(pgdat, NR_ACTIVE_FILE)), nid, K(node_page_state(pgdat, NR_INACTIVE_ANON) + @@ -461,16 +467,11 @@ static ssize_t node_read_meminfo(struct device *dev, nid, K(sunreclaimable) #ifdef CONFIG_TRANSPARENT_HUGEPAGE , - nid, K(node_page_state(pgdat, NR_ANON_THPS) * - HPAGE_PMD_NR), - nid, K(node_page_state(pgdat, NR_SHMEM_THPS) * - HPAGE_PMD_NR), - nid, K(node_page_state(pgdat, NR_SHMEM_PMDMAPPED) * - HPAGE_PMD_NR), - nid, K(node_page_state(pgdat, NR_FILE_THPS) * - HPAGE_PMD_NR), - nid, K(node_page_state(pgdat, NR_FILE_PMDMAPPED) * - HPAGE_PMD_NR) + nid, K(node_page_state(pgdat, NR_ANON_THPS)), + nid, K(node_page_state(pgdat, NR_SHMEM_THPS)), + nid, K(node_page_state(pgdat, NR_SHMEM_PMDMAPPED)), + nid, K(node_page_state(pgdat, NR_FILE_THPS)), + nid, K(node_page_state(pgdat, NR_FILE_PMDMAPPED)) #endif ); len += hugetlb_report_node_meminfo(buf, len, nid); @@ -519,10 +520,14 @@ static ssize_t node_read_vmstat(struct device *dev, sum_zone_numa_state(nid, i)); #endif - for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) - len += sysfs_emit_at(buf, len, "%s %lu\n", - node_stat_name(i), - node_page_state_pages(pgdat, i)); + for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) { + unsigned long pages = node_page_state_pages(pgdat, i); + + if (vmstat_item_print_in_thp(i)) + pages /= HPAGE_PMD_NR; + len += sysfs_emit_at(buf, len, "%s %lu\n", node_stat_name(i), + pages); + } return len; } diff --git a/drivers/base/platform.c b/drivers/base/platform.c index 8456d8384ac8e14842c603554516aede25239f6c..6e1f8e0b661cef101dc6d4a01ad56beea41345d5 100644 --- a/drivers/base/platform.c +++ b/drivers/base/platform.c @@ -573,7 +573,7 @@ static void platform_device_release(struct device *dev) struct platform_object *pa = container_of(dev, struct platform_object, pdev.dev); - of_device_node_put(&pa->pdev.dev); + of_node_put(pa->pdev.dev.of_node); kfree(pa->pdev.dev.platform_data); kfree(pa->pdev.mfd_cell); kfree(pa->pdev.resource); @@ -1463,13 +1463,16 @@ static int platform_remove(struct device *_dev) { struct platform_driver *drv = to_platform_driver(_dev->driver); struct platform_device *dev = to_platform_device(_dev); - int ret = 0; - if (drv->remove) - ret = drv->remove(dev); + if (drv->remove) { + int ret = drv->remove(dev); + + if (ret) + dev_warn(_dev, "remove callback returned a non-zero value. This will be ignored.\n"); + } dev_pm_domain_detach(_dev, true); - return ret; + return 0; } static void platform_shutdown(struct device *_dev) diff --git a/drivers/base/power/domain.c b/drivers/base/power/domain.c index aaf6c83b5cf616364ed33bfb2884498293b67fdd..78c310d3179d36b67c59a62e6ceda8d2f0f774d4 100644 --- a/drivers/base/power/domain.c +++ b/drivers/base/power/domain.c @@ -2196,6 +2196,7 @@ static int genpd_add_provider(struct device_node *np, genpd_xlate_t xlate, cp->node = of_node_get(np); cp->data = data; cp->xlate = xlate; + fwnode_dev_initialized(&np->fwnode, true); mutex_lock(&of_genpd_mutex); list_add(&cp->link, &of_genpd_providers); @@ -2385,6 +2386,7 @@ void of_genpd_del_provider(struct device_node *np) } } + fwnode_dev_initialized(&cp->node->fwnode, false); list_del(&cp->link); of_node_put(cp->node); kfree(cp); diff --git a/drivers/base/regmap/regcache.c b/drivers/base/regmap/regcache.c index 7f4b3b62492ca3849fb08b0bbe888d8b548c12a7..f2469d3435ca384db5f014c031ca3bd2e4687dd9 100644 --- a/drivers/base/regmap/regcache.c +++ b/drivers/base/regmap/regcache.c @@ -68,7 +68,7 @@ static int regcache_hw_init(struct regmap *map) map->cache_bypass = cache_bypass; if (ret == 0) { map->reg_defaults_raw = tmp_buf; - map->cache_free = 1; + map->cache_free = true; } else { kfree(tmp_buf); } diff --git a/drivers/base/regmap/regmap-sdw-mbq.c b/drivers/base/regmap/regmap-sdw-mbq.c index 8ce30650b97c7b19b291463d6da92e5062ce99e3..fe3ac26b66ad73a415b2fb4eb60dc067db67ccf2 100644 --- a/drivers/base/regmap/regmap-sdw-mbq.c +++ b/drivers/base/regmap/regmap-sdw-mbq.c @@ -15,11 +15,11 @@ static int regmap_sdw_mbq_write(void *context, unsigned int reg, unsigned int va struct sdw_slave *slave = dev_to_sdw_dev(dev); int ret; - ret = sdw_write(slave, SDW_SDCA_MBQ_CTL(reg), (val >> 8) & 0xff); + ret = sdw_write_no_pm(slave, SDW_SDCA_MBQ_CTL(reg), (val >> 8) & 0xff); if (ret < 0) return ret; - return sdw_write(slave, reg, val & 0xff); + return sdw_write_no_pm(slave, reg, val & 0xff); } static int regmap_sdw_mbq_read(void *context, unsigned int reg, unsigned int *val) @@ -29,11 +29,11 @@ static int regmap_sdw_mbq_read(void *context, unsigned int reg, unsigned int *va int read0; int read1; - read0 = sdw_read(slave, reg); + read0 = sdw_read_no_pm(slave, reg); if (read0 < 0) return read0; - read1 = sdw_read(slave, SDW_SDCA_MBQ_CTL(reg)); + read1 = sdw_read_no_pm(slave, SDW_SDCA_MBQ_CTL(reg)); if (read1 < 0) return read1; @@ -98,4 +98,4 @@ struct regmap *__devm_regmap_init_sdw_mbq(struct sdw_slave *sdw, EXPORT_SYMBOL_GPL(__devm_regmap_init_sdw_mbq); MODULE_DESCRIPTION("Regmap SoundWire MBQ Module"); -MODULE_LICENSE("GPL v2"); +MODULE_LICENSE("GPL"); diff --git a/drivers/base/regmap/regmap-sdw.c b/drivers/base/regmap/regmap-sdw.c index c83be26434e7621d49bacc34386690afff766fa3..966de8a136d9004247b94d5a99bb5f8af1a26ac4 100644 --- a/drivers/base/regmap/regmap-sdw.c +++ b/drivers/base/regmap/regmap-sdw.c @@ -13,7 +13,7 @@ static int regmap_sdw_write(void *context, unsigned int reg, unsigned int val) struct device *dev = context; struct sdw_slave *slave = dev_to_sdw_dev(dev); - return sdw_write(slave, reg, val); + return sdw_write_no_pm(slave, reg, val); } static int regmap_sdw_read(void *context, unsigned int reg, unsigned int *val) @@ -22,7 +22,7 @@ static int regmap_sdw_read(void *context, unsigned int reg, unsigned int *val) struct sdw_slave *slave = dev_to_sdw_dev(dev); int read; - read = sdw_read(slave, reg); + read = sdw_read_no_pm(slave, reg); if (read < 0) return read; diff --git a/drivers/base/test/Makefile b/drivers/base/test/Makefile index 3ca56367c84b729a7b7f3d0e7df7b61c9b652a46..2f15fae8625f191d42fd80f594d86fcb5a8fe4c5 100644 --- a/drivers/base/test/Makefile +++ b/drivers/base/test/Makefile @@ -2,3 +2,4 @@ obj-$(CONFIG_TEST_ASYNC_DRIVER_PROBE) += test_async_driver_probe.o obj-$(CONFIG_KUNIT_DRIVER_PE_TEST) += property-entry-test.o +CFLAGS_REMOVE_property-entry-test.o += -fplugin-arg-structleak_plugin-byref -fplugin-arg-structleak_plugin-byref-all diff --git a/drivers/block/Kconfig b/drivers/block/Kconfig index 2779e85795a70a2392900d74dd145ed2fd4a3d99..fd236158f32d995f69781a8c0208913bcbb2276c 100644 --- a/drivers/block/Kconfig +++ b/drivers/block/Kconfig @@ -66,6 +66,12 @@ config AMIGA_Z2RAM To compile this driver as a module, choose M here: the module will be called z2ram. +config N64CART + bool "N64 cart support" + depends on MACH_NINTENDO64 + help + Support for the N64 cart. + config CDROM tristate select BLK_SCSI_REQUEST diff --git a/drivers/block/Makefile b/drivers/block/Makefile index b501b8728fb9621eeba888872ad5905354c0addb..e3e3f1c79a827e829a20da9af1f593b069c5aa58 100644 --- a/drivers/block/Makefile +++ b/drivers/block/Makefile @@ -17,6 +17,7 @@ obj-$(CONFIG_PS3_DISK) += ps3disk.o obj-$(CONFIG_PS3_VRAM) += ps3vram.o obj-$(CONFIG_ATARI_FLOPPY) += ataflop.o obj-$(CONFIG_AMIGA_Z2RAM) += z2ram.o +obj-$(CONFIG_N64CART) += n64cart.o obj-$(CONFIG_BLK_DEV_RAM) += brd.o obj-$(CONFIG_BLK_DEV_LOOP) += loop.o obj-$(CONFIG_XILINX_SYSACE) += xsysace.o diff --git a/drivers/block/loop.c b/drivers/block/loop.c index 578fc034db3f483c9224e222f35fb4c4a341e2a6..a370cde3ddd49ad7e8c8d005279f904fa793bb57 100644 --- a/drivers/block/loop.c +++ b/drivers/block/loop.c @@ -663,7 +663,7 @@ static inline int is_loop_device(struct file *file) { struct inode *i = file->f_mapping->host; - return i && S_ISBLK(i->i_mode) && MAJOR(i->i_rdev) == LOOP_MAJOR; + return i && S_ISBLK(i->i_mode) && imajor(i) == LOOP_MAJOR; } static int loop_validate_file(struct file *file, struct block_device *bdev) @@ -1212,6 +1212,9 @@ static int __loop_clr_fd(struct loop_device *lo, bool release) goto out_unlock; } + if (test_bit(QUEUE_FLAG_WC, &lo->lo_queue->queue_flags)) + blk_queue_write_cache(lo->lo_queue, false, false); + /* freeze request queue during the transition */ blk_mq_freeze_queue(lo->lo_queue); diff --git a/drivers/block/n64cart.c b/drivers/block/n64cart.c new file mode 100644 index 0000000000000000000000000000000000000000..47bdf324e962e4efd2d268dcf8714f9640bb703b --- /dev/null +++ b/drivers/block/n64cart.c @@ -0,0 +1,178 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Support for the N64 cart. + * + * Copyright (c) 2021 Lauri Kasanen + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt +#include +#include +#include +#include +#include +#include + +enum { + PI_DRAM_REG = 0, + PI_CART_REG, + PI_READ_REG, + PI_WRITE_REG, + PI_STATUS_REG, +}; + +#define PI_STATUS_DMA_BUSY (1 << 0) +#define PI_STATUS_IO_BUSY (1 << 1) + +#define CART_DOMAIN 0x10000000 +#define CART_MAX 0x1FFFFFFF + +#define MIN_ALIGNMENT 8 + +static u32 __iomem *reg_base; + +static unsigned int start; +module_param(start, uint, 0); +MODULE_PARM_DESC(start, "Start address of the cart block data"); + +static unsigned int size; +module_param(size, uint, 0); +MODULE_PARM_DESC(size, "Size of the cart block data, in bytes"); + +static void n64cart_write_reg(const u8 reg, const u32 value) +{ + writel(value, reg_base + reg); +} + +static u32 n64cart_read_reg(const u8 reg) +{ + return readl(reg_base + reg); +} + +static void n64cart_wait_dma(void) +{ + while (n64cart_read_reg(PI_STATUS_REG) & + (PI_STATUS_DMA_BUSY | PI_STATUS_IO_BUSY)) + cpu_relax(); +} + +/* + * Process a single bvec of a bio. + */ +static bool n64cart_do_bvec(struct device *dev, struct bio_vec *bv, u32 pos) +{ + dma_addr_t dma_addr; + const u32 bstart = pos + start; + + /* Alignment check */ + WARN_ON_ONCE((bv->bv_offset & (MIN_ALIGNMENT - 1)) || + (bv->bv_len & (MIN_ALIGNMENT - 1))); + + dma_addr = dma_map_bvec(dev, bv, DMA_FROM_DEVICE, 0); + if (dma_mapping_error(dev, dma_addr)) + return false; + + n64cart_wait_dma(); + + n64cart_write_reg(PI_DRAM_REG, dma_addr + bv->bv_offset); + n64cart_write_reg(PI_CART_REG, (bstart | CART_DOMAIN) & CART_MAX); + n64cart_write_reg(PI_WRITE_REG, bv->bv_len - 1); + + n64cart_wait_dma(); + + dma_unmap_page(dev, dma_addr, bv->bv_len, DMA_FROM_DEVICE); + return true; +} + +static blk_qc_t n64cart_submit_bio(struct bio *bio) +{ + struct bio_vec bvec; + struct bvec_iter iter; + struct device *dev = bio->bi_disk->private_data; + u32 pos = bio->bi_iter.bi_sector << SECTOR_SHIFT; + + bio_for_each_segment(bvec, bio, iter) { + if (!n64cart_do_bvec(dev, &bvec, pos)) + goto io_error; + pos += bvec.bv_len; + } + + bio_endio(bio); + return BLK_QC_T_NONE; +io_error: + bio_io_error(bio); + return BLK_QC_T_NONE; +} + +static const struct block_device_operations n64cart_fops = { + .owner = THIS_MODULE, + .submit_bio = n64cart_submit_bio, +}; + +/* + * The target device is embedded and RAM-constrained. We save RAM + * by initializing in __init code that gets dropped late in boot. + * For the same reason there is no module or unloading support. + */ +static int __init n64cart_probe(struct platform_device *pdev) +{ + struct gendisk *disk; + + if (!start || !size) { + pr_err("start or size not specified\n"); + return -ENODEV; + } + + if (size & 4095) { + pr_err("size must be a multiple of 4K\n"); + return -ENODEV; + } + + reg_base = devm_platform_ioremap_resource(pdev, 0); + if (!reg_base) + return -EINVAL; + + disk = alloc_disk(0); + if (!disk) + return -ENOMEM; + + disk->queue = blk_alloc_queue(NUMA_NO_NODE); + if (!disk->queue) + return -ENOMEM; + + disk->first_minor = 0; + disk->flags = GENHD_FL_NO_PART_SCAN | GENHD_FL_EXT_DEVT; + disk->fops = &n64cart_fops; + disk->private_data = &pdev->dev; + strcpy(disk->disk_name, "n64cart"); + + set_capacity(disk, size >> SECTOR_SHIFT); + set_disk_ro(disk, 1); + + blk_queue_flag_set(QUEUE_FLAG_NONROT, disk->queue); + blk_queue_physical_block_size(disk->queue, 4096); + blk_queue_logical_block_size(disk->queue, 4096); + + add_disk(disk); + + pr_info("n64cart: %u kb disk\n", size / 1024); + + return 0; +} + +static struct platform_driver n64cart_driver = { + .driver = { + .name = "n64cart", + }, +}; + +static int __init n64cart_init(void) +{ + return platform_driver_probe(&n64cart_driver, n64cart_probe); +} + +module_init(n64cart_init); + +MODULE_AUTHOR("Lauri Kasanen "); +MODULE_DESCRIPTION("Driver for the N64 cart"); +MODULE_LICENSE("GPL"); diff --git a/drivers/block/nbd.c b/drivers/block/nbd.c index 8b9622eb0a21b280bc30d60182fec9c521be8537..4ff71b579cfcc8051f911fdef19a35a653509c0c 100644 --- a/drivers/block/nbd.c +++ b/drivers/block/nbd.c @@ -78,8 +78,7 @@ struct link_dead_args { #define NBD_RT_HAS_PID_FILE 3 #define NBD_RT_HAS_CONFIG_REF 4 #define NBD_RT_BOUND 5 -#define NBD_RT_DESTROY_ON_DISCONNECT 6 -#define NBD_RT_DISCONNECT_ON_CLOSE 7 +#define NBD_RT_DISCONNECT_ON_CLOSE 6 #define NBD_DESTROY_ON_DISCONNECT 0 #define NBD_DISCONNECT_REQUESTED 1 @@ -1904,12 +1903,21 @@ static int nbd_genl_connect(struct sk_buff *skb, struct genl_info *info) if (info->attrs[NBD_ATTR_CLIENT_FLAGS]) { u64 flags = nla_get_u64(info->attrs[NBD_ATTR_CLIENT_FLAGS]); if (flags & NBD_CFLAG_DESTROY_ON_DISCONNECT) { - set_bit(NBD_RT_DESTROY_ON_DISCONNECT, - &config->runtime_flags); - set_bit(NBD_DESTROY_ON_DISCONNECT, &nbd->flags); - put_dev = true; + /* + * We have 1 ref to keep the device around, and then 1 + * ref for our current operation here, which will be + * inherited by the config. If we already have + * DESTROY_ON_DISCONNECT set then we know we don't have + * that extra ref already held so we don't need the + * put_dev. + */ + if (!test_and_set_bit(NBD_DESTROY_ON_DISCONNECT, + &nbd->flags)) + put_dev = true; } else { - clear_bit(NBD_DESTROY_ON_DISCONNECT, &nbd->flags); + if (test_and_clear_bit(NBD_DESTROY_ON_DISCONNECT, + &nbd->flags)) + refcount_inc(&nbd->refs); } if (flags & NBD_CFLAG_DISCONNECT_ON_CLOSE) { set_bit(NBD_RT_DISCONNECT_ON_CLOSE, @@ -2080,15 +2088,13 @@ static int nbd_genl_reconfigure(struct sk_buff *skb, struct genl_info *info) if (info->attrs[NBD_ATTR_CLIENT_FLAGS]) { u64 flags = nla_get_u64(info->attrs[NBD_ATTR_CLIENT_FLAGS]); if (flags & NBD_CFLAG_DESTROY_ON_DISCONNECT) { - if (!test_and_set_bit(NBD_RT_DESTROY_ON_DISCONNECT, - &config->runtime_flags)) + if (!test_and_set_bit(NBD_DESTROY_ON_DISCONNECT, + &nbd->flags)) put_dev = true; - set_bit(NBD_DESTROY_ON_DISCONNECT, &nbd->flags); } else { - if (test_and_clear_bit(NBD_RT_DESTROY_ON_DISCONNECT, - &config->runtime_flags)) + if (test_and_clear_bit(NBD_DESTROY_ON_DISCONNECT, + &nbd->flags)) refcount_inc(&nbd->refs); - clear_bit(NBD_DESTROY_ON_DISCONNECT, &nbd->flags); } if (flags & NBD_CFLAG_DISCONNECT_ON_CLOSE) { diff --git a/drivers/block/virtio_blk.c b/drivers/block/virtio_blk.c index b0285db7cf4f66ae4cf1ea9a2d8d192a7fa98148..b9fa3ef5b57c0b0b0d10aed51779f75ebd04ba27 100644 --- a/drivers/block/virtio_blk.c +++ b/drivers/block/virtio_blk.c @@ -705,6 +705,7 @@ static int virtblk_probe(struct virtio_device *vdev) u32 v, blk_size, max_size, sg_elems, opt_io_size; u16 min_io_size; u8 physical_block_exp, alignment_offset; + unsigned int queue_depth; if (!vdev->config->get) { dev_err(&vdev->dev, "%s failure: config access disabled\n", @@ -756,16 +757,18 @@ static int virtblk_probe(struct virtio_device *vdev) } /* Default queue sizing is to fill the ring. */ - if (!virtblk_queue_depth) { - virtblk_queue_depth = vblk->vqs[0].vq->num_free; + if (likely(!virtblk_queue_depth)) { + queue_depth = vblk->vqs[0].vq->num_free; /* ... but without indirect descs, we use 2 descs per req */ if (!virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC)) - virtblk_queue_depth /= 2; + queue_depth /= 2; + } else { + queue_depth = virtblk_queue_depth; } memset(&vblk->tag_set, 0, sizeof(vblk->tag_set)); vblk->tag_set.ops = &virtio_mq_ops; - vblk->tag_set.queue_depth = virtblk_queue_depth; + vblk->tag_set.queue_depth = queue_depth; vblk->tag_set.numa_node = NUMA_NO_NODE; vblk->tag_set.flags = BLK_MQ_F_SHOULD_MERGE; vblk->tag_set.cmd_size = diff --git a/drivers/block/xen-blkback/blkback.c b/drivers/block/xen-blkback/blkback.c index da16121140cab23b8840e2aaa72131c795df8099..1cdf09ff67b68c5d704d002ae59a3a6a0c9ba7c2 100644 --- a/drivers/block/xen-blkback/blkback.c +++ b/drivers/block/xen-blkback/blkback.c @@ -1326,9 +1326,7 @@ static int dispatch_rw_block_io(struct xen_blkif_ring *ring, pages[i]->page, seg[i].nsec << 9, seg[i].offset) == 0)) { - - int nr_iovecs = min_t(int, (nseg-i), BIO_MAX_PAGES); - bio = bio_alloc(GFP_KERNEL, nr_iovecs); + bio = bio_alloc(GFP_KERNEL, bio_max_segs(nseg - i)); if (unlikely(bio == NULL)) goto fail_put_bio; diff --git a/drivers/block/zram/zram_drv.c b/drivers/block/zram/zram_drv.c index d7018543842e026ab4d8d5c83ba14367a7de7c3e..a711a2e2a7946bc466913c38585b867b62151748 100644 --- a/drivers/block/zram/zram_drv.c +++ b/drivers/block/zram/zram_drv.c @@ -1081,7 +1081,7 @@ static ssize_t mm_stat_show(struct device *dev, zram->limit_pages << PAGE_SHIFT, max_used << PAGE_SHIFT, (u64)atomic64_read(&zram->stats.same_pages), - pool_stats.pages_compacted, + atomic_long_read(&pool_stats.pages_compacted), (u64)atomic64_read(&zram->stats.huge_pages), (u64)atomic64_read(&zram->stats.huge_pages_since)); up_read(&zram->init_lock); diff --git a/drivers/bus/fsl-mc/Kconfig b/drivers/bus/fsl-mc/Kconfig index c23c77c9b7057e9cc8f47f2ae07a42095ff1d3f4..b1fd55901c504d298ce368f03cafd50e2adf5ada 100644 --- a/drivers/bus/fsl-mc/Kconfig +++ b/drivers/bus/fsl-mc/Kconfig @@ -14,3 +14,10 @@ config FSL_MC_BUS architecture. The fsl-mc bus driver handles discovery of DPAA2 objects (which are represented as Linux devices) and binding objects to drivers. + +config FSL_MC_UAPI_SUPPORT + bool "Management Complex (MC) userspace support" + depends on FSL_MC_BUS + help + Provides userspace support for interrogating, creating, destroying or + configuring DPAA2 objects exported by the Management Complex. diff --git a/drivers/bus/fsl-mc/Makefile b/drivers/bus/fsl-mc/Makefile index 3c518c7e8374b140381058ae4c5ee8248b976e6e..4ae292a30e5304464ebcc523afef85ac8eff4ffc 100644 --- a/drivers/bus/fsl-mc/Makefile +++ b/drivers/bus/fsl-mc/Makefile @@ -16,3 +16,6 @@ mc-bus-driver-objs := fsl-mc-bus.o \ fsl-mc-allocator.o \ fsl-mc-msi.o \ dpmcp.o + +# MC userspace support +obj-$(CONFIG_FSL_MC_UAPI_SUPPORT) += fsl-mc-uapi.o diff --git a/drivers/bus/fsl-mc/dprc-driver.c b/drivers/bus/fsl-mc/dprc-driver.c index 68488a7ad0d6a8c9ee20df4abed6d95fff9cab0b..e3e2ae41c22b3f0e059382992a3034b894b442b7 100644 --- a/drivers/bus/fsl-mc/dprc-driver.c +++ b/drivers/bus/fsl-mc/dprc-driver.c @@ -237,8 +237,8 @@ static void dprc_add_new_devices(struct fsl_mc_device *mc_bus_dev, * populated before they can get allocation requests from probe callbacks * of the device drivers for the non-allocatable devices. */ -static int dprc_scan_objects(struct fsl_mc_device *mc_bus_dev, - bool alloc_interrupts) +int dprc_scan_objects(struct fsl_mc_device *mc_bus_dev, + bool alloc_interrupts) { int num_child_objects; int dprc_get_obj_failures; @@ -458,8 +458,9 @@ static irqreturn_t dprc_irq0_handler_thread(int irq_num, void *arg) /* * Disable and clear interrupt for a given DPRC object */ -static int disable_dprc_irq(struct fsl_mc_device *mc_dev) +int disable_dprc_irq(struct fsl_mc_device *mc_dev) { + struct fsl_mc_bus *mc_bus = to_fsl_mc_bus(mc_dev); int error; struct fsl_mc_io *mc_io = mc_dev->mc_io; @@ -496,9 +497,18 @@ static int disable_dprc_irq(struct fsl_mc_device *mc_dev) return error; } + mc_bus->irq_enabled = 0; + return 0; } +int get_dprc_irq_state(struct fsl_mc_device *mc_dev) +{ + struct fsl_mc_bus *mc_bus = to_fsl_mc_bus(mc_dev); + + return mc_bus->irq_enabled; +} + static int register_dprc_irq_handler(struct fsl_mc_device *mc_dev) { int error; @@ -525,8 +535,9 @@ static int register_dprc_irq_handler(struct fsl_mc_device *mc_dev) return 0; } -static int enable_dprc_irq(struct fsl_mc_device *mc_dev) +int enable_dprc_irq(struct fsl_mc_device *mc_dev) { + struct fsl_mc_bus *mc_bus = to_fsl_mc_bus(mc_dev); int error; /* @@ -554,6 +565,8 @@ static int enable_dprc_irq(struct fsl_mc_device *mc_dev) return error; } + mc_bus->irq_enabled = 1; + return 0; } @@ -603,6 +616,7 @@ int dprc_setup(struct fsl_mc_device *mc_dev) struct irq_domain *mc_msi_domain; bool mc_io_created = false; bool msi_domain_set = false; + bool uapi_created = false; u16 major_ver, minor_ver; size_t region_size; int error; @@ -635,6 +649,11 @@ int dprc_setup(struct fsl_mc_device *mc_dev) return error; mc_io_created = true; + } else { + error = fsl_mc_uapi_create_device_file(mc_bus); + if (error < 0) + return -EPROBE_DEFER; + uapi_created = true; } mc_msi_domain = fsl_mc_find_msi_domain(&mc_dev->dev); @@ -692,6 +711,9 @@ int dprc_setup(struct fsl_mc_device *mc_dev) mc_dev->mc_io = NULL; } + if (uapi_created) + fsl_mc_uapi_remove_device_file(mc_bus); + return error; } EXPORT_SYMBOL_GPL(dprc_setup); @@ -763,6 +785,7 @@ static void dprc_teardown_irq(struct fsl_mc_device *mc_dev) int dprc_cleanup(struct fsl_mc_device *mc_dev) { + struct fsl_mc_bus *mc_bus = to_fsl_mc_bus(mc_dev); int error; /* this function should be called only for DPRCs, it @@ -793,6 +816,8 @@ int dprc_cleanup(struct fsl_mc_device *mc_dev) if (!fsl_mc_is_root_dprc(&mc_dev->dev)) { fsl_destroy_mc_io(mc_dev->mc_io); mc_dev->mc_io = NULL; + } else { + fsl_mc_uapi_remove_device_file(mc_bus); } return 0; diff --git a/drivers/bus/fsl-mc/fsl-mc-bus.c b/drivers/bus/fsl-mc/fsl-mc-bus.c index 8af978bd0000c2a2302e609dc49851fe9528d733..380ad1fdb74564b4f63a0150672d5d20afc5b045 100644 --- a/drivers/bus/fsl-mc/fsl-mc-bus.c +++ b/drivers/bus/fsl-mc/fsl-mc-bus.c @@ -41,7 +41,7 @@ struct fsl_mc { struct fsl_mc_device *root_mc_bus_dev; u8 num_translation_ranges; struct fsl_mc_addr_translation_range *translation_ranges; - void *fsl_mc_regs; + void __iomem *fsl_mc_regs; }; /** @@ -208,12 +208,108 @@ static struct attribute *fsl_mc_dev_attrs[] = { ATTRIBUTE_GROUPS(fsl_mc_dev); +static int scan_fsl_mc_bus(struct device *dev, void *data) +{ + struct fsl_mc_device *root_mc_dev; + struct fsl_mc_bus *root_mc_bus; + + if (!fsl_mc_is_root_dprc(dev)) + goto exit; + + root_mc_dev = to_fsl_mc_device(dev); + root_mc_bus = to_fsl_mc_bus(root_mc_dev); + mutex_lock(&root_mc_bus->scan_mutex); + dprc_scan_objects(root_mc_dev, NULL); + mutex_unlock(&root_mc_bus->scan_mutex); + +exit: + return 0; +} + +static ssize_t rescan_store(struct bus_type *bus, + const char *buf, size_t count) +{ + unsigned long val; + + if (kstrtoul(buf, 0, &val) < 0) + return -EINVAL; + + if (val) + bus_for_each_dev(bus, NULL, NULL, scan_fsl_mc_bus); + + return count; +} +static BUS_ATTR_WO(rescan); + +static int fsl_mc_bus_set_autorescan(struct device *dev, void *data) +{ + struct fsl_mc_device *root_mc_dev; + unsigned long val; + char *buf = data; + + if (!fsl_mc_is_root_dprc(dev)) + goto exit; + + root_mc_dev = to_fsl_mc_device(dev); + + if (kstrtoul(buf, 0, &val) < 0) + return -EINVAL; + + if (val) + enable_dprc_irq(root_mc_dev); + else + disable_dprc_irq(root_mc_dev); + +exit: + return 0; +} + +static int fsl_mc_bus_get_autorescan(struct device *dev, void *data) +{ + struct fsl_mc_device *root_mc_dev; + char *buf = data; + + if (!fsl_mc_is_root_dprc(dev)) + goto exit; + + root_mc_dev = to_fsl_mc_device(dev); + + sprintf(buf, "%d\n", get_dprc_irq_state(root_mc_dev)); +exit: + return 0; +} + +static ssize_t autorescan_store(struct bus_type *bus, + const char *buf, size_t count) +{ + bus_for_each_dev(bus, NULL, (void *)buf, fsl_mc_bus_set_autorescan); + + return count; +} + +static ssize_t autorescan_show(struct bus_type *bus, char *buf) +{ + bus_for_each_dev(bus, NULL, (void *)buf, fsl_mc_bus_get_autorescan); + return strlen(buf); +} + +static BUS_ATTR_RW(autorescan); + +static struct attribute *fsl_mc_bus_attrs[] = { + &bus_attr_rescan.attr, + &bus_attr_autorescan.attr, + NULL, +}; + +ATTRIBUTE_GROUPS(fsl_mc_bus); + struct bus_type fsl_mc_bus_type = { .name = "fsl-mc", .match = fsl_mc_bus_match, .uevent = fsl_mc_bus_uevent, .dma_configure = fsl_mc_dma_configure, .dev_groups = fsl_mc_dev_groups, + .bus_groups = fsl_mc_bus_groups, }; EXPORT_SYMBOL_GPL(fsl_mc_bus_type); @@ -292,6 +388,11 @@ struct device_type fsl_mc_bus_dpdmai_type = { }; EXPORT_SYMBOL_GPL(fsl_mc_bus_dpdmai_type); +struct device_type fsl_mc_bus_dpdbg_type = { + .name = "fsl_mc_bus_dpdbg" +}; +EXPORT_SYMBOL_GPL(fsl_mc_bus_dpdbg_type); + static struct device_type *fsl_mc_get_device_type(const char *type) { static const struct { @@ -313,6 +414,7 @@ static struct device_type *fsl_mc_get_device_type(const char *type) { &fsl_mc_bus_dpaiop_type, "dpaiop" }, { &fsl_mc_bus_dpci_type, "dpci" }, { &fsl_mc_bus_dpdmai_type, "dpdmai" }, + { &fsl_mc_bus_dpdbg_type, "dpdbg" }, { NULL, NULL } }; int i; diff --git a/drivers/bus/fsl-mc/fsl-mc-private.h b/drivers/bus/fsl-mc/fsl-mc-private.h index c932387641fa1cd767a3d257b48e25ebbf30d599..1958fa065360521eb88bf8bcc45af4a90146e060 100644 --- a/drivers/bus/fsl-mc/fsl-mc-private.h +++ b/drivers/bus/fsl-mc/fsl-mc-private.h @@ -10,6 +10,8 @@ #include #include +#include +#include /* * Data Path Management Complex (DPMNG) General API @@ -542,6 +544,22 @@ struct fsl_mc_resource_pool { struct fsl_mc_bus *mc_bus; }; +/** + * struct fsl_mc_uapi - information associated with a device file + * @misc: struct miscdevice linked to the root dprc + * @device: newly created device in /dev + * @mutex: mutex lock to serialize the open/release operations + * @local_instance_in_use: local MC I/O instance in use or not + * @static_mc_io: pointer to the static MC I/O object + */ +struct fsl_mc_uapi { + struct miscdevice misc; + struct device *device; + struct mutex mutex; /* serialize open/release operations */ + u32 local_instance_in_use; + struct fsl_mc_io *static_mc_io; +}; + /** * struct fsl_mc_bus - logical bus that corresponds to a physical DPRC * @mc_dev: fsl-mc device for the bus device itself. @@ -551,6 +569,7 @@ struct fsl_mc_resource_pool { * @irq_resources: Pointer to array of IRQ objects for the IRQ pool * @scan_mutex: Serializes bus scanning * @dprc_attr: DPRC attributes + * @uapi_misc: struct that abstracts the interaction with userspace */ struct fsl_mc_bus { struct fsl_mc_device mc_dev; @@ -558,6 +577,8 @@ struct fsl_mc_bus { struct fsl_mc_device_irq *irq_resources; struct mutex scan_mutex; /* serializes bus scanning */ struct dprc_attributes dprc_attr; + struct fsl_mc_uapi uapi_misc; + int irq_enabled; }; #define to_fsl_mc_bus(_mc_dev) \ @@ -574,6 +595,9 @@ int __init dprc_driver_init(void); void dprc_driver_exit(void); +int dprc_scan_objects(struct fsl_mc_device *mc_bus_dev, + bool alloc_interrupts); + int __init fsl_mc_allocator_driver_init(void); void fsl_mc_allocator_driver_exit(void); @@ -612,4 +636,29 @@ void fsl_mc_get_root_dprc(struct device *dev, struct fsl_mc_device *fsl_mc_device_lookup(struct fsl_mc_obj_desc *obj_desc, struct fsl_mc_device *mc_bus_dev); +u16 mc_cmd_hdr_read_cmdid(struct fsl_mc_command *cmd); + +#ifdef CONFIG_FSL_MC_UAPI_SUPPORT + +int fsl_mc_uapi_create_device_file(struct fsl_mc_bus *mc_bus); + +void fsl_mc_uapi_remove_device_file(struct fsl_mc_bus *mc_bus); + +#else + +static inline int fsl_mc_uapi_create_device_file(struct fsl_mc_bus *mc_bus) +{ + return 0; +} + +static inline void fsl_mc_uapi_remove_device_file(struct fsl_mc_bus *mc_bus) +{ +} + +#endif + +int disable_dprc_irq(struct fsl_mc_device *mc_dev); +int enable_dprc_irq(struct fsl_mc_device *mc_dev); +int get_dprc_irq_state(struct fsl_mc_device *mc_dev); + #endif /* _FSL_MC_PRIVATE_H_ */ diff --git a/drivers/bus/fsl-mc/fsl-mc-uapi.c b/drivers/bus/fsl-mc/fsl-mc-uapi.c new file mode 100644 index 0000000000000000000000000000000000000000..9c4c1395fcdbf26a8c4ddaae0a948260e9e76d3d --- /dev/null +++ b/drivers/bus/fsl-mc/fsl-mc-uapi.c @@ -0,0 +1,597 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Management Complex (MC) userspace support + * + * Copyright 2021 NXP + * + */ + +#include +#include +#include +#include + +#include "fsl-mc-private.h" + +struct uapi_priv_data { + struct fsl_mc_uapi *uapi; + struct fsl_mc_io *mc_io; +}; + +struct fsl_mc_cmd_desc { + u16 cmdid_value; + u16 cmdid_mask; + int size; + bool token; + int flags; +}; + +#define FSL_MC_CHECK_MODULE_ID BIT(0) +#define FSL_MC_CAP_NET_ADMIN_NEEDED BIT(1) + +enum fsl_mc_cmd_index { + DPDBG_DUMP = 0, + DPDBG_SET, + DPRC_GET_CONTAINER_ID, + DPRC_CREATE_CONT, + DPRC_DESTROY_CONT, + DPRC_ASSIGN, + DPRC_UNASSIGN, + DPRC_GET_OBJ_COUNT, + DPRC_GET_OBJ, + DPRC_GET_RES_COUNT, + DPRC_GET_RES_IDS, + DPRC_SET_OBJ_LABEL, + DPRC_SET_LOCKED, + DPRC_CONNECT, + DPRC_DISCONNECT, + DPRC_GET_POOL, + DPRC_GET_POOL_COUNT, + DPRC_GET_CONNECTION, + DPCI_GET_LINK_STATE, + DPCI_GET_PEER_ATTR, + DPAIOP_GET_SL_VERSION, + DPAIOP_GET_STATE, + DPMNG_GET_VERSION, + DPSECI_GET_TX_QUEUE, + DPMAC_GET_COUNTER, + DPMAC_GET_MAC_ADDR, + DPNI_SET_PRIM_MAC, + DPNI_GET_PRIM_MAC, + DPNI_GET_STATISTICS, + DPNI_GET_LINK_STATE, + DPNI_GET_MAX_FRAME_LENGTH, + DPSW_GET_TAILDROP, + DPSW_SET_TAILDROP, + DPSW_IF_GET_COUNTER, + DPSW_IF_GET_MAX_FRAME_LENGTH, + DPDMUX_GET_COUNTER, + DPDMUX_IF_GET_MAX_FRAME_LENGTH, + GET_ATTR, + GET_IRQ_MASK, + GET_IRQ_STATUS, + CLOSE, + OPEN, + GET_API_VERSION, + DESTROY, + CREATE, +}; + +static struct fsl_mc_cmd_desc fsl_mc_accepted_cmds[] = { + [DPDBG_DUMP] = { + .cmdid_value = 0x1300, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 28, + }, + [DPDBG_SET] = { + .cmdid_value = 0x1400, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 28, + }, + [DPRC_GET_CONTAINER_ID] = { + .cmdid_value = 0x8300, + .cmdid_mask = 0xFFF0, + .token = false, + .size = 8, + }, + [DPRC_CREATE_CONT] = { + .cmdid_value = 0x1510, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 40, + .flags = FSL_MC_CAP_NET_ADMIN_NEEDED, + }, + [DPRC_DESTROY_CONT] = { + .cmdid_value = 0x1520, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 12, + .flags = FSL_MC_CAP_NET_ADMIN_NEEDED, + }, + [DPRC_ASSIGN] = { + .cmdid_value = 0x1570, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 40, + .flags = FSL_MC_CAP_NET_ADMIN_NEEDED, + }, + [DPRC_UNASSIGN] = { + .cmdid_value = 0x1580, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 40, + .flags = FSL_MC_CAP_NET_ADMIN_NEEDED, + }, + [DPRC_GET_OBJ_COUNT] = { + .cmdid_value = 0x1590, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 16, + }, + [DPRC_GET_OBJ] = { + .cmdid_value = 0x15A0, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 12, + }, + [DPRC_GET_RES_COUNT] = { + .cmdid_value = 0x15B0, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 32, + }, + [DPRC_GET_RES_IDS] = { + .cmdid_value = 0x15C0, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 40, + }, + [DPRC_SET_OBJ_LABEL] = { + .cmdid_value = 0x1610, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 48, + .flags = FSL_MC_CAP_NET_ADMIN_NEEDED, + }, + [DPRC_SET_LOCKED] = { + .cmdid_value = 0x16B0, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 16, + .flags = FSL_MC_CAP_NET_ADMIN_NEEDED, + }, + [DPRC_CONNECT] = { + .cmdid_value = 0x1670, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 56, + .flags = FSL_MC_CAP_NET_ADMIN_NEEDED, + }, + [DPRC_DISCONNECT] = { + .cmdid_value = 0x1680, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 32, + .flags = FSL_MC_CAP_NET_ADMIN_NEEDED, + }, + [DPRC_GET_POOL] = { + .cmdid_value = 0x1690, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 12, + }, + [DPRC_GET_POOL_COUNT] = { + .cmdid_value = 0x16A0, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 8, + }, + [DPRC_GET_CONNECTION] = { + .cmdid_value = 0x16C0, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 32, + }, + + [DPCI_GET_LINK_STATE] = { + .cmdid_value = 0x0E10, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 8, + }, + [DPCI_GET_PEER_ATTR] = { + .cmdid_value = 0x0E20, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 8, + }, + [DPAIOP_GET_SL_VERSION] = { + .cmdid_value = 0x2820, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 8, + }, + [DPAIOP_GET_STATE] = { + .cmdid_value = 0x2830, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 8, + }, + [DPMNG_GET_VERSION] = { + .cmdid_value = 0x8310, + .cmdid_mask = 0xFFF0, + .token = false, + .size = 8, + }, + [DPSECI_GET_TX_QUEUE] = { + .cmdid_value = 0x1970, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 14, + }, + [DPMAC_GET_COUNTER] = { + .cmdid_value = 0x0c40, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 9, + }, + [DPMAC_GET_MAC_ADDR] = { + .cmdid_value = 0x0c50, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 8, + }, + [DPNI_SET_PRIM_MAC] = { + .cmdid_value = 0x2240, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 16, + .flags = FSL_MC_CAP_NET_ADMIN_NEEDED, + }, + [DPNI_GET_PRIM_MAC] = { + .cmdid_value = 0x2250, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 8, + }, + [DPNI_GET_STATISTICS] = { + .cmdid_value = 0x25D0, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 10, + }, + [DPNI_GET_LINK_STATE] = { + .cmdid_value = 0x2150, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 8, + }, + [DPNI_GET_MAX_FRAME_LENGTH] = { + .cmdid_value = 0x2170, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 8, + }, + [DPSW_GET_TAILDROP] = { + .cmdid_value = 0x0A80, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 14, + }, + [DPSW_SET_TAILDROP] = { + .cmdid_value = 0x0A90, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 24, + .flags = FSL_MC_CAP_NET_ADMIN_NEEDED, + }, + [DPSW_IF_GET_COUNTER] = { + .cmdid_value = 0x0340, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 11, + }, + [DPSW_IF_GET_MAX_FRAME_LENGTH] = { + .cmdid_value = 0x0450, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 10, + }, + [DPDMUX_GET_COUNTER] = { + .cmdid_value = 0x0b20, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 11, + }, + [DPDMUX_IF_GET_MAX_FRAME_LENGTH] = { + .cmdid_value = 0x0a20, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 10, + }, + [GET_ATTR] = { + .cmdid_value = 0x0040, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 8, + }, + [GET_IRQ_MASK] = { + .cmdid_value = 0x0150, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 13, + }, + [GET_IRQ_STATUS] = { + .cmdid_value = 0x0160, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 13, + }, + [CLOSE] = { + .cmdid_value = 0x8000, + .cmdid_mask = 0xFFF0, + .token = true, + .size = 8, + }, + + /* Common commands amongst all types of objects. Must be checked last. */ + [OPEN] = { + .cmdid_value = 0x8000, + .cmdid_mask = 0xFC00, + .token = false, + .size = 12, + .flags = FSL_MC_CHECK_MODULE_ID, + }, + [GET_API_VERSION] = { + .cmdid_value = 0xA000, + .cmdid_mask = 0xFC00, + .token = false, + .size = 8, + .flags = FSL_MC_CHECK_MODULE_ID, + }, + [DESTROY] = { + .cmdid_value = 0x9800, + .cmdid_mask = 0xFC00, + .token = true, + .size = 12, + .flags = FSL_MC_CHECK_MODULE_ID | FSL_MC_CAP_NET_ADMIN_NEEDED, + }, + [CREATE] = { + .cmdid_value = 0x9000, + .cmdid_mask = 0xFC00, + .token = true, + .size = 64, + .flags = FSL_MC_CHECK_MODULE_ID | FSL_MC_CAP_NET_ADMIN_NEEDED, + }, +}; + +#define FSL_MC_NUM_ACCEPTED_CMDS ARRAY_SIZE(fsl_mc_accepted_cmds) + +#define FSL_MC_MAX_MODULE_ID 0x10 + +static int fsl_mc_command_check(struct fsl_mc_device *mc_dev, + struct fsl_mc_command *mc_cmd) +{ + struct fsl_mc_cmd_desc *desc = NULL; + int mc_cmd_max_size, i; + bool token_provided; + u16 cmdid, module_id; + char *mc_cmd_end; + char sum = 0; + + /* Check if this is an accepted MC command */ + cmdid = mc_cmd_hdr_read_cmdid(mc_cmd); + for (i = 0; i < FSL_MC_NUM_ACCEPTED_CMDS; i++) { + desc = &fsl_mc_accepted_cmds[i]; + if ((cmdid & desc->cmdid_mask) == desc->cmdid_value) + break; + } + if (i == FSL_MC_NUM_ACCEPTED_CMDS) { + dev_err(&mc_dev->dev, "MC command 0x%04x: cmdid not accepted\n", cmdid); + return -EACCES; + } + + /* Check if the size of the command is honored. Anything beyond the + * last valid byte of the command should be zeroed. + */ + mc_cmd_max_size = sizeof(*mc_cmd); + mc_cmd_end = ((char *)mc_cmd) + desc->size; + for (i = desc->size; i < mc_cmd_max_size; i++) + sum |= *mc_cmd_end++; + if (sum) { + dev_err(&mc_dev->dev, "MC command 0x%04x: garbage beyond max size of %d bytes!\n", + cmdid, desc->size); + return -EACCES; + } + + /* Some MC commands request a token to be passed so that object + * identification is possible. Check if the token passed in the command + * is as expected. + */ + token_provided = mc_cmd_hdr_read_token(mc_cmd) ? true : false; + if (token_provided != desc->token) { + dev_err(&mc_dev->dev, "MC command 0x%04x: token 0x%04x is invalid!\n", + cmdid, mc_cmd_hdr_read_token(mc_cmd)); + return -EACCES; + } + + /* If needed, check if the module ID passed is valid */ + if (desc->flags & FSL_MC_CHECK_MODULE_ID) { + /* The module ID is represented by bits [4:9] from the cmdid */ + module_id = (cmdid & GENMASK(9, 4)) >> 4; + if (module_id == 0 || module_id > FSL_MC_MAX_MODULE_ID) { + dev_err(&mc_dev->dev, "MC command 0x%04x: unknown module ID 0x%x\n", + cmdid, module_id); + return -EACCES; + } + } + + /* Some commands alter how hardware resources are managed. For these + * commands, check for CAP_NET_ADMIN. + */ + if (desc->flags & FSL_MC_CAP_NET_ADMIN_NEEDED) { + if (!capable(CAP_NET_ADMIN)) { + dev_err(&mc_dev->dev, "MC command 0x%04x: needs CAP_NET_ADMIN!\n", + cmdid); + return -EPERM; + } + } + + return 0; +} + +static int fsl_mc_uapi_send_command(struct fsl_mc_device *mc_dev, unsigned long arg, + struct fsl_mc_io *mc_io) +{ + struct fsl_mc_command mc_cmd; + int error; + + error = copy_from_user(&mc_cmd, (void __user *)arg, sizeof(mc_cmd)); + if (error) + return -EFAULT; + + error = fsl_mc_command_check(mc_dev, &mc_cmd); + if (error) + return error; + + error = mc_send_command(mc_io, &mc_cmd); + if (error) + return error; + + error = copy_to_user((void __user *)arg, &mc_cmd, sizeof(mc_cmd)); + if (error) + return -EFAULT; + + return 0; +} + +static int fsl_mc_uapi_dev_open(struct inode *inode, struct file *filep) +{ + struct fsl_mc_device *root_mc_device; + struct uapi_priv_data *priv_data; + struct fsl_mc_io *dynamic_mc_io; + struct fsl_mc_uapi *mc_uapi; + struct fsl_mc_bus *mc_bus; + int error; + + priv_data = kzalloc(sizeof(*priv_data), GFP_KERNEL); + if (!priv_data) + return -ENOMEM; + + mc_uapi = container_of(filep->private_data, struct fsl_mc_uapi, misc); + mc_bus = container_of(mc_uapi, struct fsl_mc_bus, uapi_misc); + root_mc_device = &mc_bus->mc_dev; + + mutex_lock(&mc_uapi->mutex); + + if (!mc_uapi->local_instance_in_use) { + priv_data->mc_io = mc_uapi->static_mc_io; + mc_uapi->local_instance_in_use = 1; + } else { + error = fsl_mc_portal_allocate(root_mc_device, 0, + &dynamic_mc_io); + if (error) { + dev_dbg(&root_mc_device->dev, + "Could not allocate MC portal\n"); + goto error_portal_allocate; + } + + priv_data->mc_io = dynamic_mc_io; + } + priv_data->uapi = mc_uapi; + filep->private_data = priv_data; + + mutex_unlock(&mc_uapi->mutex); + + return 0; + +error_portal_allocate: + mutex_unlock(&mc_uapi->mutex); + kfree(priv_data); + + return error; +} + +static int fsl_mc_uapi_dev_release(struct inode *inode, struct file *filep) +{ + struct uapi_priv_data *priv_data; + struct fsl_mc_uapi *mc_uapi; + struct fsl_mc_io *mc_io; + + priv_data = filep->private_data; + mc_uapi = priv_data->uapi; + mc_io = priv_data->mc_io; + + mutex_lock(&mc_uapi->mutex); + + if (mc_io == mc_uapi->static_mc_io) + mc_uapi->local_instance_in_use = 0; + else + fsl_mc_portal_free(mc_io); + + kfree(filep->private_data); + filep->private_data = NULL; + + mutex_unlock(&mc_uapi->mutex); + + return 0; +} + +static long fsl_mc_uapi_dev_ioctl(struct file *file, + unsigned int cmd, + unsigned long arg) +{ + struct uapi_priv_data *priv_data = file->private_data; + struct fsl_mc_device *root_mc_device; + struct fsl_mc_bus *mc_bus; + int error; + + mc_bus = container_of(priv_data->uapi, struct fsl_mc_bus, uapi_misc); + root_mc_device = &mc_bus->mc_dev; + + switch (cmd) { + case FSL_MC_SEND_MC_COMMAND: + error = fsl_mc_uapi_send_command(root_mc_device, arg, priv_data->mc_io); + break; + default: + dev_dbg(&root_mc_device->dev, "unexpected ioctl call number\n"); + error = -EINVAL; + } + + return error; +} + +static const struct file_operations fsl_mc_uapi_dev_fops = { + .owner = THIS_MODULE, + .open = fsl_mc_uapi_dev_open, + .release = fsl_mc_uapi_dev_release, + .unlocked_ioctl = fsl_mc_uapi_dev_ioctl, +}; + +int fsl_mc_uapi_create_device_file(struct fsl_mc_bus *mc_bus) +{ + struct fsl_mc_device *mc_dev = &mc_bus->mc_dev; + struct fsl_mc_uapi *mc_uapi = &mc_bus->uapi_misc; + int error; + + mc_uapi->misc.minor = MISC_DYNAMIC_MINOR; + mc_uapi->misc.name = dev_name(&mc_dev->dev); + mc_uapi->misc.fops = &fsl_mc_uapi_dev_fops; + + error = misc_register(&mc_uapi->misc); + if (error) + return error; + + mc_uapi->static_mc_io = mc_bus->mc_dev.mc_io; + + mutex_init(&mc_uapi->mutex); + + return 0; +} + +void fsl_mc_uapi_remove_device_file(struct fsl_mc_bus *mc_bus) +{ + misc_deregister(&mc_bus->uapi_misc.misc); +} diff --git a/drivers/bus/fsl-mc/mc-sys.c b/drivers/bus/fsl-mc/mc-sys.c index 85a0225db522a167cb1d5439e87d9eeb8cfc2e3a..b291b35e3884bbf03f7ed902374bf352fcf58ff8 100644 --- a/drivers/bus/fsl-mc/mc-sys.c +++ b/drivers/bus/fsl-mc/mc-sys.c @@ -35,7 +35,7 @@ static enum mc_cmd_status mc_cmd_hdr_read_status(struct fsl_mc_command *cmd) return (enum mc_cmd_status)hdr->status; } -static u16 mc_cmd_hdr_read_cmdid(struct fsl_mc_command *cmd) +u16 mc_cmd_hdr_read_cmdid(struct fsl_mc_command *cmd) { struct mc_cmd_header *hdr = (struct mc_cmd_header *)&cmd->header; u16 cmd_id = le16_to_cpu(hdr->cmd_id); diff --git a/drivers/bus/mhi/core/init.c b/drivers/bus/mhi/core/init.c index f0697f433c2f1bb8d32ad71ca24148b5a6d3cbfa..be4eebb0971bc18270371f9c10dd9071f9b80cbf 100644 --- a/drivers/bus/mhi/core/init.c +++ b/drivers/bus/mhi/core/init.c @@ -151,12 +151,17 @@ int mhi_init_irq_setup(struct mhi_controller *mhi_cntrl) { struct mhi_event *mhi_event = mhi_cntrl->mhi_event; struct device *dev = &mhi_cntrl->mhi_dev->dev; + unsigned long irq_flags = IRQF_SHARED | IRQF_NO_SUSPEND; int i, ret; + /* if controller driver has set irq_flags, use it */ + if (mhi_cntrl->irq_flags) + irq_flags = mhi_cntrl->irq_flags; + /* Setup BHI_INTVEC IRQ */ ret = request_threaded_irq(mhi_cntrl->irq[0], mhi_intvec_handler, mhi_intvec_threaded_handler, - IRQF_SHARED | IRQF_NO_SUSPEND, + irq_flags, "bhi", mhi_cntrl); if (ret) return ret; @@ -174,7 +179,7 @@ int mhi_init_irq_setup(struct mhi_controller *mhi_cntrl) ret = request_irq(mhi_cntrl->irq[mhi_event->irq], mhi_irq_handler, - IRQF_SHARED | IRQF_NO_SUSPEND, + irq_flags, "mhi", mhi_event); if (ret) { dev_err(dev, "Error requesting irq:%d for ev:%d\n", @@ -552,6 +557,9 @@ void mhi_deinit_chan_ctxt(struct mhi_controller *mhi_cntrl, tre_ring = &mhi_chan->tre_ring; chan_ctxt = &mhi_cntrl->mhi_ctxt->chan_ctxt[mhi_chan->chan]; + if (!chan_ctxt->rbase) /* Already uninitialized */ + return; + mhi_free_coherent(mhi_cntrl, tre_ring->alloc_size, tre_ring->pre_aligned, tre_ring->dma_handle); vfree(buf_ring->base); diff --git a/drivers/bus/mhi/core/main.c b/drivers/bus/mhi/core/main.c index 1202433ecf9817b46c068b398825349d8b99706c..4e0131b94056ba8532f591960fee4e4c527570c2 100644 --- a/drivers/bus/mhi/core/main.c +++ b/drivers/bus/mhi/core/main.c @@ -111,7 +111,14 @@ void mhi_ring_chan_db(struct mhi_controller *mhi_cntrl, dma_addr_t db; db = ring->iommu_base + (ring->wp - ring->base); + + /* + * Writes to the new ring element must be visible to the hardware + * before letting h/w know there is new element to fetch. + */ + dma_wmb(); *ring->ctxt_wp = db; + mhi_chan->db_cfg.process_db(mhi_cntrl, &mhi_chan->db_cfg, ring->db_addr, db); } @@ -135,6 +142,19 @@ enum mhi_state mhi_get_mhi_state(struct mhi_controller *mhi_cntrl) } EXPORT_SYMBOL_GPL(mhi_get_mhi_state); +void mhi_soc_reset(struct mhi_controller *mhi_cntrl) +{ + if (mhi_cntrl->reset) { + mhi_cntrl->reset(mhi_cntrl); + return; + } + + /* Generic MHI SoC reset */ + mhi_write_reg(mhi_cntrl, mhi_cntrl->regs, MHI_SOC_RESET_REQ_OFFSET, + MHI_SOC_RESET_REQ); +} +EXPORT_SYMBOL_GPL(mhi_soc_reset); + int mhi_map_single_no_bb(struct mhi_controller *mhi_cntrl, struct mhi_buf_info *buf_info) { @@ -959,118 +979,88 @@ static bool mhi_is_ring_full(struct mhi_controller *mhi_cntrl, return (tmp == ring->rp); } -int mhi_queue_skb(struct mhi_device *mhi_dev, enum dma_data_direction dir, - struct sk_buff *skb, size_t len, enum mhi_flags mflags) +static int mhi_queue(struct mhi_device *mhi_dev, struct mhi_buf_info *buf_info, + enum dma_data_direction dir, enum mhi_flags mflags) { struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl; struct mhi_chan *mhi_chan = (dir == DMA_TO_DEVICE) ? mhi_dev->ul_chan : mhi_dev->dl_chan; struct mhi_ring *tre_ring = &mhi_chan->tre_ring; - struct mhi_buf_info buf_info = { }; + unsigned long flags; int ret; - /* If MHI host pre-allocates buffers then client drivers cannot queue */ - if (mhi_chan->pre_alloc) - return -EINVAL; + if (unlikely(MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state))) + return -EIO; - if (mhi_is_ring_full(mhi_cntrl, tre_ring)) - return -ENOMEM; + read_lock_irqsave(&mhi_cntrl->pm_lock, flags); - read_lock_bh(&mhi_cntrl->pm_lock); - if (unlikely(MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state))) { - read_unlock_bh(&mhi_cntrl->pm_lock); - return -EIO; + ret = mhi_is_ring_full(mhi_cntrl, tre_ring); + if (unlikely(ret)) { + ret = -ENOMEM; + goto exit_unlock; } - /* we're in M3 or transitioning to M3 */ + ret = mhi_gen_tre(mhi_cntrl, mhi_chan, buf_info, mflags); + if (unlikely(ret)) + goto exit_unlock; + + /* trigger M3 exit if necessary */ if (MHI_PM_IN_SUSPEND_STATE(mhi_cntrl->pm_state)) mhi_trigger_resume(mhi_cntrl); - /* Toggle wake to exit out of M2 */ + /* Assert dev_wake (to exit/prevent M1/M2)*/ mhi_cntrl->wake_toggle(mhi_cntrl); - buf_info.v_addr = skb->data; - buf_info.cb_buf = skb; - buf_info.len = len; - - ret = mhi_gen_tre(mhi_cntrl, mhi_chan, &buf_info, mflags); - if (unlikely(ret)) { - read_unlock_bh(&mhi_cntrl->pm_lock); - return ret; - } - if (mhi_chan->dir == DMA_TO_DEVICE) atomic_inc(&mhi_cntrl->pending_pkts); - if (likely(MHI_DB_ACCESS_VALID(mhi_cntrl))) { - read_lock_bh(&mhi_chan->lock); - mhi_ring_chan_db(mhi_cntrl, mhi_chan); - read_unlock_bh(&mhi_chan->lock); + if (unlikely(!MHI_DB_ACCESS_VALID(mhi_cntrl))) { + ret = -EIO; + goto exit_unlock; } - read_unlock_bh(&mhi_cntrl->pm_lock); + mhi_ring_chan_db(mhi_cntrl, mhi_chan); - return 0; +exit_unlock: + read_unlock_irqrestore(&mhi_cntrl->pm_lock, flags); + + return ret; } -EXPORT_SYMBOL_GPL(mhi_queue_skb); -int mhi_queue_dma(struct mhi_device *mhi_dev, enum dma_data_direction dir, - struct mhi_buf *mhi_buf, size_t len, enum mhi_flags mflags) +int mhi_queue_skb(struct mhi_device *mhi_dev, enum dma_data_direction dir, + struct sk_buff *skb, size_t len, enum mhi_flags mflags) { - struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl; struct mhi_chan *mhi_chan = (dir == DMA_TO_DEVICE) ? mhi_dev->ul_chan : mhi_dev->dl_chan; - struct device *dev = &mhi_cntrl->mhi_dev->dev; - struct mhi_ring *tre_ring = &mhi_chan->tre_ring; struct mhi_buf_info buf_info = { }; - int ret; - - /* If MHI host pre-allocates buffers then client drivers cannot queue */ - if (mhi_chan->pre_alloc) - return -EINVAL; - if (mhi_is_ring_full(mhi_cntrl, tre_ring)) - return -ENOMEM; - - read_lock_bh(&mhi_cntrl->pm_lock); - if (unlikely(MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state))) { - dev_err(dev, "MHI is not in activate state, PM state: %s\n", - to_mhi_pm_state_str(mhi_cntrl->pm_state)); - read_unlock_bh(&mhi_cntrl->pm_lock); + buf_info.v_addr = skb->data; + buf_info.cb_buf = skb; + buf_info.len = len; - return -EIO; - } + if (unlikely(mhi_chan->pre_alloc)) + return -EINVAL; - /* we're in M3 or transitioning to M3 */ - if (MHI_PM_IN_SUSPEND_STATE(mhi_cntrl->pm_state)) - mhi_trigger_resume(mhi_cntrl); + return mhi_queue(mhi_dev, &buf_info, dir, mflags); +} +EXPORT_SYMBOL_GPL(mhi_queue_skb); - /* Toggle wake to exit out of M2 */ - mhi_cntrl->wake_toggle(mhi_cntrl); +int mhi_queue_dma(struct mhi_device *mhi_dev, enum dma_data_direction dir, + struct mhi_buf *mhi_buf, size_t len, enum mhi_flags mflags) +{ + struct mhi_chan *mhi_chan = (dir == DMA_TO_DEVICE) ? mhi_dev->ul_chan : + mhi_dev->dl_chan; + struct mhi_buf_info buf_info = { }; buf_info.p_addr = mhi_buf->dma_addr; buf_info.cb_buf = mhi_buf; buf_info.pre_mapped = true; buf_info.len = len; - ret = mhi_gen_tre(mhi_cntrl, mhi_chan, &buf_info, mflags); - if (unlikely(ret)) { - read_unlock_bh(&mhi_cntrl->pm_lock); - return ret; - } - - if (mhi_chan->dir == DMA_TO_DEVICE) - atomic_inc(&mhi_cntrl->pending_pkts); - - if (likely(MHI_DB_ACCESS_VALID(mhi_cntrl))) { - read_lock_bh(&mhi_chan->lock); - mhi_ring_chan_db(mhi_cntrl, mhi_chan); - read_unlock_bh(&mhi_chan->lock); - } - - read_unlock_bh(&mhi_cntrl->pm_lock); + if (unlikely(mhi_chan->pre_alloc)) + return -EINVAL; - return 0; + return mhi_queue(mhi_dev, &buf_info, dir, mflags); } EXPORT_SYMBOL_GPL(mhi_queue_dma); @@ -1124,57 +1114,13 @@ int mhi_gen_tre(struct mhi_controller *mhi_cntrl, struct mhi_chan *mhi_chan, int mhi_queue_buf(struct mhi_device *mhi_dev, enum dma_data_direction dir, void *buf, size_t len, enum mhi_flags mflags) { - struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl; - struct mhi_chan *mhi_chan = (dir == DMA_TO_DEVICE) ? mhi_dev->ul_chan : - mhi_dev->dl_chan; - struct mhi_ring *tre_ring; struct mhi_buf_info buf_info = { }; - unsigned long flags; - int ret; - - /* - * this check here only as a guard, it's always - * possible mhi can enter error while executing rest of function, - * which is not fatal so we do not need to hold pm_lock - */ - if (unlikely(MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state))) - return -EIO; - - tre_ring = &mhi_chan->tre_ring; - if (mhi_is_ring_full(mhi_cntrl, tre_ring)) - return -ENOMEM; buf_info.v_addr = buf; buf_info.cb_buf = buf; buf_info.len = len; - ret = mhi_gen_tre(mhi_cntrl, mhi_chan, &buf_info, mflags); - if (unlikely(ret)) - return ret; - - read_lock_irqsave(&mhi_cntrl->pm_lock, flags); - - /* we're in M3 or transitioning to M3 */ - if (MHI_PM_IN_SUSPEND_STATE(mhi_cntrl->pm_state)) - mhi_trigger_resume(mhi_cntrl); - - /* Toggle wake to exit out of M2 */ - mhi_cntrl->wake_toggle(mhi_cntrl); - - if (mhi_chan->dir == DMA_TO_DEVICE) - atomic_inc(&mhi_cntrl->pending_pkts); - - if (likely(MHI_DB_ACCESS_VALID(mhi_cntrl))) { - unsigned long flags; - - read_lock_irqsave(&mhi_chan->lock, flags); - mhi_ring_chan_db(mhi_cntrl, mhi_chan); - read_unlock_irqrestore(&mhi_chan->lock, flags); - } - - read_unlock_irqrestore(&mhi_cntrl->pm_lock, flags); - - return 0; + return mhi_queue(mhi_dev, &buf_info, dir, mflags); } EXPORT_SYMBOL_GPL(mhi_queue_buf); diff --git a/drivers/bus/mhi/pci_generic.c b/drivers/bus/mhi/pci_generic.c index f5bee76ea0618ef02277f736f9989b188afd4d29..20673a4b4a3c657ed97d59cbc2bac29af35ce962 100644 --- a/drivers/bus/mhi/pci_generic.c +++ b/drivers/bus/mhi/pci_generic.c @@ -8,13 +8,21 @@ * Copyright (C) 2020 Linaro Ltd */ +#include +#include #include #include #include #include +#include +#include #define MHI_PCI_DEFAULT_BAR_NUM 0 +#define MHI_POST_RESET_DELAY_MS 500 + +#define HEALTH_CHECK_PERIOD (HZ * 2) + /** * struct mhi_pci_dev_info - MHI PCI device specific information * @config: MHI controller configuration @@ -76,6 +84,36 @@ struct mhi_pci_dev_info { .offload_channel = false, \ } +#define MHI_CHANNEL_CONFIG_HW_UL(ch_num, ch_name, el_count, ev_ring) \ + { \ + .num = ch_num, \ + .name = ch_name, \ + .num_elements = el_count, \ + .event_ring = ev_ring, \ + .dir = DMA_TO_DEVICE, \ + .ee_mask = BIT(MHI_EE_AMSS), \ + .pollcfg = 0, \ + .doorbell = MHI_DB_BRST_ENABLE, \ + .lpm_notify = false, \ + .offload_channel = false, \ + .doorbell_mode_switch = true, \ + } \ + +#define MHI_CHANNEL_CONFIG_HW_DL(ch_num, ch_name, el_count, ev_ring) \ + { \ + .num = ch_num, \ + .name = ch_name, \ + .num_elements = el_count, \ + .event_ring = ev_ring, \ + .dir = DMA_FROM_DEVICE, \ + .ee_mask = BIT(MHI_EE_AMSS), \ + .pollcfg = 0, \ + .doorbell = MHI_DB_BRST_ENABLE, \ + .lpm_notify = false, \ + .offload_channel = false, \ + .doorbell_mode_switch = true, \ + } + #define MHI_EVENT_CONFIG_DATA(ev_ring) \ { \ .num_elements = 128, \ @@ -91,8 +129,8 @@ struct mhi_pci_dev_info { #define MHI_EVENT_CONFIG_HW_DATA(ev_ring, ch_num) \ { \ - .num_elements = 128, \ - .irq_moderation_ms = 5, \ + .num_elements = 2048, \ + .irq_moderation_ms = 1, \ .irq = (ev_ring) + 1, \ .priority = 1, \ .mode = MHI_DB_BRST_DISABLE, \ @@ -104,27 +142,31 @@ struct mhi_pci_dev_info { } static const struct mhi_channel_config modem_qcom_v1_mhi_channels[] = { + MHI_CHANNEL_CONFIG_UL(4, "DIAG", 16, 1), + MHI_CHANNEL_CONFIG_DL(5, "DIAG", 16, 1), MHI_CHANNEL_CONFIG_UL(12, "MBIM", 4, 0), MHI_CHANNEL_CONFIG_DL(13, "MBIM", 4, 0), MHI_CHANNEL_CONFIG_UL(14, "QMI", 4, 0), MHI_CHANNEL_CONFIG_DL(15, "QMI", 4, 0), MHI_CHANNEL_CONFIG_UL(20, "IPCR", 8, 0), MHI_CHANNEL_CONFIG_DL(21, "IPCR", 8, 0), - MHI_CHANNEL_CONFIG_UL(100, "IP_HW0", 128, 1), - MHI_CHANNEL_CONFIG_DL(101, "IP_HW0", 128, 2), + MHI_CHANNEL_CONFIG_HW_UL(100, "IP_HW0", 128, 2), + MHI_CHANNEL_CONFIG_HW_DL(101, "IP_HW0", 128, 3), }; -static const struct mhi_event_config modem_qcom_v1_mhi_events[] = { +static struct mhi_event_config modem_qcom_v1_mhi_events[] = { /* first ring is control+data ring */ MHI_EVENT_CONFIG_CTRL(0), + /* DIAG dedicated event ring */ + MHI_EVENT_CONFIG_DATA(1), /* Hardware channels request dedicated hardware event rings */ - MHI_EVENT_CONFIG_HW_DATA(1, 100), - MHI_EVENT_CONFIG_HW_DATA(2, 101) + MHI_EVENT_CONFIG_HW_DATA(2, 100), + MHI_EVENT_CONFIG_HW_DATA(3, 101) }; -static const struct mhi_controller_config modem_qcom_v1_mhiv_config = { +static struct mhi_controller_config modem_qcom_v1_mhiv_config = { .max_channels = 128, - .timeout_ms = 5000, + .timeout_ms = 8000, .num_channels = ARRAY_SIZE(modem_qcom_v1_mhi_channels), .ch_cfg = modem_qcom_v1_mhi_channels, .num_events = ARRAY_SIZE(modem_qcom_v1_mhi_events), @@ -147,6 +189,18 @@ static const struct pci_device_id mhi_pci_id_table[] = { }; MODULE_DEVICE_TABLE(pci, mhi_pci_id_table); +enum mhi_pci_device_status { + MHI_PCI_DEV_STARTED, +}; + +struct mhi_pci_device { + struct mhi_controller mhi_cntrl; + struct pci_saved_state *pci_state; + struct work_struct recovery_work; + struct timer_list health_check_timer; + unsigned long status; +}; + static int mhi_pci_read_reg(struct mhi_controller *mhi_cntrl, void __iomem *addr, u32 *out) { @@ -163,7 +217,31 @@ static void mhi_pci_write_reg(struct mhi_controller *mhi_cntrl, static void mhi_pci_status_cb(struct mhi_controller *mhi_cntrl, enum mhi_callback cb) { + struct pci_dev *pdev = to_pci_dev(mhi_cntrl->cntrl_dev); + /* Nothing to do for now */ + switch (cb) { + case MHI_CB_FATAL_ERROR: + case MHI_CB_SYS_ERROR: + dev_warn(&pdev->dev, "firmware crashed (%u)\n", cb); + break; + default: + break; + } +} + +static bool mhi_pci_is_alive(struct mhi_controller *mhi_cntrl) +{ + struct pci_dev *pdev = to_pci_dev(mhi_cntrl->cntrl_dev); + u16 vendor = 0; + + if (pci_read_config_word(pdev, PCI_VENDOR_ID, &vendor)) + return false; + + if (vendor == (u16) ~0 || vendor == 0) + return false; + + return true; } static int mhi_pci_claim(struct mhi_controller *mhi_cntrl, @@ -227,8 +305,12 @@ static int mhi_pci_get_irqs(struct mhi_controller *mhi_cntrl, } if (nr_vectors < mhi_cntrl->nr_irqs) { - dev_warn(&pdev->dev, "Not enough MSI vectors (%d/%d), use shared MSI\n", - nr_vectors, mhi_cntrl_config->num_events); + dev_warn(&pdev->dev, "using shared MSI\n"); + + /* Patch msi vectors, use only one (shared) */ + for (i = 0; i < mhi_cntrl_config->num_events; i++) + mhi_cntrl_config->event_cfg[i].irq = 0; + mhi_cntrl->nr_irqs = 1; } irq = devm_kcalloc(&pdev->dev, mhi_cntrl->nr_irqs, sizeof(int), GFP_KERNEL); @@ -257,20 +339,89 @@ static void mhi_pci_runtime_put(struct mhi_controller *mhi_cntrl) /* no PM for now */ } +static void mhi_pci_recovery_work(struct work_struct *work) +{ + struct mhi_pci_device *mhi_pdev = container_of(work, struct mhi_pci_device, + recovery_work); + struct mhi_controller *mhi_cntrl = &mhi_pdev->mhi_cntrl; + struct pci_dev *pdev = to_pci_dev(mhi_cntrl->cntrl_dev); + int err; + + dev_warn(&pdev->dev, "device recovery started\n"); + + del_timer(&mhi_pdev->health_check_timer); + + /* Clean up MHI state */ + if (test_and_clear_bit(MHI_PCI_DEV_STARTED, &mhi_pdev->status)) { + mhi_power_down(mhi_cntrl, false); + mhi_unprepare_after_power_down(mhi_cntrl); + } + + /* Check if we can recover without full reset */ + pci_set_power_state(pdev, PCI_D0); + pci_load_saved_state(pdev, mhi_pdev->pci_state); + pci_restore_state(pdev); + + if (!mhi_pci_is_alive(mhi_cntrl)) + goto err_try_reset; + + err = mhi_prepare_for_power_up(mhi_cntrl); + if (err) + goto err_try_reset; + + err = mhi_sync_power_up(mhi_cntrl); + if (err) + goto err_unprepare; + + dev_dbg(&pdev->dev, "Recovery completed\n"); + + set_bit(MHI_PCI_DEV_STARTED, &mhi_pdev->status); + mod_timer(&mhi_pdev->health_check_timer, jiffies + HEALTH_CHECK_PERIOD); + return; + +err_unprepare: + mhi_unprepare_after_power_down(mhi_cntrl); +err_try_reset: + if (pci_reset_function(pdev)) + dev_err(&pdev->dev, "Recovery failed\n"); +} + +static void health_check(struct timer_list *t) +{ + struct mhi_pci_device *mhi_pdev = from_timer(mhi_pdev, t, health_check_timer); + struct mhi_controller *mhi_cntrl = &mhi_pdev->mhi_cntrl; + + if (!mhi_pci_is_alive(mhi_cntrl)) { + dev_err(mhi_cntrl->cntrl_dev, "Device died\n"); + queue_work(system_long_wq, &mhi_pdev->recovery_work); + return; + } + + /* reschedule in two seconds */ + mod_timer(&mhi_pdev->health_check_timer, jiffies + HEALTH_CHECK_PERIOD); +} + static int mhi_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) { const struct mhi_pci_dev_info *info = (struct mhi_pci_dev_info *) id->driver_data; const struct mhi_controller_config *mhi_cntrl_config; + struct mhi_pci_device *mhi_pdev; struct mhi_controller *mhi_cntrl; int err; dev_dbg(&pdev->dev, "MHI PCI device found: %s\n", info->name); - mhi_cntrl = mhi_alloc_controller(); - if (!mhi_cntrl) + /* mhi_pdev.mhi_cntrl must be zero-initialized */ + mhi_pdev = devm_kzalloc(&pdev->dev, sizeof(*mhi_pdev), GFP_KERNEL); + if (!mhi_pdev) return -ENOMEM; + INIT_WORK(&mhi_pdev->recovery_work, mhi_pci_recovery_work); + timer_setup(&mhi_pdev->health_check_timer, health_check, 0); + mhi_cntrl_config = info->config; + mhi_cntrl = &mhi_pdev->mhi_cntrl; + mhi_cntrl->cntrl_dev = &pdev->dev; mhi_cntrl->iova_start = 0; mhi_cntrl->iova_stop = (dma_addr_t)DMA_BIT_MASK(info->dma_data_width); @@ -285,17 +436,23 @@ static int mhi_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) err = mhi_pci_claim(mhi_cntrl, info->bar_num, DMA_BIT_MASK(info->dma_data_width)); if (err) - goto err_release; + return err; err = mhi_pci_get_irqs(mhi_cntrl, mhi_cntrl_config); if (err) - goto err_release; + return err; + + pci_set_drvdata(pdev, mhi_pdev); + + /* Have stored pci confspace at hand for restore in sudden PCI error */ + pci_save_state(pdev); + mhi_pdev->pci_state = pci_store_saved_state(pdev); - pci_set_drvdata(pdev, mhi_cntrl); + pci_enable_pcie_error_reporting(pdev); err = mhi_register_controller(mhi_cntrl, mhi_cntrl_config); if (err) - goto err_release; + return err; /* MHI bus does not power up the controller by default */ err = mhi_prepare_for_power_up(mhi_cntrl); @@ -310,33 +467,209 @@ static int mhi_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) goto err_unprepare; } + set_bit(MHI_PCI_DEV_STARTED, &mhi_pdev->status); + + /* start health check */ + mod_timer(&mhi_pdev->health_check_timer, jiffies + HEALTH_CHECK_PERIOD); + return 0; err_unprepare: mhi_unprepare_after_power_down(mhi_cntrl); err_unregister: mhi_unregister_controller(mhi_cntrl); -err_release: - mhi_free_controller(mhi_cntrl); return err; } static void mhi_pci_remove(struct pci_dev *pdev) { - struct mhi_controller *mhi_cntrl = pci_get_drvdata(pdev); + struct mhi_pci_device *mhi_pdev = pci_get_drvdata(pdev); + struct mhi_controller *mhi_cntrl = &mhi_pdev->mhi_cntrl; + + del_timer(&mhi_pdev->health_check_timer); + cancel_work_sync(&mhi_pdev->recovery_work); + + if (test_and_clear_bit(MHI_PCI_DEV_STARTED, &mhi_pdev->status)) { + mhi_power_down(mhi_cntrl, true); + mhi_unprepare_after_power_down(mhi_cntrl); + } - mhi_power_down(mhi_cntrl, true); - mhi_unprepare_after_power_down(mhi_cntrl); mhi_unregister_controller(mhi_cntrl); - mhi_free_controller(mhi_cntrl); } +static void mhi_pci_reset_prepare(struct pci_dev *pdev) +{ + struct mhi_pci_device *mhi_pdev = pci_get_drvdata(pdev); + struct mhi_controller *mhi_cntrl = &mhi_pdev->mhi_cntrl; + + dev_info(&pdev->dev, "reset\n"); + + del_timer(&mhi_pdev->health_check_timer); + + /* Clean up MHI state */ + if (test_and_clear_bit(MHI_PCI_DEV_STARTED, &mhi_pdev->status)) { + mhi_power_down(mhi_cntrl, false); + mhi_unprepare_after_power_down(mhi_cntrl); + } + + /* cause internal device reset */ + mhi_soc_reset(mhi_cntrl); + + /* Be sure device reset has been executed */ + msleep(MHI_POST_RESET_DELAY_MS); +} + +static void mhi_pci_reset_done(struct pci_dev *pdev) +{ + struct mhi_pci_device *mhi_pdev = pci_get_drvdata(pdev); + struct mhi_controller *mhi_cntrl = &mhi_pdev->mhi_cntrl; + int err; + + /* Restore initial known working PCI state */ + pci_load_saved_state(pdev, mhi_pdev->pci_state); + pci_restore_state(pdev); + + /* Is device status available ? */ + if (!mhi_pci_is_alive(mhi_cntrl)) { + dev_err(&pdev->dev, "reset failed\n"); + return; + } + + err = mhi_prepare_for_power_up(mhi_cntrl); + if (err) { + dev_err(&pdev->dev, "failed to prepare MHI controller\n"); + return; + } + + err = mhi_sync_power_up(mhi_cntrl); + if (err) { + dev_err(&pdev->dev, "failed to power up MHI controller\n"); + mhi_unprepare_after_power_down(mhi_cntrl); + return; + } + + set_bit(MHI_PCI_DEV_STARTED, &mhi_pdev->status); + mod_timer(&mhi_pdev->health_check_timer, jiffies + HEALTH_CHECK_PERIOD); +} + +static pci_ers_result_t mhi_pci_error_detected(struct pci_dev *pdev, + pci_channel_state_t state) +{ + struct mhi_pci_device *mhi_pdev = pci_get_drvdata(pdev); + struct mhi_controller *mhi_cntrl = &mhi_pdev->mhi_cntrl; + + dev_err(&pdev->dev, "PCI error detected, state = %u\n", state); + + if (state == pci_channel_io_perm_failure) + return PCI_ERS_RESULT_DISCONNECT; + + /* Clean up MHI state */ + if (test_and_clear_bit(MHI_PCI_DEV_STARTED, &mhi_pdev->status)) { + mhi_power_down(mhi_cntrl, false); + mhi_unprepare_after_power_down(mhi_cntrl); + } else { + /* Nothing to do */ + return PCI_ERS_RESULT_RECOVERED; + } + + pci_disable_device(pdev); + + return PCI_ERS_RESULT_NEED_RESET; +} + +static pci_ers_result_t mhi_pci_slot_reset(struct pci_dev *pdev) +{ + if (pci_enable_device(pdev)) { + dev_err(&pdev->dev, "Cannot re-enable PCI device after reset.\n"); + return PCI_ERS_RESULT_DISCONNECT; + } + + return PCI_ERS_RESULT_RECOVERED; +} + +static void mhi_pci_io_resume(struct pci_dev *pdev) +{ + struct mhi_pci_device *mhi_pdev = pci_get_drvdata(pdev); + + dev_err(&pdev->dev, "PCI slot reset done\n"); + + queue_work(system_long_wq, &mhi_pdev->recovery_work); +} + +static const struct pci_error_handlers mhi_pci_err_handler = { + .error_detected = mhi_pci_error_detected, + .slot_reset = mhi_pci_slot_reset, + .resume = mhi_pci_io_resume, + .reset_prepare = mhi_pci_reset_prepare, + .reset_done = mhi_pci_reset_done, +}; + +static int __maybe_unused mhi_pci_suspend(struct device *dev) +{ + struct pci_dev *pdev = to_pci_dev(dev); + struct mhi_pci_device *mhi_pdev = dev_get_drvdata(dev); + struct mhi_controller *mhi_cntrl = &mhi_pdev->mhi_cntrl; + + del_timer(&mhi_pdev->health_check_timer); + cancel_work_sync(&mhi_pdev->recovery_work); + + /* Transition to M3 state */ + mhi_pm_suspend(mhi_cntrl); + + pci_save_state(pdev); + pci_disable_device(pdev); + pci_wake_from_d3(pdev, true); + pci_set_power_state(pdev, PCI_D3hot); + + return 0; +} + +static int __maybe_unused mhi_pci_resume(struct device *dev) +{ + struct pci_dev *pdev = to_pci_dev(dev); + struct mhi_pci_device *mhi_pdev = dev_get_drvdata(dev); + struct mhi_controller *mhi_cntrl = &mhi_pdev->mhi_cntrl; + int err; + + pci_set_power_state(pdev, PCI_D0); + pci_restore_state(pdev); + pci_set_master(pdev); + + err = pci_enable_device(pdev); + if (err) + goto err_recovery; + + /* Exit M3, transition to M0 state */ + err = mhi_pm_resume(mhi_cntrl); + if (err) { + dev_err(&pdev->dev, "failed to resume device: %d\n", err); + goto err_recovery; + } + + /* Resume health check */ + mod_timer(&mhi_pdev->health_check_timer, jiffies + HEALTH_CHECK_PERIOD); + + return 0; + +err_recovery: + /* The device may have loose power or crashed, try recovering it */ + queue_work(system_long_wq, &mhi_pdev->recovery_work); + + return err; +} + +static const struct dev_pm_ops mhi_pci_pm_ops = { + SET_SYSTEM_SLEEP_PM_OPS(mhi_pci_suspend, mhi_pci_resume) +}; + static struct pci_driver mhi_pci_driver = { .name = "mhi-pci-generic", .id_table = mhi_pci_id_table, .probe = mhi_pci_probe, - .remove = mhi_pci_remove + .remove = mhi_pci_remove, + .err_handler = &mhi_pci_err_handler, + .driver.pm = &mhi_pci_pm_ops }; module_pci_driver(mhi_pci_driver); diff --git a/drivers/char/hw_random/ingenic-trng.c b/drivers/char/hw_random/ingenic-trng.c index 954a8411d67d2fb885c36b111f510067b1c16add..0eb80f786f4dd57506f0a004b729e3d832e85798 100644 --- a/drivers/char/hw_random/ingenic-trng.c +++ b/drivers/char/hw_random/ingenic-trng.c @@ -113,13 +113,17 @@ static int ingenic_trng_probe(struct platform_device *pdev) ret = hwrng_register(&trng->rng); if (ret) { dev_err(&pdev->dev, "Failed to register hwrng\n"); - return ret; + goto err_unprepare_clk; } platform_set_drvdata(pdev, trng); dev_info(&pdev->dev, "Ingenic DTRNG driver registered\n"); return 0; + +err_unprepare_clk: + clk_disable_unprepare(trng->clk); + return ret; } static int ingenic_trng_remove(struct platform_device *pdev) diff --git a/drivers/char/hw_random/iproc-rng200.c b/drivers/char/hw_random/iproc-rng200.c index 01583faf9893e8772fe7dc08e8b5b858e25813c1..a43743887db1988106b0219433d62d25d5eefe88 100644 --- a/drivers/char/hw_random/iproc-rng200.c +++ b/drivers/char/hw_random/iproc-rng200.c @@ -28,7 +28,6 @@ #define RNG_CTRL_OFFSET 0x00 #define RNG_CTRL_RNG_RBGEN_MASK 0x00001FFF #define RNG_CTRL_RNG_RBGEN_ENABLE 0x00000001 -#define RNG_CTRL_RNG_RBGEN_DISABLE 0x00000000 #define RNG_SOFT_RESET_OFFSET 0x04 #define RNG_SOFT_RESET 0x00000001 @@ -54,15 +53,24 @@ struct iproc_rng200_dev { #define to_rng_priv(rng) container_of(rng, struct iproc_rng200_dev, rng) -static void iproc_rng200_restart(void __iomem *rng_base) +static void iproc_rng200_enable_set(void __iomem *rng_base, bool enable) { - uint32_t val; + u32 val; - /* Disable RBG */ val = ioread32(rng_base + RNG_CTRL_OFFSET); val &= ~RNG_CTRL_RNG_RBGEN_MASK; - val |= RNG_CTRL_RNG_RBGEN_DISABLE; + + if (enable) + val |= RNG_CTRL_RNG_RBGEN_ENABLE; + iowrite32(val, rng_base + RNG_CTRL_OFFSET); +} + +static void iproc_rng200_restart(void __iomem *rng_base) +{ + uint32_t val; + + iproc_rng200_enable_set(rng_base, false); /* Clear all interrupt status */ iowrite32(0xFFFFFFFFUL, rng_base + RNG_INT_STATUS_OFFSET); @@ -84,11 +92,7 @@ static void iproc_rng200_restart(void __iomem *rng_base) val &= ~RBG_SOFT_RESET; iowrite32(val, rng_base + RBG_SOFT_RESET_OFFSET); - /* Enable RBG */ - val = ioread32(rng_base + RNG_CTRL_OFFSET); - val &= ~RNG_CTRL_RNG_RBGEN_MASK; - val |= RNG_CTRL_RNG_RBGEN_ENABLE; - iowrite32(val, rng_base + RNG_CTRL_OFFSET); + iproc_rng200_enable_set(rng_base, true); } static int iproc_rng200_read(struct hwrng *rng, void *buf, size_t max, @@ -155,13 +159,8 @@ static int iproc_rng200_read(struct hwrng *rng, void *buf, size_t max, static int iproc_rng200_init(struct hwrng *rng) { struct iproc_rng200_dev *priv = to_rng_priv(rng); - uint32_t val; - /* Setup RNG. */ - val = ioread32(priv->base + RNG_CTRL_OFFSET); - val &= ~RNG_CTRL_RNG_RBGEN_MASK; - val |= RNG_CTRL_RNG_RBGEN_ENABLE; - iowrite32(val, priv->base + RNG_CTRL_OFFSET); + iproc_rng200_enable_set(priv->base, true); return 0; } @@ -169,13 +168,8 @@ static int iproc_rng200_init(struct hwrng *rng) static void iproc_rng200_cleanup(struct hwrng *rng) { struct iproc_rng200_dev *priv = to_rng_priv(rng); - uint32_t val; - /* Disable RNG hardware */ - val = ioread32(priv->base + RNG_CTRL_OFFSET); - val &= ~RNG_CTRL_RNG_RBGEN_MASK; - val |= RNG_CTRL_RNG_RBGEN_DISABLE; - iowrite32(val, priv->base + RNG_CTRL_OFFSET); + iproc_rng200_enable_set(priv->base, false); } static int iproc_rng200_probe(struct platform_device *pdev) diff --git a/drivers/char/hw_random/nomadik-rng.c b/drivers/char/hw_random/nomadik-rng.c index b0ded41eb865f56483c2fa80dbf7361832680b3c..67947a19aa225e8cd7b05622479452e8b5651bce 100644 --- a/drivers/char/hw_random/nomadik-rng.c +++ b/drivers/char/hw_random/nomadik-rng.c @@ -69,11 +69,10 @@ static int nmk_rng_probe(struct amba_device *dev, const struct amba_id *id) return ret; } -static int nmk_rng_remove(struct amba_device *dev) +static void nmk_rng_remove(struct amba_device *dev) { amba_release_regions(dev); clk_disable(rng_clk); - return 0; } static const struct amba_id nmk_rng_ids[] = { diff --git a/drivers/char/hw_random/optee-rng.c b/drivers/char/hw_random/optee-rng.c index a99d8294998182f874427a73ef2c98c1ea55b709..135a82590923d97f6801802fdcafb442b75768f5 100644 --- a/drivers/char/hw_random/optee-rng.c +++ b/drivers/char/hw_random/optee-rng.c @@ -243,7 +243,7 @@ static int optee_rng_probe(struct device *dev) if (err) goto out_sess; - err = hwrng_register(&pvt_data.optee_rng); + err = devm_hwrng_register(dev, &pvt_data.optee_rng); if (err) { dev_err(dev, "hwrng registration failed (%d)\n", err); goto out_sess; @@ -263,7 +263,6 @@ static int optee_rng_probe(struct device *dev) static int optee_rng_remove(struct device *dev) { - hwrng_unregister(&pvt_data.optee_rng); tee_client_close_session(pvt_data.ctx, pvt_data.session_id); tee_client_close_context(pvt_data.ctx); diff --git a/drivers/char/hw_random/timeriomem-rng.c b/drivers/char/hw_random/timeriomem-rng.c index e262445fed5f5353460e8dd0a3f7553f7d7bfafe..8ea1fc831eb7bac773848fc39ea9f63742217ba3 100644 --- a/drivers/char/hw_random/timeriomem-rng.c +++ b/drivers/char/hw_random/timeriomem-rng.c @@ -69,7 +69,7 @@ static int timeriomem_rng_read(struct hwrng *hwrng, void *data, */ if (retval > 0) usleep_range(period_us, - period_us + min(1, period_us / 100)); + period_us + max(1, period_us / 100)); *(u32 *)data = readl(priv->io_base); retval += sizeof(u32); @@ -169,7 +169,7 @@ static int timeriomem_rng_probe(struct platform_device *pdev) priv->present = 1; complete(&priv->completion); - err = hwrng_register(&priv->rng_ops); + err = devm_hwrng_register(&pdev->dev, &priv->rng_ops); if (err) { dev_err(&pdev->dev, "problem registering\n"); return err; @@ -185,7 +185,6 @@ static int timeriomem_rng_remove(struct platform_device *pdev) { struct timeriomem_rng_private *priv = platform_get_drvdata(pdev); - hwrng_unregister(&priv->rng_ops); hrtimer_cancel(&priv->timer); return 0; diff --git a/drivers/char/ipmi/ipmb_dev_int.c b/drivers/char/ipmi/ipmb_dev_int.c index 382b28f1cf2f6daf922236e827a2abce1bc73940..49b8f22fdcf03b8883e378d28e07ce13172df854 100644 --- a/drivers/char/ipmi/ipmb_dev_int.c +++ b/drivers/char/ipmi/ipmb_dev_int.c @@ -137,7 +137,7 @@ static ssize_t ipmb_write(struct file *file, const char __user *buf, { struct ipmb_dev *ipmb_dev = to_ipmb_dev(file); u8 rq_sa, netf_rq_lun, msg_len; - union i2c_smbus_data data; + struct i2c_client *temp_client; u8 msg[MAX_MSG_LEN]; ssize_t ret; @@ -160,21 +160,21 @@ static ssize_t ipmb_write(struct file *file, const char __user *buf, } /* - * subtract rq_sa and netf_rq_lun from the length of the msg passed to - * i2c_smbus_xfer + * subtract rq_sa and netf_rq_lun from the length of the msg. Fill the + * temporary client. Note that its use is an exception for IPMI. */ msg_len = msg[IPMB_MSG_LEN_IDX] - SMBUS_MSG_HEADER_LENGTH; - if (msg_len > I2C_SMBUS_BLOCK_MAX) - msg_len = I2C_SMBUS_BLOCK_MAX; + temp_client = kmemdup(ipmb_dev->client, sizeof(*temp_client), GFP_KERNEL); + if (!temp_client) + return -ENOMEM; + + temp_client->addr = rq_sa; - data.block[0] = msg_len; - memcpy(&data.block[1], msg + SMBUS_MSG_IDX_OFFSET, msg_len); - ret = i2c_smbus_xfer(ipmb_dev->client->adapter, rq_sa, - ipmb_dev->client->flags, - I2C_SMBUS_WRITE, netf_rq_lun, - I2C_SMBUS_BLOCK_DATA, &data); + ret = i2c_smbus_write_block_data(temp_client, netf_rq_lun, msg_len, + msg + SMBUS_MSG_IDX_OFFSET); + kfree(temp_client); - return ret ? : count; + return ret < 0 ? ret : count; } static __poll_t ipmb_poll(struct file *file, poll_table *wait) diff --git a/drivers/char/mem.c b/drivers/char/mem.c index 887ffca3f47f8a7c1c63dcbf5f4d3d2536b0fcb7..869b9f5e8e0315d4e497fcfec20a0e48c7414f85 100644 --- a/drivers/char/mem.c +++ b/drivers/char/mem.c @@ -31,9 +31,6 @@ #include #include #include -#include -#include -#include #ifdef CONFIG_IA64 # include @@ -829,42 +826,6 @@ static loff_t memory_lseek(struct file *file, loff_t offset, int orig) return ret; } -static struct inode *devmem_inode; - -#ifdef CONFIG_IO_STRICT_DEVMEM -void revoke_devmem(struct resource *res) -{ - /* pairs with smp_store_release() in devmem_init_inode() */ - struct inode *inode = smp_load_acquire(&devmem_inode); - - /* - * Check that the initialization has completed. Losing the race - * is ok because it means drivers are claiming resources before - * the fs_initcall level of init and prevent /dev/mem from - * establishing mappings. - */ - if (!inode) - return; - - /* - * The expectation is that the driver has successfully marked - * the resource busy by this point, so devmem_is_allowed() - * should start returning false, however for performance this - * does not iterate the entire resource range. - */ - if (devmem_is_allowed(PHYS_PFN(res->start)) && - devmem_is_allowed(PHYS_PFN(res->end))) { - /* - * *cringe* iomem=relaxed says "go ahead, what's the - * worst that can happen?" - */ - return; - } - - unmap_mapping_range(inode->i_mapping, res->start, resource_size(res), 1); -} -#endif - static int open_port(struct inode *inode, struct file *filp) { int rc; @@ -884,8 +845,7 @@ static int open_port(struct inode *inode, struct file *filp) * revocations when drivers want to take over a /dev/mem mapped * range. */ - inode->i_mapping = devmem_inode->i_mapping; - filp->f_mapping = inode->i_mapping; + filp->f_mapping = iomem_get_mapping(); return 0; } @@ -1017,48 +977,6 @@ static char *mem_devnode(struct device *dev, umode_t *mode) static struct class *mem_class; -static int devmem_fs_init_fs_context(struct fs_context *fc) -{ - return init_pseudo(fc, DEVMEM_MAGIC) ? 0 : -ENOMEM; -} - -static struct file_system_type devmem_fs_type = { - .name = "devmem", - .owner = THIS_MODULE, - .init_fs_context = devmem_fs_init_fs_context, - .kill_sb = kill_anon_super, -}; - -static int devmem_init_inode(void) -{ - static struct vfsmount *devmem_vfs_mount; - static int devmem_fs_cnt; - struct inode *inode; - int rc; - - rc = simple_pin_fs(&devmem_fs_type, &devmem_vfs_mount, &devmem_fs_cnt); - if (rc < 0) { - pr_err("Cannot mount /dev/mem pseudo filesystem: %d\n", rc); - return rc; - } - - inode = alloc_anon_inode(devmem_vfs_mount->mnt_sb); - if (IS_ERR(inode)) { - rc = PTR_ERR(inode); - pr_err("Cannot allocate inode for /dev/mem: %d\n", rc); - simple_release_fs(&devmem_vfs_mount, &devmem_fs_cnt); - return rc; - } - - /* - * Publish /dev/mem initialized. - * Pairs with smp_load_acquire() in revoke_devmem(). - */ - smp_store_release(&devmem_inode, inode); - - return 0; -} - static int __init chr_dev_init(void) { int minor; @@ -1080,8 +998,6 @@ static int __init chr_dev_init(void) */ if ((minor == DEVPORT_MINOR) && !arch_has_dev_port()) continue; - if ((minor == DEVMEM_MINOR) && devmem_init_inode() != 0) - continue; device_create(mem_class, NULL, MKDEV(MEM_MAJOR, minor), NULL, devlist[minor].name); diff --git a/drivers/char/random.c b/drivers/char/random.c index 84e24986a97a3c876d5572ccf9a34f9d8b9e3088..0fe9e200e4c8401681954e7f59cd785205b2924b 100644 --- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -1959,7 +1959,7 @@ static long random_ioctl(struct file *f, unsigned int cmd, unsigned long arg) return -EPERM; if (crng_init < 2) return -ENODATA; - crng_reseed(&primary_crng, NULL); + crng_reseed(&primary_crng, &input_pool); crng_global_init_time = jiffies - 1; return 0; default: diff --git a/drivers/char/tpm/Kconfig b/drivers/char/tpm/Kconfig index a18c314da21118657c822eecdee370deaf7680a3..4308f9ca7a43d1bd53dfc4cb9b1129f221216cb0 100644 --- a/drivers/char/tpm/Kconfig +++ b/drivers/char/tpm/Kconfig @@ -86,6 +86,16 @@ config TCG_TIS_SYNQUACER To compile this driver as a module, choose M here; the module will be called tpm_tis_synquacer. +config TCG_TIS_I2C_CR50 + tristate "TPM Interface Specification 2.0 Interface (I2C - CR50)" + depends on I2C + select TCG_CR50 + help + This is a driver for the Google cr50 I2C TPM interface which is a + custom microcontroller and requires a custom i2c protocol interface + to handle the limitations of the hardware. To compile this driver + as a module, choose M here; the module will be called tcg_tis_i2c_cr50. + config TCG_TIS_I2C_ATMEL tristate "TPM Interface Specification 1.2 Interface (I2C - Atmel)" depends on I2C diff --git a/drivers/char/tpm/Makefile b/drivers/char/tpm/Makefile index 84db4fb3a9c92db1111a1d0ed51cb67490e2bf22..66d39ea6bd10a333b20f53a1f57b5bf954283e9a 100644 --- a/drivers/char/tpm/Makefile +++ b/drivers/char/tpm/Makefile @@ -27,6 +27,8 @@ obj-$(CONFIG_TCG_TIS_SPI) += tpm_tis_spi.o tpm_tis_spi-y := tpm_tis_spi_main.o tpm_tis_spi-$(CONFIG_TCG_TIS_SPI_CR50) += tpm_tis_spi_cr50.o +obj-$(CONFIG_TCG_TIS_I2C_CR50) += tpm_tis_i2c_cr50.o + obj-$(CONFIG_TCG_TIS_I2C_ATMEL) += tpm_i2c_atmel.o obj-$(CONFIG_TCG_TIS_I2C_INFINEON) += tpm_i2c_infineon.o obj-$(CONFIG_TCG_TIS_I2C_NUVOTON) += tpm_i2c_nuvoton.o diff --git a/drivers/char/tpm/eventlog/tpm1.c b/drivers/char/tpm/eventlog/tpm1.c index 2c96977ad080a698a9128a8040423094e3cf9219..8aa9057601d6d6fb943c9c42268fb8d396775dd2 100644 --- a/drivers/char/tpm/eventlog/tpm1.c +++ b/drivers/char/tpm/eventlog/tpm1.c @@ -210,6 +210,7 @@ static int get_event_name(char *dest, struct tcpa_event *event, default: break; } + break; default: break; } diff --git a/drivers/char/tpm/tpm-chip.c b/drivers/char/tpm/tpm-chip.c index ddaeceb7e109105297f6baf53d74138911246fed..19e23fcc6bc8c1bdca3a26ab07333e744ef8451e 100644 --- a/drivers/char/tpm/tpm-chip.c +++ b/drivers/char/tpm/tpm-chip.c @@ -278,6 +278,8 @@ static void tpm_devs_release(struct device *dev) { struct tpm_chip *chip = container_of(dev, struct tpm_chip, devs); + dump_stack(); + /* release the master device reference */ put_device(&chip->dev); } diff --git a/drivers/char/tpm/tpm-dev-common.c b/drivers/char/tpm/tpm-dev-common.c index 1784530b8387bb46bec8694a49ddb5385833be80..c08cbb306636badabbb04c0333172814215fe876 100644 --- a/drivers/char/tpm/tpm-dev-common.c +++ b/drivers/char/tpm/tpm-dev-common.c @@ -20,7 +20,6 @@ #include "tpm-dev.h" static struct workqueue_struct *tpm_dev_wq; -static DEFINE_MUTEX(tpm_dev_wq_lock); static ssize_t tpm_dev_transmit(struct tpm_chip *chip, struct tpm_space *space, u8 *buf, size_t bufsiz) diff --git a/drivers/char/tpm/tpm-sysfs.c b/drivers/char/tpm/tpm-sysfs.c index e2ff0b273a0f1de9533a892e175e498d308414d9..63f03cfb8e6ad3e8f28c6869535a2749913b845d 100644 --- a/drivers/char/tpm/tpm-sysfs.c +++ b/drivers/char/tpm/tpm-sysfs.c @@ -337,11 +337,190 @@ static const struct attribute_group tpm2_dev_group = { .attrs = tpm2_dev_attrs, }; +struct tpm_pcr_attr { + int alg_id; + int pcr; + struct device_attribute attr; +}; + +#define to_tpm_pcr_attr(a) container_of(a, struct tpm_pcr_attr, attr) + +static ssize_t pcr_value_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct tpm_pcr_attr *ha = to_tpm_pcr_attr(attr); + struct tpm_chip *chip = to_tpm_chip(dev); + struct tpm_digest digest; + int i; + int digest_size = 0; + int rc; + char *str = buf; + + for (i = 0; i < chip->nr_allocated_banks; i++) + if (ha->alg_id == chip->allocated_banks[i].alg_id) + digest_size = chip->allocated_banks[i].digest_size; + /* should never happen */ + if (!digest_size) + return -EINVAL; + + digest.alg_id = ha->alg_id; + rc = tpm_pcr_read(chip, ha->pcr, &digest); + if (rc) + return rc; + for (i = 0; i < digest_size; i++) + str += sprintf(str, "%02X", digest.digest[i]); + str += sprintf(str, "\n"); + + return str - buf; +} + +/* + * The following set of defines represents all the magic to build + * the per hash attribute groups for displaying each bank of PCRs. + * The only slight problem with this approach is that every PCR is + * hard coded to be present, so you don't know if an PCR is missing + * until a cat of the file returns -EINVAL + * + * Also note you must ignore checkpatch warnings in this macro + * code. This is deep macro magic that checkpatch.pl doesn't + * understand. + */ + +/* Note, this must match TPM2_PLATFORM_PCR which is fixed at 24. */ +#define _TPM_HELPER(_alg, _hash, F) \ + F(_alg, _hash, 0) \ + F(_alg, _hash, 1) \ + F(_alg, _hash, 2) \ + F(_alg, _hash, 3) \ + F(_alg, _hash, 4) \ + F(_alg, _hash, 5) \ + F(_alg, _hash, 6) \ + F(_alg, _hash, 7) \ + F(_alg, _hash, 8) \ + F(_alg, _hash, 9) \ + F(_alg, _hash, 10) \ + F(_alg, _hash, 11) \ + F(_alg, _hash, 12) \ + F(_alg, _hash, 13) \ + F(_alg, _hash, 14) \ + F(_alg, _hash, 15) \ + F(_alg, _hash, 16) \ + F(_alg, _hash, 17) \ + F(_alg, _hash, 18) \ + F(_alg, _hash, 19) \ + F(_alg, _hash, 20) \ + F(_alg, _hash, 21) \ + F(_alg, _hash, 22) \ + F(_alg, _hash, 23) + +/* ignore checkpatch warning about trailing ; in macro. */ +#define PCR_ATTR(_alg, _hash, _pcr) \ + static struct tpm_pcr_attr dev_attr_pcr_##_hash##_##_pcr = { \ + .alg_id = _alg, \ + .pcr = _pcr, \ + .attr = { \ + .attr = { \ + .name = __stringify(_pcr), \ + .mode = 0444 \ + }, \ + .show = pcr_value_show \ + } \ + }; + +#define PCR_ATTRS(_alg, _hash) \ + _TPM_HELPER(_alg, _hash, PCR_ATTR) + +/* ignore checkpatch warning about trailing , in macro. */ +#define PCR_ATTR_VAL(_alg, _hash, _pcr) \ + &dev_attr_pcr_##_hash##_##_pcr.attr.attr, + +#define PCR_ATTR_GROUP_ARRAY(_alg, _hash) \ + static struct attribute *pcr_group_attrs_##_hash[] = { \ + _TPM_HELPER(_alg, _hash, PCR_ATTR_VAL) \ + NULL \ + } + +#define PCR_ATTR_GROUP(_alg, _hash) \ + static struct attribute_group pcr_group_##_hash = { \ + .name = "pcr-" __stringify(_hash), \ + .attrs = pcr_group_attrs_##_hash \ + } + +#define PCR_ATTR_BUILD(_alg, _hash) \ + PCR_ATTRS(_alg, _hash) \ + PCR_ATTR_GROUP_ARRAY(_alg, _hash); \ + PCR_ATTR_GROUP(_alg, _hash) +/* + * End of macro structure to build an attribute group containing 24 + * PCR value files for each supported hash algorithm + */ + +/* + * The next set of macros implements the cleverness for each hash to + * build a static attribute group called pcr_group_ which can be + * added to chip->groups[]. + * + * The first argument is the TPM algorithm id and the second is the + * hash used as both the suffix and the group name. Note: the group + * name is a directory in the top level tpm class with the name + * pcr-, so it must not clash with any other names already + * in the sysfs directory. + */ +PCR_ATTR_BUILD(TPM_ALG_SHA1, sha1); +PCR_ATTR_BUILD(TPM_ALG_SHA256, sha256); +PCR_ATTR_BUILD(TPM_ALG_SHA384, sha384); +PCR_ATTR_BUILD(TPM_ALG_SHA512, sha512); +PCR_ATTR_BUILD(TPM_ALG_SM3_256, sm3); + + void tpm_sysfs_add_device(struct tpm_chip *chip) { + int i; + WARN_ON(chip->groups_cnt != 0); + if (chip->flags & TPM_CHIP_FLAG_TPM2) chip->groups[chip->groups_cnt++] = &tpm2_dev_group; else chip->groups[chip->groups_cnt++] = &tpm1_dev_group; + + /* add one group for each bank hash */ + for (i = 0; i < chip->nr_allocated_banks; i++) { + switch (chip->allocated_banks[i].alg_id) { + case TPM_ALG_SHA1: + chip->groups[chip->groups_cnt++] = &pcr_group_sha1; + break; + case TPM_ALG_SHA256: + chip->groups[chip->groups_cnt++] = &pcr_group_sha256; + break; + case TPM_ALG_SHA384: + chip->groups[chip->groups_cnt++] = &pcr_group_sha384; + break; + case TPM_ALG_SHA512: + chip->groups[chip->groups_cnt++] = &pcr_group_sha512; + break; + case TPM_ALG_SM3_256: + chip->groups[chip->groups_cnt++] = &pcr_group_sm3; + break; + default: + /* + * If triggers, send a patch to add both a + * PCR_ATTR_BUILD() macro above for the + * missing algorithm as well as an additional + * case in this switch statement. + */ + dev_err(&chip->dev, + "TPM with unsupported bank algorithm 0x%04x", + chip->allocated_banks[i].alg_id); + break; + } + } + + /* + * This will only trigger if someone has added an additional + * hash to the tpm_algorithms enum without incrementing + * TPM_MAX_HASHES. + */ + WARN_ON(chip->groups_cnt > TPM_MAX_HASHES + 1); } diff --git a/drivers/char/tpm/tpm.h b/drivers/char/tpm/tpm.h index 947d1db0a5ccf756172cfb9275ed9e046d4cdd38..283f78211c3a7bc09092085d12834c48571f93f3 100644 --- a/drivers/char/tpm/tpm.h +++ b/drivers/char/tpm/tpm.h @@ -164,8 +164,6 @@ extern const struct file_operations tpmrm_fops; extern struct idr dev_nums_idr; ssize_t tpm_transmit(struct tpm_chip *chip, u8 *buf, size_t bufsiz); -ssize_t tpm_transmit_cmd(struct tpm_chip *chip, struct tpm_buf *buf, - size_t min_rsp_body_length, const char *desc); int tpm_get_timeouts(struct tpm_chip *); int tpm_auto_startup(struct tpm_chip *chip); @@ -194,8 +192,6 @@ static inline void tpm_msleep(unsigned int delay_msec) int tpm_chip_start(struct tpm_chip *chip); void tpm_chip_stop(struct tpm_chip *chip); struct tpm_chip *tpm_find_get_ops(struct tpm_chip *chip); -__must_check int tpm_try_get_ops(struct tpm_chip *chip); -void tpm_put_ops(struct tpm_chip *chip); struct tpm_chip *tpm_chip_alloc(struct device *dev, const struct tpm_class_ops *ops); diff --git a/drivers/char/tpm/tpm_ppi.c b/drivers/char/tpm/tpm_ppi.c index b2dab941cb7fb5722a813ee6d538fda7bd827703..40018a73b3cb7e93aa206b012d0afca8a72ce176 100644 --- a/drivers/char/tpm/tpm_ppi.c +++ b/drivers/char/tpm/tpm_ppi.c @@ -358,7 +358,7 @@ static struct attribute *ppi_attrs[] = { &dev_attr_tcg_operations.attr, &dev_attr_vs_operations.attr, NULL, }; -static struct attribute_group ppi_attr_grp = { +static const struct attribute_group ppi_attr_grp = { .name = "ppi", .attrs = ppi_attrs }; diff --git a/drivers/char/tpm/tpm_tis_core.c b/drivers/char/tpm/tpm_tis_core.c index 92c51c6cfd1b781efde95cea0e9aacba3776924c..431919d5f48af168029741f1afb1dfb90df21231 100644 --- a/drivers/char/tpm/tpm_tis_core.c +++ b/drivers/char/tpm/tpm_tis_core.c @@ -125,7 +125,8 @@ static bool check_locality(struct tpm_chip *chip, int l) if (rc < 0) return false; - if ((access & (TPM_ACCESS_ACTIVE_LOCALITY | TPM_ACCESS_VALID)) == + if ((access & (TPM_ACCESS_ACTIVE_LOCALITY | TPM_ACCESS_VALID + | TPM_ACCESS_REQUEST_USE)) == (TPM_ACCESS_ACTIVE_LOCALITY | TPM_ACCESS_VALID)) { priv->locality = l; return true; @@ -134,58 +135,13 @@ static bool check_locality(struct tpm_chip *chip, int l) return false; } -static bool locality_inactive(struct tpm_chip *chip, int l) -{ - struct tpm_tis_data *priv = dev_get_drvdata(&chip->dev); - int rc; - u8 access; - - rc = tpm_tis_read8(priv, TPM_ACCESS(l), &access); - if (rc < 0) - return false; - - if ((access & (TPM_ACCESS_VALID | TPM_ACCESS_ACTIVE_LOCALITY)) - == TPM_ACCESS_VALID) - return true; - - return false; -} - static int release_locality(struct tpm_chip *chip, int l) { struct tpm_tis_data *priv = dev_get_drvdata(&chip->dev); - unsigned long stop, timeout; - long rc; tpm_tis_write8(priv, TPM_ACCESS(l), TPM_ACCESS_ACTIVE_LOCALITY); - stop = jiffies + chip->timeout_a; - - if (chip->flags & TPM_CHIP_FLAG_IRQ) { -again: - timeout = stop - jiffies; - if ((long)timeout <= 0) - return -1; - - rc = wait_event_interruptible_timeout(priv->int_queue, - (locality_inactive(chip, l)), - timeout); - - if (rc > 0) - return 0; - - if (rc == -ERESTARTSYS && freezing(current)) { - clear_thread_flag(TIF_SIGPENDING); - goto again; - } - } else { - do { - if (locality_inactive(chip, l)) - return 0; - tpm_msleep(TPM_TIMEOUT); - } while (time_before(jiffies, stop)); - } - return -1; + return 0; } static int request_locality(struct tpm_chip *chip, int l) diff --git a/drivers/char/tpm/tpm_tis_i2c_cr50.c b/drivers/char/tpm/tpm_tis_i2c_cr50.c new file mode 100644 index 0000000000000000000000000000000000000000..ec9a65e7887dd5d49ef789e1e27f19374a8d07cd --- /dev/null +++ b/drivers/char/tpm/tpm_tis_i2c_cr50.c @@ -0,0 +1,790 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright 2020 Google Inc. + * + * Based on Infineon TPM driver by Peter Huewe. + * + * cr50 is a firmware for H1 secure modules that requires special + * handling for the I2C interface. + * + * - Use an interrupt for transaction status instead of hardcoded delays. + * - Must use write+wait+read read protocol. + * - All 4 bytes of status register must be read/written at once. + * - Burst count max is 63 bytes, and burst count behaves slightly differently + * than other I2C TPMs. + * - When reading from FIFO the full burstcnt must be read instead of just + * reading header and determining the remainder. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "tpm_tis_core.h" + +#define TPM_CR50_MAX_BUFSIZE 64 +#define TPM_CR50_TIMEOUT_SHORT_MS 2 /* Short timeout during transactions */ +#define TPM_CR50_TIMEOUT_NOIRQ_MS 20 /* Timeout for TPM ready without IRQ */ +#define TPM_CR50_I2C_DID_VID 0x00281ae0L /* Device and vendor ID reg value */ +#define TPM_CR50_I2C_MAX_RETRIES 3 /* Max retries due to I2C errors */ +#define TPM_CR50_I2C_RETRY_DELAY_LO 55 /* Min usecs between retries on I2C */ +#define TPM_CR50_I2C_RETRY_DELAY_HI 65 /* Max usecs between retries on I2C */ + +#define TPM_I2C_ACCESS(l) (0x0000 | ((l) << 4)) +#define TPM_I2C_STS(l) (0x0001 | ((l) << 4)) +#define TPM_I2C_DATA_FIFO(l) (0x0005 | ((l) << 4)) +#define TPM_I2C_DID_VID(l) (0x0006 | ((l) << 4)) + +/** + * struct tpm_i2c_cr50_priv_data - Driver private data. + * @irq: Irq number used for this chip. + * If irq <= 0, then a fixed timeout is used instead of waiting for irq. + * @tpm_ready: Struct used by irq handler to signal R/W readiness. + * @buf: Buffer used for i2c writes, with i2c address prepended to content. + * + * Private driver struct used by kernel threads and interrupt context. + */ +struct tpm_i2c_cr50_priv_data { + int irq; + struct completion tpm_ready; + u8 buf[TPM_CR50_MAX_BUFSIZE]; +}; + +/** + * tpm_cr50_i2c_int_handler() - cr50 interrupt handler. + * @dummy: Unused parameter. + * @tpm_info: TPM chip information. + * + * The cr50 interrupt handler signals waiting threads that the + * interrupt has been asserted. It does not do any interrupt triggered + * processing but is instead used to avoid fixed delays. + * + * Return: + * IRQ_HANDLED signifies irq was handled by this device. + */ +static irqreturn_t tpm_cr50_i2c_int_handler(int dummy, void *tpm_info) +{ + struct tpm_chip *chip = tpm_info; + struct tpm_i2c_cr50_priv_data *priv = dev_get_drvdata(&chip->dev); + + complete(&priv->tpm_ready); + + return IRQ_HANDLED; +} + +/** + * tpm_cr50_i2c_wait_tpm_ready() - Wait for tpm to signal ready. + * @chip: A TPM chip. + * + * Wait for completion interrupt if available, otherwise use a fixed + * delay for the TPM to be ready. + * + * Return: + * - 0: Success. + * - -errno: A POSIX error code. + */ +static int tpm_cr50_i2c_wait_tpm_ready(struct tpm_chip *chip) +{ + struct tpm_i2c_cr50_priv_data *priv = dev_get_drvdata(&chip->dev); + + /* Use a safe fixed delay if interrupt is not supported */ + if (priv->irq <= 0) { + msleep(TPM_CR50_TIMEOUT_NOIRQ_MS); + return 0; + } + + /* Wait for interrupt to indicate TPM is ready to respond */ + if (!wait_for_completion_timeout(&priv->tpm_ready, + msecs_to_jiffies(chip->timeout_a))) { + dev_warn(&chip->dev, "Timeout waiting for TPM ready\n"); + return -ETIMEDOUT; + } + + return 0; +} + +/** + * tpm_cr50_i2c_enable_tpm_irq() - Enable TPM irq. + * @chip: A TPM chip. + */ +static void tpm_cr50_i2c_enable_tpm_irq(struct tpm_chip *chip) +{ + struct tpm_i2c_cr50_priv_data *priv = dev_get_drvdata(&chip->dev); + + if (priv->irq > 0) { + reinit_completion(&priv->tpm_ready); + enable_irq(priv->irq); + } +} + +/** + * tpm_cr50_i2c_disable_tpm_irq() - Disable TPM irq. + * @chip: A TPM chip. + */ +static void tpm_cr50_i2c_disable_tpm_irq(struct tpm_chip *chip) +{ + struct tpm_i2c_cr50_priv_data *priv = dev_get_drvdata(&chip->dev); + + if (priv->irq > 0) + disable_irq(priv->irq); +} + +/** + * tpm_cr50_i2c_transfer_message() - Transfer a message over i2c. + * @dev: Device information. + * @adapter: I2C adapter. + * @msg: Message to transfer. + * + * Call unlocked i2c transfer routine with the provided parameters and + * retry in case of bus errors. + * + * Return: + * - 0: Success. + * - -errno: A POSIX error code. + */ +static int tpm_cr50_i2c_transfer_message(struct device *dev, + struct i2c_adapter *adapter, + struct i2c_msg *msg) +{ + unsigned int try; + int rc; + + for (try = 0; try < TPM_CR50_I2C_MAX_RETRIES; try++) { + rc = __i2c_transfer(adapter, msg, 1); + if (rc == 1) + return 0; /* Successfully transferred the message */ + if (try) + dev_warn(dev, "i2c transfer failed (attempt %d/%d): %d\n", + try + 1, TPM_CR50_I2C_MAX_RETRIES, rc); + usleep_range(TPM_CR50_I2C_RETRY_DELAY_LO, TPM_CR50_I2C_RETRY_DELAY_HI); + } + + /* No i2c message transferred */ + return -EIO; +} + +/** + * tpm_cr50_i2c_read() - Read from TPM register. + * @chip: A TPM chip. + * @addr: Register address to read from. + * @buffer: Read destination, provided by caller. + * @len: Number of bytes to read. + * + * Sends the register address byte to the TPM, then waits until TPM + * is ready via interrupt signal or timeout expiration, then 'len' + * bytes are read from TPM response into the provided 'buffer'. + * + * Return: + * - 0: Success. + * - -errno: A POSIX error code. + */ +static int tpm_cr50_i2c_read(struct tpm_chip *chip, u8 addr, u8 *buffer, size_t len) +{ + struct i2c_client *client = to_i2c_client(chip->dev.parent); + struct i2c_msg msg_reg_addr = { + .addr = client->addr, + .len = 1, + .buf = &addr + }; + struct i2c_msg msg_response = { + .addr = client->addr, + .flags = I2C_M_RD, + .len = len, + .buf = buffer + }; + int rc; + + i2c_lock_bus(client->adapter, I2C_LOCK_SEGMENT); + + /* Prepare for completion interrupt */ + tpm_cr50_i2c_enable_tpm_irq(chip); + + /* Send the register address byte to the TPM */ + rc = tpm_cr50_i2c_transfer_message(&chip->dev, client->adapter, &msg_reg_addr); + if (rc < 0) + goto out; + + /* Wait for TPM to be ready with response data */ + rc = tpm_cr50_i2c_wait_tpm_ready(chip); + if (rc < 0) + goto out; + + /* Read response data from the TPM */ + rc = tpm_cr50_i2c_transfer_message(&chip->dev, client->adapter, &msg_response); + +out: + tpm_cr50_i2c_disable_tpm_irq(chip); + i2c_unlock_bus(client->adapter, I2C_LOCK_SEGMENT); + + if (rc < 0) + return rc; + + return 0; +} + +/** + * tpm_cr50_i2c_write()- Write to TPM register. + * @chip: A TPM chip. + * @addr: Register address to write to. + * @buffer: Data to write. + * @len: Number of bytes to write. + * + * The provided address is prepended to the data in 'buffer', the + * cobined address+data is sent to the TPM, then wait for TPM to + * indicate it is done writing. + * + * Return: + * - 0: Success. + * - -errno: A POSIX error code. + */ +static int tpm_cr50_i2c_write(struct tpm_chip *chip, u8 addr, u8 *buffer, + size_t len) +{ + struct tpm_i2c_cr50_priv_data *priv = dev_get_drvdata(&chip->dev); + struct i2c_client *client = to_i2c_client(chip->dev.parent); + struct i2c_msg msg = { + .addr = client->addr, + .len = len + 1, + .buf = priv->buf + }; + int rc; + + if (len > TPM_CR50_MAX_BUFSIZE - 1) + return -EINVAL; + + /* Prepend the 'register address' to the buffer */ + priv->buf[0] = addr; + memcpy(priv->buf + 1, buffer, len); + + i2c_lock_bus(client->adapter, I2C_LOCK_SEGMENT); + + /* Prepare for completion interrupt */ + tpm_cr50_i2c_enable_tpm_irq(chip); + + /* Send write request buffer with address */ + rc = tpm_cr50_i2c_transfer_message(&chip->dev, client->adapter, &msg); + if (rc < 0) + goto out; + + /* Wait for TPM to be ready, ignore timeout */ + tpm_cr50_i2c_wait_tpm_ready(chip); + +out: + tpm_cr50_i2c_disable_tpm_irq(chip); + i2c_unlock_bus(client->adapter, I2C_LOCK_SEGMENT); + + if (rc < 0) + return rc; + + return 0; +} + +/** + * tpm_cr50_check_locality() - Verify TPM locality 0 is active. + * @chip: A TPM chip. + * + * Return: + * - 0: Success. + * - -errno: A POSIX error code. + */ +static int tpm_cr50_check_locality(struct tpm_chip *chip) +{ + u8 mask = TPM_ACCESS_VALID | TPM_ACCESS_ACTIVE_LOCALITY; + u8 buf; + int rc; + + rc = tpm_cr50_i2c_read(chip, TPM_I2C_ACCESS(0), &buf, sizeof(buf)); + if (rc < 0) + return rc; + + if ((buf & mask) == mask) + return 0; + + return -EIO; +} + +/** + * tpm_cr50_release_locality() - Release TPM locality. + * @chip: A TPM chip. + * @force: Flag to force release if set. + */ +static void tpm_cr50_release_locality(struct tpm_chip *chip, bool force) +{ + u8 mask = TPM_ACCESS_VALID | TPM_ACCESS_REQUEST_PENDING; + u8 addr = TPM_I2C_ACCESS(0); + u8 buf; + + if (tpm_cr50_i2c_read(chip, addr, &buf, sizeof(buf)) < 0) + return; + + if (force || (buf & mask) == mask) { + buf = TPM_ACCESS_ACTIVE_LOCALITY; + tpm_cr50_i2c_write(chip, addr, &buf, sizeof(buf)); + } +} + +/** + * tpm_cr50_request_locality() - Request TPM locality 0. + * @chip: A TPM chip. + * + * Return: + * - 0: Success. + * - -errno: A POSIX error code. + */ +static int tpm_cr50_request_locality(struct tpm_chip *chip) +{ + u8 buf = TPM_ACCESS_REQUEST_USE; + unsigned long stop; + int rc; + + if (!tpm_cr50_check_locality(chip)) + return 0; + + rc = tpm_cr50_i2c_write(chip, TPM_I2C_ACCESS(0), &buf, sizeof(buf)); + if (rc < 0) + return rc; + + stop = jiffies + chip->timeout_a; + do { + if (!tpm_cr50_check_locality(chip)) + return 0; + + msleep(TPM_CR50_TIMEOUT_SHORT_MS); + } while (time_before(jiffies, stop)); + + return -ETIMEDOUT; +} + +/** + * tpm_cr50_i2c_tis_status() - Read cr50 tis status. + * @chip: A TPM chip. + * + * cr50 requires all 4 bytes of status register to be read. + * + * Return: + * TPM status byte. + */ +static u8 tpm_cr50_i2c_tis_status(struct tpm_chip *chip) +{ + u8 buf[4]; + + if (tpm_cr50_i2c_read(chip, TPM_I2C_STS(0), buf, sizeof(buf)) < 0) + return 0; + + return buf[0]; +} + +/** + * tpm_cr50_i2c_tis_set_ready() - Set status register to ready. + * @chip: A TPM chip. + * + * cr50 requires all 4 bytes of status register to be written. + */ +static void tpm_cr50_i2c_tis_set_ready(struct tpm_chip *chip) +{ + u8 buf[4] = { TPM_STS_COMMAND_READY }; + + tpm_cr50_i2c_write(chip, TPM_I2C_STS(0), buf, sizeof(buf)); + msleep(TPM_CR50_TIMEOUT_SHORT_MS); +} + +/** + * tpm_cr50_i2c_get_burst_and_status() - Get burst count and status. + * @chip: A TPM chip. + * @mask: Status mask. + * @burst: Return value for burst. + * @status: Return value for status. + * + * cr50 uses bytes 3:2 of status register for burst count and + * all 4 bytes must be read. + * + * Return: + * - 0: Success. + * - -errno: A POSIX error code. + */ +static int tpm_cr50_i2c_get_burst_and_status(struct tpm_chip *chip, u8 mask, + size_t *burst, u32 *status) +{ + unsigned long stop; + u8 buf[4]; + + *status = 0; + + /* wait for burstcount */ + stop = jiffies + chip->timeout_b; + + do { + if (tpm_cr50_i2c_read(chip, TPM_I2C_STS(0), buf, sizeof(buf)) < 0) { + msleep(TPM_CR50_TIMEOUT_SHORT_MS); + continue; + } + + *status = *buf; + *burst = le16_to_cpup((__le16 *)(buf + 1)); + + if ((*status & mask) == mask && + *burst > 0 && *burst <= TPM_CR50_MAX_BUFSIZE - 1) + return 0; + + msleep(TPM_CR50_TIMEOUT_SHORT_MS); + } while (time_before(jiffies, stop)); + + dev_err(&chip->dev, "Timeout reading burst and status\n"); + return -ETIMEDOUT; +} + +/** + * tpm_cr50_i2c_tis_recv() - TPM reception callback. + * @chip: A TPM chip. + * @buf: Reception buffer. + * @buf_len: Buffer length to read. + * + * Return: + * - >= 0: Number of read bytes. + * - -errno: A POSIX error code. + */ +static int tpm_cr50_i2c_tis_recv(struct tpm_chip *chip, u8 *buf, size_t buf_len) +{ + + u8 mask = TPM_STS_VALID | TPM_STS_DATA_AVAIL; + size_t burstcnt, cur, len, expected; + u8 addr = TPM_I2C_DATA_FIFO(0); + u32 status; + int rc; + + if (buf_len < TPM_HEADER_SIZE) + return -EINVAL; + + rc = tpm_cr50_i2c_get_burst_and_status(chip, mask, &burstcnt, &status); + if (rc < 0) + goto out_err; + + if (burstcnt > buf_len || burstcnt < TPM_HEADER_SIZE) { + dev_err(&chip->dev, + "Unexpected burstcnt: %zu (max=%zu, min=%d)\n", + burstcnt, buf_len, TPM_HEADER_SIZE); + rc = -EIO; + goto out_err; + } + + /* Read first chunk of burstcnt bytes */ + rc = tpm_cr50_i2c_read(chip, addr, buf, burstcnt); + if (rc < 0) { + dev_err(&chip->dev, "Read of first chunk failed\n"); + goto out_err; + } + + /* Determine expected data in the return buffer */ + expected = be32_to_cpup((__be32 *)(buf + 2)); + if (expected > buf_len) { + dev_err(&chip->dev, "Buffer too small to receive i2c data\n"); + goto out_err; + } + + /* Now read the rest of the data */ + cur = burstcnt; + while (cur < expected) { + /* Read updated burst count and check status */ + rc = tpm_cr50_i2c_get_burst_and_status(chip, mask, &burstcnt, &status); + if (rc < 0) + goto out_err; + + len = min_t(size_t, burstcnt, expected - cur); + rc = tpm_cr50_i2c_read(chip, addr, buf + cur, len); + if (rc < 0) { + dev_err(&chip->dev, "Read failed\n"); + goto out_err; + } + + cur += len; + } + + /* Ensure TPM is done reading data */ + rc = tpm_cr50_i2c_get_burst_and_status(chip, TPM_STS_VALID, &burstcnt, &status); + if (rc < 0) + goto out_err; + if (status & TPM_STS_DATA_AVAIL) { + dev_err(&chip->dev, "Data still available\n"); + rc = -EIO; + goto out_err; + } + + tpm_cr50_release_locality(chip, false); + return cur; + +out_err: + /* Abort current transaction if still pending */ + if (tpm_cr50_i2c_tis_status(chip) & TPM_STS_COMMAND_READY) + tpm_cr50_i2c_tis_set_ready(chip); + + tpm_cr50_release_locality(chip, false); + return rc; +} + +/** + * tpm_cr50_i2c_tis_send() - TPM transmission callback. + * @chip: A TPM chip. + * @buf: Buffer to send. + * @len: Buffer length. + * + * Return: + * - 0: Success. + * - -errno: A POSIX error code. + */ +static int tpm_cr50_i2c_tis_send(struct tpm_chip *chip, u8 *buf, size_t len) +{ + size_t burstcnt, limit, sent = 0; + u8 tpm_go[4] = { TPM_STS_GO }; + unsigned long stop; + u32 status; + int rc; + + rc = tpm_cr50_request_locality(chip); + if (rc < 0) + return rc; + + /* Wait until TPM is ready for a command */ + stop = jiffies + chip->timeout_b; + while (!(tpm_cr50_i2c_tis_status(chip) & TPM_STS_COMMAND_READY)) { + if (time_after(jiffies, stop)) { + rc = -ETIMEDOUT; + goto out_err; + } + + tpm_cr50_i2c_tis_set_ready(chip); + } + + while (len > 0) { + u8 mask = TPM_STS_VALID; + + /* Wait for data if this is not the first chunk */ + if (sent > 0) + mask |= TPM_STS_DATA_EXPECT; + + /* Read burst count and check status */ + rc = tpm_cr50_i2c_get_burst_and_status(chip, mask, &burstcnt, &status); + if (rc < 0) + goto out_err; + + /* + * Use burstcnt - 1 to account for the address byte + * that is inserted by tpm_cr50_i2c_write() + */ + limit = min_t(size_t, burstcnt - 1, len); + rc = tpm_cr50_i2c_write(chip, TPM_I2C_DATA_FIFO(0), &buf[sent], limit); + if (rc < 0) { + dev_err(&chip->dev, "Write failed\n"); + goto out_err; + } + + sent += limit; + len -= limit; + } + + /* Ensure TPM is not expecting more data */ + rc = tpm_cr50_i2c_get_burst_and_status(chip, TPM_STS_VALID, &burstcnt, &status); + if (rc < 0) + goto out_err; + if (status & TPM_STS_DATA_EXPECT) { + dev_err(&chip->dev, "Data still expected\n"); + rc = -EIO; + goto out_err; + } + + /* Start the TPM command */ + rc = tpm_cr50_i2c_write(chip, TPM_I2C_STS(0), tpm_go, + sizeof(tpm_go)); + if (rc < 0) { + dev_err(&chip->dev, "Start command failed\n"); + goto out_err; + } + return 0; + +out_err: + /* Abort current transaction if still pending */ + if (tpm_cr50_i2c_tis_status(chip) & TPM_STS_COMMAND_READY) + tpm_cr50_i2c_tis_set_ready(chip); + + tpm_cr50_release_locality(chip, false); + return rc; +} + +/** + * tpm_cr50_i2c_req_canceled() - Callback to notify a request cancel. + * @chip: A TPM chip. + * @status: Status given by the cancel callback. + * + * Return: + * True if command is ready, False otherwise. + */ +static bool tpm_cr50_i2c_req_canceled(struct tpm_chip *chip, u8 status) +{ + return status == TPM_STS_COMMAND_READY; +} + +static const struct tpm_class_ops cr50_i2c = { + .flags = TPM_OPS_AUTO_STARTUP, + .status = &tpm_cr50_i2c_tis_status, + .recv = &tpm_cr50_i2c_tis_recv, + .send = &tpm_cr50_i2c_tis_send, + .cancel = &tpm_cr50_i2c_tis_set_ready, + .req_complete_mask = TPM_STS_DATA_AVAIL | TPM_STS_VALID, + .req_complete_val = TPM_STS_DATA_AVAIL | TPM_STS_VALID, + .req_canceled = &tpm_cr50_i2c_req_canceled, +}; + +static const struct i2c_device_id cr50_i2c_table[] = { + {"cr50_i2c", 0}, + {} +}; +MODULE_DEVICE_TABLE(i2c, cr50_i2c_table); + +#ifdef CONFIG_ACPI +static const struct acpi_device_id cr50_i2c_acpi_id[] = { + { "GOOG0005", 0 }, + {} +}; +MODULE_DEVICE_TABLE(acpi, cr50_i2c_acpi_id); +#endif + +#ifdef CONFIG_OF +static const struct of_device_id of_cr50_i2c_match[] = { + { .compatible = "google,cr50", }, + {} +}; +MODULE_DEVICE_TABLE(of, of_cr50_i2c_match); +#endif + +/** + * tpm_cr50_i2c_probe() - Driver probe function. + * @client: I2C client information. + * @id: I2C device id. + * + * Return: + * - 0: Success. + * - -errno: A POSIX error code. + */ +static int tpm_cr50_i2c_probe(struct i2c_client *client, + const struct i2c_device_id *id) +{ + struct tpm_i2c_cr50_priv_data *priv; + struct device *dev = &client->dev; + struct tpm_chip *chip; + u32 vendor; + u8 buf[4]; + int rc; + + if (!i2c_check_functionality(client->adapter, I2C_FUNC_I2C)) + return -ENODEV; + + chip = tpmm_chip_alloc(dev, &cr50_i2c); + if (IS_ERR(chip)) + return PTR_ERR(chip); + + priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + + /* cr50 is a TPM 2.0 chip */ + chip->flags |= TPM_CHIP_FLAG_TPM2; + chip->flags |= TPM_CHIP_FLAG_FIRMWARE_POWER_MANAGED; + + /* Default timeouts */ + chip->timeout_a = msecs_to_jiffies(TIS_SHORT_TIMEOUT); + chip->timeout_b = msecs_to_jiffies(TIS_LONG_TIMEOUT); + chip->timeout_c = msecs_to_jiffies(TIS_SHORT_TIMEOUT); + chip->timeout_d = msecs_to_jiffies(TIS_SHORT_TIMEOUT); + + dev_set_drvdata(&chip->dev, priv); + init_completion(&priv->tpm_ready); + + if (client->irq > 0) { + rc = devm_request_irq(dev, client->irq, tpm_cr50_i2c_int_handler, + IRQF_TRIGGER_FALLING | IRQF_ONESHOT, + dev->driver->name, chip); + if (rc < 0) { + dev_err(dev, "Failed to probe IRQ %d\n", client->irq); + return rc; + } + + disable_irq(client->irq); + priv->irq = client->irq; + } else { + dev_warn(dev, "No IRQ, will use %ums delay for TPM ready\n", + TPM_CR50_TIMEOUT_NOIRQ_MS); + } + + rc = tpm_cr50_request_locality(chip); + if (rc < 0) { + dev_err(dev, "Could not request locality\n"); + return rc; + } + + /* Read four bytes from DID_VID register */ + rc = tpm_cr50_i2c_read(chip, TPM_I2C_DID_VID(0), buf, sizeof(buf)); + if (rc < 0) { + dev_err(dev, "Could not read vendor id\n"); + tpm_cr50_release_locality(chip, true); + return rc; + } + + vendor = le32_to_cpup((__le32 *)buf); + if (vendor != TPM_CR50_I2C_DID_VID) { + dev_err(dev, "Vendor ID did not match! ID was %08x\n", vendor); + tpm_cr50_release_locality(chip, true); + return -ENODEV; + } + + dev_info(dev, "cr50 TPM 2.0 (i2c 0x%02x irq %d id 0x%x)\n", + client->addr, client->irq, vendor >> 16); + + return tpm_chip_register(chip); +} + +/** + * tpm_cr50_i2c_remove() - Driver remove function. + * @client: I2C client information. + * + * Return: + * - 0: Success. + * - -errno: A POSIX error code. + */ +static int tpm_cr50_i2c_remove(struct i2c_client *client) +{ + struct tpm_chip *chip = i2c_get_clientdata(client); + struct device *dev = &client->dev; + + if (!chip) { + dev_err(dev, "Could not get client data at remove\n"); + return -ENODEV; + } + + tpm_chip_unregister(chip); + tpm_cr50_release_locality(chip, true); + + return 0; +} + +static SIMPLE_DEV_PM_OPS(cr50_i2c_pm, tpm_pm_suspend, tpm_pm_resume); + +static struct i2c_driver cr50_i2c_driver = { + .id_table = cr50_i2c_table, + .probe = tpm_cr50_i2c_probe, + .remove = tpm_cr50_i2c_remove, + .driver = { + .name = "cr50_i2c", + .pm = &cr50_i2c_pm, + .acpi_match_table = ACPI_PTR(cr50_i2c_acpi_id), + .of_match_table = of_match_ptr(of_cr50_i2c_match), + }, +}; + +module_i2c_driver(cr50_i2c_driver); + +MODULE_DESCRIPTION("cr50 TPM I2C Driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/Kconfig b/drivers/clk/Kconfig index 85856cff506c01660841d71840225f37fc21df91..a588d56502d40a07ae65654752161fbf58dc33ab 100644 --- a/drivers/clk/Kconfig +++ b/drivers/clk/Kconfig @@ -247,7 +247,8 @@ config CLK_TWL6040 config COMMON_CLK_AXI_CLKGEN tristate "AXI clkgen driver" - depends on ARCH_ZYNQ || MICROBLAZE || COMPILE_TEST + depends on HAS_IOMEM || COMPILE_TEST + depends on OF help Support for the Analog Devices axi-clkgen pcore clock generator for Xilinx FPGAs. It is commonly used in Analog Devices' reference designs. @@ -368,6 +369,13 @@ config COMMON_CLK_FIXED_MMIO help Support for Memory Mapped IO Fixed clocks +config COMMON_CLK_K210 + bool "Clock driver for the Canaan Kendryte K210 SoC" + depends on OF && RISCV && SOC_CANAAN + default SOC_CANAAN + help + Support for the Canaan Kendryte K210 RISC-V SoC clocks. + source "drivers/clk/actions/Kconfig" source "drivers/clk/analogbits/Kconfig" source "drivers/clk/baikal-t1/Kconfig" @@ -379,6 +387,7 @@ source "drivers/clk/ingenic/Kconfig" source "drivers/clk/keystone/Kconfig" source "drivers/clk/mediatek/Kconfig" source "drivers/clk/meson/Kconfig" +source "drivers/clk/mstar/Kconfig" source "drivers/clk/mvebu/Kconfig" source "drivers/clk/qcom/Kconfig" source "drivers/clk/renesas/Kconfig" @@ -392,6 +401,7 @@ source "drivers/clk/tegra/Kconfig" source "drivers/clk/ti/Kconfig" source "drivers/clk/uniphier/Kconfig" source "drivers/clk/x86/Kconfig" +source "drivers/clk/xilinx/Kconfig" source "drivers/clk/zynqmp/Kconfig" endif diff --git a/drivers/clk/Makefile b/drivers/clk/Makefile index dbdc590e7de3aca81ef39256b690b11e26dfc760..b22ae4f81e0b33c375e1c28f9caa8247b662512a 100644 --- a/drivers/clk/Makefile +++ b/drivers/clk/Makefile @@ -27,7 +27,6 @@ obj-$(CONFIG_COMMON_CLK_CDCE706) += clk-cdce706.o obj-$(CONFIG_COMMON_CLK_CDCE925) += clk-cdce925.o obj-$(CONFIG_ARCH_CLPS711X) += clk-clps711x.o obj-$(CONFIG_COMMON_CLK_CS2000_CP) += clk-cs2000-cp.o -obj-$(CONFIG_ARCH_EFM32) += clk-efm32gg.o obj-$(CONFIG_ARCH_SPARX5) += clk-sparx5.o obj-$(CONFIG_COMMON_CLK_FIXED_MMIO) += clk-fixed-mmio.o obj-$(CONFIG_COMMON_CLK_FSL_FLEXSPI) += clk-fsl-flexspi.o @@ -37,6 +36,7 @@ obj-$(CONFIG_COMMON_CLK_ASPEED) += clk-aspeed.o obj-$(CONFIG_MACH_ASPEED_G6) += clk-ast2600.o obj-$(CONFIG_ARCH_HIGHBANK) += clk-highbank.o obj-$(CONFIG_CLK_HSDK) += clk-hsdk-pll.o +obj-$(CONFIG_COMMON_CLK_K210) += clk-k210.o obj-$(CONFIG_COMMON_CLK_LOCHNAGAR) += clk-lochnagar.o obj-$(CONFIG_COMMON_CLK_MAX77686) += clk-max77686.o obj-$(CONFIG_COMMON_CLK_MAX9485) += clk-max9485.o @@ -63,9 +63,7 @@ obj-$(CONFIG_COMMON_CLK_SI570) += clk-si570.o obj-$(CONFIG_COMMON_CLK_STM32F) += clk-stm32f4.o obj-$(CONFIG_COMMON_CLK_STM32H7) += clk-stm32h7.o obj-$(CONFIG_COMMON_CLK_STM32MP157) += clk-stm32mp1.o -obj-$(CONFIG_ARCH_TANGO) += clk-tango4.o obj-$(CONFIG_CLK_TWL6040) += clk-twl6040.o -obj-$(CONFIG_ARCH_U300) += clk-u300.o obj-$(CONFIG_ARCH_VT8500) += clk-vt8500.o obj-$(CONFIG_COMMON_CLK_VC5) += clk-versaclock5.o obj-$(CONFIG_COMMON_CLK_WM831X) += clk-wm831x.o @@ -95,6 +93,7 @@ obj-$(CONFIG_MACH_PIC32) += microchip/ ifeq ($(CONFIG_COMMON_CLK), y) obj-$(CONFIG_ARCH_MMP) += mmp/ endif +obj-y += mstar/ obj-y += mvebu/ obj-$(CONFIG_ARCH_MXS) += mxs/ obj-$(CONFIG_COMMON_CLK_NXP) += nxp/ @@ -105,7 +104,6 @@ obj-y += renesas/ obj-$(CONFIG_ARCH_ROCKCHIP) += rockchip/ obj-$(CONFIG_COMMON_CLK_SAMSUNG) += samsung/ obj-$(CONFIG_CLK_SIFIVE) += sifive/ -obj-$(CONFIG_ARCH_SIRF) += sirf/ obj-$(CONFIG_ARCH_SOCFPGA) += socfpga/ obj-$(CONFIG_ARCH_AGILEX) += socfpga/ obj-$(CONFIG_ARCH_STRATIX10) += socfpga/ @@ -122,6 +120,6 @@ obj-y += versatile/ ifeq ($(CONFIG_COMMON_CLK), y) obj-$(CONFIG_X86) += x86/ endif -obj-$(CONFIG_ARCH_ZX) += zte/ +obj-y += xilinx/ obj-$(CONFIG_ARCH_ZYNQ) += zynq/ obj-$(CONFIG_COMMON_CLK_ZYNQMP) += zynqmp/ diff --git a/drivers/clk/at91/at91rm9200.c b/drivers/clk/at91/at91rm9200.c index 0fad1009f3152aec25a65df615242df1390e23cb..428a6f4b9ebc56e054601e6f1a079fa01272aab2 100644 --- a/drivers/clk/at91/at91rm9200.c +++ b/drivers/clk/at91/at91rm9200.c @@ -215,5 +215,4 @@ static void __init at91rm9200_pmc_setup(struct device_node *np) * deferring properly. Once this is fixed, this can be switched to a platform * driver. */ -CLK_OF_DECLARE_DRIVER(at91rm9200_pmc, "atmel,at91rm9200-pmc", - at91rm9200_pmc_setup); +CLK_OF_DECLARE(at91rm9200_pmc, "atmel,at91rm9200-pmc", at91rm9200_pmc_setup); diff --git a/drivers/clk/at91/at91sam9260.c b/drivers/clk/at91/at91sam9260.c index ceb5495f723af8377da1954f79f85a679ee787a7..b29843bea278b3a04dc7e1a42e8abf4ad9385533 100644 --- a/drivers/clk/at91/at91sam9260.c +++ b/drivers/clk/at91/at91sam9260.c @@ -491,26 +491,26 @@ static void __init at91sam9260_pmc_setup(struct device_node *np) { at91sam926x_pmc_setup(np, &at91sam9260_data); } -CLK_OF_DECLARE_DRIVER(at91sam9260_pmc, "atmel,at91sam9260-pmc", - at91sam9260_pmc_setup); + +CLK_OF_DECLARE(at91sam9260_pmc, "atmel,at91sam9260-pmc", at91sam9260_pmc_setup); static void __init at91sam9261_pmc_setup(struct device_node *np) { at91sam926x_pmc_setup(np, &at91sam9261_data); } -CLK_OF_DECLARE_DRIVER(at91sam9261_pmc, "atmel,at91sam9261-pmc", - at91sam9261_pmc_setup); + +CLK_OF_DECLARE(at91sam9261_pmc, "atmel,at91sam9261-pmc", at91sam9261_pmc_setup); static void __init at91sam9263_pmc_setup(struct device_node *np) { at91sam926x_pmc_setup(np, &at91sam9263_data); } -CLK_OF_DECLARE_DRIVER(at91sam9263_pmc, "atmel,at91sam9263-pmc", - at91sam9263_pmc_setup); + +CLK_OF_DECLARE(at91sam9263_pmc, "atmel,at91sam9263-pmc", at91sam9263_pmc_setup); static void __init at91sam9g20_pmc_setup(struct device_node *np) { at91sam926x_pmc_setup(np, &at91sam9g20_data); } -CLK_OF_DECLARE_DRIVER(at91sam9g20_pmc, "atmel,at91sam9g20-pmc", - at91sam9g20_pmc_setup); + +CLK_OF_DECLARE(at91sam9g20_pmc, "atmel,at91sam9g20-pmc", at91sam9g20_pmc_setup); diff --git a/drivers/clk/at91/at91sam9g45.c b/drivers/clk/at91/at91sam9g45.c index 0214333dedd33f3f1cbf8776f55f59bfc3f0c862..15da0dfe3ef25fe2f59e1ca9a0c60df4dc5ffdfa 100644 --- a/drivers/clk/at91/at91sam9g45.c +++ b/drivers/clk/at91/at91sam9g45.c @@ -228,5 +228,4 @@ static void __init at91sam9g45_pmc_setup(struct device_node *np) * The TCB is used as the clocksource so its clock is needed early. This means * this can't be a platform driver. */ -CLK_OF_DECLARE_DRIVER(at91sam9g45_pmc, "atmel,at91sam9g45-pmc", - at91sam9g45_pmc_setup); +CLK_OF_DECLARE(at91sam9g45_pmc, "atmel,at91sam9g45-pmc", at91sam9g45_pmc_setup); diff --git a/drivers/clk/at91/at91sam9n12.c b/drivers/clk/at91/at91sam9n12.c index f9db5316a7f1bad47c333956f36f57ed23cc20e5..7fe435f4b46b71bfca8949377cf32111c6814930 100644 --- a/drivers/clk/at91/at91sam9n12.c +++ b/drivers/clk/at91/at91sam9n12.c @@ -255,5 +255,4 @@ static void __init at91sam9n12_pmc_setup(struct device_node *np) * The TCB is used as the clocksource so its clock is needed early. This means * this can't be a platform driver. */ -CLK_OF_DECLARE_DRIVER(at91sam9n12_pmc, "atmel,at91sam9n12-pmc", - at91sam9n12_pmc_setup); +CLK_OF_DECLARE(at91sam9n12_pmc, "atmel,at91sam9n12-pmc", at91sam9n12_pmc_setup); diff --git a/drivers/clk/at91/at91sam9rl.c b/drivers/clk/at91/at91sam9rl.c index 66736e03cfef49539fc79059b43a1dbec7d797a7..ecbabf5162bd6a715b65fb258a0ae6f0da52d147 100644 --- a/drivers/clk/at91/at91sam9rl.c +++ b/drivers/clk/at91/at91sam9rl.c @@ -186,4 +186,5 @@ static void __init at91sam9rl_pmc_setup(struct device_node *np) err_free: kfree(at91sam9rl_pmc); } -CLK_OF_DECLARE_DRIVER(at91sam9rl_pmc, "atmel,at91sam9rl-pmc", at91sam9rl_pmc_setup); + +CLK_OF_DECLARE(at91sam9rl_pmc, "atmel,at91sam9rl-pmc", at91sam9rl_pmc_setup); diff --git a/drivers/clk/at91/at91sam9x5.c b/drivers/clk/at91/at91sam9x5.c index 79b9d3667228dc51a5fc4ec49a5bb1f16487212e..5cce48c64ea2d334403305b0855c7a82b158cb3a 100644 --- a/drivers/clk/at91/at91sam9x5.c +++ b/drivers/clk/at91/at91sam9x5.c @@ -302,33 +302,33 @@ static void __init at91sam9g15_pmc_setup(struct device_node *np) { at91sam9x5_pmc_setup(np, at91sam9g15_periphck, true); } -CLK_OF_DECLARE_DRIVER(at91sam9g15_pmc, "atmel,at91sam9g15-pmc", - at91sam9g15_pmc_setup); + +CLK_OF_DECLARE(at91sam9g15_pmc, "atmel,at91sam9g15-pmc", at91sam9g15_pmc_setup); static void __init at91sam9g25_pmc_setup(struct device_node *np) { at91sam9x5_pmc_setup(np, at91sam9g25_periphck, false); } -CLK_OF_DECLARE_DRIVER(at91sam9g25_pmc, "atmel,at91sam9g25-pmc", - at91sam9g25_pmc_setup); + +CLK_OF_DECLARE(at91sam9g25_pmc, "atmel,at91sam9g25-pmc", at91sam9g25_pmc_setup); static void __init at91sam9g35_pmc_setup(struct device_node *np) { at91sam9x5_pmc_setup(np, at91sam9g35_periphck, true); } -CLK_OF_DECLARE_DRIVER(at91sam9g35_pmc, "atmel,at91sam9g35-pmc", - at91sam9g35_pmc_setup); + +CLK_OF_DECLARE(at91sam9g35_pmc, "atmel,at91sam9g35-pmc", at91sam9g35_pmc_setup); static void __init at91sam9x25_pmc_setup(struct device_node *np) { at91sam9x5_pmc_setup(np, at91sam9x25_periphck, false); } -CLK_OF_DECLARE_DRIVER(at91sam9x25_pmc, "atmel,at91sam9x25-pmc", - at91sam9x25_pmc_setup); + +CLK_OF_DECLARE(at91sam9x25_pmc, "atmel,at91sam9x25-pmc", at91sam9x25_pmc_setup); static void __init at91sam9x35_pmc_setup(struct device_node *np) { at91sam9x5_pmc_setup(np, at91sam9x35_periphck, true); } -CLK_OF_DECLARE_DRIVER(at91sam9x35_pmc, "atmel,at91sam9x35-pmc", - at91sam9x35_pmc_setup); + +CLK_OF_DECLARE(at91sam9x35_pmc, "atmel,at91sam9x35-pmc", at91sam9x35_pmc_setup); diff --git a/drivers/clk/at91/sama5d2.c b/drivers/clk/at91/sama5d2.c index 9a5cbc7cd55a4f8ced9aa5d39a6104d6cc8e3369..3d1f78176c3e594db643bfbae4ef2331b6c77f4f 100644 --- a/drivers/clk/at91/sama5d2.c +++ b/drivers/clk/at91/sama5d2.c @@ -372,4 +372,5 @@ static void __init sama5d2_pmc_setup(struct device_node *np) err_free: kfree(sama5d2_pmc); } -CLK_OF_DECLARE_DRIVER(sama5d2_pmc, "atmel,sama5d2-pmc", sama5d2_pmc_setup); + +CLK_OF_DECLARE(sama5d2_pmc, "atmel,sama5d2-pmc", sama5d2_pmc_setup); diff --git a/drivers/clk/at91/sama5d3.c b/drivers/clk/at91/sama5d3.c index 87009ee8effc991c06f9a348c6cbb969c2acef06..d376257807d26281e1877ec8abed113a9bd23c36 100644 --- a/drivers/clk/at91/sama5d3.c +++ b/drivers/clk/at91/sama5d3.c @@ -255,4 +255,4 @@ static void __init sama5d3_pmc_setup(struct device_node *np) * The TCB is used as the clocksource so its clock is needed early. This means * this can't be a platform driver. */ -CLK_OF_DECLARE_DRIVER(sama5d3_pmc, "atmel,sama5d3-pmc", sama5d3_pmc_setup); +CLK_OF_DECLARE(sama5d3_pmc, "atmel,sama5d3-pmc", sama5d3_pmc_setup); diff --git a/drivers/clk/at91/sama5d4.c b/drivers/clk/at91/sama5d4.c index 57fff790188b6aa8a5e25aee173692ecd7885660..5cbaac68da447edd4a4a1b8f4e9507d446fc1d69 100644 --- a/drivers/clk/at91/sama5d4.c +++ b/drivers/clk/at91/sama5d4.c @@ -286,4 +286,5 @@ static void __init sama5d4_pmc_setup(struct device_node *np) err_free: kfree(sama5d4_pmc); } -CLK_OF_DECLARE_DRIVER(sama5d4_pmc, "atmel,sama5d4-pmc", sama5d4_pmc_setup); + +CLK_OF_DECLARE(sama5d4_pmc, "atmel,sama5d4-pmc", sama5d4_pmc_setup); diff --git a/drivers/clk/bcm/clk-iproc-pll.c b/drivers/clk/bcm/clk-iproc-pll.c index 274441e2ddb2891d916f2747ff5e8a55b3fd27e9..33da30f99c79b48e5f04ace387bcfc1b0625ec6b 100644 --- a/drivers/clk/bcm/clk-iproc-pll.c +++ b/drivers/clk/bcm/clk-iproc-pll.c @@ -704,7 +704,7 @@ static const struct clk_ops iproc_clk_ops = { .set_rate = iproc_clk_set_rate, }; -/** +/* * Some PLLs require the PLL SW override bit to be set before changes can be * applied to the PLL */ diff --git a/drivers/clk/clk-ast2600.c b/drivers/clk/clk-ast2600.c index 177368cac6dd6afa25993bc206dd42ac65fbcae0..a55b37fc2c8bdd564652dbccccbb4f6536074bb9 100644 --- a/drivers/clk/clk-ast2600.c +++ b/drivers/clk/clk-ast2600.c @@ -17,7 +17,8 @@ #define ASPEED_G6_NUM_CLKS 71 -#define ASPEED_G6_SILICON_REV 0x004 +#define ASPEED_G6_SILICON_REV 0x014 +#define CHIP_REVISION_ID GENMASK(23, 16) #define ASPEED_G6_RESET_CTRL 0x040 #define ASPEED_G6_RESET_CTRL2 0x050 @@ -190,18 +191,34 @@ static struct clk_hw *ast2600_calc_pll(const char *name, u32 val) static struct clk_hw *ast2600_calc_apll(const char *name, u32 val) { unsigned int mult, div; + u32 chip_id = readl(scu_g6_base + ASPEED_G6_SILICON_REV); - if (val & BIT(20)) { - /* Pass through mode */ - mult = div = 1; + if (((chip_id & CHIP_REVISION_ID) >> 16) >= 2) { + if (val & BIT(24)) { + /* Pass through mode */ + mult = div = 1; + } else { + /* F = 25Mhz * [(m + 1) / (n + 1)] / (p + 1) */ + u32 m = val & 0x1fff; + u32 n = (val >> 13) & 0x3f; + u32 p = (val >> 19) & 0xf; + + mult = (m + 1); + div = (n + 1) * (p + 1); + } } else { - /* F = 25Mhz * (2-od) * [(m + 2) / (n + 1)] */ - u32 m = (val >> 5) & 0x3f; - u32 od = (val >> 4) & 0x1; - u32 n = val & 0xf; + if (val & BIT(20)) { + /* Pass through mode */ + mult = div = 1; + } else { + /* F = 25Mhz * (2-od) * [(m + 2) / (n + 1)] */ + u32 m = (val >> 5) & 0x3f; + u32 od = (val >> 4) & 0x1; + u32 n = val & 0xf; - mult = (2 - od) * (m + 2); - div = n + 1; + mult = (2 - od) * (m + 2); + div = n + 1; + } } return clk_hw_register_fixed_factor(NULL, name, "clkin", 0, mult, div); diff --git a/drivers/clk/clk-axi-clkgen.c b/drivers/clk/clk-axi-clkgen.c index ad86e031ba3ee248dbdd8490effd8b823626d6bd..ac6ff736ac8f6cd598d33554d4c1c3943bdbd2d5 100644 --- a/drivers/clk/clk-axi-clkgen.c +++ b/drivers/clk/clk-axi-clkgen.c @@ -108,6 +108,13 @@ static uint32_t axi_clkgen_lookup_lock(unsigned int m) return 0x1f1f00fa; } +static const struct axi_clkgen_limits axi_clkgen_zynqmp_default_limits = { + .fpfd_min = 10000, + .fpfd_max = 450000, + .fvco_min = 800000, + .fvco_max = 1600000, +}; + static const struct axi_clkgen_limits axi_clkgen_zynq_default_limits = { .fpfd_min = 10000, .fpfd_max = 300000, @@ -503,7 +510,6 @@ static int axi_clkgen_probe(struct platform_device *pdev) struct clk_init_data init; const char *parent_names[2]; const char *clk_name; - struct resource *mem; unsigned int i; int ret; @@ -515,8 +521,7 @@ static int axi_clkgen_probe(struct platform_device *pdev) if (!axi_clkgen) return -ENOMEM; - mem = platform_get_resource(pdev, IORESOURCE_MEM, 0); - axi_clkgen->base = devm_ioremap_resource(&pdev->dev, mem); + axi_clkgen->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(axi_clkgen->base)) return PTR_ERR(axi_clkgen->base); @@ -560,6 +565,10 @@ static int axi_clkgen_remove(struct platform_device *pdev) } static const struct of_device_id axi_clkgen_ids[] = { + { + .compatible = "adi,zynqmp-axi-clkgen-2.00.a", + .data = &axi_clkgen_zynqmp_default_limits, + }, { .compatible = "adi,axi-clkgen-2.00.a", .data = &axi_clkgen_zynq_default_limits, diff --git a/drivers/clk/clk-bd718x7.c b/drivers/clk/clk-bd718x7.c index b52e8d6f660c4a9e71c4904331b285aa0077af7a..17d90e09f1c0f2480a2046d797c82831a0cc0c88 100644 --- a/drivers/clk/clk-bd718x7.c +++ b/drivers/clk/clk-bd718x7.c @@ -31,12 +31,12 @@ struct bd718xx_clk { u8 reg; u8 mask; struct platform_device *pdev; - struct rohm_regmap_dev *mfd; + struct regmap *regmap; }; static int bd71837_clk_set(struct bd718xx_clk *c, unsigned int status) { - return regmap_update_bits(c->mfd->regmap, c->reg, c->mask, status); + return regmap_update_bits(c->regmap, c->reg, c->mask, status); } static void bd71837_clk_disable(struct clk_hw *hw) @@ -62,7 +62,7 @@ static int bd71837_clk_is_enabled(struct clk_hw *hw) int rval; struct bd718xx_clk *c = container_of(hw, struct bd718xx_clk, hw); - rval = regmap_read(c->mfd->regmap, c->reg, &enabled); + rval = regmap_read(c->regmap, c->reg, &enabled); if (rval) return rval; @@ -82,7 +82,6 @@ static int bd71837_clk_probe(struct platform_device *pdev) int rval = -ENOMEM; const char *parent_clk; struct device *parent = pdev->dev.parent; - struct rohm_regmap_dev *mfd = dev_get_drvdata(parent); struct clk_init_data init = { .name = "bd718xx-32k-out", .ops = &bd71837_clk_ops, @@ -93,6 +92,10 @@ static int bd71837_clk_probe(struct platform_device *pdev) if (!c) return -ENOMEM; + c->regmap = dev_get_regmap(pdev->dev.parent, NULL); + if (!c->regmap) + return -ENODEV; + init.num_parents = 1; parent_clk = of_clk_get_parent_name(parent->of_node, 0); @@ -119,7 +122,6 @@ static int bd71837_clk_probe(struct platform_device *pdev) dev_err(&pdev->dev, "Unknown clk chip\n"); return -EINVAL; } - c->mfd = mfd; c->pdev = pdev; c->hw.init = &init; diff --git a/drivers/clk/clk-divider.c b/drivers/clk/clk-divider.c index c499799693cccc8d45d6a821a27a7f2720f3e556..344997203f0e700df7aa7d35073ef27168c0f941 100644 --- a/drivers/clk/clk-divider.c +++ b/drivers/clk/clk-divider.c @@ -494,8 +494,13 @@ struct clk_hw *__clk_hw_register_divider(struct device *dev, else init.ops = &clk_divider_ops; init.flags = flags; - init.parent_names = (parent_name ? &parent_name: NULL); - init.num_parents = (parent_name ? 1 : 0); + init.parent_names = parent_name ? &parent_name : NULL; + init.parent_hws = parent_hw ? &parent_hw : NULL; + init.parent_data = parent_data; + if (parent_name || parent_hw || parent_data) + init.num_parents = 1; + else + init.num_parents = 0; /* struct clk_divider assignments */ div->reg = reg; diff --git a/drivers/clk/clk-efm32gg.c b/drivers/clk/clk-efm32gg.c deleted file mode 100644 index 85beaacb40885fe92776f9ce43aa0ed95d58a6c9..0000000000000000000000000000000000000000 --- a/drivers/clk/clk-efm32gg.c +++ /dev/null @@ -1,84 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Copyright (C) 2013 Pengutronix - * Uwe Kleine-Koenig - */ -#include -#include -#include -#include -#include - -#include - -#define CMU_HFPERCLKEN0 0x44 -#define CMU_MAX_CLKS 37 - -static struct clk_hw_onecell_data *clk_data; - -static void __init efm32gg_cmu_init(struct device_node *np) -{ - int i; - void __iomem *base; - struct clk_hw **hws; - - clk_data = kzalloc(struct_size(clk_data, hws, CMU_MAX_CLKS), - GFP_KERNEL); - - if (!clk_data) - return; - - hws = clk_data->hws; - - for (i = 0; i < CMU_MAX_CLKS; ++i) - hws[i] = ERR_PTR(-ENOENT); - - base = of_iomap(np, 0); - if (!base) { - pr_warn("Failed to map address range for efm32gg,cmu node\n"); - return; - } - - hws[clk_HFXO] = clk_hw_register_fixed_rate(NULL, "HFXO", NULL, 0, - 48000000); - - hws[clk_HFPERCLKUSART0] = clk_hw_register_gate(NULL, "HFPERCLK.USART0", - "HFXO", 0, base + CMU_HFPERCLKEN0, 0, 0, NULL); - hws[clk_HFPERCLKUSART1] = clk_hw_register_gate(NULL, "HFPERCLK.USART1", - "HFXO", 0, base + CMU_HFPERCLKEN0, 1, 0, NULL); - hws[clk_HFPERCLKUSART2] = clk_hw_register_gate(NULL, "HFPERCLK.USART2", - "HFXO", 0, base + CMU_HFPERCLKEN0, 2, 0, NULL); - hws[clk_HFPERCLKUART0] = clk_hw_register_gate(NULL, "HFPERCLK.UART0", - "HFXO", 0, base + CMU_HFPERCLKEN0, 3, 0, NULL); - hws[clk_HFPERCLKUART1] = clk_hw_register_gate(NULL, "HFPERCLK.UART1", - "HFXO", 0, base + CMU_HFPERCLKEN0, 4, 0, NULL); - hws[clk_HFPERCLKTIMER0] = clk_hw_register_gate(NULL, "HFPERCLK.TIMER0", - "HFXO", 0, base + CMU_HFPERCLKEN0, 5, 0, NULL); - hws[clk_HFPERCLKTIMER1] = clk_hw_register_gate(NULL, "HFPERCLK.TIMER1", - "HFXO", 0, base + CMU_HFPERCLKEN0, 6, 0, NULL); - hws[clk_HFPERCLKTIMER2] = clk_hw_register_gate(NULL, "HFPERCLK.TIMER2", - "HFXO", 0, base + CMU_HFPERCLKEN0, 7, 0, NULL); - hws[clk_HFPERCLKTIMER3] = clk_hw_register_gate(NULL, "HFPERCLK.TIMER3", - "HFXO", 0, base + CMU_HFPERCLKEN0, 8, 0, NULL); - hws[clk_HFPERCLKACMP0] = clk_hw_register_gate(NULL, "HFPERCLK.ACMP0", - "HFXO", 0, base + CMU_HFPERCLKEN0, 9, 0, NULL); - hws[clk_HFPERCLKACMP1] = clk_hw_register_gate(NULL, "HFPERCLK.ACMP1", - "HFXO", 0, base + CMU_HFPERCLKEN0, 10, 0, NULL); - hws[clk_HFPERCLKI2C0] = clk_hw_register_gate(NULL, "HFPERCLK.I2C0", - "HFXO", 0, base + CMU_HFPERCLKEN0, 11, 0, NULL); - hws[clk_HFPERCLKI2C1] = clk_hw_register_gate(NULL, "HFPERCLK.I2C1", - "HFXO", 0, base + CMU_HFPERCLKEN0, 12, 0, NULL); - hws[clk_HFPERCLKGPIO] = clk_hw_register_gate(NULL, "HFPERCLK.GPIO", - "HFXO", 0, base + CMU_HFPERCLKEN0, 13, 0, NULL); - hws[clk_HFPERCLKVCMP] = clk_hw_register_gate(NULL, "HFPERCLK.VCMP", - "HFXO", 0, base + CMU_HFPERCLKEN0, 14, 0, NULL); - hws[clk_HFPERCLKPRS] = clk_hw_register_gate(NULL, "HFPERCLK.PRS", - "HFXO", 0, base + CMU_HFPERCLKEN0, 15, 0, NULL); - hws[clk_HFPERCLKADC0] = clk_hw_register_gate(NULL, "HFPERCLK.ADC0", - "HFXO", 0, base + CMU_HFPERCLKEN0, 16, 0, NULL); - hws[clk_HFPERCLKDAC0] = clk_hw_register_gate(NULL, "HFPERCLK.DAC0", - "HFXO", 0, base + CMU_HFPERCLKEN0, 17, 0, NULL); - - of_clk_add_hw_provider(np, of_clk_hw_onecell_get, clk_data); -} -CLK_OF_DECLARE(efm32ggcmu, "efm32gg,cmu", efm32gg_cmu_init); diff --git a/drivers/clk/clk-fixed-factor.c b/drivers/clk/clk-fixed-factor.c index 910e6e74ae90a9b86659cfba0ff8609097ac4c78..4f7bf3929d6d9cfac23b40ff04b1315cd7307a14 100644 --- a/drivers/clk/clk-fixed-factor.c +++ b/drivers/clk/clk-fixed-factor.c @@ -64,10 +64,16 @@ const struct clk_ops clk_fixed_factor_ops = { }; EXPORT_SYMBOL_GPL(clk_fixed_factor_ops); +static void devm_clk_hw_register_fixed_factor_release(struct device *dev, void *res) +{ + clk_hw_unregister_fixed_factor(&((struct clk_fixed_factor *)res)->hw); +} + static struct clk_hw * __clk_hw_register_fixed_factor(struct device *dev, struct device_node *np, const char *name, const char *parent_name, int index, - unsigned long flags, unsigned int mult, unsigned int div) + unsigned long flags, unsigned int mult, unsigned int div, + bool devm) { struct clk_fixed_factor *fix; struct clk_init_data init = { }; @@ -75,7 +81,15 @@ __clk_hw_register_fixed_factor(struct device *dev, struct device_node *np, struct clk_hw *hw; int ret; - fix = kmalloc(sizeof(*fix), GFP_KERNEL); + /* You can't use devm without a dev */ + if (devm && !dev) + return ERR_PTR(-EINVAL); + + if (devm) + fix = devres_alloc(devm_clk_hw_register_fixed_factor_release, + sizeof(*fix), GFP_KERNEL); + else + fix = kmalloc(sizeof(*fix), GFP_KERNEL); if (!fix) return ERR_PTR(-ENOMEM); @@ -99,9 +113,13 @@ __clk_hw_register_fixed_factor(struct device *dev, struct device_node *np, else ret = of_clk_hw_register(np, hw); if (ret) { - kfree(fix); + if (devm) + devres_free(fix); + else + kfree(fix); hw = ERR_PTR(ret); - } + } else if (devm) + devres_add(dev, fix); return hw; } @@ -111,7 +129,7 @@ struct clk_hw *clk_hw_register_fixed_factor(struct device *dev, unsigned int mult, unsigned int div) { return __clk_hw_register_fixed_factor(dev, NULL, name, parent_name, -1, - flags, mult, div); + flags, mult, div, false); } EXPORT_SYMBOL_GPL(clk_hw_register_fixed_factor); @@ -153,6 +171,15 @@ void clk_hw_unregister_fixed_factor(struct clk_hw *hw) } EXPORT_SYMBOL_GPL(clk_hw_unregister_fixed_factor); +struct clk_hw *devm_clk_hw_register_fixed_factor(struct device *dev, + const char *name, const char *parent_name, unsigned long flags, + unsigned int mult, unsigned int div) +{ + return __clk_hw_register_fixed_factor(dev, NULL, name, parent_name, -1, + flags, mult, div, true); +} +EXPORT_SYMBOL_GPL(devm_clk_hw_register_fixed_factor); + #ifdef CONFIG_OF static const struct of_device_id set_rate_parent_matches[] = { { .compatible = "allwinner,sun4i-a10-pll3-2x-clk" }, @@ -185,7 +212,7 @@ static struct clk_hw *_of_fixed_factor_clk_setup(struct device_node *node) flags |= CLK_SET_RATE_PARENT; hw = __clk_hw_register_fixed_factor(NULL, node, clk_name, NULL, 0, - flags, mult, div); + flags, mult, div, false); if (IS_ERR(hw)) { /* * Clear OF_POPULATED flag so that clock registration can be diff --git a/drivers/clk/clk-fixed-mmio.c b/drivers/clk/clk-fixed-mmio.c index 51f26619b6a23383bbf745a250d74d479146ea40..5225d17d6b3f39210dc19de7ffb861f4544e124a 100644 --- a/drivers/clk/clk-fixed-mmio.c +++ b/drivers/clk/clk-fixed-mmio.c @@ -55,7 +55,7 @@ static void __init of_fixed_mmio_clk_setup(struct device_node *node) } CLK_OF_DECLARE(fixed_mmio_clk, "fixed-mmio-clock", of_fixed_mmio_clk_setup); -/** +/* * This is not executed when of_fixed_mmio_clk_setup succeeded. */ static int of_fixed_mmio_clk_probe(struct platform_device *pdev) diff --git a/drivers/clk/clk-k210.c b/drivers/clk/clk-k210.c new file mode 100644 index 0000000000000000000000000000000000000000..6c84abf5b2e36dc5b849b71825bf14286008a027 --- /dev/null +++ b/drivers/clk/clk-k210.c @@ -0,0 +1,1007 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2019-20 Sean Anderson + * Copyright (c) 2019 Western Digital Corporation or its affiliates. + */ +#define pr_fmt(fmt) "k210-clk: " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +struct k210_sysclk; + +struct k210_clk { + int id; + struct k210_sysclk *ksc; + struct clk_hw hw; +}; + +struct k210_clk_cfg { + const char *name; + u8 gate_reg; + u8 gate_bit; + u8 div_reg; + u8 div_shift; + u8 div_width; + u8 div_type; + u8 mux_reg; + u8 mux_bit; +}; + +enum k210_clk_div_type { + K210_DIV_NONE, + K210_DIV_ONE_BASED, + K210_DIV_DOUBLE_ONE_BASED, + K210_DIV_POWER_OF_TWO, +}; + +#define K210_GATE(_reg, _bit) \ + .gate_reg = (_reg), \ + .gate_bit = (_bit) + +#define K210_DIV(_reg, _shift, _width, _type) \ + .div_reg = (_reg), \ + .div_shift = (_shift), \ + .div_width = (_width), \ + .div_type = (_type) + +#define K210_MUX(_reg, _bit) \ + .mux_reg = (_reg), \ + .mux_bit = (_bit) + +static struct k210_clk_cfg k210_clk_cfgs[K210_NUM_CLKS] = { + /* Gated clocks, no mux, no divider */ + [K210_CLK_CPU] = { + .name = "cpu", + K210_GATE(K210_SYSCTL_EN_CENT, 0) + }, + [K210_CLK_DMA] = { + .name = "dma", + K210_GATE(K210_SYSCTL_EN_PERI, 1) + }, + [K210_CLK_FFT] = { + .name = "fft", + K210_GATE(K210_SYSCTL_EN_PERI, 4) + }, + [K210_CLK_GPIO] = { + .name = "gpio", + K210_GATE(K210_SYSCTL_EN_PERI, 5) + }, + [K210_CLK_UART1] = { + .name = "uart1", + K210_GATE(K210_SYSCTL_EN_PERI, 16) + }, + [K210_CLK_UART2] = { + .name = "uart2", + K210_GATE(K210_SYSCTL_EN_PERI, 17) + }, + [K210_CLK_UART3] = { + .name = "uart3", + K210_GATE(K210_SYSCTL_EN_PERI, 18) + }, + [K210_CLK_FPIOA] = { + .name = "fpioa", + K210_GATE(K210_SYSCTL_EN_PERI, 20) + }, + [K210_CLK_SHA] = { + .name = "sha", + K210_GATE(K210_SYSCTL_EN_PERI, 26) + }, + [K210_CLK_AES] = { + .name = "aes", + K210_GATE(K210_SYSCTL_EN_PERI, 19) + }, + [K210_CLK_OTP] = { + .name = "otp", + K210_GATE(K210_SYSCTL_EN_PERI, 27) + }, + [K210_CLK_RTC] = { + .name = "rtc", + K210_GATE(K210_SYSCTL_EN_PERI, 29) + }, + + /* Gated divider clocks */ + [K210_CLK_SRAM0] = { + .name = "sram0", + K210_GATE(K210_SYSCTL_EN_CENT, 1), + K210_DIV(K210_SYSCTL_THR0, 0, 4, K210_DIV_ONE_BASED) + }, + [K210_CLK_SRAM1] = { + .name = "sram1", + K210_GATE(K210_SYSCTL_EN_CENT, 2), + K210_DIV(K210_SYSCTL_THR0, 4, 4, K210_DIV_ONE_BASED) + }, + [K210_CLK_ROM] = { + .name = "rom", + K210_GATE(K210_SYSCTL_EN_PERI, 0), + K210_DIV(K210_SYSCTL_THR0, 16, 4, K210_DIV_ONE_BASED) + }, + [K210_CLK_DVP] = { + .name = "dvp", + K210_GATE(K210_SYSCTL_EN_PERI, 3), + K210_DIV(K210_SYSCTL_THR0, 12, 4, K210_DIV_ONE_BASED) + }, + [K210_CLK_APB0] = { + .name = "apb0", + K210_GATE(K210_SYSCTL_EN_CENT, 3), + K210_DIV(K210_SYSCTL_SEL0, 3, 3, K210_DIV_ONE_BASED) + }, + [K210_CLK_APB1] = { + .name = "apb1", + K210_GATE(K210_SYSCTL_EN_CENT, 4), + K210_DIV(K210_SYSCTL_SEL0, 6, 3, K210_DIV_ONE_BASED) + }, + [K210_CLK_APB2] = { + .name = "apb2", + K210_GATE(K210_SYSCTL_EN_CENT, 5), + K210_DIV(K210_SYSCTL_SEL0, 9, 3, K210_DIV_ONE_BASED) + }, + [K210_CLK_AI] = { + .name = "ai", + K210_GATE(K210_SYSCTL_EN_PERI, 2), + K210_DIV(K210_SYSCTL_THR0, 8, 4, K210_DIV_ONE_BASED) + }, + [K210_CLK_SPI0] = { + .name = "spi0", + K210_GATE(K210_SYSCTL_EN_PERI, 6), + K210_DIV(K210_SYSCTL_THR1, 0, 8, K210_DIV_DOUBLE_ONE_BASED) + }, + [K210_CLK_SPI1] = { + .name = "spi1", + K210_GATE(K210_SYSCTL_EN_PERI, 7), + K210_DIV(K210_SYSCTL_THR1, 8, 8, K210_DIV_DOUBLE_ONE_BASED) + }, + [K210_CLK_SPI2] = { + .name = "spi2", + K210_GATE(K210_SYSCTL_EN_PERI, 8), + K210_DIV(K210_SYSCTL_THR1, 16, 8, K210_DIV_DOUBLE_ONE_BASED) + }, + [K210_CLK_I2C0] = { + .name = "i2c0", + K210_GATE(K210_SYSCTL_EN_PERI, 13), + K210_DIV(K210_SYSCTL_THR5, 8, 8, K210_DIV_DOUBLE_ONE_BASED) + }, + [K210_CLK_I2C1] = { + .name = "i2c1", + K210_GATE(K210_SYSCTL_EN_PERI, 14), + K210_DIV(K210_SYSCTL_THR5, 16, 8, K210_DIV_DOUBLE_ONE_BASED) + }, + [K210_CLK_I2C2] = { + .name = "i2c2", + K210_GATE(K210_SYSCTL_EN_PERI, 15), + K210_DIV(K210_SYSCTL_THR5, 24, 8, K210_DIV_DOUBLE_ONE_BASED) + }, + [K210_CLK_WDT0] = { + .name = "wdt0", + K210_GATE(K210_SYSCTL_EN_PERI, 24), + K210_DIV(K210_SYSCTL_THR6, 0, 8, K210_DIV_DOUBLE_ONE_BASED) + }, + [K210_CLK_WDT1] = { + .name = "wdt1", + K210_GATE(K210_SYSCTL_EN_PERI, 25), + K210_DIV(K210_SYSCTL_THR6, 8, 8, K210_DIV_DOUBLE_ONE_BASED) + }, + [K210_CLK_I2S0] = { + .name = "i2s0", + K210_GATE(K210_SYSCTL_EN_PERI, 10), + K210_DIV(K210_SYSCTL_THR3, 0, 16, K210_DIV_DOUBLE_ONE_BASED) + }, + [K210_CLK_I2S1] = { + .name = "i2s1", + K210_GATE(K210_SYSCTL_EN_PERI, 11), + K210_DIV(K210_SYSCTL_THR3, 16, 16, K210_DIV_DOUBLE_ONE_BASED) + }, + [K210_CLK_I2S2] = { + .name = "i2s2", + K210_GATE(K210_SYSCTL_EN_PERI, 12), + K210_DIV(K210_SYSCTL_THR4, 0, 16, K210_DIV_DOUBLE_ONE_BASED) + }, + + /* Divider clocks, no gate, no mux */ + [K210_CLK_I2S0_M] = { + .name = "i2s0_m", + K210_DIV(K210_SYSCTL_THR4, 16, 8, K210_DIV_DOUBLE_ONE_BASED) + }, + [K210_CLK_I2S1_M] = { + .name = "i2s1_m", + K210_DIV(K210_SYSCTL_THR4, 24, 8, K210_DIV_DOUBLE_ONE_BASED) + }, + [K210_CLK_I2S2_M] = { + .name = "i2s2_m", + K210_DIV(K210_SYSCTL_THR4, 0, 8, K210_DIV_DOUBLE_ONE_BASED) + }, + + /* Muxed gated divider clocks */ + [K210_CLK_SPI3] = { + .name = "spi3", + K210_GATE(K210_SYSCTL_EN_PERI, 9), + K210_DIV(K210_SYSCTL_THR1, 24, 8, K210_DIV_DOUBLE_ONE_BASED), + K210_MUX(K210_SYSCTL_SEL0, 12) + }, + [K210_CLK_TIMER0] = { + .name = "timer0", + K210_GATE(K210_SYSCTL_EN_PERI, 21), + K210_DIV(K210_SYSCTL_THR2, 0, 8, K210_DIV_DOUBLE_ONE_BASED), + K210_MUX(K210_SYSCTL_SEL0, 13) + }, + [K210_CLK_TIMER1] = { + .name = "timer1", + K210_GATE(K210_SYSCTL_EN_PERI, 22), + K210_DIV(K210_SYSCTL_THR2, 8, 8, K210_DIV_DOUBLE_ONE_BASED), + K210_MUX(K210_SYSCTL_SEL0, 14) + }, + [K210_CLK_TIMER2] = { + .name = "timer2", + K210_GATE(K210_SYSCTL_EN_PERI, 23), + K210_DIV(K210_SYSCTL_THR2, 16, 8, K210_DIV_DOUBLE_ONE_BASED), + K210_MUX(K210_SYSCTL_SEL0, 15) + }, +}; + +/* + * PLL control register bits. + */ +#define K210_PLL_CLKR GENMASK(3, 0) +#define K210_PLL_CLKF GENMASK(9, 4) +#define K210_PLL_CLKOD GENMASK(13, 10) +#define K210_PLL_BWADJ GENMASK(19, 14) +#define K210_PLL_RESET (1 << 20) +#define K210_PLL_PWRD (1 << 21) +#define K210_PLL_INTFB (1 << 22) +#define K210_PLL_BYPASS (1 << 23) +#define K210_PLL_TEST (1 << 24) +#define K210_PLL_EN (1 << 25) +#define K210_PLL_SEL GENMASK(27, 26) /* PLL2 only */ + +/* + * PLL lock register bits. + */ +#define K210_PLL_LOCK 0 +#define K210_PLL_CLEAR_SLIP 2 +#define K210_PLL_TEST_OUT 3 + +/* + * Clock selector register bits. + */ +#define K210_ACLK_SEL BIT(0) +#define K210_ACLK_DIV GENMASK(2, 1) + +/* + * PLLs. + */ +enum k210_pll_id { + K210_PLL0, K210_PLL1, K210_PLL2, K210_PLL_NUM +}; + +struct k210_pll { + enum k210_pll_id id; + struct k210_sysclk *ksc; + void __iomem *base; + void __iomem *reg; + void __iomem *lock; + u8 lock_shift; + u8 lock_width; + struct clk_hw hw; +}; +#define to_k210_pll(_hw) container_of(_hw, struct k210_pll, hw) + +/* + * PLLs configuration: by default PLL0 runs at 780 MHz and PLL1 at 299 MHz. + * The first 2 SRAM banks depend on ACLK/CPU clock which is by default PLL0 + * rate divided by 2. Set PLL1 to 390 MHz so that the third SRAM bank has the + * same clock as the first 2. + */ +struct k210_pll_cfg { + u32 reg; + u8 lock_shift; + u8 lock_width; + u32 r; + u32 f; + u32 od; + u32 bwadj; +}; + +static struct k210_pll_cfg k210_plls_cfg[] = { + { K210_SYSCTL_PLL0, 0, 2, 0, 59, 1, 59 }, /* 780 MHz */ + { K210_SYSCTL_PLL1, 8, 1, 0, 59, 3, 59 }, /* 390 MHz */ + { K210_SYSCTL_PLL2, 16, 1, 0, 22, 1, 22 }, /* 299 MHz */ +}; + +/** + * struct k210_sysclk - sysclk driver data + * @regs: system controller registers start address + * @clk_lock: clock setting spinlock + * @plls: SoC PLLs descriptors + * @aclk: ACLK clock + * @clks: All other clocks + */ +struct k210_sysclk { + void __iomem *regs; + spinlock_t clk_lock; + struct k210_pll plls[K210_PLL_NUM]; + struct clk_hw aclk; + struct k210_clk clks[K210_NUM_CLKS]; +}; + +#define to_k210_sysclk(_hw) container_of(_hw, struct k210_sysclk, aclk) + +/* + * Set ACLK parent selector: 0 for IN0, 1 for PLL0. + */ +static void k210_aclk_set_selector(void __iomem *regs, u8 sel) +{ + u32 reg = readl(regs + K210_SYSCTL_SEL0); + + if (sel) + reg |= K210_ACLK_SEL; + else + reg &= K210_ACLK_SEL; + writel(reg, regs + K210_SYSCTL_SEL0); +} + +static void k210_init_pll(void __iomem *regs, enum k210_pll_id pllid, + struct k210_pll *pll) +{ + pll->id = pllid; + pll->reg = regs + k210_plls_cfg[pllid].reg; + pll->lock = regs + K210_SYSCTL_PLL_LOCK; + pll->lock_shift = k210_plls_cfg[pllid].lock_shift; + pll->lock_width = k210_plls_cfg[pllid].lock_width; +} + +static void k210_pll_wait_for_lock(struct k210_pll *pll) +{ + u32 reg, mask = GENMASK(pll->lock_shift + pll->lock_width - 1, + pll->lock_shift); + + while (true) { + reg = readl(pll->lock); + if ((reg & mask) == mask) + break; + + reg |= BIT(pll->lock_shift + K210_PLL_CLEAR_SLIP); + writel(reg, pll->lock); + } +} + +static bool k210_pll_hw_is_enabled(struct k210_pll *pll) +{ + u32 reg = readl(pll->reg); + u32 mask = K210_PLL_PWRD | K210_PLL_EN; + + if (reg & K210_PLL_RESET) + return false; + + return (reg & mask) == mask; +} + +static void k210_pll_enable_hw(void __iomem *regs, struct k210_pll *pll) +{ + struct k210_pll_cfg *pll_cfg = &k210_plls_cfg[pll->id]; + u32 reg; + + if (k210_pll_hw_is_enabled(pll)) + return; + + /* + * For PLL0, we need to re-parent ACLK to IN0 to keep the CPU cores and + * SRAM running. + */ + if (pll->id == K210_PLL0) + k210_aclk_set_selector(regs, 0); + + /* Set PLL factors */ + reg = readl(pll->reg); + reg &= ~GENMASK(19, 0); + reg |= FIELD_PREP(K210_PLL_CLKR, pll_cfg->r); + reg |= FIELD_PREP(K210_PLL_CLKF, pll_cfg->f); + reg |= FIELD_PREP(K210_PLL_CLKOD, pll_cfg->od); + reg |= FIELD_PREP(K210_PLL_BWADJ, pll_cfg->bwadj); + reg |= K210_PLL_PWRD; + writel(reg, pll->reg); + + /* + * Reset the PLL: ensure reset is low before asserting it. + * The magic NOPs come from the Kendryte reference SDK. + */ + reg &= ~K210_PLL_RESET; + writel(reg, pll->reg); + reg |= K210_PLL_RESET; + writel(reg, pll->reg); + nop(); + nop(); + reg &= ~K210_PLL_RESET; + writel(reg, pll->reg); + + k210_pll_wait_for_lock(pll); + + reg &= ~K210_PLL_BYPASS; + reg |= K210_PLL_EN; + writel(reg, pll->reg); + + if (pll->id == K210_PLL0) + k210_aclk_set_selector(regs, 1); +} + +static int k210_pll_enable(struct clk_hw *hw) +{ + struct k210_pll *pll = to_k210_pll(hw); + struct k210_sysclk *ksc = pll->ksc; + unsigned long flags; + + spin_lock_irqsave(&ksc->clk_lock, flags); + + k210_pll_enable_hw(ksc->regs, pll); + + spin_unlock_irqrestore(&ksc->clk_lock, flags); + + return 0; +} + +static void k210_pll_disable(struct clk_hw *hw) +{ + struct k210_pll *pll = to_k210_pll(hw); + struct k210_sysclk *ksc = pll->ksc; + unsigned long flags; + u32 reg; + + /* + * Bypassing before powering off is important so child clocks do not + * stop working. This is especially important for pll0, the indirect + * parent of the cpu clock. + */ + spin_lock_irqsave(&ksc->clk_lock, flags); + reg = readl(pll->reg); + reg |= K210_PLL_BYPASS; + writel(reg, pll->reg); + + reg &= ~K210_PLL_PWRD; + reg &= ~K210_PLL_EN; + writel(reg, pll->reg); + spin_unlock_irqrestore(&ksc->clk_lock, flags); +} + +static int k210_pll_is_enabled(struct clk_hw *hw) +{ + return k210_pll_hw_is_enabled(to_k210_pll(hw)); +} + +static unsigned long k210_pll_get_rate(struct clk_hw *hw, + unsigned long parent_rate) +{ + struct k210_pll *pll = to_k210_pll(hw); + u32 reg = readl(pll->reg); + u32 r, f, od; + + if (reg & K210_PLL_BYPASS) + return parent_rate; + + if (!(reg & K210_PLL_PWRD)) + return 0; + + r = FIELD_GET(K210_PLL_CLKR, reg) + 1; + f = FIELD_GET(K210_PLL_CLKF, reg) + 1; + od = FIELD_GET(K210_PLL_CLKOD, reg) + 1; + + return (u64)parent_rate * f / (r * od); +} + +static const struct clk_ops k210_pll_ops = { + .enable = k210_pll_enable, + .disable = k210_pll_disable, + .is_enabled = k210_pll_is_enabled, + .recalc_rate = k210_pll_get_rate, +}; + +static int k210_pll2_set_parent(struct clk_hw *hw, u8 index) +{ + struct k210_pll *pll = to_k210_pll(hw); + struct k210_sysclk *ksc = pll->ksc; + unsigned long flags; + u32 reg; + + spin_lock_irqsave(&ksc->clk_lock, flags); + + reg = readl(pll->reg); + reg &= ~K210_PLL_SEL; + reg |= FIELD_PREP(K210_PLL_SEL, index); + writel(reg, pll->reg); + + spin_unlock_irqrestore(&ksc->clk_lock, flags); + + return 0; +} + +static u8 k210_pll2_get_parent(struct clk_hw *hw) +{ + struct k210_pll *pll = to_k210_pll(hw); + u32 reg = readl(pll->reg); + + return FIELD_GET(K210_PLL_SEL, reg); +} + +static const struct clk_ops k210_pll2_ops = { + .enable = k210_pll_enable, + .disable = k210_pll_disable, + .is_enabled = k210_pll_is_enabled, + .recalc_rate = k210_pll_get_rate, + .set_parent = k210_pll2_set_parent, + .get_parent = k210_pll2_get_parent, +}; + +static int __init k210_register_pll(struct device_node *np, + struct k210_sysclk *ksc, + enum k210_pll_id pllid, const char *name, + int num_parents, const struct clk_ops *ops) +{ + struct k210_pll *pll = &ksc->plls[pllid]; + struct clk_init_data init = {}; + const struct clk_parent_data parent_data[] = { + { /* .index = 0 for in0 */ }, + { .hw = &ksc->plls[K210_PLL0].hw }, + { .hw = &ksc->plls[K210_PLL1].hw }, + }; + + init.name = name; + init.parent_data = parent_data; + init.num_parents = num_parents; + init.ops = ops; + + pll->hw.init = &init; + pll->ksc = ksc; + + return of_clk_hw_register(np, &pll->hw); +} + +static int __init k210_register_plls(struct device_node *np, + struct k210_sysclk *ksc) +{ + int i, ret; + + for (i = 0; i < K210_PLL_NUM; i++) + k210_init_pll(ksc->regs, i, &ksc->plls[i]); + + /* PLL0 and PLL1 only have IN0 as parent */ + ret = k210_register_pll(np, ksc, K210_PLL0, "pll0", 1, &k210_pll_ops); + if (ret) { + pr_err("%pOFP: register PLL0 failed\n", np); + return ret; + } + ret = k210_register_pll(np, ksc, K210_PLL1, "pll1", 1, &k210_pll_ops); + if (ret) { + pr_err("%pOFP: register PLL1 failed\n", np); + return ret; + } + + /* PLL2 has IN0, PLL0 and PLL1 as parents */ + ret = k210_register_pll(np, ksc, K210_PLL2, "pll2", 3, &k210_pll2_ops); + if (ret) { + pr_err("%pOFP: register PLL2 failed\n", np); + return ret; + } + + return 0; +} + +static int k210_aclk_set_parent(struct clk_hw *hw, u8 index) +{ + struct k210_sysclk *ksc = to_k210_sysclk(hw); + unsigned long flags; + + spin_lock_irqsave(&ksc->clk_lock, flags); + + k210_aclk_set_selector(ksc->regs, index); + + spin_unlock_irqrestore(&ksc->clk_lock, flags); + + return 0; +} + +static u8 k210_aclk_get_parent(struct clk_hw *hw) +{ + struct k210_sysclk *ksc = to_k210_sysclk(hw); + u32 sel; + + sel = readl(ksc->regs + K210_SYSCTL_SEL0) & K210_ACLK_SEL; + + return sel ? 1 : 0; +} + +static unsigned long k210_aclk_get_rate(struct clk_hw *hw, + unsigned long parent_rate) +{ + struct k210_sysclk *ksc = to_k210_sysclk(hw); + u32 reg = readl(ksc->regs + K210_SYSCTL_SEL0); + unsigned int shift; + + if (!(reg & 0x1)) + return parent_rate; + + shift = FIELD_GET(K210_ACLK_DIV, reg); + + return parent_rate / (2UL << shift); +} + +static const struct clk_ops k210_aclk_ops = { + .set_parent = k210_aclk_set_parent, + .get_parent = k210_aclk_get_parent, + .recalc_rate = k210_aclk_get_rate, +}; + +/* + * ACLK has IN0 and PLL0 as parents. + */ +static int __init k210_register_aclk(struct device_node *np, + struct k210_sysclk *ksc) +{ + struct clk_init_data init = {}; + const struct clk_parent_data parent_data[] = { + { /* .index = 0 for in0 */ }, + { .hw = &ksc->plls[K210_PLL0].hw }, + }; + int ret; + + init.name = "aclk"; + init.parent_data = parent_data; + init.num_parents = 2; + init.ops = &k210_aclk_ops; + ksc->aclk.init = &init; + + ret = of_clk_hw_register(np, &ksc->aclk); + if (ret) { + pr_err("%pOFP: register aclk failed\n", np); + return ret; + } + + return 0; +} + +#define to_k210_clk(_hw) container_of(_hw, struct k210_clk, hw) + +static int k210_clk_enable(struct clk_hw *hw) +{ + struct k210_clk *kclk = to_k210_clk(hw); + struct k210_sysclk *ksc = kclk->ksc; + struct k210_clk_cfg *cfg = &k210_clk_cfgs[kclk->id]; + unsigned long flags; + u32 reg; + + if (!cfg->gate_reg) + return 0; + + spin_lock_irqsave(&ksc->clk_lock, flags); + reg = readl(ksc->regs + cfg->gate_reg); + reg |= BIT(cfg->gate_bit); + writel(reg, ksc->regs + cfg->gate_reg); + spin_unlock_irqrestore(&ksc->clk_lock, flags); + + return 0; +} + +static void k210_clk_disable(struct clk_hw *hw) +{ + struct k210_clk *kclk = to_k210_clk(hw); + struct k210_sysclk *ksc = kclk->ksc; + struct k210_clk_cfg *cfg = &k210_clk_cfgs[kclk->id]; + unsigned long flags; + u32 reg; + + if (!cfg->gate_reg) + return; + + spin_lock_irqsave(&ksc->clk_lock, flags); + reg = readl(ksc->regs + cfg->gate_reg); + reg &= ~BIT(cfg->gate_bit); + writel(reg, ksc->regs + cfg->gate_reg); + spin_unlock_irqrestore(&ksc->clk_lock, flags); +} + +static int k210_clk_set_parent(struct clk_hw *hw, u8 index) +{ + struct k210_clk *kclk = to_k210_clk(hw); + struct k210_sysclk *ksc = kclk->ksc; + struct k210_clk_cfg *cfg = &k210_clk_cfgs[kclk->id]; + unsigned long flags; + u32 reg; + + spin_lock_irqsave(&ksc->clk_lock, flags); + reg = readl(ksc->regs + cfg->mux_reg); + if (index) + reg |= BIT(cfg->mux_bit); + else + reg &= ~BIT(cfg->mux_bit); + spin_unlock_irqrestore(&ksc->clk_lock, flags); + + return 0; +} + +static u8 k210_clk_get_parent(struct clk_hw *hw) +{ + struct k210_clk *kclk = to_k210_clk(hw); + struct k210_sysclk *ksc = kclk->ksc; + struct k210_clk_cfg *cfg = &k210_clk_cfgs[kclk->id]; + unsigned long flags; + u32 reg, idx; + + spin_lock_irqsave(&ksc->clk_lock, flags); + reg = readl(ksc->regs + cfg->mux_reg); + idx = (reg & BIT(cfg->mux_bit)) ? 1 : 0; + spin_unlock_irqrestore(&ksc->clk_lock, flags); + + return idx; +} + +static unsigned long k210_clk_get_rate(struct clk_hw *hw, + unsigned long parent_rate) +{ + struct k210_clk *kclk = to_k210_clk(hw); + struct k210_sysclk *ksc = kclk->ksc; + struct k210_clk_cfg *cfg = &k210_clk_cfgs[kclk->id]; + u32 reg, div_val; + + if (!cfg->div_reg) + return parent_rate; + + reg = readl(ksc->regs + cfg->div_reg); + div_val = (reg >> cfg->div_shift) & GENMASK(cfg->div_width - 1, 0); + + switch (cfg->div_type) { + case K210_DIV_ONE_BASED: + return parent_rate / (div_val + 1); + case K210_DIV_DOUBLE_ONE_BASED: + return parent_rate / ((div_val + 1) * 2); + case K210_DIV_POWER_OF_TWO: + return parent_rate / (2UL << div_val); + case K210_DIV_NONE: + default: + return 0; + } +} + +static const struct clk_ops k210_clk_mux_ops = { + .enable = k210_clk_enable, + .disable = k210_clk_disable, + .set_parent = k210_clk_set_parent, + .get_parent = k210_clk_get_parent, + .recalc_rate = k210_clk_get_rate, +}; + +static const struct clk_ops k210_clk_ops = { + .enable = k210_clk_enable, + .disable = k210_clk_disable, + .recalc_rate = k210_clk_get_rate, +}; + +static void __init k210_register_clk(struct device_node *np, + struct k210_sysclk *ksc, int id, + const struct clk_parent_data *parent_data, + int num_parents, unsigned long flags) +{ + struct k210_clk *kclk = &ksc->clks[id]; + struct clk_init_data init = {}; + int ret; + + init.name = k210_clk_cfgs[id].name; + init.flags = flags; + init.parent_data = parent_data; + init.num_parents = num_parents; + if (num_parents > 1) + init.ops = &k210_clk_mux_ops; + else + init.ops = &k210_clk_ops; + + kclk->id = id; + kclk->ksc = ksc; + kclk->hw.init = &init; + + ret = of_clk_hw_register(np, &kclk->hw); + if (ret) { + pr_err("%pOFP: register clock %s failed\n", + np, k210_clk_cfgs[id].name); + kclk->id = -1; + } +} + +/* + * All muxed clocks have IN0 and PLL0 as parents. + */ +static inline void __init k210_register_mux_clk(struct device_node *np, + struct k210_sysclk *ksc, int id) +{ + const struct clk_parent_data parent_data[2] = { + { /* .index = 0 for in0 */ }, + { .hw = &ksc->plls[K210_PLL0].hw } + }; + + k210_register_clk(np, ksc, id, parent_data, 2, 0); +} + +static inline void __init k210_register_in0_child(struct device_node *np, + struct k210_sysclk *ksc, int id) +{ + const struct clk_parent_data parent_data = { + /* .index = 0 for in0 */ + }; + + k210_register_clk(np, ksc, id, &parent_data, 1, 0); +} + +static inline void __init k210_register_pll_child(struct device_node *np, + struct k210_sysclk *ksc, int id, + enum k210_pll_id pllid, + unsigned long flags) +{ + const struct clk_parent_data parent_data = { + .hw = &ksc->plls[pllid].hw, + }; + + k210_register_clk(np, ksc, id, &parent_data, 1, flags); +} + +static inline void __init k210_register_aclk_child(struct device_node *np, + struct k210_sysclk *ksc, int id, + unsigned long flags) +{ + const struct clk_parent_data parent_data = { + .hw = &ksc->aclk, + }; + + k210_register_clk(np, ksc, id, &parent_data, 1, flags); +} + +static inline void __init k210_register_clk_child(struct device_node *np, + struct k210_sysclk *ksc, int id, + int parent_id) +{ + const struct clk_parent_data parent_data = { + .hw = &ksc->clks[parent_id].hw, + }; + + k210_register_clk(np, ksc, id, &parent_data, 1, 0); +} + +static struct clk_hw *k210_clk_hw_onecell_get(struct of_phandle_args *clkspec, + void *data) +{ + struct k210_sysclk *ksc = data; + unsigned int idx = clkspec->args[0]; + + if (idx >= K210_NUM_CLKS) + return ERR_PTR(-EINVAL); + + return &ksc->clks[idx].hw; +} + +static void __init k210_clk_init(struct device_node *np) +{ + struct device_node *sysctl_np; + struct k210_sysclk *ksc; + int i, ret; + + ksc = kzalloc(sizeof(*ksc), GFP_KERNEL); + if (!ksc) + return; + + spin_lock_init(&ksc->clk_lock); + sysctl_np = of_get_parent(np); + ksc->regs = of_iomap(sysctl_np, 0); + of_node_put(sysctl_np); + if (!ksc->regs) { + pr_err("%pOFP: failed to map registers\n", np); + return; + } + + ret = k210_register_plls(np, ksc); + if (ret) + return; + + ret = k210_register_aclk(np, ksc); + if (ret) + return; + + /* + * Critical clocks: there are no consumers of the SRAM clocks, + * including the AI clock for the third SRAM bank. The CPU clock + * is only referenced by the uarths serial device and so would be + * disabled if the serial console is disabled to switch to another + * console. Mark all these clocks as critical so that they are never + * disabled by the core clock management. + */ + k210_register_aclk_child(np, ksc, K210_CLK_CPU, CLK_IS_CRITICAL); + k210_register_aclk_child(np, ksc, K210_CLK_SRAM0, CLK_IS_CRITICAL); + k210_register_aclk_child(np, ksc, K210_CLK_SRAM1, CLK_IS_CRITICAL); + k210_register_pll_child(np, ksc, K210_CLK_AI, K210_PLL1, + CLK_IS_CRITICAL); + + /* Clocks with aclk as source */ + k210_register_aclk_child(np, ksc, K210_CLK_DMA, 0); + k210_register_aclk_child(np, ksc, K210_CLK_FFT, 0); + k210_register_aclk_child(np, ksc, K210_CLK_ROM, 0); + k210_register_aclk_child(np, ksc, K210_CLK_DVP, 0); + k210_register_aclk_child(np, ksc, K210_CLK_APB0, 0); + k210_register_aclk_child(np, ksc, K210_CLK_APB1, 0); + k210_register_aclk_child(np, ksc, K210_CLK_APB2, 0); + + /* Clocks with PLL0 as source */ + k210_register_pll_child(np, ksc, K210_CLK_SPI0, K210_PLL0, 0); + k210_register_pll_child(np, ksc, K210_CLK_SPI1, K210_PLL0, 0); + k210_register_pll_child(np, ksc, K210_CLK_SPI2, K210_PLL0, 0); + k210_register_pll_child(np, ksc, K210_CLK_I2C0, K210_PLL0, 0); + k210_register_pll_child(np, ksc, K210_CLK_I2C1, K210_PLL0, 0); + k210_register_pll_child(np, ksc, K210_CLK_I2C2, K210_PLL0, 0); + + /* Clocks with PLL2 as source */ + k210_register_pll_child(np, ksc, K210_CLK_I2S0, K210_PLL2, 0); + k210_register_pll_child(np, ksc, K210_CLK_I2S1, K210_PLL2, 0); + k210_register_pll_child(np, ksc, K210_CLK_I2S2, K210_PLL2, 0); + k210_register_pll_child(np, ksc, K210_CLK_I2S0_M, K210_PLL2, 0); + k210_register_pll_child(np, ksc, K210_CLK_I2S1_M, K210_PLL2, 0); + k210_register_pll_child(np, ksc, K210_CLK_I2S2_M, K210_PLL2, 0); + + /* Clocks with IN0 as source */ + k210_register_in0_child(np, ksc, K210_CLK_WDT0); + k210_register_in0_child(np, ksc, K210_CLK_WDT1); + k210_register_in0_child(np, ksc, K210_CLK_RTC); + + /* Clocks with APB0 as source */ + k210_register_clk_child(np, ksc, K210_CLK_GPIO, K210_CLK_APB0); + k210_register_clk_child(np, ksc, K210_CLK_UART1, K210_CLK_APB0); + k210_register_clk_child(np, ksc, K210_CLK_UART2, K210_CLK_APB0); + k210_register_clk_child(np, ksc, K210_CLK_UART3, K210_CLK_APB0); + k210_register_clk_child(np, ksc, K210_CLK_FPIOA, K210_CLK_APB0); + k210_register_clk_child(np, ksc, K210_CLK_SHA, K210_CLK_APB0); + + /* Clocks with APB1 as source */ + k210_register_clk_child(np, ksc, K210_CLK_AES, K210_CLK_APB1); + k210_register_clk_child(np, ksc, K210_CLK_OTP, K210_CLK_APB1); + + /* Mux clocks with in0 or pll0 as source */ + k210_register_mux_clk(np, ksc, K210_CLK_SPI3); + k210_register_mux_clk(np, ksc, K210_CLK_TIMER0); + k210_register_mux_clk(np, ksc, K210_CLK_TIMER1); + k210_register_mux_clk(np, ksc, K210_CLK_TIMER2); + + /* Check for registration errors */ + for (i = 0; i < K210_NUM_CLKS; i++) { + if (ksc->clks[i].id != i) + return; + } + + ret = of_clk_add_hw_provider(np, k210_clk_hw_onecell_get, ksc); + if (ret) { + pr_err("%pOFP: add clock provider failed %d\n", np, ret); + return; + } + + pr_info("%pOFP: CPU running at %lu MHz\n", + np, clk_hw_get_rate(&ksc->clks[K210_CLK_CPU].hw) / 1000000); +} + +CLK_OF_DECLARE(k210_clk, "canaan,k210-clk", k210_clk_init); + +/* + * Enable PLL1 to be able to use the AI SRAM. + */ +void __init k210_clk_early_init(void __iomem *regs) +{ + struct k210_pll pll1; + + /* Make sure ACLK selector is set to PLL0 */ + k210_aclk_set_selector(regs, 1); + + /* Startup PLL1 to enable the aisram bank for general memory use */ + k210_init_pll(regs, K210_PLL1, &pll1); + k210_pll_enable_hw(regs, &pll1); +} diff --git a/drivers/clk/clk-npcm7xx.c b/drivers/clk/clk-npcm7xx.c index 27a86b7a34dbf64df553ac8283904310cc2da24c..e677bb5a784b96ac2bb71c90e2a365e0d316f46e 100644 --- a/drivers/clk/clk-npcm7xx.c +++ b/drivers/clk/clk-npcm7xx.c @@ -361,13 +361,6 @@ static const struct npcm7xx_clk_mux_data npcm7xx_muxes[] __initconst = { dvcssel_mux_parents, ARRAY_SIZE(dvcssel_mux_parents), 0, -1}, }; -/* fixed ratio dividers (no register): */ -static const struct npcm7xx_clk_div_fixed_data npcm7xx_divs_fx[] __initconst = { - { 1, 2, NPCM7XX_CLK_S_MC, NPCM7XX_CLK_S_MC_MUX, 0, NPCM7XX_CLK_MC}, - { 1, 2, NPCM7XX_CLK_S_PLL1_DIV2, NPCM7XX_CLK_S_PLL1, 0, -1}, - { 1, 2, NPCM7XX_CLK_S_PLL2_DIV2, NPCM7XX_CLK_S_PLL2, 0, -1}, -}; - /* configurable dividers: */ static const struct npcm7xx_clk_div_data npcm7xx_divs[] __initconst = { {NPCM7XX_CLKDIV1, 28, 3, NPCM7XX_CLK_S_ADC, @@ -435,107 +428,6 @@ static const struct npcm7xx_clk_div_data npcm7xx_divs[] __initconst = { }; -static const struct npcm7xx_clk_gate_data npcm7xx_gates[] __initconst = { - {NPCM7XX_CLKEN1, 31, "smb1-gate", NPCM7XX_CLK_S_APB2, 0}, - {NPCM7XX_CLKEN1, 30, "smb0-gate", NPCM7XX_CLK_S_APB2, 0}, - {NPCM7XX_CLKEN1, 29, "smb7-gate", NPCM7XX_CLK_S_APB2, 0}, - {NPCM7XX_CLKEN1, 28, "smb6-gate", NPCM7XX_CLK_S_APB2, 0}, - {NPCM7XX_CLKEN1, 27, "adc-gate", NPCM7XX_CLK_S_APB1, 0}, - {NPCM7XX_CLKEN1, 26, "wdt-gate", NPCM7XX_CLK_S_TIMER, 0}, - {NPCM7XX_CLKEN1, 25, "usbdev3-gate", NPCM7XX_CLK_S_AHB, 0}, - {NPCM7XX_CLKEN1, 24, "usbdev6-gate", NPCM7XX_CLK_S_AHB, 0}, - {NPCM7XX_CLKEN1, 23, "usbdev5-gate", NPCM7XX_CLK_S_AHB, 0}, - {NPCM7XX_CLKEN1, 22, "usbdev4-gate", NPCM7XX_CLK_S_AHB, 0}, - {NPCM7XX_CLKEN1, 21, "emc2-gate", NPCM7XX_CLK_S_AHB, 0}, - {NPCM7XX_CLKEN1, 20, "timer5_9-gate", NPCM7XX_CLK_S_APB1, 0}, - {NPCM7XX_CLKEN1, 19, "timer0_4-gate", NPCM7XX_CLK_S_APB1, 0}, - {NPCM7XX_CLKEN1, 18, "pwmm0-gate", NPCM7XX_CLK_S_APB3, 0}, - {NPCM7XX_CLKEN1, 17, "huart-gate", NPCM7XX_CLK_S_UART, 0}, - {NPCM7XX_CLKEN1, 16, "smb5-gate", NPCM7XX_CLK_S_APB2, 0}, - {NPCM7XX_CLKEN1, 15, "smb4-gate", NPCM7XX_CLK_S_APB2, 0}, - {NPCM7XX_CLKEN1, 14, "smb3-gate", NPCM7XX_CLK_S_APB2, 0}, - {NPCM7XX_CLKEN1, 13, "smb2-gate", NPCM7XX_CLK_S_APB2, 0}, - {NPCM7XX_CLKEN1, 12, "mc-gate", NPCM7XX_CLK_S_MC, 0}, - {NPCM7XX_CLKEN1, 11, "uart01-gate", NPCM7XX_CLK_S_APB1, 0}, - {NPCM7XX_CLKEN1, 10, "aes-gate", NPCM7XX_CLK_S_AHB, 0}, - {NPCM7XX_CLKEN1, 9, "peci-gate", NPCM7XX_CLK_S_APB3, 0}, - {NPCM7XX_CLKEN1, 8, "usbdev2-gate", NPCM7XX_CLK_S_AHB, 0}, - {NPCM7XX_CLKEN1, 7, "uart23-gate", NPCM7XX_CLK_S_APB1, 0}, - {NPCM7XX_CLKEN1, 6, "emc1-gate", NPCM7XX_CLK_S_AHB, 0}, - {NPCM7XX_CLKEN1, 5, "usbdev1-gate", NPCM7XX_CLK_S_AHB, 0}, - {NPCM7XX_CLKEN1, 4, "shm-gate", NPCM7XX_CLK_S_AHB, 0}, - /* bit 3 is reserved */ - {NPCM7XX_CLKEN1, 2, "kcs-gate", NPCM7XX_CLK_S_APB1, 0}, - {NPCM7XX_CLKEN1, 1, "spi3-gate", NPCM7XX_CLK_S_AHB, 0}, - {NPCM7XX_CLKEN1, 0, "spi0-gate", NPCM7XX_CLK_S_AHB, 0}, - - {NPCM7XX_CLKEN2, 31, "cp-gate", NPCM7XX_CLK_S_AHB, 0}, - {NPCM7XX_CLKEN2, 30, "tock-gate", NPCM7XX_CLK_S_TOCK, 0}, - /* bit 29 is reserved */ - {NPCM7XX_CLKEN2, 28, "gmac1-gate", NPCM7XX_CLK_S_AHB, 0}, - {NPCM7XX_CLKEN2, 27, "usbif-gate", NPCM7XX_CLK_S_USBIF, 0}, - {NPCM7XX_CLKEN2, 26, "usbhost-gate", NPCM7XX_CLK_S_AHB, 0}, - {NPCM7XX_CLKEN2, 25, "gmac2-gate", NPCM7XX_CLK_S_AHB, 0}, - /* bit 24 is reserved */ - {NPCM7XX_CLKEN2, 23, "pspi2-gate", NPCM7XX_CLK_S_APB5, 0}, - {NPCM7XX_CLKEN2, 22, "pspi1-gate", NPCM7XX_CLK_S_APB5, 0}, - {NPCM7XX_CLKEN2, 21, "3des-gate", NPCM7XX_CLK_S_AHB, 0}, - /* bit 20 is reserved */ - {NPCM7XX_CLKEN2, 19, "siox2-gate", NPCM7XX_CLK_S_APB3, 0}, - {NPCM7XX_CLKEN2, 18, "siox1-gate", NPCM7XX_CLK_S_APB3, 0}, - /* bit 17 is reserved */ - {NPCM7XX_CLKEN2, 16, "fuse-gate", NPCM7XX_CLK_S_APB4, 0}, - /* bit 15 is reserved */ - {NPCM7XX_CLKEN2, 14, "vcd-gate", NPCM7XX_CLK_S_AHB, 0}, - {NPCM7XX_CLKEN2, 13, "ece-gate", NPCM7XX_CLK_S_AHB, 0}, - {NPCM7XX_CLKEN2, 12, "vdma-gate", NPCM7XX_CLK_S_AHB, 0}, - {NPCM7XX_CLKEN2, 11, "ahbpcibrg-gate", NPCM7XX_CLK_S_AHB, 0}, - {NPCM7XX_CLKEN2, 10, "gfxsys-gate", NPCM7XX_CLK_S_APB1, 0}, - {NPCM7XX_CLKEN2, 9, "sdhc-gate", NPCM7XX_CLK_S_AHB, 0}, - {NPCM7XX_CLKEN2, 8, "mmc-gate", NPCM7XX_CLK_S_AHB, 0}, - {NPCM7XX_CLKEN2, 7, "mft7-gate", NPCM7XX_CLK_S_APB4, 0}, - {NPCM7XX_CLKEN2, 6, "mft6-gate", NPCM7XX_CLK_S_APB4, 0}, - {NPCM7XX_CLKEN2, 5, "mft5-gate", NPCM7XX_CLK_S_APB4, 0}, - {NPCM7XX_CLKEN2, 4, "mft4-gate", NPCM7XX_CLK_S_APB4, 0}, - {NPCM7XX_CLKEN2, 3, "mft3-gate", NPCM7XX_CLK_S_APB4, 0}, - {NPCM7XX_CLKEN2, 2, "mft2-gate", NPCM7XX_CLK_S_APB4, 0}, - {NPCM7XX_CLKEN2, 1, "mft1-gate", NPCM7XX_CLK_S_APB4, 0}, - {NPCM7XX_CLKEN2, 0, "mft0-gate", NPCM7XX_CLK_S_APB4, 0}, - - {NPCM7XX_CLKEN3, 31, "gpiom7-gate", NPCM7XX_CLK_S_APB1, 0}, - {NPCM7XX_CLKEN3, 30, "gpiom6-gate", NPCM7XX_CLK_S_APB1, 0}, - {NPCM7XX_CLKEN3, 29, "gpiom5-gate", NPCM7XX_CLK_S_APB1, 0}, - {NPCM7XX_CLKEN3, 28, "gpiom4-gate", NPCM7XX_CLK_S_APB1, 0}, - {NPCM7XX_CLKEN3, 27, "gpiom3-gate", NPCM7XX_CLK_S_APB1, 0}, - {NPCM7XX_CLKEN3, 26, "gpiom2-gate", NPCM7XX_CLK_S_APB1, 0}, - {NPCM7XX_CLKEN3, 25, "gpiom1-gate", NPCM7XX_CLK_S_APB1, 0}, - {NPCM7XX_CLKEN3, 24, "gpiom0-gate", NPCM7XX_CLK_S_APB1, 0}, - {NPCM7XX_CLKEN3, 23, "espi-gate", NPCM7XX_CLK_S_APB2, 0}, - {NPCM7XX_CLKEN3, 22, "smb11-gate", NPCM7XX_CLK_S_APB2, 0}, - {NPCM7XX_CLKEN3, 21, "smb10-gate", NPCM7XX_CLK_S_APB2, 0}, - {NPCM7XX_CLKEN3, 20, "smb9-gate", NPCM7XX_CLK_S_APB2, 0}, - {NPCM7XX_CLKEN3, 19, "smb8-gate", NPCM7XX_CLK_S_APB2, 0}, - {NPCM7XX_CLKEN3, 18, "smb15-gate", NPCM7XX_CLK_S_APB2, 0}, - {NPCM7XX_CLKEN3, 17, "rng-gate", NPCM7XX_CLK_S_APB1, 0}, - {NPCM7XX_CLKEN3, 16, "timer10_14-gate", NPCM7XX_CLK_S_APB1, 0}, - {NPCM7XX_CLKEN3, 15, "pcirc-gate", NPCM7XX_CLK_S_AHB, 0}, - {NPCM7XX_CLKEN3, 14, "sececc-gate", NPCM7XX_CLK_S_AHB, 0}, - {NPCM7XX_CLKEN3, 13, "sha-gate", NPCM7XX_CLK_S_AHB, 0}, - {NPCM7XX_CLKEN3, 12, "smb14-gate", NPCM7XX_CLK_S_APB2, 0}, - /* bit 11 is reserved */ - /* bit 10 is reserved */ - {NPCM7XX_CLKEN3, 9, "pcimbx-gate", NPCM7XX_CLK_S_AHB, 0}, - /* bit 8 is reserved */ - {NPCM7XX_CLKEN3, 7, "usbdev9-gate", NPCM7XX_CLK_S_AHB, 0}, - {NPCM7XX_CLKEN3, 6, "usbdev8-gate", NPCM7XX_CLK_S_AHB, 0}, - {NPCM7XX_CLKEN3, 5, "usbdev7-gate", NPCM7XX_CLK_S_AHB, 0}, - {NPCM7XX_CLKEN3, 4, "usbdev0-gate", NPCM7XX_CLK_S_AHB, 0}, - {NPCM7XX_CLKEN3, 3, "smb13-gate", NPCM7XX_CLK_S_APB2, 0}, - {NPCM7XX_CLKEN3, 2, "spix-gate", NPCM7XX_CLK_S_AHB, 0}, - {NPCM7XX_CLKEN3, 1, "smb12-gate", NPCM7XX_CLK_S_APB2, 0}, - {NPCM7XX_CLKEN3, 0, "pwmm1-gate", NPCM7XX_CLK_S_APB3, 0}, -}; - static DEFINE_SPINLOCK(npcm7xx_clk_lock); static void __init npcm7xx_clk_init(struct device_node *clk_np) diff --git a/drivers/clk/clk-qoriq.c b/drivers/clk/clk-qoriq.c index 70aa521e7e7fa52f89a4180169a8583a2c3326fa..88898b97a443122193c105526b236b7c8507d49f 100644 --- a/drivers/clk/clk-qoriq.c +++ b/drivers/clk/clk-qoriq.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only /* * Copyright 2013 Freescale Semiconductor, Inc. + * Copyright 2021 NXP * * clock driver for Freescale QorIQ SoCs. */ @@ -564,7 +565,9 @@ static const struct clockgen_chipinfo chipinfo[] = { .cmux_to_group = { 0, 1, 1, 1, -1 }, - .pll_mask = 0x3f, + .pll_mask = BIT(PLATFORM_PLL) | + BIT(CGA_PLL1) | BIT(CGA_PLL2) | BIT(CGA_PLL3) | + BIT(CGB_PLL1) | BIT(CGB_PLL2), .flags = CG_PLL_8BIT, }, { @@ -580,7 +583,9 @@ static const struct clockgen_chipinfo chipinfo[] = { .cmux_to_group = { 0, 1, 1, 1, -1 }, - .pll_mask = 0x3f, + .pll_mask = BIT(PLATFORM_PLL) | + BIT(CGA_PLL1) | BIT(CGA_PLL2) | BIT(CGA_PLL3) | + BIT(CGB_PLL1) | BIT(CGB_PLL2), .flags = CG_PLL_8BIT, }, { @@ -591,7 +596,8 @@ static const struct clockgen_chipinfo chipinfo[] = { .cmux_to_group = { 0, -1 }, - .pll_mask = 0x03, + .pll_mask = BIT(PLATFORM_PLL) | + BIT(CGA_PLL1) | BIT(CGA_PLL2), }, { .compat = "fsl,ls1028a-clockgen", @@ -605,7 +611,8 @@ static const struct clockgen_chipinfo chipinfo[] = { .cmux_to_group = { 0, 0, 0, 0, -1 }, - .pll_mask = 0x07, + .pll_mask = BIT(PLATFORM_PLL) | + BIT(CGA_PLL1) | BIT(CGA_PLL2), .flags = CG_VER3 | CG_LITTLE_ENDIAN, }, { @@ -620,7 +627,8 @@ static const struct clockgen_chipinfo chipinfo[] = { .cmux_to_group = { 0, -1 }, - .pll_mask = 0x07, + .pll_mask = BIT(PLATFORM_PLL) | + BIT(CGA_PLL1) | BIT(CGA_PLL2), .flags = CG_PLL_8BIT, }, { @@ -635,7 +643,8 @@ static const struct clockgen_chipinfo chipinfo[] = { .cmux_to_group = { 0, -1 }, - .pll_mask = 0x07, + .pll_mask = BIT(PLATFORM_PLL) | + BIT(CGA_PLL1) | BIT(CGA_PLL2), .flags = CG_PLL_8BIT, }, { @@ -649,7 +658,8 @@ static const struct clockgen_chipinfo chipinfo[] = { .cmux_to_group = { 0, 0, -1 }, - .pll_mask = 0x07, + .pll_mask = BIT(PLATFORM_PLL) | + BIT(CGA_PLL1) | BIT(CGA_PLL2), .flags = CG_VER3 | CG_LITTLE_ENDIAN, }, { @@ -660,7 +670,7 @@ static const struct clockgen_chipinfo chipinfo[] = { .cmux_to_group = { 0, -1 }, - .pll_mask = 0x03, + .pll_mask = BIT(PLATFORM_PLL) | BIT(CGA_PLL1), }, { .compat = "fsl,ls2080a-clockgen", @@ -670,7 +680,9 @@ static const struct clockgen_chipinfo chipinfo[] = { .cmux_to_group = { 0, 0, 1, 1, -1 }, - .pll_mask = 0x37, + .pll_mask = BIT(PLATFORM_PLL) | + BIT(CGA_PLL1) | BIT(CGA_PLL2) | + BIT(CGB_PLL1) | BIT(CGB_PLL2), .flags = CG_VER3 | CG_LITTLE_ENDIAN, }, { @@ -681,7 +693,9 @@ static const struct clockgen_chipinfo chipinfo[] = { .cmux_to_group = { 0, 0, 0, 0, 1, 1, 1, 1, -1 }, - .pll_mask = 0x37, + .pll_mask = BIT(PLATFORM_PLL) | + BIT(CGA_PLL1) | BIT(CGA_PLL2) | + BIT(CGB_PLL1) | BIT(CGB_PLL2), .flags = CG_VER3 | CG_LITTLE_ENDIAN, }, { @@ -694,7 +708,8 @@ static const struct clockgen_chipinfo chipinfo[] = { .cmux_to_group = { 0, 0, 1, 1, -1 }, - .pll_mask = 0x07, + .pll_mask = BIT(PLATFORM_PLL) | + BIT(CGA_PLL1) | BIT(CGA_PLL2), }, { .compat = "fsl,p3041-clockgen", @@ -706,7 +721,8 @@ static const struct clockgen_chipinfo chipinfo[] = { .cmux_to_group = { 0, 0, 1, 1, -1 }, - .pll_mask = 0x07, + .pll_mask = BIT(PLATFORM_PLL) | + BIT(CGA_PLL1) | BIT(CGA_PLL2), }, { .compat = "fsl,p4080-clockgen", @@ -718,7 +734,9 @@ static const struct clockgen_chipinfo chipinfo[] = { .cmux_to_group = { 0, 0, 0, 0, 1, 1, 1, 1, -1 }, - .pll_mask = 0x1f, + .pll_mask = BIT(PLATFORM_PLL) | + BIT(CGA_PLL1) | BIT(CGA_PLL2) | + BIT(CGA_PLL3) | BIT(CGA_PLL4), }, { .compat = "fsl,p5020-clockgen", @@ -730,7 +748,8 @@ static const struct clockgen_chipinfo chipinfo[] = { .cmux_to_group = { 0, 1, -1 }, - .pll_mask = 0x07, + .pll_mask = BIT(PLATFORM_PLL) | + BIT(CGA_PLL1) | BIT(CGA_PLL2), }, { .compat = "fsl,p5040-clockgen", @@ -742,7 +761,8 @@ static const struct clockgen_chipinfo chipinfo[] = { .cmux_to_group = { 0, 0, 1, 1, -1 }, - .pll_mask = 0x0f, + .pll_mask = BIT(PLATFORM_PLL) | + BIT(CGA_PLL1) | BIT(CGA_PLL2) | BIT(CGA_PLL3), }, { .compat = "fsl,t1023-clockgen", @@ -757,7 +777,7 @@ static const struct clockgen_chipinfo chipinfo[] = { .cmux_to_group = { 0, 0, -1 }, - .pll_mask = 0x03, + .pll_mask = BIT(PLATFORM_PLL) | BIT(CGA_PLL1), .flags = CG_PLL_8BIT, }, { @@ -770,7 +790,8 @@ static const struct clockgen_chipinfo chipinfo[] = { .cmux_to_group = { 0, 0, 0, 0, -1 }, - .pll_mask = 0x07, + .pll_mask = BIT(PLATFORM_PLL) | + BIT(CGA_PLL1) | BIT(CGA_PLL2), .flags = CG_PLL_8BIT, }, { @@ -786,7 +807,8 @@ static const struct clockgen_chipinfo chipinfo[] = { .cmux_to_group = { 0, -1 }, - .pll_mask = 0x07, + .pll_mask = BIT(PLATFORM_PLL) | + BIT(CGA_PLL1) | BIT(CGA_PLL2), .flags = CG_PLL_8BIT, }, { @@ -802,7 +824,9 @@ static const struct clockgen_chipinfo chipinfo[] = { .cmux_to_group = { 0, 0, 1, -1 }, - .pll_mask = 0x3f, + .pll_mask = BIT(PLATFORM_PLL) | + BIT(CGA_PLL1) | BIT(CGA_PLL2) | BIT(CGA_PLL3) | + BIT(CGB_PLL1) | BIT(CGB_PLL2), .flags = CG_PLL_8BIT, }, {}, diff --git a/drivers/clk/clk-si570.c b/drivers/clk/clk-si570.c index 34b25609f55fa473c8c251c642b6331b1813128d..eea50121718a69996287483ea37f62476a99ba7b 100644 --- a/drivers/clk/clk-si570.c +++ b/drivers/clk/clk-si570.c @@ -4,7 +4,7 @@ * * Copyright (C) 2010, 2011 Ericsson AB. * Copyright (C) 2011 Guenter Roeck. - * Copyright (C) 2011 - 2013 Xilinx Inc. + * Copyright (C) 2011 - 2021 Xilinx Inc. * * Author: Guenter Roeck * Sören Brinkmann @@ -123,14 +123,18 @@ static int si570_get_divs(struct clk_si570 *data, u64 *rfreq, * si570_get_defaults() - Get default values * @data: Driver data structure * @fout: Factory frequency output + * @skip_recall: If true, don't recall NVM into RAM * Returns 0 on success, negative errno otherwise. */ -static int si570_get_defaults(struct clk_si570 *data, u64 fout) +static int si570_get_defaults(struct clk_si570 *data, u64 fout, + bool skip_recall) { int err; u64 fdco; - regmap_write(data->regmap, SI570_REG_CONTROL, SI570_CNTRL_RECALL); + if (!skip_recall) + regmap_write(data->regmap, SI570_REG_CONTROL, + SI570_CNTRL_RECALL); err = si570_get_divs(data, &data->rfreq, &data->n1, &data->hs_div); if (err) @@ -400,6 +404,7 @@ static int si570_probe(struct i2c_client *client, struct clk_si570 *data; struct clk_init_data init; u32 initial_fout, factory_fout, stability; + bool skip_recall; int err; enum clk_si570_variant variant = id->driver_data; @@ -441,6 +446,9 @@ static int si570_probe(struct i2c_client *client, return err; } + skip_recall = of_property_read_bool(client->dev.of_node, + "silabs,skip-recall"); + data->regmap = devm_regmap_init_i2c(client, &si570_regmap_config); if (IS_ERR(data->regmap)) { dev_err(&client->dev, "failed to allocate register map\n"); @@ -448,7 +456,7 @@ static int si570_probe(struct i2c_client *client, } i2c_set_clientdata(client, data); - err = si570_get_defaults(data, factory_fout); + err = si570_get_defaults(data, factory_fout, skip_recall); if (err) return err; diff --git a/drivers/clk/clk-tango4.c b/drivers/clk/clk-tango4.c deleted file mode 100644 index fe12a43f7a4008011aa9c8af774150abbeb19354..0000000000000000000000000000000000000000 --- a/drivers/clk/clk-tango4.c +++ /dev/null @@ -1,85 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -#include -#include -#include -#include -#include - -#define CLK_COUNT 4 /* cpu_clk, sys_clk, usb_clk, sdio_clk */ -static struct clk *clks[CLK_COUNT]; -static struct clk_onecell_data clk_data = { clks, CLK_COUNT }; - -#define SYSCLK_DIV 0x20 -#define CPUCLK_DIV 0x24 -#define DIV_BYPASS BIT(23) - -/*** CLKGEN_PLL ***/ -#define extract_pll_n(val) ((val >> 0) & ((1u << 7) - 1)) -#define extract_pll_k(val) ((val >> 13) & ((1u << 3) - 1)) -#define extract_pll_m(val) ((val >> 16) & ((1u << 3) - 1)) -#define extract_pll_isel(val) ((val >> 24) & ((1u << 3) - 1)) - -static void __init make_pll(int idx, const char *parent, void __iomem *base) -{ - char name[8]; - u32 val, mul, div; - - sprintf(name, "pll%d", idx); - val = readl(base + idx * 8); - mul = extract_pll_n(val) + 1; - div = (extract_pll_m(val) + 1) << extract_pll_k(val); - clk_register_fixed_factor(NULL, name, parent, 0, mul, div); - if (extract_pll_isel(val) != 1) - panic("%s: input not set to XTAL_IN\n", name); -} - -static void __init make_cd(int idx, void __iomem *base) -{ - char name[8]; - u32 val, mul, div; - - sprintf(name, "cd%d", idx); - val = readl(base + idx * 8); - mul = 1 << 27; - div = (2 << 27) + val; - clk_register_fixed_factor(NULL, name, "pll2", 0, mul, div); - if (val > 0xf0000000) - panic("%s: unsupported divider %x\n", name, val); -} - -static void __init tango4_clkgen_setup(struct device_node *np) -{ - struct clk **pp = clk_data.clks; - void __iomem *base = of_iomap(np, 0); - const char *parent = of_clk_get_parent_name(np, 0); - - if (!base) - panic("%pOFn: invalid address\n", np); - - if (readl(base + CPUCLK_DIV) & DIV_BYPASS) - panic("%pOFn: unsupported cpuclk setup\n", np); - - if (readl(base + SYSCLK_DIV) & DIV_BYPASS) - panic("%pOFn: unsupported sysclk setup\n", np); - - writel(0x100, base + CPUCLK_DIV); /* disable frequency ramping */ - - make_pll(0, parent, base); - make_pll(1, parent, base); - make_pll(2, parent, base); - make_cd(2, base + 0x80); - make_cd(6, base + 0x80); - - pp[0] = clk_register_divider(NULL, "cpu_clk", "pll0", 0, - base + CPUCLK_DIV, 8, 8, CLK_DIVIDER_ONE_BASED, NULL); - pp[1] = clk_register_fixed_factor(NULL, "sys_clk", "pll1", 0, 1, 4); - pp[2] = clk_register_fixed_factor(NULL, "usb_clk", "cd2", 0, 1, 2); - pp[3] = clk_register_fixed_factor(NULL, "sdio_clk", "cd6", 0, 1, 2); - - if (IS_ERR(pp[0]) || IS_ERR(pp[1]) || IS_ERR(pp[2]) || IS_ERR(pp[3])) - panic("%pOFn: clk registration failed\n", np); - - if (of_clk_add_provider(np, of_clk_src_onecell_get, &clk_data)) - panic("%pOFn: clk provider registration failed\n", np); -} -CLK_OF_DECLARE(tango4_clkgen, "sigma,tango4-clkgen", tango4_clkgen_setup); diff --git a/drivers/clk/clk-u300.c b/drivers/clk/clk-u300.c deleted file mode 100644 index e228c07c4c6eec787d122637f0ccf30575b06107..0000000000000000000000000000000000000000 --- a/drivers/clk/clk-u300.c +++ /dev/null @@ -1,1199 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * U300 clock implementation - * Copyright (C) 2007-2012 ST-Ericsson AB - * Author: Linus Walleij - * Author: Jonas Aaberg - */ -#include -#include -#include -#include -#include -#include -#include -#include - -/* APP side SYSCON registers */ -/* CLK Control Register 16bit (R/W) */ -#define U300_SYSCON_CCR (0x0000) -#define U300_SYSCON_CCR_I2S1_USE_VCXO (0x0040) -#define U300_SYSCON_CCR_I2S0_USE_VCXO (0x0020) -#define U300_SYSCON_CCR_TURN_VCXO_ON (0x0008) -#define U300_SYSCON_CCR_CLKING_PERFORMANCE_MASK (0x0007) -#define U300_SYSCON_CCR_CLKING_PERFORMANCE_LOW_POWER (0x04) -#define U300_SYSCON_CCR_CLKING_PERFORMANCE_LOW (0x03) -#define U300_SYSCON_CCR_CLKING_PERFORMANCE_INTERMEDIATE (0x02) -#define U300_SYSCON_CCR_CLKING_PERFORMANCE_HIGH (0x01) -#define U300_SYSCON_CCR_CLKING_PERFORMANCE_BEST (0x00) -/* CLK Status Register 16bit (R/W) */ -#define U300_SYSCON_CSR (0x0004) -#define U300_SYSCON_CSR_PLL208_LOCK_IND (0x0002) -#define U300_SYSCON_CSR_PLL13_LOCK_IND (0x0001) -/* Reset lines for SLOW devices 16bit (R/W) */ -#define U300_SYSCON_RSR (0x0014) -#define U300_SYSCON_RSR_PPM_RESET_EN (0x0200) -#define U300_SYSCON_RSR_ACC_TMR_RESET_EN (0x0100) -#define U300_SYSCON_RSR_APP_TMR_RESET_EN (0x0080) -#define U300_SYSCON_RSR_RTC_RESET_EN (0x0040) -#define U300_SYSCON_RSR_KEYPAD_RESET_EN (0x0020) -#define U300_SYSCON_RSR_GPIO_RESET_EN (0x0010) -#define U300_SYSCON_RSR_EH_RESET_EN (0x0008) -#define U300_SYSCON_RSR_BTR_RESET_EN (0x0004) -#define U300_SYSCON_RSR_UART_RESET_EN (0x0002) -#define U300_SYSCON_RSR_SLOW_BRIDGE_RESET_EN (0x0001) -/* Reset lines for FAST devices 16bit (R/W) */ -#define U300_SYSCON_RFR (0x0018) -#define U300_SYSCON_RFR_UART1_RESET_ENABLE (0x0080) -#define U300_SYSCON_RFR_SPI_RESET_ENABLE (0x0040) -#define U300_SYSCON_RFR_MMC_RESET_ENABLE (0x0020) -#define U300_SYSCON_RFR_PCM_I2S1_RESET_ENABLE (0x0010) -#define U300_SYSCON_RFR_PCM_I2S0_RESET_ENABLE (0x0008) -#define U300_SYSCON_RFR_I2C1_RESET_ENABLE (0x0004) -#define U300_SYSCON_RFR_I2C0_RESET_ENABLE (0x0002) -#define U300_SYSCON_RFR_FAST_BRIDGE_RESET_ENABLE (0x0001) -/* Reset lines for the rest of the peripherals 16bit (R/W) */ -#define U300_SYSCON_RRR (0x001c) -#define U300_SYSCON_RRR_CDS_RESET_EN (0x4000) -#define U300_SYSCON_RRR_ISP_RESET_EN (0x2000) -#define U300_SYSCON_RRR_INTCON_RESET_EN (0x1000) -#define U300_SYSCON_RRR_MSPRO_RESET_EN (0x0800) -#define U300_SYSCON_RRR_XGAM_RESET_EN (0x0100) -#define U300_SYSCON_RRR_XGAM_VC_SYNC_RESET_EN (0x0080) -#define U300_SYSCON_RRR_NANDIF_RESET_EN (0x0040) -#define U300_SYSCON_RRR_EMIF_RESET_EN (0x0020) -#define U300_SYSCON_RRR_DMAC_RESET_EN (0x0010) -#define U300_SYSCON_RRR_CPU_RESET_EN (0x0008) -#define U300_SYSCON_RRR_APEX_RESET_EN (0x0004) -#define U300_SYSCON_RRR_AHB_RESET_EN (0x0002) -#define U300_SYSCON_RRR_AAIF_RESET_EN (0x0001) -/* Clock enable for SLOW peripherals 16bit (R/W) */ -#define U300_SYSCON_CESR (0x0020) -#define U300_SYSCON_CESR_PPM_CLK_EN (0x0200) -#define U300_SYSCON_CESR_ACC_TMR_CLK_EN (0x0100) -#define U300_SYSCON_CESR_APP_TMR_CLK_EN (0x0080) -#define U300_SYSCON_CESR_KEYPAD_CLK_EN (0x0040) -#define U300_SYSCON_CESR_GPIO_CLK_EN (0x0010) -#define U300_SYSCON_CESR_EH_CLK_EN (0x0008) -#define U300_SYSCON_CESR_BTR_CLK_EN (0x0004) -#define U300_SYSCON_CESR_UART_CLK_EN (0x0002) -#define U300_SYSCON_CESR_SLOW_BRIDGE_CLK_EN (0x0001) -/* Clock enable for FAST peripherals 16bit (R/W) */ -#define U300_SYSCON_CEFR (0x0024) -#define U300_SYSCON_CEFR_UART1_CLK_EN (0x0200) -#define U300_SYSCON_CEFR_I2S1_CORE_CLK_EN (0x0100) -#define U300_SYSCON_CEFR_I2S0_CORE_CLK_EN (0x0080) -#define U300_SYSCON_CEFR_SPI_CLK_EN (0x0040) -#define U300_SYSCON_CEFR_MMC_CLK_EN (0x0020) -#define U300_SYSCON_CEFR_I2S1_CLK_EN (0x0010) -#define U300_SYSCON_CEFR_I2S0_CLK_EN (0x0008) -#define U300_SYSCON_CEFR_I2C1_CLK_EN (0x0004) -#define U300_SYSCON_CEFR_I2C0_CLK_EN (0x0002) -#define U300_SYSCON_CEFR_FAST_BRIDGE_CLK_EN (0x0001) -/* Clock enable for the rest of the peripherals 16bit (R/W) */ -#define U300_SYSCON_CERR (0x0028) -#define U300_SYSCON_CERR_CDS_CLK_EN (0x2000) -#define U300_SYSCON_CERR_ISP_CLK_EN (0x1000) -#define U300_SYSCON_CERR_MSPRO_CLK_EN (0x0800) -#define U300_SYSCON_CERR_AHB_SUBSYS_BRIDGE_CLK_EN (0x0400) -#define U300_SYSCON_CERR_SEMI_CLK_EN (0x0200) -#define U300_SYSCON_CERR_XGAM_CLK_EN (0x0100) -#define U300_SYSCON_CERR_VIDEO_ENC_CLK_EN (0x0080) -#define U300_SYSCON_CERR_NANDIF_CLK_EN (0x0040) -#define U300_SYSCON_CERR_EMIF_CLK_EN (0x0020) -#define U300_SYSCON_CERR_DMAC_CLK_EN (0x0010) -#define U300_SYSCON_CERR_CPU_CLK_EN (0x0008) -#define U300_SYSCON_CERR_APEX_CLK_EN (0x0004) -#define U300_SYSCON_CERR_AHB_CLK_EN (0x0002) -#define U300_SYSCON_CERR_AAIF_CLK_EN (0x0001) -/* Single block clock enable 16bit (-/W) */ -#define U300_SYSCON_SBCER (0x002c) -#define U300_SYSCON_SBCER_PPM_CLK_EN (0x0009) -#define U300_SYSCON_SBCER_ACC_TMR_CLK_EN (0x0008) -#define U300_SYSCON_SBCER_APP_TMR_CLK_EN (0x0007) -#define U300_SYSCON_SBCER_KEYPAD_CLK_EN (0x0006) -#define U300_SYSCON_SBCER_GPIO_CLK_EN (0x0004) -#define U300_SYSCON_SBCER_EH_CLK_EN (0x0003) -#define U300_SYSCON_SBCER_BTR_CLK_EN (0x0002) -#define U300_SYSCON_SBCER_UART_CLK_EN (0x0001) -#define U300_SYSCON_SBCER_SLOW_BRIDGE_CLK_EN (0x0000) -#define U300_SYSCON_SBCER_UART1_CLK_EN (0x0019) -#define U300_SYSCON_SBCER_I2S1_CORE_CLK_EN (0x0018) -#define U300_SYSCON_SBCER_I2S0_CORE_CLK_EN (0x0017) -#define U300_SYSCON_SBCER_SPI_CLK_EN (0x0016) -#define U300_SYSCON_SBCER_MMC_CLK_EN (0x0015) -#define U300_SYSCON_SBCER_I2S1_CLK_EN (0x0014) -#define U300_SYSCON_SBCER_I2S0_CLK_EN (0x0013) -#define U300_SYSCON_SBCER_I2C1_CLK_EN (0x0012) -#define U300_SYSCON_SBCER_I2C0_CLK_EN (0x0011) -#define U300_SYSCON_SBCER_FAST_BRIDGE_CLK_EN (0x0010) -#define U300_SYSCON_SBCER_CDS_CLK_EN (0x002D) -#define U300_SYSCON_SBCER_ISP_CLK_EN (0x002C) -#define U300_SYSCON_SBCER_MSPRO_CLK_EN (0x002B) -#define U300_SYSCON_SBCER_AHB_SUBSYS_BRIDGE_CLK_EN (0x002A) -#define U300_SYSCON_SBCER_SEMI_CLK_EN (0x0029) -#define U300_SYSCON_SBCER_XGAM_CLK_EN (0x0028) -#define U300_SYSCON_SBCER_VIDEO_ENC_CLK_EN (0x0027) -#define U300_SYSCON_SBCER_NANDIF_CLK_EN (0x0026) -#define U300_SYSCON_SBCER_EMIF_CLK_EN (0x0025) -#define U300_SYSCON_SBCER_DMAC_CLK_EN (0x0024) -#define U300_SYSCON_SBCER_CPU_CLK_EN (0x0023) -#define U300_SYSCON_SBCER_APEX_CLK_EN (0x0022) -#define U300_SYSCON_SBCER_AHB_CLK_EN (0x0021) -#define U300_SYSCON_SBCER_AAIF_CLK_EN (0x0020) -/* Single block clock disable 16bit (-/W) */ -#define U300_SYSCON_SBCDR (0x0030) -/* Same values as above for SBCER */ -/* Clock force SLOW peripherals 16bit (R/W) */ -#define U300_SYSCON_CFSR (0x003c) -#define U300_SYSCON_CFSR_PPM_CLK_FORCE_EN (0x0200) -#define U300_SYSCON_CFSR_ACC_TMR_CLK_FORCE_EN (0x0100) -#define U300_SYSCON_CFSR_APP_TMR_CLK_FORCE_EN (0x0080) -#define U300_SYSCON_CFSR_KEYPAD_CLK_FORCE_EN (0x0020) -#define U300_SYSCON_CFSR_GPIO_CLK_FORCE_EN (0x0010) -#define U300_SYSCON_CFSR_EH_CLK_FORCE_EN (0x0008) -#define U300_SYSCON_CFSR_BTR_CLK_FORCE_EN (0x0004) -#define U300_SYSCON_CFSR_UART_CLK_FORCE_EN (0x0002) -#define U300_SYSCON_CFSR_SLOW_BRIDGE_CLK_FORCE_EN (0x0001) -/* Clock force FAST peripherals 16bit (R/W) */ -#define U300_SYSCON_CFFR (0x40) -/* Values not defined. Define if you want to use them. */ -/* Clock force the rest of the peripherals 16bit (R/W) */ -#define U300_SYSCON_CFRR (0x44) -#define U300_SYSCON_CFRR_CDS_CLK_FORCE_EN (0x2000) -#define U300_SYSCON_CFRR_ISP_CLK_FORCE_EN (0x1000) -#define U300_SYSCON_CFRR_MSPRO_CLK_FORCE_EN (0x0800) -#define U300_SYSCON_CFRR_AHB_SUBSYS_BRIDGE_CLK_FORCE_EN (0x0400) -#define U300_SYSCON_CFRR_SEMI_CLK_FORCE_EN (0x0200) -#define U300_SYSCON_CFRR_XGAM_CLK_FORCE_EN (0x0100) -#define U300_SYSCON_CFRR_VIDEO_ENC_CLK_FORCE_EN (0x0080) -#define U300_SYSCON_CFRR_NANDIF_CLK_FORCE_EN (0x0040) -#define U300_SYSCON_CFRR_EMIF_CLK_FORCE_EN (0x0020) -#define U300_SYSCON_CFRR_DMAC_CLK_FORCE_EN (0x0010) -#define U300_SYSCON_CFRR_CPU_CLK_FORCE_EN (0x0008) -#define U300_SYSCON_CFRR_APEX_CLK_FORCE_EN (0x0004) -#define U300_SYSCON_CFRR_AHB_CLK_FORCE_EN (0x0002) -#define U300_SYSCON_CFRR_AAIF_CLK_FORCE_EN (0x0001) -/* PLL208 Frequency Control 16bit (R/W) */ -#define U300_SYSCON_PFCR (0x48) -#define U300_SYSCON_PFCR_DPLL_MULT_NUM (0x000F) -/* Power Management Control 16bit (R/W) */ -#define U300_SYSCON_PMCR (0x50) -#define U300_SYSCON_PMCR_DCON_ENABLE (0x0002) -#define U300_SYSCON_PMCR_PWR_MGNT_ENABLE (0x0001) -/* Reset Out 16bit (R/W) */ -#define U300_SYSCON_RCR (0x6c) -#define U300_SYSCON_RCR_RESOUT0_RST_N_DISABLE (0x0001) -/* EMIF Slew Rate Control 16bit (R/W) */ -#define U300_SYSCON_SRCLR (0x70) -#define U300_SYSCON_SRCLR_MASK (0x03FF) -#define U300_SYSCON_SRCLR_VALUE (0x03FF) -#define U300_SYSCON_SRCLR_EMIF_1_SLRC_5_B (0x0200) -#define U300_SYSCON_SRCLR_EMIF_1_SLRC_5_A (0x0100) -#define U300_SYSCON_SRCLR_EMIF_1_SLRC_4_B (0x0080) -#define U300_SYSCON_SRCLR_EMIF_1_SLRC_4_A (0x0040) -#define U300_SYSCON_SRCLR_EMIF_1_SLRC_3_B (0x0020) -#define U300_SYSCON_SRCLR_EMIF_1_SLRC_3_A (0x0010) -#define U300_SYSCON_SRCLR_EMIF_1_SLRC_2_B (0x0008) -#define U300_SYSCON_SRCLR_EMIF_1_SLRC_2_A (0x0004) -#define U300_SYSCON_SRCLR_EMIF_1_SLRC_1_B (0x0002) -#define U300_SYSCON_SRCLR_EMIF_1_SLRC_1_A (0x0001) -/* EMIF Clock Control Register 16bit (R/W) */ -#define U300_SYSCON_ECCR (0x0078) -#define U300_SYSCON_ECCR_MASK (0x000F) -#define U300_SYSCON_ECCR_EMIF_1_STATIC_CLK_EN_N_DISABLE (0x0008) -#define U300_SYSCON_ECCR_EMIF_1_RET_OUT_CLK_EN_N_DISABLE (0x0004) -#define U300_SYSCON_ECCR_EMIF_MEMCLK_RET_EN_N_DISABLE (0x0002) -#define U300_SYSCON_ECCR_EMIF_SDRCLK_RET_EN_N_DISABLE (0x0001) -/* MMC/MSPRO frequency divider register 0 16bit (R/W) */ -#define U300_SYSCON_MMF0R (0x90) -#define U300_SYSCON_MMF0R_MASK (0x00FF) -#define U300_SYSCON_MMF0R_FREQ_0_HIGH_MASK (0x00F0) -#define U300_SYSCON_MMF0R_FREQ_0_LOW_MASK (0x000F) -/* MMC/MSPRO frequency divider register 1 16bit (R/W) */ -#define U300_SYSCON_MMF1R (0x94) -#define U300_SYSCON_MMF1R_MASK (0x00FF) -#define U300_SYSCON_MMF1R_FREQ_1_HIGH_MASK (0x00F0) -#define U300_SYSCON_MMF1R_FREQ_1_LOW_MASK (0x000F) -/* Clock control for the MMC and MSPRO blocks 16bit (R/W) */ -#define U300_SYSCON_MMCR (0x9C) -#define U300_SYSCON_MMCR_MASK (0x0003) -#define U300_SYSCON_MMCR_MMC_FB_CLK_SEL_ENABLE (0x0002) -#define U300_SYSCON_MMCR_MSPRO_FREQSEL_ENABLE (0x0001) -/* SYS_0_CLK_CONTROL first clock control 16bit (R/W) */ -#define U300_SYSCON_S0CCR (0x120) -#define U300_SYSCON_S0CCR_FIELD_MASK (0x43FF) -#define U300_SYSCON_S0CCR_CLOCK_REQ (0x4000) -#define U300_SYSCON_S0CCR_CLOCK_REQ_MONITOR (0x2000) -#define U300_SYSCON_S0CCR_CLOCK_INV (0x0200) -#define U300_SYSCON_S0CCR_CLOCK_FREQ_MASK (0x01E0) -#define U300_SYSCON_S0CCR_CLOCK_SELECT_MASK (0x001E) -#define U300_SYSCON_S0CCR_CLOCK_ENABLE (0x0001) -#define U300_SYSCON_S0CCR_SEL_MCLK (0x8 << 1) -#define U300_SYSCON_S0CCR_SEL_ACC_FSM_CLK (0xA << 1) -#define U300_SYSCON_S0CCR_SEL_PLL60_48_CLK (0xC << 1) -#define U300_SYSCON_S0CCR_SEL_PLL60_60_CLK (0xD << 1) -#define U300_SYSCON_S0CCR_SEL_ACC_PLL208_CLK (0xE << 1) -#define U300_SYSCON_S0CCR_SEL_APP_PLL13_CLK (0x0 << 1) -#define U300_SYSCON_S0CCR_SEL_APP_FSM_CLK (0x2 << 1) -#define U300_SYSCON_S0CCR_SEL_RTC_CLK (0x4 << 1) -#define U300_SYSCON_S0CCR_SEL_APP_PLL208_CLK (0x6 << 1) -/* SYS_1_CLK_CONTROL second clock control 16 bit (R/W) */ -#define U300_SYSCON_S1CCR (0x124) -#define U300_SYSCON_S1CCR_FIELD_MASK (0x43FF) -#define U300_SYSCON_S1CCR_CLOCK_REQ (0x4000) -#define U300_SYSCON_S1CCR_CLOCK_REQ_MONITOR (0x2000) -#define U300_SYSCON_S1CCR_CLOCK_INV (0x0200) -#define U300_SYSCON_S1CCR_CLOCK_FREQ_MASK (0x01E0) -#define U300_SYSCON_S1CCR_CLOCK_SELECT_MASK (0x001E) -#define U300_SYSCON_S1CCR_CLOCK_ENABLE (0x0001) -#define U300_SYSCON_S1CCR_SEL_MCLK (0x8 << 1) -#define U300_SYSCON_S1CCR_SEL_ACC_FSM_CLK (0xA << 1) -#define U300_SYSCON_S1CCR_SEL_PLL60_48_CLK (0xC << 1) -#define U300_SYSCON_S1CCR_SEL_PLL60_60_CLK (0xD << 1) -#define U300_SYSCON_S1CCR_SEL_ACC_PLL208_CLK (0xE << 1) -#define U300_SYSCON_S1CCR_SEL_ACC_PLL13_CLK (0x0 << 1) -#define U300_SYSCON_S1CCR_SEL_APP_FSM_CLK (0x2 << 1) -#define U300_SYSCON_S1CCR_SEL_RTC_CLK (0x4 << 1) -#define U300_SYSCON_S1CCR_SEL_APP_PLL208_CLK (0x6 << 1) -/* SYS_2_CLK_CONTROL third clock control 16 bit (R/W) */ -#define U300_SYSCON_S2CCR (0x128) -#define U300_SYSCON_S2CCR_FIELD_MASK (0xC3FF) -#define U300_SYSCON_S2CCR_CLK_STEAL (0x8000) -#define U300_SYSCON_S2CCR_CLOCK_REQ (0x4000) -#define U300_SYSCON_S2CCR_CLOCK_REQ_MONITOR (0x2000) -#define U300_SYSCON_S2CCR_CLOCK_INV (0x0200) -#define U300_SYSCON_S2CCR_CLOCK_FREQ_MASK (0x01E0) -#define U300_SYSCON_S2CCR_CLOCK_SELECT_MASK (0x001E) -#define U300_SYSCON_S2CCR_CLOCK_ENABLE (0x0001) -#define U300_SYSCON_S2CCR_SEL_MCLK (0x8 << 1) -#define U300_SYSCON_S2CCR_SEL_ACC_FSM_CLK (0xA << 1) -#define U300_SYSCON_S2CCR_SEL_PLL60_48_CLK (0xC << 1) -#define U300_SYSCON_S2CCR_SEL_PLL60_60_CLK (0xD << 1) -#define U300_SYSCON_S2CCR_SEL_ACC_PLL208_CLK (0xE << 1) -#define U300_SYSCON_S2CCR_SEL_ACC_PLL13_CLK (0x0 << 1) -#define U300_SYSCON_S2CCR_SEL_APP_FSM_CLK (0x2 << 1) -#define U300_SYSCON_S2CCR_SEL_RTC_CLK (0x4 << 1) -#define U300_SYSCON_S2CCR_SEL_APP_PLL208_CLK (0x6 << 1) -/* SC_PLL_IRQ_CONTROL 16bit (R/W) */ -#define U300_SYSCON_PICR (0x0130) -#define U300_SYSCON_PICR_MASK (0x00FF) -#define U300_SYSCON_PICR_FORCE_PLL208_LOCK_LOW_ENABLE (0x0080) -#define U300_SYSCON_PICR_FORCE_PLL208_LOCK_HIGH_ENABLE (0x0040) -#define U300_SYSCON_PICR_FORCE_PLL13_LOCK_LOW_ENABLE (0x0020) -#define U300_SYSCON_PICR_FORCE_PLL13_LOCK_HIGH_ENABLE (0x0010) -#define U300_SYSCON_PICR_IRQMASK_PLL13_UNLOCK_ENABLE (0x0008) -#define U300_SYSCON_PICR_IRQMASK_PLL13_LOCK_ENABLE (0x0004) -#define U300_SYSCON_PICR_IRQMASK_PLL208_UNLOCK_ENABLE (0x0002) -#define U300_SYSCON_PICR_IRQMASK_PLL208_LOCK_ENABLE (0x0001) -/* SC_PLL_IRQ_STATUS 16 bit (R/-) */ -#define U300_SYSCON_PISR (0x0134) -#define U300_SYSCON_PISR_MASK (0x000F) -#define U300_SYSCON_PISR_PLL13_UNLOCK_IND (0x0008) -#define U300_SYSCON_PISR_PLL13_LOCK_IND (0x0004) -#define U300_SYSCON_PISR_PLL208_UNLOCK_IND (0x0002) -#define U300_SYSCON_PISR_PLL208_LOCK_IND (0x0001) -/* SC_PLL_IRQ_CLEAR 16 bit (-/W) */ -#define U300_SYSCON_PICLR (0x0138) -#define U300_SYSCON_PICLR_MASK (0x000F) -#define U300_SYSCON_PICLR_RWMASK (0x0000) -#define U300_SYSCON_PICLR_PLL13_UNLOCK_SC (0x0008) -#define U300_SYSCON_PICLR_PLL13_LOCK_SC (0x0004) -#define U300_SYSCON_PICLR_PLL208_UNLOCK_SC (0x0002) -#define U300_SYSCON_PICLR_PLL208_LOCK_SC (0x0001) -/* Clock activity observability register 0 */ -#define U300_SYSCON_C0OAR (0x140) -#define U300_SYSCON_C0OAR_MASK (0xFFFF) -#define U300_SYSCON_C0OAR_VALUE (0xFFFF) -#define U300_SYSCON_C0OAR_BT_H_CLK (0x8000) -#define U300_SYSCON_C0OAR_ASPB_P_CLK (0x4000) -#define U300_SYSCON_C0OAR_APP_SEMI_H_CLK (0x2000) -#define U300_SYSCON_C0OAR_APP_SEMI_CLK (0x1000) -#define U300_SYSCON_C0OAR_APP_MMC_MSPRO_CLK (0x0800) -#define U300_SYSCON_C0OAR_APP_I2S1_CLK (0x0400) -#define U300_SYSCON_C0OAR_APP_I2S0_CLK (0x0200) -#define U300_SYSCON_C0OAR_APP_CPU_CLK (0x0100) -#define U300_SYSCON_C0OAR_APP_52_CLK (0x0080) -#define U300_SYSCON_C0OAR_APP_208_CLK (0x0040) -#define U300_SYSCON_C0OAR_APP_104_CLK (0x0020) -#define U300_SYSCON_C0OAR_APEX_CLK (0x0010) -#define U300_SYSCON_C0OAR_AHPB_M_H_CLK (0x0008) -#define U300_SYSCON_C0OAR_AHB_CLK (0x0004) -#define U300_SYSCON_C0OAR_AFPB_P_CLK (0x0002) -#define U300_SYSCON_C0OAR_AAIF_CLK (0x0001) -/* Clock activity observability register 1 */ -#define U300_SYSCON_C1OAR (0x144) -#define U300_SYSCON_C1OAR_MASK (0x3FFE) -#define U300_SYSCON_C1OAR_VALUE (0x3FFE) -#define U300_SYSCON_C1OAR_NFIF_F_CLK (0x2000) -#define U300_SYSCON_C1OAR_MSPRO_CLK (0x1000) -#define U300_SYSCON_C1OAR_MMC_P_CLK (0x0800) -#define U300_SYSCON_C1OAR_MMC_CLK (0x0400) -#define U300_SYSCON_C1OAR_KP_P_CLK (0x0200) -#define U300_SYSCON_C1OAR_I2C1_P_CLK (0x0100) -#define U300_SYSCON_C1OAR_I2C0_P_CLK (0x0080) -#define U300_SYSCON_C1OAR_GPIO_CLK (0x0040) -#define U300_SYSCON_C1OAR_EMIF_MPMC_CLK (0x0020) -#define U300_SYSCON_C1OAR_EMIF_H_CLK (0x0010) -#define U300_SYSCON_C1OAR_EVHIST_CLK (0x0008) -#define U300_SYSCON_C1OAR_PPM_CLK (0x0004) -#define U300_SYSCON_C1OAR_DMA_CLK (0x0002) -/* Clock activity observability register 2 */ -#define U300_SYSCON_C2OAR (0x148) -#define U300_SYSCON_C2OAR_MASK (0x0FFF) -#define U300_SYSCON_C2OAR_VALUE (0x0FFF) -#define U300_SYSCON_C2OAR_XGAM_CDI_CLK (0x0800) -#define U300_SYSCON_C2OAR_XGAM_CLK (0x0400) -#define U300_SYSCON_C2OAR_VC_H_CLK (0x0200) -#define U300_SYSCON_C2OAR_VC_CLK (0x0100) -#define U300_SYSCON_C2OAR_UA_P_CLK (0x0080) -#define U300_SYSCON_C2OAR_TMR1_CLK (0x0040) -#define U300_SYSCON_C2OAR_TMR0_CLK (0x0020) -#define U300_SYSCON_C2OAR_SPI_P_CLK (0x0010) -#define U300_SYSCON_C2OAR_PCM_I2S1_CORE_CLK (0x0008) -#define U300_SYSCON_C2OAR_PCM_I2S1_CLK (0x0004) -#define U300_SYSCON_C2OAR_PCM_I2S0_CORE_CLK (0x0002) -#define U300_SYSCON_C2OAR_PCM_I2S0_CLK (0x0001) - - -/* - * The clocking hierarchy currently looks like this. - * NOTE: the idea is NOT to show how the clocks are routed on the chip! - * The ideas is to show dependencies, so a clock higher up in the - * hierarchy has to be on in order for another clock to be on. Now, - * both CPU and DMA can actually be on top of the hierarchy, and that - * is not modeled currently. Instead we have the backbone AMBA bus on - * top. This bus cannot be programmed in any way but conceptually it - * needs to be active for the bridges and devices to transport data. - * - * Please be aware that a few clocks are hw controlled, which mean that - * the hw itself can turn on/off or change the rate of the clock when - * needed! - * - * AMBA bus - * | - * +- CPU - * +- FSMC NANDIF NAND Flash interface - * +- SEMI Shared Memory interface - * +- ISP Image Signal Processor (U335 only) - * +- CDS (U335 only) - * +- DMA Direct Memory Access Controller - * +- AAIF APP/ACC Interface (Mobile Scalable Link, MSL) - * +- APEX - * +- VIDEO_ENC AVE2/3 Video Encoder - * +- XGAM Graphics Accelerator Controller - * +- AHB - * | - * +- ahb:0 AHB Bridge - * | | - * | +- ahb:1 INTCON Interrupt controller - * | +- ahb:3 MSPRO Memory Stick Pro controller - * | +- ahb:4 EMIF External Memory interface - * | - * +- fast:0 FAST bridge - * | | - * | +- fast:1 MMCSD MMC/SD card reader controller - * | +- fast:2 I2S0 PCM I2S channel 0 controller - * | +- fast:3 I2S1 PCM I2S channel 1 controller - * | +- fast:4 I2C0 I2C channel 0 controller - * | +- fast:5 I2C1 I2C channel 1 controller - * | +- fast:6 SPI SPI controller - * | +- fast:7 UART1 Secondary UART (U335 only) - * | - * +- slow:0 SLOW bridge - * | - * +- slow:1 SYSCON (not possible to control) - * +- slow:2 WDOG Watchdog - * +- slow:3 UART0 primary UART - * +- slow:4 TIMER_APP Application timer - used in Linux - * +- slow:5 KEYPAD controller - * +- slow:6 GPIO controller - * +- slow:7 RTC controller - * +- slow:8 BT Bus Tracer (not used currently) - * +- slow:9 EH Event Handler (not used currently) - * +- slow:a TIMER_ACC Access style timer (not used currently) - * +- slow:b PPM (U335 only, what is that?) - */ - -/* Global syscon virtual base */ -static void __iomem *syscon_vbase; - -/** - * struct clk_syscon - U300 syscon clock - * @hw: corresponding clock hardware entry - * @hw_ctrld: whether this clock is hardware controlled (for refcount etc) - * and does not need any magic pokes to be enabled/disabled - * @reset: state holder, whether this block's reset line is asserted or not - * @res_reg: reset line enable/disable flag register - * @res_bit: bit for resetting or taking this consumer out of reset - * @en_reg: clock line enable/disable flag register - * @en_bit: bit for enabling/disabling this consumer clock line - * @clk_val: magic value to poke in the register to enable/disable - * this one clock - */ -struct clk_syscon { - struct clk_hw hw; - bool hw_ctrld; - bool reset; - void __iomem *res_reg; - u8 res_bit; - void __iomem *en_reg; - u8 en_bit; - u16 clk_val; -}; - -#define to_syscon(_hw) container_of(_hw, struct clk_syscon, hw) - -static DEFINE_SPINLOCK(syscon_resetreg_lock); - -/* - * Reset control functions. We remember if a block has been - * taken out of reset and don't remove the reset assertion again - * and vice versa. Currently we only remove resets so the - * enablement function is defined out. - */ -static void syscon_block_reset_enable(struct clk_syscon *sclk) -{ - unsigned long iflags; - u16 val; - - /* Not all blocks support resetting */ - if (!sclk->res_reg) - return; - spin_lock_irqsave(&syscon_resetreg_lock, iflags); - val = readw(sclk->res_reg); - val |= BIT(sclk->res_bit); - writew(val, sclk->res_reg); - spin_unlock_irqrestore(&syscon_resetreg_lock, iflags); - sclk->reset = true; -} - -static void syscon_block_reset_disable(struct clk_syscon *sclk) -{ - unsigned long iflags; - u16 val; - - /* Not all blocks support resetting */ - if (!sclk->res_reg) - return; - spin_lock_irqsave(&syscon_resetreg_lock, iflags); - val = readw(sclk->res_reg); - val &= ~BIT(sclk->res_bit); - writew(val, sclk->res_reg); - spin_unlock_irqrestore(&syscon_resetreg_lock, iflags); - sclk->reset = false; -} - -static int syscon_clk_prepare(struct clk_hw *hw) -{ - struct clk_syscon *sclk = to_syscon(hw); - - /* If the block is in reset, bring it out */ - if (sclk->reset) - syscon_block_reset_disable(sclk); - return 0; -} - -static void syscon_clk_unprepare(struct clk_hw *hw) -{ - struct clk_syscon *sclk = to_syscon(hw); - - /* Please don't force the console into reset */ - if (sclk->clk_val == U300_SYSCON_SBCER_UART_CLK_EN) - return; - /* When unpreparing, force block into reset */ - if (!sclk->reset) - syscon_block_reset_enable(sclk); -} - -static int syscon_clk_enable(struct clk_hw *hw) -{ - struct clk_syscon *sclk = to_syscon(hw); - - /* Don't touch the hardware controlled clocks */ - if (sclk->hw_ctrld) - return 0; - /* These cannot be controlled */ - if (sclk->clk_val == 0xFFFFU) - return 0; - - writew(sclk->clk_val, syscon_vbase + U300_SYSCON_SBCER); - return 0; -} - -static void syscon_clk_disable(struct clk_hw *hw) -{ - struct clk_syscon *sclk = to_syscon(hw); - - /* Don't touch the hardware controlled clocks */ - if (sclk->hw_ctrld) - return; - if (sclk->clk_val == 0xFFFFU) - return; - /* Please don't disable the console port */ - if (sclk->clk_val == U300_SYSCON_SBCER_UART_CLK_EN) - return; - - writew(sclk->clk_val, syscon_vbase + U300_SYSCON_SBCDR); -} - -static int syscon_clk_is_enabled(struct clk_hw *hw) -{ - struct clk_syscon *sclk = to_syscon(hw); - u16 val; - - /* If no enable register defined, it's always-on */ - if (!sclk->en_reg) - return 1; - - val = readw(sclk->en_reg); - val &= BIT(sclk->en_bit); - - return val ? 1 : 0; -} - -static u16 syscon_get_perf(void) -{ - u16 val; - - val = readw(syscon_vbase + U300_SYSCON_CCR); - val &= U300_SYSCON_CCR_CLKING_PERFORMANCE_MASK; - return val; -} - -static unsigned long -syscon_clk_recalc_rate(struct clk_hw *hw, - unsigned long parent_rate) -{ - struct clk_syscon *sclk = to_syscon(hw); - u16 perf = syscon_get_perf(); - - switch (sclk->clk_val) { - case U300_SYSCON_SBCER_FAST_BRIDGE_CLK_EN: - case U300_SYSCON_SBCER_I2C0_CLK_EN: - case U300_SYSCON_SBCER_I2C1_CLK_EN: - case U300_SYSCON_SBCER_MMC_CLK_EN: - case U300_SYSCON_SBCER_SPI_CLK_EN: - /* The FAST clocks have one progression */ - switch (perf) { - case U300_SYSCON_CCR_CLKING_PERFORMANCE_LOW_POWER: - case U300_SYSCON_CCR_CLKING_PERFORMANCE_LOW: - return 13000000; - default: - return parent_rate; /* 26 MHz */ - } - case U300_SYSCON_SBCER_DMAC_CLK_EN: - case U300_SYSCON_SBCER_NANDIF_CLK_EN: - case U300_SYSCON_SBCER_XGAM_CLK_EN: - /* AMBA interconnect peripherals */ - switch (perf) { - case U300_SYSCON_CCR_CLKING_PERFORMANCE_LOW_POWER: - case U300_SYSCON_CCR_CLKING_PERFORMANCE_LOW: - return 6500000; - case U300_SYSCON_CCR_CLKING_PERFORMANCE_INTERMEDIATE: - return 26000000; - default: - return parent_rate; /* 52 MHz */ - } - case U300_SYSCON_SBCER_SEMI_CLK_EN: - case U300_SYSCON_SBCER_EMIF_CLK_EN: - /* EMIF speeds */ - switch (perf) { - case U300_SYSCON_CCR_CLKING_PERFORMANCE_LOW_POWER: - case U300_SYSCON_CCR_CLKING_PERFORMANCE_LOW: - return 13000000; - case U300_SYSCON_CCR_CLKING_PERFORMANCE_INTERMEDIATE: - return 52000000; - default: - return 104000000; - } - case U300_SYSCON_SBCER_CPU_CLK_EN: - /* And the fast CPU clock */ - switch (perf) { - case U300_SYSCON_CCR_CLKING_PERFORMANCE_LOW_POWER: - case U300_SYSCON_CCR_CLKING_PERFORMANCE_LOW: - return 13000000; - case U300_SYSCON_CCR_CLKING_PERFORMANCE_INTERMEDIATE: - return 52000000; - case U300_SYSCON_CCR_CLKING_PERFORMANCE_HIGH: - return 104000000; - default: - return parent_rate; /* 208 MHz */ - } - default: - /* - * The SLOW clocks and default just inherit the rate of - * their parent (typically PLL13 13 MHz). - */ - return parent_rate; - } -} - -static long -syscon_clk_round_rate(struct clk_hw *hw, unsigned long rate, - unsigned long *prate) -{ - struct clk_syscon *sclk = to_syscon(hw); - - if (sclk->clk_val != U300_SYSCON_SBCER_CPU_CLK_EN) - return *prate; - /* We really only support setting the rate of the CPU clock */ - if (rate <= 13000000) - return 13000000; - if (rate <= 52000000) - return 52000000; - if (rate <= 104000000) - return 104000000; - return 208000000; -} - -static int syscon_clk_set_rate(struct clk_hw *hw, unsigned long rate, - unsigned long parent_rate) -{ - struct clk_syscon *sclk = to_syscon(hw); - u16 val; - - /* We only support setting the rate of the CPU clock */ - if (sclk->clk_val != U300_SYSCON_SBCER_CPU_CLK_EN) - return -EINVAL; - switch (rate) { - case 13000000: - val = U300_SYSCON_CCR_CLKING_PERFORMANCE_LOW_POWER; - break; - case 52000000: - val = U300_SYSCON_CCR_CLKING_PERFORMANCE_INTERMEDIATE; - break; - case 104000000: - val = U300_SYSCON_CCR_CLKING_PERFORMANCE_HIGH; - break; - case 208000000: - val = U300_SYSCON_CCR_CLKING_PERFORMANCE_BEST; - break; - default: - return -EINVAL; - } - val |= readw(syscon_vbase + U300_SYSCON_CCR) & - ~U300_SYSCON_CCR_CLKING_PERFORMANCE_MASK ; - writew(val, syscon_vbase + U300_SYSCON_CCR); - return 0; -} - -static const struct clk_ops syscon_clk_ops = { - .prepare = syscon_clk_prepare, - .unprepare = syscon_clk_unprepare, - .enable = syscon_clk_enable, - .disable = syscon_clk_disable, - .is_enabled = syscon_clk_is_enabled, - .recalc_rate = syscon_clk_recalc_rate, - .round_rate = syscon_clk_round_rate, - .set_rate = syscon_clk_set_rate, -}; - -static struct clk_hw * __init -syscon_clk_register(struct device *dev, const char *name, - const char *parent_name, unsigned long flags, - bool hw_ctrld, - void __iomem *res_reg, u8 res_bit, - void __iomem *en_reg, u8 en_bit, - u16 clk_val) -{ - struct clk_hw *hw; - struct clk_syscon *sclk; - struct clk_init_data init; - int ret; - - sclk = kzalloc(sizeof(*sclk), GFP_KERNEL); - if (!sclk) - return ERR_PTR(-ENOMEM); - - init.name = name; - init.ops = &syscon_clk_ops; - init.flags = flags; - init.parent_names = (parent_name ? &parent_name : NULL); - init.num_parents = (parent_name ? 1 : 0); - sclk->hw.init = &init; - sclk->hw_ctrld = hw_ctrld; - /* Assume the block is in reset at registration */ - sclk->reset = true; - sclk->res_reg = res_reg; - sclk->res_bit = res_bit; - sclk->en_reg = en_reg; - sclk->en_bit = en_bit; - sclk->clk_val = clk_val; - - hw = &sclk->hw; - ret = clk_hw_register(dev, hw); - if (ret) { - kfree(sclk); - hw = ERR_PTR(ret); - } - - return hw; -} - -#define U300_CLK_TYPE_SLOW 0 -#define U300_CLK_TYPE_FAST 1 -#define U300_CLK_TYPE_REST 2 - -/** - * struct u300_clock - defines the bits and pieces for a certain clock - * @type: the clock type, slow fast or rest - * @id: the bit in the slow/fast/rest register for this clock - * @hw_ctrld: whether the clock is hardware controlled - * @clk_val: a value to poke in the one-write enable/disable registers - */ -struct u300_clock { - u8 type; - u8 id; - bool hw_ctrld; - u16 clk_val; -}; - -static struct u300_clock const u300_clk_lookup[] __initconst = { - { - .type = U300_CLK_TYPE_REST, - .id = 3, - .hw_ctrld = true, - .clk_val = U300_SYSCON_SBCER_CPU_CLK_EN, - }, - { - .type = U300_CLK_TYPE_REST, - .id = 4, - .hw_ctrld = true, - .clk_val = U300_SYSCON_SBCER_DMAC_CLK_EN, - }, - { - .type = U300_CLK_TYPE_REST, - .id = 5, - .hw_ctrld = false, - .clk_val = U300_SYSCON_SBCER_EMIF_CLK_EN, - }, - { - .type = U300_CLK_TYPE_REST, - .id = 6, - .hw_ctrld = false, - .clk_val = U300_SYSCON_SBCER_NANDIF_CLK_EN, - }, - { - .type = U300_CLK_TYPE_REST, - .id = 8, - .hw_ctrld = true, - .clk_val = U300_SYSCON_SBCER_XGAM_CLK_EN, - }, - { - .type = U300_CLK_TYPE_REST, - .id = 9, - .hw_ctrld = false, - .clk_val = U300_SYSCON_SBCER_SEMI_CLK_EN, - }, - { - .type = U300_CLK_TYPE_REST, - .id = 10, - .hw_ctrld = true, - .clk_val = U300_SYSCON_SBCER_AHB_SUBSYS_BRIDGE_CLK_EN, - }, - { - .type = U300_CLK_TYPE_REST, - .id = 12, - .hw_ctrld = false, - /* INTCON: cannot be enabled, just taken out of reset */ - .clk_val = 0xFFFFU, - }, - { - .type = U300_CLK_TYPE_FAST, - .id = 0, - .hw_ctrld = true, - .clk_val = U300_SYSCON_SBCER_FAST_BRIDGE_CLK_EN, - }, - { - .type = U300_CLK_TYPE_FAST, - .id = 1, - .hw_ctrld = false, - .clk_val = U300_SYSCON_SBCER_I2C0_CLK_EN, - }, - { - .type = U300_CLK_TYPE_FAST, - .id = 2, - .hw_ctrld = false, - .clk_val = U300_SYSCON_SBCER_I2C1_CLK_EN, - }, - { - .type = U300_CLK_TYPE_FAST, - .id = 5, - .hw_ctrld = false, - .clk_val = U300_SYSCON_SBCER_MMC_CLK_EN, - }, - { - .type = U300_CLK_TYPE_FAST, - .id = 6, - .hw_ctrld = false, - .clk_val = U300_SYSCON_SBCER_SPI_CLK_EN, - }, - { - .type = U300_CLK_TYPE_SLOW, - .id = 0, - .hw_ctrld = true, - .clk_val = U300_SYSCON_SBCER_SLOW_BRIDGE_CLK_EN, - }, - { - .type = U300_CLK_TYPE_SLOW, - .id = 1, - .hw_ctrld = false, - .clk_val = U300_SYSCON_SBCER_UART_CLK_EN, - }, - { - .type = U300_CLK_TYPE_SLOW, - .id = 4, - .hw_ctrld = false, - .clk_val = U300_SYSCON_SBCER_GPIO_CLK_EN, - }, - { - .type = U300_CLK_TYPE_SLOW, - .id = 6, - .hw_ctrld = true, - /* No clock enable register bit */ - .clk_val = 0xFFFFU, - }, - { - .type = U300_CLK_TYPE_SLOW, - .id = 7, - .hw_ctrld = false, - .clk_val = U300_SYSCON_SBCER_APP_TMR_CLK_EN, - }, - { - .type = U300_CLK_TYPE_SLOW, - .id = 8, - .hw_ctrld = false, - .clk_val = U300_SYSCON_SBCER_ACC_TMR_CLK_EN, - }, -}; - -static void __init of_u300_syscon_clk_init(struct device_node *np) -{ - struct clk_hw *hw = ERR_PTR(-EINVAL); - const char *clk_name = np->name; - const char *parent_name; - void __iomem *res_reg; - void __iomem *en_reg; - u32 clk_type; - u32 clk_id; - int i; - - if (of_property_read_u32(np, "clock-type", &clk_type)) { - pr_err("%s: syscon clock \"%s\" missing clock-type property\n", - __func__, clk_name); - return; - } - if (of_property_read_u32(np, "clock-id", &clk_id)) { - pr_err("%s: syscon clock \"%s\" missing clock-id property\n", - __func__, clk_name); - return; - } - parent_name = of_clk_get_parent_name(np, 0); - - switch (clk_type) { - case U300_CLK_TYPE_SLOW: - res_reg = syscon_vbase + U300_SYSCON_RSR; - en_reg = syscon_vbase + U300_SYSCON_CESR; - break; - case U300_CLK_TYPE_FAST: - res_reg = syscon_vbase + U300_SYSCON_RFR; - en_reg = syscon_vbase + U300_SYSCON_CEFR; - break; - case U300_CLK_TYPE_REST: - res_reg = syscon_vbase + U300_SYSCON_RRR; - en_reg = syscon_vbase + U300_SYSCON_CERR; - break; - default: - pr_err("unknown clock type %x specified\n", clk_type); - return; - } - - for (i = 0; i < ARRAY_SIZE(u300_clk_lookup); i++) { - const struct u300_clock *u3clk = &u300_clk_lookup[i]; - - if (u3clk->type == clk_type && u3clk->id == clk_id) - hw = syscon_clk_register(NULL, clk_name, parent_name, - 0, u3clk->hw_ctrld, - res_reg, u3clk->id, - en_reg, u3clk->id, - u3clk->clk_val); - } - - if (!IS_ERR(hw)) { - of_clk_add_hw_provider(np, of_clk_hw_simple_get, hw); - - /* - * Some few system clocks - device tree does not - * represent clocks without a corresponding device node. - * for now we add these three clocks here. - */ - if (clk_type == U300_CLK_TYPE_REST && clk_id == 5) - clk_hw_register_clkdev(hw, NULL, "pl172"); - if (clk_type == U300_CLK_TYPE_REST && clk_id == 9) - clk_hw_register_clkdev(hw, NULL, "semi"); - if (clk_type == U300_CLK_TYPE_REST && clk_id == 12) - clk_hw_register_clkdev(hw, NULL, "intcon"); - } -} - -/** - * struct clk_mclk - U300 MCLK clock (MMC/SD clock) - * @hw: corresponding clock hardware entry - * @is_mspro: if this is the memory stick clock rather than MMC/SD - */ -struct clk_mclk { - struct clk_hw hw; - bool is_mspro; -}; - -#define to_mclk(_hw) container_of(_hw, struct clk_mclk, hw) - -static int mclk_clk_prepare(struct clk_hw *hw) -{ - struct clk_mclk *mclk = to_mclk(hw); - u16 val; - - /* The MMC and MSPRO clocks need some special set-up */ - if (!mclk->is_mspro) { - /* Set default MMC clock divisor to 18.9 MHz */ - writew(0x0054U, syscon_vbase + U300_SYSCON_MMF0R); - val = readw(syscon_vbase + U300_SYSCON_MMCR); - /* Disable the MMC feedback clock */ - val &= ~U300_SYSCON_MMCR_MMC_FB_CLK_SEL_ENABLE; - /* Disable MSPRO frequency */ - val &= ~U300_SYSCON_MMCR_MSPRO_FREQSEL_ENABLE; - writew(val, syscon_vbase + U300_SYSCON_MMCR); - } else { - val = readw(syscon_vbase + U300_SYSCON_MMCR); - /* Disable the MMC feedback clock */ - val &= ~U300_SYSCON_MMCR_MMC_FB_CLK_SEL_ENABLE; - /* Enable MSPRO frequency */ - val |= U300_SYSCON_MMCR_MSPRO_FREQSEL_ENABLE; - writew(val, syscon_vbase + U300_SYSCON_MMCR); - } - - return 0; -} - -static unsigned long -mclk_clk_recalc_rate(struct clk_hw *hw, - unsigned long parent_rate) -{ - u16 perf = syscon_get_perf(); - - switch (perf) { - case U300_SYSCON_CCR_CLKING_PERFORMANCE_LOW_POWER: - /* - * Here, the 208 MHz PLL gets shut down and the always - * on 13 MHz PLL used for RTC etc kicks into use - * instead. - */ - return 13000000; - case U300_SYSCON_CCR_CLKING_PERFORMANCE_LOW: - case U300_SYSCON_CCR_CLKING_PERFORMANCE_INTERMEDIATE: - case U300_SYSCON_CCR_CLKING_PERFORMANCE_HIGH: - case U300_SYSCON_CCR_CLKING_PERFORMANCE_BEST: - { - /* - * This clock is under program control. The register is - * divided in two nybbles, bit 7-4 gives cycles-1 to count - * high, bit 3-0 gives cycles-1 to count low. Distribute - * these with no more than 1 cycle difference between - * low and high and add low and high to get the actual - * divisor. The base PLL is 208 MHz. Writing 0x00 will - * divide by 1 and 1 so the highest frequency possible - * is 104 MHz. - * - * e.g. 0x54 => - * f = 208 / ((5+1) + (4+1)) = 208 / 11 = 18.9 MHz - */ - u16 val = readw(syscon_vbase + U300_SYSCON_MMF0R) & - U300_SYSCON_MMF0R_MASK; - switch (val) { - case 0x0054: - return 18900000; - case 0x0044: - return 20800000; - case 0x0043: - return 23100000; - case 0x0033: - return 26000000; - case 0x0032: - return 29700000; - case 0x0022: - return 34700000; - case 0x0021: - return 41600000; - case 0x0011: - return 52000000; - case 0x0000: - return 104000000; - default: - break; - } - } - default: - break; - } - return parent_rate; -} - -static long -mclk_clk_round_rate(struct clk_hw *hw, unsigned long rate, - unsigned long *prate) -{ - if (rate <= 18900000) - return 18900000; - if (rate <= 20800000) - return 20800000; - if (rate <= 23100000) - return 23100000; - if (rate <= 26000000) - return 26000000; - if (rate <= 29700000) - return 29700000; - if (rate <= 34700000) - return 34700000; - if (rate <= 41600000) - return 41600000; - /* Highest rate */ - return 52000000; -} - -static int mclk_clk_set_rate(struct clk_hw *hw, unsigned long rate, - unsigned long parent_rate) -{ - u16 val; - u16 reg; - - switch (rate) { - case 18900000: - val = 0x0054; - break; - case 20800000: - val = 0x0044; - break; - case 23100000: - val = 0x0043; - break; - case 26000000: - val = 0x0033; - break; - case 29700000: - val = 0x0032; - break; - case 34700000: - val = 0x0022; - break; - case 41600000: - val = 0x0021; - break; - case 52000000: - val = 0x0011; - break; - case 104000000: - val = 0x0000; - break; - default: - return -EINVAL; - } - - reg = readw(syscon_vbase + U300_SYSCON_MMF0R) & - ~U300_SYSCON_MMF0R_MASK; - writew(reg | val, syscon_vbase + U300_SYSCON_MMF0R); - return 0; -} - -static const struct clk_ops mclk_ops = { - .prepare = mclk_clk_prepare, - .recalc_rate = mclk_clk_recalc_rate, - .round_rate = mclk_clk_round_rate, - .set_rate = mclk_clk_set_rate, -}; - -static struct clk_hw * __init -mclk_clk_register(struct device *dev, const char *name, - const char *parent_name, bool is_mspro) -{ - struct clk_hw *hw; - struct clk_mclk *mclk; - struct clk_init_data init; - int ret; - - mclk = kzalloc(sizeof(*mclk), GFP_KERNEL); - if (!mclk) - return ERR_PTR(-ENOMEM); - - init.name = "mclk"; - init.ops = &mclk_ops; - init.flags = 0; - init.parent_names = (parent_name ? &parent_name : NULL); - init.num_parents = (parent_name ? 1 : 0); - mclk->hw.init = &init; - mclk->is_mspro = is_mspro; - - hw = &mclk->hw; - ret = clk_hw_register(dev, hw); - if (ret) { - kfree(mclk); - hw = ERR_PTR(ret); - } - - return hw; -} - -static void __init of_u300_syscon_mclk_init(struct device_node *np) -{ - struct clk_hw *hw; - const char *clk_name = np->name; - const char *parent_name; - - parent_name = of_clk_get_parent_name(np, 0); - hw = mclk_clk_register(NULL, clk_name, parent_name, false); - if (!IS_ERR(hw)) - of_clk_add_hw_provider(np, of_clk_hw_simple_get, hw); -} - -static const struct of_device_id u300_clk_match[] __initconst = { - { - .compatible = "fixed-clock", - .data = of_fixed_clk_setup, - }, - { - .compatible = "fixed-factor-clock", - .data = of_fixed_factor_clk_setup, - }, - { - .compatible = "stericsson,u300-syscon-clk", - .data = of_u300_syscon_clk_init, - }, - { - .compatible = "stericsson,u300-syscon-mclk", - .data = of_u300_syscon_mclk_init, - }, - {} -}; - - -void __init u300_clk_init(void __iomem *base) -{ - u16 val; - - syscon_vbase = base; - - /* Set system to run at PLL208, max performance, a known state. */ - val = readw(syscon_vbase + U300_SYSCON_CCR); - val &= ~U300_SYSCON_CCR_CLKING_PERFORMANCE_MASK; - writew(val, syscon_vbase + U300_SYSCON_CCR); - /* Wait for the PLL208 to lock if not locked in yet */ - while (!(readw(syscon_vbase + U300_SYSCON_CSR) & - U300_SYSCON_CSR_PLL208_LOCK_IND)); - - /* Power management enable */ - val = readw(syscon_vbase + U300_SYSCON_PMCR); - val |= U300_SYSCON_PMCR_PWR_MGNT_ENABLE; - writew(val, syscon_vbase + U300_SYSCON_PMCR); - - of_clk_init(u300_clk_match); -} diff --git a/drivers/clk/clk-versaclock5.c b/drivers/clk/clk-versaclock5.c index 43db67337bc0682475c467de2ba2b05b76c106b7..344cd6c6118873cd9665c5afca8275dd92523df2 100644 --- a/drivers/clk/clk-versaclock5.c +++ b/drivers/clk/clk-versaclock5.c @@ -759,6 +759,63 @@ static int vc5_update_power(struct device_node *np_output, return 0; } +static int vc5_map_cap_value(u32 femtofarads) +{ + int mapped_value; + + /* + * The datasheet explicitly states 9000 - 25000 with 0.5pF + * steps, but the Programmer's guide shows the steps are 0.430pF. + * After getting feedback from Renesas, the .5pF steps were the + * goal, but 430nF was the actual values. + * Because of this, the actual range goes to 22760 instead of 25000 + */ + if (femtofarads < 9000 || femtofarads > 22760) + return -EINVAL; + + /* + * The Programmer's guide shows XTAL[5:0] but in reality, + * XTAL[0] and XTAL[1] are both LSB which makes the math + * strange. With clarfication from Renesas, setting the + * values should be simpler by ignoring XTAL[0] + */ + mapped_value = DIV_ROUND_CLOSEST(femtofarads - 9000, 430); + + /* + * Since the calculation ignores XTAL[0], there is one + * special case where mapped_value = 32. In reality, this means + * the real mapped value should be 111111b. In other cases, + * the mapped_value needs to be shifted 1 to the left. + */ + if (mapped_value > 31) + mapped_value = 0x3f; + else + mapped_value <<= 1; + + return mapped_value; +} +static int vc5_update_cap_load(struct device_node *node, struct vc5_driver_data *vc5) +{ + u32 value; + int mapped_value; + + if (!of_property_read_u32(node, "idt,xtal-load-femtofarads", &value)) { + mapped_value = vc5_map_cap_value(value); + if (mapped_value < 0) + return mapped_value; + + /* + * The mapped_value is really the high 6 bits of + * VC5_XTAL_X1_LOAD_CAP and VC5_XTAL_X2_LOAD_CAP, so + * shift the value 2 places. + */ + regmap_update_bits(vc5->regmap, VC5_XTAL_X1_LOAD_CAP, ~0x03, mapped_value << 2); + regmap_update_bits(vc5->regmap, VC5_XTAL_X2_LOAD_CAP, ~0x03, mapped_value << 2); + } + + return 0; +} + static int vc5_update_slew(struct device_node *np_output, struct vc5_out_data *clk_out) { @@ -884,6 +941,13 @@ static int vc5_probe(struct i2c_client *client, const struct i2c_device_id *id) return -EINVAL; } + /* Configure Optional Loading Capacitance for external XTAL */ + if (!(vc5->chip_info->flags & VC5_HAS_INTERNAL_XTAL)) { + ret = vc5_update_cap_load(client->dev.of_node, vc5); + if (ret) + goto err_clk_register; + } + init.name = kasprintf(GFP_KERNEL, "%pOFn.mux", client->dev.of_node); init.ops = &vc5_mux_ops; init.flags = 0; diff --git a/drivers/clk/clk-xgene.c b/drivers/clk/clk-xgene.c index 3fd53057c01fe2b6f3a5422e3640403d83b2849c..857217cbcef87580dd104fd77ec995b5062f7d3b 100644 --- a/drivers/clk/clk-xgene.c +++ b/drivers/clk/clk-xgene.c @@ -206,17 +206,16 @@ static void xgene_pcppllclk_init(struct device_node *np) * @hw: handle between common and hardware-specific interfaces * @reg: register containing the fractional scale multiplier (scaler) * @shift: shift to the unit bit field + * @mask: mask to the unit bit field * @denom: 1/denominator unit * @lock: register lock - * Flags: - * XGENE_CLK_PMD_SCALE_INVERTED - By default the scaler is the value read + * @flags: XGENE_CLK_PMD_SCALE_INVERTED - By default the scaler is the value read * from the register plus one. For example, * 0 for (0 + 1) / denom, * 1 for (1 + 1) / denom and etc. * If this flag is set, it is * 0 for (denom - 0) / denom, * 1 for (denom - 1) / denom and etc. - * */ struct xgene_clk_pmd { struct clk_hw hw; diff --git a/drivers/clk/clk.c b/drivers/clk/clk.c index 3d751ae5bc703280eb2918afea367277641cf933..5052541a0986658248867a20a531cbac673dd1d1 100644 --- a/drivers/clk/clk.c +++ b/drivers/clk/clk.c @@ -4576,6 +4576,8 @@ int of_clk_add_provider(struct device_node *np, if (ret < 0) of_clk_del_provider(np); + fwnode_dev_initialized(&np->fwnode, true); + return ret; } EXPORT_SYMBOL_GPL(of_clk_add_provider); @@ -4693,6 +4695,7 @@ void of_clk_del_provider(struct device_node *np) list_for_each_entry(cp, &of_clk_providers, link) { if (cp->node == np) { list_del(&cp->link); + fwnode_dev_initialized(&np->fwnode, false); of_node_put(cp->node); kfree(cp); break; diff --git a/drivers/clk/imx/clk-imx31.c b/drivers/clk/imx/clk-imx31.c index 7b13fb57d842815df0f0133bb341427d78f200df..c44e18c6f63f79e4aa2dcfd4c7dc6a9743d3558e 100644 --- a/drivers/clk/imx/clk-imx31.c +++ b/drivers/clk/imx/clk-imx31.c @@ -51,16 +51,6 @@ enum mx31_clks { static struct clk *clk[clk_max]; static struct clk_onecell_data clk_data; -static struct clk ** const uart_clks[] __initconst = { - &clk[ipg], - &clk[uart1_gate], - &clk[uart2_gate], - &clk[uart3_gate], - &clk[uart4_gate], - &clk[uart5_gate], - NULL -}; - static void __init _mx31_clocks_init(void __iomem *base, unsigned long fref) { clk[dummy] = imx_clk_fixed("dummy", 0); diff --git a/drivers/clk/imx/clk-imx6q.c b/drivers/clk/imx/clk-imx6q.c index b2ff187cedabc19228247a4b7f6ba59c88def423..521d6136d22c5f36c63b8c0e50de5e65161153dc 100644 --- a/drivers/clk/imx/clk-imx6q.c +++ b/drivers/clk/imx/clk-imx6q.c @@ -338,10 +338,10 @@ static void init_ldb_clks(struct device_node *np, void __iomem *ccm_base) of_assigned_ldb_sels(np, &sel[0][3], &sel[1][3]); for (i = 0; i < 2; i++) { - /* Warn if a glitch might have been introduced already */ + /* Print a notice if a glitch might have been introduced already */ if (sel[i][0] != 3) { - pr_warn("ccm: ldb_di%d_sel already changed from reset value: %d\n", - i, sel[i][0]); + pr_notice("ccm: possible glitch: ldb_di%d_sel already changed from reset value: %d\n", + i, sel[i][0]); } if (sel[i][0] == sel[i][3]) diff --git a/drivers/clk/imx/clk-imx6sl.c b/drivers/clk/imx/clk-imx6sl.c index 2f9361946a0e16d0153e7df40977b1b1649acb19..29eab05c9068984cbd9c232cf4daa718604fbac6 100644 --- a/drivers/clk/imx/clk-imx6sl.c +++ b/drivers/clk/imx/clk-imx6sl.c @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include diff --git a/drivers/clk/imx/clk-imx8mm.c b/drivers/clk/imx/clk-imx8mm.c index 7c905861af5dc528ab05f48ae68e7a9dabc8e21e..6a01eec36dd04e0fcef628c3dac9f6e58b7875fe 100644 --- a/drivers/clk/imx/clk-imx8mm.c +++ b/drivers/clk/imx/clk-imx8mm.c @@ -288,6 +288,11 @@ static const char *imx8mm_clko1_sels[] = {"osc_24m", "sys_pll1_800m", "dummy", " static const char *imx8mm_clko2_sels[] = {"osc_24m", "sys_pll2_200m", "sys_pll1_400m", "sys_pll2_166m", "sys_pll3_out", "audio_pll1_out", "video_pll1_out", "osc_32k", }; +static const char * const clkout_sels[] = {"audio_pll1_out", "audio_pll2_out", "video_pll1_out", + "dummy", "dummy", "gpu_pll_out", "vpu_pll_out", + "arm_pll_out", "sys_pll1", "sys_pll2", "sys_pll3", + "dummy", "dummy", "osc_24m", "dummy", "osc_32k"}; + static struct clk_hw_onecell_data *clk_hw_data; static struct clk_hw **hws; @@ -410,6 +415,13 @@ static int imx8mm_clocks_probe(struct platform_device *pdev) hws[IMX8MM_SYS_PLL2_500M] = imx_clk_hw_fixed_factor("sys_pll2_500m", "sys_pll2_500m_cg", 1, 2); hws[IMX8MM_SYS_PLL2_1000M] = imx_clk_hw_fixed_factor("sys_pll2_1000m", "sys_pll2_out", 1, 1); + hws[IMX8MM_CLK_CLKOUT1_SEL] = imx_clk_hw_mux("clkout1_sel", base + 0x128, 4, 4, clkout_sels, ARRAY_SIZE(clkout_sels)); + hws[IMX8MM_CLK_CLKOUT1_DIV] = imx_clk_hw_divider("clkout1_div", "clkout1_sel", base + 0x128, 0, 4); + hws[IMX8MM_CLK_CLKOUT1] = imx_clk_hw_gate("clkout1", "clkout1_div", base + 0x128, 8); + hws[IMX8MM_CLK_CLKOUT2_SEL] = imx_clk_hw_mux("clkout2_sel", base + 0x128, 20, 4, clkout_sels, ARRAY_SIZE(clkout_sels)); + hws[IMX8MM_CLK_CLKOUT2_DIV] = imx_clk_hw_divider("clkout2_div", "clkout2_sel", base + 0x128, 16, 4); + hws[IMX8MM_CLK_CLKOUT2] = imx_clk_hw_gate("clkout2", "clkout2_div", base + 0x128, 24); + np = dev->of_node; base = devm_platform_ioremap_resource(pdev, 0); if (WARN_ON(IS_ERR(base))) diff --git a/drivers/clk/imx/clk-imx8mn.c b/drivers/clk/imx/clk-imx8mn.c index 3c21db942d5bc6dc471703b62169f57ccee863c0..324c5fd0aa04fce971234f00cddfaa358a49f844 100644 --- a/drivers/clk/imx/clk-imx8mn.c +++ b/drivers/clk/imx/clk-imx8mn.c @@ -281,6 +281,11 @@ static const char * const imx8mn_clko2_sels[] = {"osc_24m", "sys_pll2_200m", "sy "sys_pll2_166m", "sys_pll3_out", "audio_pll1_out", "video_pll1_out", "osc_32k", }; +static const char * const clkout_sels[] = {"audio_pll1_out", "audio_pll2_out", "video_pll1_out", + "dummy", "dummy", "gpu_pll_out", "dummy", + "arm_pll_out", "sys_pll1", "sys_pll2", "sys_pll3", + "dummy", "dummy", "osc_24m", "dummy", "osc_32k"}; + static struct clk_hw_onecell_data *clk_hw_data; static struct clk_hw **hws; @@ -405,6 +410,13 @@ static int imx8mn_clocks_probe(struct platform_device *pdev) hws[IMX8MN_SYS_PLL2_500M] = imx_clk_hw_fixed_factor("sys_pll2_500m", "sys_pll2_500m_cg", 1, 2); hws[IMX8MN_SYS_PLL2_1000M] = imx_clk_hw_fixed_factor("sys_pll2_1000m", "sys_pll2_out", 1, 1); + hws[IMX8MN_CLK_CLKOUT1_SEL] = imx_clk_hw_mux("clkout1_sel", base + 0x128, 4, 4, clkout_sels, ARRAY_SIZE(clkout_sels)); + hws[IMX8MN_CLK_CLKOUT1_DIV] = imx_clk_hw_divider("clkout1_div", "clkout1_sel", base + 0x128, 0, 4); + hws[IMX8MN_CLK_CLKOUT1] = imx_clk_hw_gate("clkout1", "clkout1_div", base + 0x128, 8); + hws[IMX8MN_CLK_CLKOUT2_SEL] = imx_clk_hw_mux("clkout2_sel", base + 0x128, 20, 4, clkout_sels, ARRAY_SIZE(clkout_sels)); + hws[IMX8MN_CLK_CLKOUT2_DIV] = imx_clk_hw_divider("clkout2_div", "clkout2_sel", base + 0x128, 16, 4); + hws[IMX8MN_CLK_CLKOUT2] = imx_clk_hw_gate("clkout2", "clkout2_div", base + 0x128, 24); + np = dev->of_node; base = devm_platform_ioremap_resource(pdev, 0); if (WARN_ON(IS_ERR(base))) { diff --git a/drivers/clk/imx/clk-imx8mq.c b/drivers/clk/imx/clk-imx8mq.c index 779ea69e639cf5bcb1afd412ee8cc4eda8c320f4..4dd4ae9d022baa315fb3c175c025efbcc6f5a6d2 100644 --- a/drivers/clk/imx/clk-imx8mq.c +++ b/drivers/clk/imx/clk-imx8mq.c @@ -270,6 +270,14 @@ static const char * const imx8mq_clko1_sels[] = {"osc_25m", "sys1_pll_800m", "os static const char * const imx8mq_clko2_sels[] = {"osc_25m", "sys2_pll_200m", "sys1_pll_400m", "sys2_pll_166m", "sys3_pll_out", "audio_pll1_out", "video_pll1_out", "ckil", }; +static const char * const pllout_monitor_sels[] = {"osc_25m", "osc_27m", "dummy", "dummy", "ckil", + "audio_pll1_out_monitor", "audio_pll2_out_monitor", + "video_pll1_out_monitor", "gpu_pll_out_monitor", + "vpu_pll_out_monitor", "arm_pll_out_monitor", + "sys_pll1_out_monitor", "sys_pll2_out_monitor", + "sys_pll3_out_monitor", "dram_pll_out_monitor", + "video_pll2_out_monitor", }; + static struct clk_hw_onecell_data *clk_hw_data; static struct clk_hw **hws; @@ -399,6 +407,20 @@ static int imx8mq_clocks_probe(struct platform_device *pdev) hws[IMX8MQ_SYS2_PLL_500M] = imx_clk_hw_fixed_factor("sys2_pll_500m", "sys2_pll_500m_cg", 1, 2); hws[IMX8MQ_SYS2_PLL_1000M] = imx_clk_hw_fixed_factor("sys2_pll_1000m", "sys2_pll_1000m_cg", 1, 1); + hws[IMX8MQ_CLK_MON_AUDIO_PLL1_DIV] = imx_clk_hw_divider("audio_pll1_out_monitor", "audio_pll1_bypass", base + 0x78, 0, 3); + hws[IMX8MQ_CLK_MON_AUDIO_PLL2_DIV] = imx_clk_hw_divider("audio_pll2_out_monitor", "audio_pll2_bypass", base + 0x78, 4, 3); + hws[IMX8MQ_CLK_MON_VIDEO_PLL1_DIV] = imx_clk_hw_divider("video_pll1_out_monitor", "video_pll1_bypass", base + 0x78, 8, 3); + hws[IMX8MQ_CLK_MON_GPU_PLL_DIV] = imx_clk_hw_divider("gpu_pll_out_monitor", "gpu_pll_bypass", base + 0x78, 12, 3); + hws[IMX8MQ_CLK_MON_VPU_PLL_DIV] = imx_clk_hw_divider("vpu_pll_out_monitor", "vpu_pll_bypass", base + 0x78, 16, 3); + hws[IMX8MQ_CLK_MON_ARM_PLL_DIV] = imx_clk_hw_divider("arm_pll_out_monitor", "arm_pll_bypass", base + 0x78, 20, 3); + hws[IMX8MQ_CLK_MON_SYS_PLL1_DIV] = imx_clk_hw_divider("sys_pll1_out_monitor", "sys1_pll_out", base + 0x7c, 0, 3); + hws[IMX8MQ_CLK_MON_SYS_PLL2_DIV] = imx_clk_hw_divider("sys_pll2_out_monitor", "sys2_pll_out", base + 0x7c, 4, 3); + hws[IMX8MQ_CLK_MON_SYS_PLL3_DIV] = imx_clk_hw_divider("sys_pll3_out_monitor", "sys3_pll_out", base + 0x7c, 8, 3); + hws[IMX8MQ_CLK_MON_DRAM_PLL_DIV] = imx_clk_hw_divider("dram_pll_out_monitor", "dram_pll_out", base + 0x7c, 12, 3); + hws[IMX8MQ_CLK_MON_VIDEO_PLL2_DIV] = imx_clk_hw_divider("video_pll2_out_monitor", "video2_pll_out", base + 0x7c, 16, 3); + hws[IMX8MQ_CLK_MON_SEL] = imx_clk_hw_mux("pllout_monitor_sel", base + 0x74, 0, 4, pllout_monitor_sels, ARRAY_SIZE(pllout_monitor_sels)); + hws[IMX8MQ_CLK_MON_CLK2_OUT] = imx_clk_hw_gate("pllout_monitor_clk2", "pllout_monitor_sel", base + 0x74, 4); + np = dev->of_node; base = devm_platform_ioremap_resource(pdev, 0); if (WARN_ON(IS_ERR(base))) diff --git a/drivers/clk/imx/clk-imx8qxp.c b/drivers/clk/imx/clk-imx8qxp.c index 5b3d4ede7c7c4685a22bdf31f8eb5570ce097bb7..fbf1170c09ed8ad4fd00f52d0b2a1caeef1e1b66 100644 --- a/drivers/clk/imx/clk-imx8qxp.c +++ b/drivers/clk/imx/clk-imx8qxp.c @@ -17,6 +17,14 @@ #include #include +static const char *dc0_sels[] = { + "clk_dummy", + "clk_dummy", + "dc0_pll0_clk", + "dc0_pll1_clk", + "dc0_bypass0_clk", +}; + static int imx8qxp_clk_probe(struct platform_device *pdev) { struct device_node *ccm_node = pdev->dev.of_node; @@ -115,12 +123,26 @@ static int imx8qxp_clk_probe(struct platform_device *pdev) clks[IMX_CONN_USB2_LPM_CLK] = imx_clk_scu("usb3_lpm_div", IMX_SC_R_USB_2, IMX_SC_PM_CLK_MISC, clk_cells); /* Display controller SS */ - clks[IMX_DC0_DISP0_CLK] = imx_clk_scu("dc0_disp0_clk", IMX_SC_R_DC_0, IMX_SC_PM_CLK_MISC0, clk_cells); - clks[IMX_DC0_DISP1_CLK] = imx_clk_scu("dc0_disp1_clk", IMX_SC_R_DC_0, IMX_SC_PM_CLK_MISC1, clk_cells); + clks[IMX_DC0_DISP0_CLK] = imx_clk_scu2("dc0_disp0_clk", dc0_sels, ARRAY_SIZE(dc0_sels), IMX_SC_R_DC_0, IMX_SC_PM_CLK_MISC0, clk_cells); + clks[IMX_DC0_DISP1_CLK] = imx_clk_scu2("dc0_disp1_clk", dc0_sels, ARRAY_SIZE(dc0_sels), IMX_SC_R_DC_0, IMX_SC_PM_CLK_MISC1, clk_cells); + clks[IMX_DC0_PLL0_CLK] = imx_clk_scu("dc0_pll0_clk", IMX_SC_R_DC_0_PLL_0, IMX_SC_PM_CLK_PLL, clk_cells); + clks[IMX_DC0_PLL1_CLK] = imx_clk_scu("dc0_pll1_clk", IMX_SC_R_DC_0_PLL_1, IMX_SC_PM_CLK_PLL, clk_cells); + clks[IMX_DC0_BYPASS0_CLK] = imx_clk_scu("dc0_bypass0_clk", IMX_SC_R_DC_0_VIDEO0, IMX_SC_PM_CLK_BYPASS, clk_cells); + clks[IMX_DC0_BYPASS1_CLK] = imx_clk_scu("dc0_bypass1_clk", IMX_SC_R_DC_0_VIDEO1, IMX_SC_PM_CLK_BYPASS, clk_cells); /* MIPI-LVDS SS */ + clks[IMX_MIPI0_LVDS_PIXEL_CLK] = imx_clk_scu("mipi0_lvds_pixel_clk", IMX_SC_R_LVDS_0, IMX_SC_PM_CLK_MISC2, clk_cells); + clks[IMX_MIPI0_LVDS_BYPASS_CLK] = imx_clk_scu("mipi0_lvds_bypass_clk", IMX_SC_R_LVDS_0, IMX_SC_PM_CLK_BYPASS, clk_cells); + clks[IMX_MIPI0_LVDS_PHY_CLK] = imx_clk_scu("mipi0_lvds_phy_clk", IMX_SC_R_LVDS_0, IMX_SC_PM_CLK_MISC3, clk_cells); clks[IMX_MIPI0_I2C0_CLK] = imx_clk_scu("mipi0_i2c0_clk", IMX_SC_R_MIPI_0_I2C_0, IMX_SC_PM_CLK_MISC2, clk_cells); clks[IMX_MIPI0_I2C1_CLK] = imx_clk_scu("mipi0_i2c1_clk", IMX_SC_R_MIPI_0_I2C_1, IMX_SC_PM_CLK_MISC2, clk_cells); + clks[IMX_MIPI0_PWM0_CLK] = imx_clk_scu("mipi0_pwm0_clk", IMX_SC_R_MIPI_0_PWM_0, IMX_SC_PM_CLK_PER, clk_cells); + clks[IMX_MIPI1_LVDS_PIXEL_CLK] = imx_clk_scu("mipi1_lvds_pixel_clk", IMX_SC_R_LVDS_1, IMX_SC_PM_CLK_MISC2, clk_cells); + clks[IMX_MIPI1_LVDS_BYPASS_CLK] = imx_clk_scu("mipi1_lvds_bypass_clk", IMX_SC_R_LVDS_1, IMX_SC_PM_CLK_BYPASS, clk_cells); + clks[IMX_MIPI1_LVDS_PHY_CLK] = imx_clk_scu("mipi1_lvds_phy_clk", IMX_SC_R_LVDS_1, IMX_SC_PM_CLK_MISC3, clk_cells); + clks[IMX_MIPI1_I2C0_CLK] = imx_clk_scu("mipi1_i2c0_clk", IMX_SC_R_MIPI_1_I2C_0, IMX_SC_PM_CLK_MISC2, clk_cells); + clks[IMX_MIPI1_I2C1_CLK] = imx_clk_scu("mipi1_i2c1_clk", IMX_SC_R_MIPI_1_I2C_1, IMX_SC_PM_CLK_MISC2, clk_cells); + clks[IMX_MIPI1_PWM0_CLK] = imx_clk_scu("mipi1_pwm0_clk", IMX_SC_R_MIPI_1_PWM_0, IMX_SC_PM_CLK_PER, clk_cells); /* MIPI CSI SS */ clks[IMX_CSI0_CORE_CLK] = imx_clk_scu("mipi_csi0_core_clk", IMX_SC_R_CSI_0, IMX_SC_PM_CLK_PER, clk_cells); diff --git a/drivers/clk/mediatek/Kconfig b/drivers/clk/mediatek/Kconfig index ce8475098b31c781d4d6ab220621368541bc07e2..886e2d9fced5c21567260f1c62a3d554fde34bfc 100644 --- a/drivers/clk/mediatek/Kconfig +++ b/drivers/clk/mediatek/Kconfig @@ -426,66 +426,77 @@ config COMMON_CLK_MT8183 config COMMON_CLK_MT8183_AUDIOSYS bool "Clock driver for MediaTek MT8183 audiosys" depends on COMMON_CLK_MT8183 + default COMMON_CLK_MT8183 help This driver supports MediaTek MT8183 audiosys clocks. config COMMON_CLK_MT8183_CAMSYS bool "Clock driver for MediaTek MT8183 camsys" depends on COMMON_CLK_MT8183 + default COMMON_CLK_MT8183 help This driver supports MediaTek MT8183 camsys clocks. config COMMON_CLK_MT8183_IMGSYS bool "Clock driver for MediaTek MT8183 imgsys" depends on COMMON_CLK_MT8183 + default COMMON_CLK_MT8183 help This driver supports MediaTek MT8183 imgsys clocks. config COMMON_CLK_MT8183_IPU_CORE0 bool "Clock driver for MediaTek MT8183 ipu_core0" depends on COMMON_CLK_MT8183 + default COMMON_CLK_MT8183 help This driver supports MediaTek MT8183 ipu_core0 clocks. config COMMON_CLK_MT8183_IPU_CORE1 bool "Clock driver for MediaTek MT8183 ipu_core1" depends on COMMON_CLK_MT8183 + default COMMON_CLK_MT8183 help This driver supports MediaTek MT8183 ipu_core1 clocks. config COMMON_CLK_MT8183_IPU_ADL bool "Clock driver for MediaTek MT8183 ipu_adl" depends on COMMON_CLK_MT8183 + default COMMON_CLK_MT8183 help This driver supports MediaTek MT8183 ipu_adl clocks. config COMMON_CLK_MT8183_IPU_CONN bool "Clock driver for MediaTek MT8183 ipu_conn" depends on COMMON_CLK_MT8183 + default COMMON_CLK_MT8183 help This driver supports MediaTek MT8183 ipu_conn clocks. config COMMON_CLK_MT8183_MFGCFG bool "Clock driver for MediaTek MT8183 mfgcfg" depends on COMMON_CLK_MT8183 + default COMMON_CLK_MT8183 help This driver supports MediaTek MT8183 mfgcfg clocks. config COMMON_CLK_MT8183_MMSYS bool "Clock driver for MediaTek MT8183 mmsys" depends on COMMON_CLK_MT8183 + default COMMON_CLK_MT8183 help This driver supports MediaTek MT8183 mmsys clocks. config COMMON_CLK_MT8183_VDECSYS bool "Clock driver for MediaTek MT8183 vdecsys" depends on COMMON_CLK_MT8183 + default COMMON_CLK_MT8183 help This driver supports MediaTek MT8183 vdecsys clocks. config COMMON_CLK_MT8183_VENCSYS bool "Clock driver for MediaTek MT8183 vencsys" depends on COMMON_CLK_MT8183 + default COMMON_CLK_MT8183 help This driver supports MediaTek MT8183 vencsys clocks. diff --git a/drivers/clk/mediatek/clk-mux.c b/drivers/clk/mediatek/clk-mux.c index dcc1352bf13cf3e120847f5589a815ead7205c75..b0c61709bacc2a57a54b0b878dbf631b0f421f1a 100644 --- a/drivers/clk/mediatek/clk-mux.c +++ b/drivers/clk/mediatek/clk-mux.c @@ -17,29 +17,36 @@ static inline struct mtk_clk_mux *to_mtk_clk_mux(struct clk_hw *hw) return container_of(hw, struct mtk_clk_mux, hw); } -static int mtk_clk_mux_enable(struct clk_hw *hw) +static int mtk_clk_mux_enable_setclr(struct clk_hw *hw) { struct mtk_clk_mux *mux = to_mtk_clk_mux(hw); - u32 mask = BIT(mux->data->gate_shift); - - return regmap_update_bits(mux->regmap, mux->data->mux_ofs, - mask, ~mask); -} + unsigned long flags = 0; -static void mtk_clk_mux_disable(struct clk_hw *hw) -{ - struct mtk_clk_mux *mux = to_mtk_clk_mux(hw); - u32 mask = BIT(mux->data->gate_shift); + if (mux->lock) + spin_lock_irqsave(mux->lock, flags); + else + __acquire(mux->lock); - regmap_update_bits(mux->regmap, mux->data->mux_ofs, mask, mask); -} + regmap_write(mux->regmap, mux->data->clr_ofs, + BIT(mux->data->gate_shift)); + + /* + * If the parent has been changed when the clock was disabled, it will + * not be effective yet. Set the update bit to ensure the mux gets + * updated. + */ + if (mux->reparent && mux->data->upd_shift >= 0) { + regmap_write(mux->regmap, mux->data->upd_ofs, + BIT(mux->data->upd_shift)); + mux->reparent = false; + } -static int mtk_clk_mux_enable_setclr(struct clk_hw *hw) -{ - struct mtk_clk_mux *mux = to_mtk_clk_mux(hw); + if (mux->lock) + spin_unlock_irqrestore(mux->lock, flags); + else + __release(mux->lock); - return regmap_write(mux->regmap, mux->data->clr_ofs, - BIT(mux->data->gate_shift)); + return 0; } static void mtk_clk_mux_disable_setclr(struct clk_hw *hw) @@ -72,28 +79,6 @@ static u8 mtk_clk_mux_get_parent(struct clk_hw *hw) return val; } -static int mtk_clk_mux_set_parent_lock(struct clk_hw *hw, u8 index) -{ - struct mtk_clk_mux *mux = to_mtk_clk_mux(hw); - u32 mask = GENMASK(mux->data->mux_width - 1, 0); - unsigned long flags = 0; - - if (mux->lock) - spin_lock_irqsave(mux->lock, flags); - else - __acquire(mux->lock); - - regmap_update_bits(mux->regmap, mux->data->mux_ofs, mask, - index << mux->data->mux_shift); - - if (mux->lock) - spin_unlock_irqrestore(mux->lock, flags); - else - __release(mux->lock); - - return 0; -} - static int mtk_clk_mux_set_parent_setclr_lock(struct clk_hw *hw, u8 index) { struct mtk_clk_mux *mux = to_mtk_clk_mux(hw); @@ -116,9 +101,11 @@ static int mtk_clk_mux_set_parent_setclr_lock(struct clk_hw *hw, u8 index) regmap_write(mux->regmap, mux->data->set_ofs, index << mux->data->mux_shift); - if (mux->data->upd_shift >= 0) + if (mux->data->upd_shift >= 0) { regmap_write(mux->regmap, mux->data->upd_ofs, BIT(mux->data->upd_shift)); + mux->reparent = true; + } } if (mux->lock) @@ -129,25 +116,7 @@ static int mtk_clk_mux_set_parent_setclr_lock(struct clk_hw *hw, u8 index) return 0; } -const struct clk_ops mtk_mux_ops = { - .get_parent = mtk_clk_mux_get_parent, - .set_parent = mtk_clk_mux_set_parent_lock, -}; - -const struct clk_ops mtk_mux_clr_set_upd_ops = { - .get_parent = mtk_clk_mux_get_parent, - .set_parent = mtk_clk_mux_set_parent_setclr_lock, -}; - -const struct clk_ops mtk_mux_gate_ops = { - .enable = mtk_clk_mux_enable, - .disable = mtk_clk_mux_disable, - .is_enabled = mtk_clk_mux_is_enabled, - .get_parent = mtk_clk_mux_get_parent, - .set_parent = mtk_clk_mux_set_parent_lock, -}; - -const struct clk_ops mtk_mux_gate_clr_set_upd_ops = { +static const struct clk_ops mtk_mux_ops = { .enable = mtk_clk_mux_enable_setclr, .disable = mtk_clk_mux_disable_setclr, .is_enabled = mtk_clk_mux_is_enabled, @@ -171,7 +140,7 @@ static struct clk *mtk_clk_register_mux(const struct mtk_mux *mux, init.flags = mux->flags | CLK_SET_RATE_PARENT; init.parent_names = mux->parent_names; init.num_parents = mux->num_parents; - init.ops = mux->ops; + init.ops = &mtk_mux_ops; clk_mux->regmap = regmap; clk_mux->data = mux; diff --git a/drivers/clk/mediatek/clk-mux.h b/drivers/clk/mediatek/clk-mux.h index 8e2f927dd2fff521ba34f38774db31241e5f7819..f1946161ade14107a836070ac7af810e46df82be 100644 --- a/drivers/clk/mediatek/clk-mux.h +++ b/drivers/clk/mediatek/clk-mux.h @@ -14,6 +14,7 @@ struct mtk_clk_mux { struct regmap *regmap; const struct mtk_mux *data; spinlock_t *lock; + bool reparent; }; struct mtk_mux { @@ -32,19 +33,12 @@ struct mtk_mux { u8 gate_shift; s8 upd_shift; - const struct clk_ops *ops; - signed char num_parents; }; -extern const struct clk_ops mtk_mux_ops; -extern const struct clk_ops mtk_mux_clr_set_upd_ops; -extern const struct clk_ops mtk_mux_gate_ops; -extern const struct clk_ops mtk_mux_gate_clr_set_upd_ops; - #define GATE_CLR_SET_UPD_FLAGS(_id, _name, _parents, _mux_ofs, \ _mux_set_ofs, _mux_clr_ofs, _shift, _width, \ - _gate, _upd_ofs, _upd, _flags, _ops) { \ + _gate, _upd_ofs, _upd, _flags) { \ .id = _id, \ .name = _name, \ .mux_ofs = _mux_ofs, \ @@ -58,7 +52,6 @@ extern const struct clk_ops mtk_mux_gate_clr_set_upd_ops; .parent_names = _parents, \ .num_parents = ARRAY_SIZE(_parents), \ .flags = _flags, \ - .ops = &_ops, \ } #define MUX_GATE_CLR_SET_UPD_FLAGS(_id, _name, _parents, _mux_ofs, \ @@ -66,8 +59,7 @@ extern const struct clk_ops mtk_mux_gate_clr_set_upd_ops; _gate, _upd_ofs, _upd, _flags) \ GATE_CLR_SET_UPD_FLAGS(_id, _name, _parents, _mux_ofs, \ _mux_set_ofs, _mux_clr_ofs, _shift, _width, \ - _gate, _upd_ofs, _upd, _flags, \ - mtk_mux_gate_clr_set_upd_ops) + _gate, _upd_ofs, _upd, _flags) \ #define MUX_GATE_CLR_SET_UPD(_id, _name, _parents, _mux_ofs, \ _mux_set_ofs, _mux_clr_ofs, _shift, _width, \ diff --git a/drivers/clk/meson/axg.c b/drivers/clk/meson/axg.c index 0e44695b8772defee1aa35b6c8647fe52fe6e720..2ad3801398dc114d660be253f1cb98dbf9f9f3be 100644 --- a/drivers/clk/meson/axg.c +++ b/drivers/clk/meson/axg.c @@ -1879,7 +1879,6 @@ static MESON_GATE(axg_mmc_pclk, HHI_GCLK_MPEG2, 11); static MESON_GATE(axg_vpu_intr, HHI_GCLK_MPEG2, 25); static MESON_GATE(axg_sec_ahb_ahb3_bridge, HHI_GCLK_MPEG2, 26); static MESON_GATE(axg_gic, HHI_GCLK_MPEG2, 30); -static MESON_GATE(axg_mipi_enable, HHI_MIPI_CNTL0, 29); /* Always On (AO) domain gates */ @@ -1974,7 +1973,6 @@ static struct clk_hw_onecell_data axg_hw_onecell_data = { [CLKID_PCIE_REF] = &axg_pcie_ref.hw, [CLKID_PCIE_CML_EN0] = &axg_pcie_cml_en0.hw, [CLKID_PCIE_CML_EN1] = &axg_pcie_cml_en1.hw, - [CLKID_MIPI_ENABLE] = &axg_mipi_enable.hw, [CLKID_GEN_CLK_SEL] = &axg_gen_clk_sel.hw, [CLKID_GEN_CLK_DIV] = &axg_gen_clk_div.hw, [CLKID_GEN_CLK] = &axg_gen_clk.hw, @@ -2115,7 +2113,6 @@ static struct clk_regmap *const axg_clk_regmaps[] = { &axg_pcie_ref, &axg_pcie_cml_en0, &axg_pcie_cml_en1, - &axg_mipi_enable, &axg_gen_clk_sel, &axg_gen_clk_div, &axg_gen_clk, diff --git a/drivers/clk/meson/axg.h b/drivers/clk/meson/axg.h index 481b307ea3cbbb95c8d929f06ea9bfe94e733a96..23ea87964af29a0b450360c7e8547b8b67bfb9a1 100644 --- a/drivers/clk/meson/axg.h +++ b/drivers/clk/meson/axg.h @@ -16,7 +16,6 @@ * Register offsets from the data sheet must be multiplied by 4 before * adding them to the base address to get the right value. */ -#define HHI_MIPI_CNTL0 0x00 #define HHI_GP0_PLL_CNTL 0x40 #define HHI_GP0_PLL_CNTL2 0x44 #define HHI_GP0_PLL_CNTL3 0x48 diff --git a/drivers/clk/meson/clk-pll.c b/drivers/clk/meson/clk-pll.c index b17a13e9337c40eb0fe677d6b283f91178ce01f9..49f27fe532139c7aa6f70b11f4a87046fc3dee0e 100644 --- a/drivers/clk/meson/clk-pll.c +++ b/drivers/clk/meson/clk-pll.c @@ -365,13 +365,14 @@ static int meson_clk_pll_set_rate(struct clk_hw *hw, unsigned long rate, { struct clk_regmap *clk = to_clk_regmap(hw); struct meson_clk_pll_data *pll = meson_clk_pll_data(clk); - unsigned int enabled, m, n, frac = 0, ret; + unsigned int enabled, m, n, frac = 0; unsigned long old_rate; + int ret; if (parent_rate == 0 || rate == 0) return -EINVAL; - old_rate = rate; + old_rate = clk_hw_get_rate(hw); ret = meson_clk_get_pll_settings(rate, parent_rate, &m, &n, pll); if (ret) @@ -393,7 +394,8 @@ static int meson_clk_pll_set_rate(struct clk_hw *hw, unsigned long rate, if (!enabled) return 0; - if (meson_clk_pll_enable(hw)) { + ret = meson_clk_pll_enable(hw); + if (ret) { pr_warn("%s: pll did not lock, trying to restore old rate %lu\n", __func__, old_rate); /* @@ -405,7 +407,7 @@ static int meson_clk_pll_set_rate(struct clk_hw *hw, unsigned long rate, meson_clk_pll_set_rate(hw, old_rate, parent_rate); } - return 0; + return ret; } /* diff --git a/drivers/clk/meson/meson8b.c b/drivers/clk/meson/meson8b.c index 862f0756b50f0c1f3ae5e1cc29ec9cf1daa470ec..a844d35b553a2b4236aa1d92fa9f4c5fca8b172f 100644 --- a/drivers/clk/meson/meson8b.c +++ b/drivers/clk/meson/meson8b.c @@ -52,15 +52,6 @@ static const struct pll_params_table sys_pll_params_table[] = { { /* sentinel */ }, }; -static struct clk_fixed_rate meson8b_xtal = { - .fixed_rate = 24000000, - .hw.init = &(struct clk_init_data){ - .name = "xtal", - .num_parents = 0, - .ops = &clk_fixed_rate_ops, - }, -}; - static struct clk_regmap meson8b_fixed_pll_dco = { .data = &(struct meson_clk_pll_data){ .en = { @@ -2715,7 +2706,6 @@ static MESON_GATE(meson8b_ao_iface, HHI_GCLK_AO, 3); static struct clk_hw_onecell_data meson8_hw_onecell_data = { .hws = { - [CLKID_XTAL] = &meson8b_xtal.hw, [CLKID_PLL_FIXED] = &meson8b_fixed_pll.hw, [CLKID_PLL_VID] = &meson8b_vid_pll.hw, [CLKID_PLL_SYS] = &meson8b_sys_pll.hw, @@ -2922,7 +2912,6 @@ static struct clk_hw_onecell_data meson8_hw_onecell_data = { static struct clk_hw_onecell_data meson8b_hw_onecell_data = { .hws = { - [CLKID_XTAL] = &meson8b_xtal.hw, [CLKID_PLL_FIXED] = &meson8b_fixed_pll.hw, [CLKID_PLL_VID] = &meson8b_vid_pll.hw, [CLKID_PLL_SYS] = &meson8b_sys_pll.hw, @@ -3140,7 +3129,6 @@ static struct clk_hw_onecell_data meson8b_hw_onecell_data = { static struct clk_hw_onecell_data meson8m2_hw_onecell_data = { .hws = { - [CLKID_XTAL] = &meson8b_xtal.hw, [CLKID_PLL_FIXED] = &meson8b_fixed_pll.hw, [CLKID_PLL_VID] = &meson8b_vid_pll.hw, [CLKID_PLL_SYS] = &meson8b_sys_pll.hw, @@ -3725,36 +3713,19 @@ static struct meson8b_nb_data meson8b_cpu_nb_data = { .nb.notifier_call = meson8b_cpu_clk_notifier_cb, }; -static const struct regmap_config clkc_regmap_config = { - .reg_bits = 32, - .val_bits = 32, - .reg_stride = 4, -}; - static void __init meson8b_clkc_init_common(struct device_node *np, struct clk_hw_onecell_data *clk_hw_onecell_data) { struct meson8b_clk_reset *rstc; const char *notifier_clk_name; struct clk *notifier_clk; - void __iomem *clk_base; struct regmap *map; int i, ret; map = syscon_node_to_regmap(of_get_parent(np)); if (IS_ERR(map)) { - pr_info("failed to get HHI regmap - Trying obsolete regs\n"); - - /* Generic clocks, PLLs and some of the reset-bits */ - clk_base = of_iomap(np, 1); - if (!clk_base) { - pr_err("%s: Unable to map clk base\n", __func__); - return; - } - - map = regmap_init_mmio(NULL, clk_base, &clkc_regmap_config); - if (IS_ERR(map)) - return; + pr_err("failed to get HHI regmap - Trying obsolete regs\n"); + return; } rstc = kzalloc(sizeof(*rstc), GFP_KERNEL); @@ -3778,16 +3749,10 @@ static void __init meson8b_clkc_init_common(struct device_node *np, meson8b_clk_regmaps[i]->map = map; /* - * always skip CLKID_UNUSED and also skip XTAL if the .dtb provides the - * XTAL clock as input. + * register all clks and start with the first used ID (which is + * CLKID_PLL_FIXED) */ - if (!IS_ERR(of_clk_get_by_name(np, "xtal"))) - i = CLKID_PLL_FIXED; - else - i = CLKID_XTAL; - - /* register all clks */ - for (; i < CLK_NR_CLKS; i++) { + for (i = CLKID_PLL_FIXED; i < CLK_NR_CLKS; i++) { /* array might be sparse */ if (!clk_hw_onecell_data->hws[i]) continue; diff --git a/drivers/clk/mstar/Kconfig b/drivers/clk/mstar/Kconfig new file mode 100644 index 0000000000000000000000000000000000000000..de37e1bce2d26cfc0d47602d170386fe240458e1 --- /dev/null +++ b/drivers/clk/mstar/Kconfig @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: GPL-2.0-only +config MSTAR_MSC313_MPLL + bool "MStar MPLL driver" + depends on ARCH_MSTARV7 || COMPILE_TEST + default ARCH_MSTARV7 + select REGMAP_MMIO + help + Support for the MPLL PLL and dividers block present on + MStar/Sigmastar SoCs. diff --git a/drivers/clk/mstar/Makefile b/drivers/clk/mstar/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..f8dcd25ede1dbb88c51b455eeac6dec765524fc1 --- /dev/null +++ b/drivers/clk/mstar/Makefile @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Makefile for mstar specific clk +# + +obj-$(CONFIG_MSTAR_MSC313_MPLL) += clk-msc313-mpll.o diff --git a/drivers/clk/mstar/clk-msc313-mpll.c b/drivers/clk/mstar/clk-msc313-mpll.c new file mode 100644 index 0000000000000000000000000000000000000000..61beb4e87525096258f3d8cf2e5548af10790b40 --- /dev/null +++ b/drivers/clk/mstar/clk-msc313-mpll.c @@ -0,0 +1,155 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * MStar MSC313 MPLL driver + * + * Copyright (C) 2020 Daniel Palmer + */ + +#include +#include +#include +#include + +#define REG_CONFIG1 0x8 +#define REG_CONFIG2 0xc + +static const struct regmap_config msc313_mpll_regmap_config = { + .reg_bits = 16, + .val_bits = 16, + .reg_stride = 4, +}; + +static const struct reg_field config1_loop_div_first = REG_FIELD(REG_CONFIG1, 8, 9); +static const struct reg_field config1_input_div_first = REG_FIELD(REG_CONFIG1, 4, 5); +static const struct reg_field config2_output_div_first = REG_FIELD(REG_CONFIG2, 12, 13); +static const struct reg_field config2_loop_div_second = REG_FIELD(REG_CONFIG2, 0, 7); + +static const unsigned int output_dividers[] = { + 2, 3, 4, 5, 6, 7, 10 +}; + +#define NUMOUTPUTS (ARRAY_SIZE(output_dividers) + 1) + +struct msc313_mpll { + struct clk_hw clk_hw; + struct regmap_field *input_div; + struct regmap_field *loop_div_first; + struct regmap_field *loop_div_second; + struct regmap_field *output_div; + struct clk_hw_onecell_data *clk_data; +}; + +#define to_mpll(_hw) container_of(_hw, struct msc313_mpll, clk_hw) + +static unsigned long msc313_mpll_recalc_rate(struct clk_hw *hw, + unsigned long parent_rate) +{ + struct msc313_mpll *mpll = to_mpll(hw); + unsigned int input_div, output_div, loop_first, loop_second; + unsigned long output_rate; + + regmap_field_read(mpll->input_div, &input_div); + regmap_field_read(mpll->output_div, &output_div); + regmap_field_read(mpll->loop_div_first, &loop_first); + regmap_field_read(mpll->loop_div_second, &loop_second); + + output_rate = parent_rate / (1 << input_div); + output_rate *= (1 << loop_first) * max(loop_second, 1U); + output_rate /= max(output_div, 1U); + + return output_rate; +} + +static const struct clk_ops msc313_mpll_ops = { + .recalc_rate = msc313_mpll_recalc_rate, +}; + +static const struct clk_parent_data mpll_parent = { + .index = 0, +}; + +static int msc313_mpll_probe(struct platform_device *pdev) +{ + void __iomem *base; + struct msc313_mpll *mpll; + struct clk_init_data clk_init = { }; + struct device *dev = &pdev->dev; + struct regmap *regmap; + char *outputname; + struct clk_hw *divhw; + int ret, i; + + mpll = devm_kzalloc(dev, sizeof(*mpll), GFP_KERNEL); + if (!mpll) + return -ENOMEM; + + base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(base)) + return PTR_ERR(base); + + regmap = devm_regmap_init_mmio(dev, base, &msc313_mpll_regmap_config); + if (IS_ERR(regmap)) + return PTR_ERR(regmap); + + mpll->input_div = devm_regmap_field_alloc(dev, regmap, config1_input_div_first); + if (IS_ERR(mpll->input_div)) + return PTR_ERR(mpll->input_div); + mpll->output_div = devm_regmap_field_alloc(dev, regmap, config2_output_div_first); + if (IS_ERR(mpll->output_div)) + return PTR_ERR(mpll->output_div); + mpll->loop_div_first = devm_regmap_field_alloc(dev, regmap, config1_loop_div_first); + if (IS_ERR(mpll->loop_div_first)) + return PTR_ERR(mpll->loop_div_first); + mpll->loop_div_second = devm_regmap_field_alloc(dev, regmap, config2_loop_div_second); + if (IS_ERR(mpll->loop_div_second)) + return PTR_ERR(mpll->loop_div_second); + + mpll->clk_data = devm_kzalloc(dev, struct_size(mpll->clk_data, hws, + ARRAY_SIZE(output_dividers)), GFP_KERNEL); + if (!mpll->clk_data) + return -ENOMEM; + + clk_init.name = dev_name(dev); + clk_init.ops = &msc313_mpll_ops; + clk_init.parent_data = &mpll_parent; + clk_init.num_parents = 1; + mpll->clk_hw.init = &clk_init; + + ret = devm_clk_hw_register(dev, &mpll->clk_hw); + if (ret) + return ret; + + mpll->clk_data->num = NUMOUTPUTS; + mpll->clk_data->hws[0] = &mpll->clk_hw; + + for (i = 0; i < ARRAY_SIZE(output_dividers); i++) { + outputname = devm_kasprintf(dev, GFP_KERNEL, "%s_div_%u", + clk_init.name, output_dividers[i]); + if (!outputname) + return -ENOMEM; + divhw = devm_clk_hw_register_fixed_factor(dev, outputname, + clk_init.name, 0, 1, output_dividers[i]); + if (IS_ERR(divhw)) + return PTR_ERR(divhw); + mpll->clk_data->hws[i + 1] = divhw; + } + + platform_set_drvdata(pdev, mpll); + + return devm_of_clk_add_hw_provider(&pdev->dev, of_clk_hw_onecell_get, + mpll->clk_data); +} + +static const struct of_device_id msc313_mpll_of_match[] = { + { .compatible = "mstar,msc313-mpll", }, + {} +}; + +static struct platform_driver msc313_mpll_driver = { + .driver = { + .name = "mstar-msc313-mpll", + .of_match_table = msc313_mpll_of_match, + }, + .probe = msc313_mpll_probe, +}; +builtin_platform_driver(msc313_mpll_driver); diff --git a/drivers/clk/mvebu/ap-cpu-clk.c b/drivers/clk/mvebu/ap-cpu-clk.c index b4259b60dcfd67b486beb3a3f630c0c2e4025df0..08ba59ec3fb17fc30dbf665f46496350029513de 100644 --- a/drivers/clk/mvebu/ap-cpu-clk.c +++ b/drivers/clk/mvebu/ap-cpu-clk.c @@ -30,7 +30,7 @@ #define APN806_MAX_DIVIDER 32 -/** +/* * struct cpu_dfs_regs: CPU DFS register mapping * @divider_reg: full integer ratio from PLL frequency to CPU clock frequency * @force_reg: request to force new ratio regardless of relation to other clocks diff --git a/drivers/clk/qcom/Kconfig b/drivers/clk/qcom/Kconfig index d32bb12cd8d0c06123e22956161836c7e8be4e48..45646b867cdbc14f2c29bc794c46bf0d691aac24 100644 --- a/drivers/clk/qcom/Kconfig +++ b/drivers/clk/qcom/Kconfig @@ -28,6 +28,14 @@ config QCOM_A53PLL Say Y if you want to support higher CPU frequencies on MSM8916 devices. +config QCOM_A7PLL + tristate "SDX55 A7 PLL" + help + Support for the A7 PLL on SDX55 devices. It provides the CPU with + frequencies above 1GHz. + Say Y if you want to support higher CPU frequencies on SDX55 + devices. + config QCOM_CLK_APCS_MSM8916 tristate "MSM8916 APCS Clock Controller" depends on QCOM_APCS_IPC || COMPILE_TEST @@ -46,6 +54,15 @@ config QCOM_CLK_APCC_MSM8996 Say Y if you want to support CPU clock scaling using CPUfreq drivers for dynamic power management. +config QCOM_CLK_APCS_SDX55 + tristate "SDX55 APCS Clock Controller" + depends on QCOM_APCS_IPC || COMPILE_TEST + help + Support for the APCS Clock Controller on SDX55 platform. The + APCS is managing the mux and divider which feeds the CPUs. + Say Y if you want to support CPU frequency scaling on devices + such as SDX55. + config QCOM_CLK_RPM tristate "RPM based Clock Controller" depends on MFD_QCOM_RPM @@ -317,6 +334,24 @@ config SC_GCC_7180 Say Y if you want to use peripheral devices such as UART, SPI, I2C, USB, UFS, SDCC, etc. +config SC_GCC_7280 + tristate "SC7280 Global Clock Controller" + select QCOM_GDSC + depends on COMMON_CLK_QCOM + help + Support for the global clock controller on SC7280 devices. + Say Y if you want to use peripheral devices such as UART, SPI, + I2C, USB, UFS, SDCC, PCIe etc. + +config SC_GCC_8180X + tristate "SC8180X Global Clock Controller" + select QCOM_GDSC + depends on COMMON_CLK_QCOM + help + Support for the global clock controller on SC8180X devices. + Say Y if you want to use peripheral devices such as UART, SPI, + I2C, USB, UFS, SDCC, etc. + config SC_LPASS_CORECC_7180 tristate "SC7180 LPASS Core Clock Controller" select SC_GCC_7180 @@ -366,6 +401,24 @@ config SDM_GCC_660 Say Y if you want to use peripheral devices such as UART, SPI, i2C, USB, UFS, SDDC, PCIe, etc. +config SDM_MMCC_660 + tristate "SDM660 Multimedia Clock Controller" + select SDM_GCC_660 + select QCOM_GDSC + help + Support for the multimedia clock controller on SDM660 devices. + Say Y if you want to support multimedia devices such as display, + graphics, video encode/decode, camera, etc. + +config SDM_GPUCC_660 + tristate "SDM660 Graphics Clock Controller" + select SDM_GCC_660 + select QCOM_GDSC + help + Support for the graphics clock controller on SDM630/636/660 devices. + Say Y if you want to support graphics controller devices and + functionality such as 3D graphics + config QCS_TURING_404 tristate "QCS404 Turing Clock Controller" help @@ -454,6 +507,14 @@ config SM_GCC_8250 Say Y if you want to use peripheral devices such as UART, SPI, I2C, USB, SD/UFS, PCIe etc. +config SM_GCC_8350 + tristate "SM8350 Global Clock Controller" + select QCOM_GDSC + help + Support for the global clock controller on SM8350 devices. + Say Y if you want to use peripheral devices such as UART, + SPI, I2C, USB, SD/UFS, PCIe etc. + config SM_GPUCC_8150 tristate "SM8150 Graphics Clock Controller" select SM_GCC_8150 diff --git a/drivers/clk/qcom/Makefile b/drivers/clk/qcom/Makefile index 9e5e0e3cb7b48e79822d9483d7bcf909e864be3b..c8291312e72307ee9a9fb3cb1662b9a3a622d1aa 100644 --- a/drivers/clk/qcom/Makefile +++ b/drivers/clk/qcom/Makefile @@ -44,8 +44,10 @@ obj-$(CONFIG_MSM_MMCC_8974) += mmcc-msm8974.o obj-$(CONFIG_MSM_MMCC_8996) += mmcc-msm8996.o obj-$(CONFIG_MSM_MMCC_8998) += mmcc-msm8998.o obj-$(CONFIG_QCOM_A53PLL) += a53-pll.o +obj-$(CONFIG_QCOM_A7PLL) += a7-pll.o obj-$(CONFIG_QCOM_CLK_APCS_MSM8916) += apcs-msm8916.o obj-$(CONFIG_QCOM_CLK_APCC_MSM8996) += clk-cpu-8996.o +obj-$(CONFIG_QCOM_CLK_APCS_SDX55) += apcs-sdx55.o obj-$(CONFIG_QCOM_CLK_RPM) += clk-rpm.o obj-$(CONFIG_QCOM_CLK_RPMH) += clk-rpmh.o obj-$(CONFIG_QCOM_CLK_SMD_RPM) += clk-smd-rpm.o @@ -55,6 +57,8 @@ obj-$(CONFIG_QCS_TURING_404) += turingcc-qcs404.o obj-$(CONFIG_SC_CAMCC_7180) += camcc-sc7180.o obj-$(CONFIG_SC_DISPCC_7180) += dispcc-sc7180.o obj-$(CONFIG_SC_GCC_7180) += gcc-sc7180.o +obj-$(CONFIG_SC_GCC_7280) += gcc-sc7280.o +obj-$(CONFIG_SC_GCC_8180X) += gcc-sc8180x.o obj-$(CONFIG_SC_GPUCC_7180) += gpucc-sc7180.o obj-$(CONFIG_SC_LPASS_CORECC_7180) += lpasscorecc-sc7180.o obj-$(CONFIG_SC_MSS_7180) += mss-sc7180.o @@ -62,6 +66,8 @@ obj-$(CONFIG_SC_VIDEOCC_7180) += videocc-sc7180.o obj-$(CONFIG_SDM_CAMCC_845) += camcc-sdm845.o obj-$(CONFIG_SDM_DISPCC_845) += dispcc-sdm845.o obj-$(CONFIG_SDM_GCC_660) += gcc-sdm660.o +obj-$(CONFIG_SDM_MMCC_660) += mmcc-sdm660.o +obj-$(CONFIG_SDM_GPUCC_660) += gpucc-sdm660.o obj-$(CONFIG_SDM_GCC_845) += gcc-sdm845.o obj-$(CONFIG_SDM_GPUCC_845) += gpucc-sdm845.o obj-$(CONFIG_SDM_LPASSCC_845) += lpasscc-sdm845.o @@ -70,6 +76,7 @@ obj-$(CONFIG_SDX_GCC_55) += gcc-sdx55.o obj-$(CONFIG_SM_DISPCC_8250) += dispcc-sm8250.o obj-$(CONFIG_SM_GCC_8150) += gcc-sm8150.o obj-$(CONFIG_SM_GCC_8250) += gcc-sm8250.o +obj-$(CONFIG_SM_GCC_8350) += gcc-sm8350.o obj-$(CONFIG_SM_GPUCC_8150) += gpucc-sm8150.o obj-$(CONFIG_SM_GPUCC_8250) += gpucc-sm8250.o obj-$(CONFIG_SM_VIDEOCC_8150) += videocc-sm8150.o diff --git a/drivers/clk/qcom/a7-pll.c b/drivers/clk/qcom/a7-pll.c new file mode 100644 index 0000000000000000000000000000000000000000..e171d3caf2cf379c69a03fa88082a952a1832678 --- /dev/null +++ b/drivers/clk/qcom/a7-pll.c @@ -0,0 +1,100 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Qualcomm A7 PLL driver + * + * Copyright (c) 2020, Linaro Limited + * Author: Manivannan Sadhasivam + */ + +#include +#include +#include +#include + +#include "clk-alpha-pll.h" + +#define LUCID_PLL_OFF_L_VAL 0x04 + +static const struct pll_vco lucid_vco[] = { + { 249600000, 2000000000, 0 }, +}; + +static struct clk_alpha_pll a7pll = { + .offset = 0x100, + .vco_table = lucid_vco, + .num_vco = ARRAY_SIZE(lucid_vco), + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID], + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "a7pll", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "bi_tcxo", + }, + .num_parents = 1, + .ops = &clk_alpha_pll_lucid_ops, + }, + }, +}; + +static const struct alpha_pll_config a7pll_config = { + .l = 0x39, + .config_ctl_val = 0x20485699, + .config_ctl_hi_val = 0x2261, + .config_ctl_hi1_val = 0x029A699C, + .user_ctl_val = 0x1, + .user_ctl_hi_val = 0x805, +}; + +static const struct regmap_config a7pll_regmap_config = { + .reg_bits = 32, + .reg_stride = 4, + .val_bits = 32, + .max_register = 0x1000, + .fast_io = true, +}; + +static int qcom_a7pll_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct regmap *regmap; + void __iomem *base; + u32 l_val; + int ret; + + base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(base)) + return PTR_ERR(base); + + regmap = devm_regmap_init_mmio(dev, base, &a7pll_regmap_config); + if (IS_ERR(regmap)) + return PTR_ERR(regmap); + + /* Configure PLL only if the l_val is zero */ + regmap_read(regmap, a7pll.offset + LUCID_PLL_OFF_L_VAL, &l_val); + if (!l_val) + clk_lucid_pll_configure(&a7pll, regmap, &a7pll_config); + + ret = devm_clk_register_regmap(dev, &a7pll.clkr); + if (ret) + return ret; + + return devm_of_clk_add_hw_provider(dev, of_clk_hw_simple_get, + &a7pll.clkr.hw); +} + +static const struct of_device_id qcom_a7pll_match_table[] = { + { .compatible = "qcom,sdx55-a7pll" }, + { } +}; + +static struct platform_driver qcom_a7pll_driver = { + .probe = qcom_a7pll_probe, + .driver = { + .name = "qcom-a7pll", + .of_match_table = qcom_a7pll_match_table, + }, +}; +module_platform_driver(qcom_a7pll_driver); + +MODULE_DESCRIPTION("Qualcomm A7 PLL Driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/clk/qcom/apcs-sdx55.c b/drivers/clk/qcom/apcs-sdx55.c new file mode 100644 index 0000000000000000000000000000000000000000..d0edabebf9c2828d9cb392e23d0a61515a416024 --- /dev/null +++ b/drivers/clk/qcom/apcs-sdx55.c @@ -0,0 +1,149 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Qualcomm SDX55 APCS clock controller driver + * + * Copyright (c) 2020, Linaro Limited + * Author: Manivannan Sadhasivam + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "clk-regmap.h" +#include "clk-regmap-mux-div.h" + +static const u32 apcs_mux_clk_parent_map[] = { 0, 1, 5 }; + +static const struct clk_parent_data pdata[] = { + { .fw_name = "ref" }, + { .fw_name = "aux" }, + { .fw_name = "pll" }, +}; + +/* + * We use the notifier function for switching to a temporary safe configuration + * (mux and divider), while the A7 PLL is reconfigured. + */ +static int a7cc_notifier_cb(struct notifier_block *nb, unsigned long event, + void *data) +{ + int ret = 0; + struct clk_regmap_mux_div *md = container_of(nb, + struct clk_regmap_mux_div, + clk_nb); + if (event == PRE_RATE_CHANGE) + /* set the mux and divider to safe frequency (400mhz) */ + ret = mux_div_set_src_div(md, 1, 2); + + return notifier_from_errno(ret); +} + +static int qcom_apcs_sdx55_clk_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct device *parent = dev->parent; + struct device *cpu_dev; + struct clk_regmap_mux_div *a7cc; + struct regmap *regmap; + struct clk_init_data init = { }; + int ret; + + regmap = dev_get_regmap(parent, NULL); + if (!regmap) { + dev_err_probe(dev, -ENODEV, "Failed to get parent regmap\n"); + return -ENODEV; + } + + a7cc = devm_kzalloc(dev, sizeof(*a7cc), GFP_KERNEL); + if (!a7cc) + return -ENOMEM; + + init.name = "a7mux"; + init.parent_data = pdata; + init.num_parents = ARRAY_SIZE(pdata); + init.ops = &clk_regmap_mux_div_ops; + + a7cc->clkr.hw.init = &init; + a7cc->clkr.regmap = regmap; + a7cc->reg_offset = 0x8; + a7cc->hid_width = 5; + a7cc->hid_shift = 0; + a7cc->src_width = 3; + a7cc->src_shift = 8; + a7cc->parent_map = apcs_mux_clk_parent_map; + + a7cc->pclk = devm_clk_get(parent, "pll"); + if (IS_ERR(a7cc->pclk)) { + ret = PTR_ERR(a7cc->pclk); + if (ret != -EPROBE_DEFER) + dev_err_probe(dev, ret, "Failed to get PLL clk\n"); + return ret; + } + + a7cc->clk_nb.notifier_call = a7cc_notifier_cb; + ret = clk_notifier_register(a7cc->pclk, &a7cc->clk_nb); + if (ret) { + dev_err_probe(dev, ret, "Failed to register clock notifier\n"); + return ret; + } + + ret = devm_clk_register_regmap(dev, &a7cc->clkr); + if (ret) { + dev_err_probe(dev, ret, "Failed to register regmap clock\n"); + goto err; + } + + ret = devm_of_clk_add_hw_provider(dev, of_clk_hw_simple_get, + &a7cc->clkr.hw); + if (ret) { + dev_err_probe(dev, ret, "Failed to add clock provider\n"); + goto err; + } + + platform_set_drvdata(pdev, a7cc); + + /* + * Attach the power domain to cpudev. Since there is no dedicated driver + * for CPUs and the SDX55 platform lacks hardware specific CPUFreq + * driver, there seems to be no better place to do this. So do it here! + */ + cpu_dev = get_cpu_device(0); + dev_pm_domain_attach(cpu_dev, true); + + return 0; + +err: + clk_notifier_unregister(a7cc->pclk, &a7cc->clk_nb); + return ret; +} + +static int qcom_apcs_sdx55_clk_remove(struct platform_device *pdev) +{ + struct device *cpu_dev = get_cpu_device(0); + struct clk_regmap_mux_div *a7cc = platform_get_drvdata(pdev); + + clk_notifier_unregister(a7cc->pclk, &a7cc->clk_nb); + dev_pm_domain_detach(cpu_dev, true); + + return 0; +} + +static struct platform_driver qcom_apcs_sdx55_clk_driver = { + .probe = qcom_apcs_sdx55_clk_probe, + .remove = qcom_apcs_sdx55_clk_remove, + .driver = { + .name = "qcom-sdx55-acps-clk", + }, +}; +module_platform_driver(qcom_apcs_sdx55_clk_driver); + +MODULE_AUTHOR("Manivannan Sadhasivam "); +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("Qualcomm SDX55 APCS clock driver"); diff --git a/drivers/clk/qcom/clk-alpha-pll.c b/drivers/clk/qcom/clk-alpha-pll.c index 21c357c26ec42a8e98d505d48dc62c4eaa569ef2..c6eb99169ddc748c61cf20c8341db4345ac1eb9f 100644 --- a/drivers/clk/qcom/clk-alpha-pll.c +++ b/drivers/clk/qcom/clk-alpha-pll.c @@ -156,6 +156,12 @@ EXPORT_SYMBOL_GPL(clk_alpha_pll_regs); /* LUCID PLL specific settings and offsets */ #define LUCID_PCAL_DONE BIT(27) +/* LUCID 5LPE PLL specific settings and offsets */ +#define LUCID_5LPE_PCAL_DONE BIT(11) +#define LUCID_5LPE_ALPHA_PLL_ACK_LATCH BIT(13) +#define LUCID_5LPE_PLL_LATCH_INPUT BIT(14) +#define LUCID_5LPE_ENABLE_VOTE_RUN BIT(21) + #define pll_alpha_width(p) \ ((PLL_ALPHA_VAL_U(p) - PLL_ALPHA_VAL(p) == 4) ? \ ALPHA_REG_BITWIDTH : ALPHA_REG_16BIT_WIDTH) @@ -777,15 +783,15 @@ static long alpha_pll_huayra_round_rate(struct clk_hw *hw, unsigned long rate, static int trion_pll_is_enabled(struct clk_alpha_pll *pll, struct regmap *regmap) { - u32 mode_regval, opmode_regval; + u32 mode_val, opmode_val; int ret; - ret = regmap_read(regmap, PLL_MODE(pll), &mode_regval); - ret |= regmap_read(regmap, PLL_OPMODE(pll), &opmode_regval); + ret = regmap_read(regmap, PLL_MODE(pll), &mode_val); + ret |= regmap_read(regmap, PLL_OPMODE(pll), &opmode_val); if (ret) return 0; - return ((opmode_regval & PLL_RUN) && (mode_regval & PLL_OUTCTRL)); + return ((opmode_val & PLL_RUN) && (mode_val & PLL_OUTCTRL)); } static int clk_trion_pll_is_enabled(struct clk_hw *hw) @@ -1445,12 +1451,12 @@ EXPORT_SYMBOL_GPL(clk_trion_pll_configure); static int __alpha_pll_trion_prepare(struct clk_hw *hw, u32 pcal_done) { struct clk_alpha_pll *pll = to_clk_alpha_pll(hw); - u32 regval; + u32 val; int ret; /* Return early if calibration is not needed. */ - regmap_read(pll->clkr.regmap, PLL_STATUS(pll), ®val); - if (regval & pcal_done) + regmap_read(pll->clkr.regmap, PLL_STATUS(pll), &val); + if (val & pcal_done) return 0; /* On/off to calibrate */ @@ -1471,12 +1477,12 @@ static int alpha_pll_lucid_prepare(struct clk_hw *hw) return __alpha_pll_trion_prepare(hw, LUCID_PCAL_DONE); } -static int alpha_pll_trion_set_rate(struct clk_hw *hw, unsigned long rate, - unsigned long prate) +static int __alpha_pll_trion_set_rate(struct clk_hw *hw, unsigned long rate, + unsigned long prate, u32 latch_bit, u32 latch_ack) { struct clk_alpha_pll *pll = to_clk_alpha_pll(hw); unsigned long rrate; - u32 regval, l, alpha_width = pll_alpha_width(pll); + u32 val, l, alpha_width = pll_alpha_width(pll); u64 a; int ret; @@ -1490,22 +1496,20 @@ static int alpha_pll_trion_set_rate(struct clk_hw *hw, unsigned long rate, regmap_write(pll->clkr.regmap, PLL_ALPHA_VAL(pll), a); /* Latch the PLL input */ - ret = regmap_update_bits(pll->clkr.regmap, PLL_MODE(pll), - PLL_UPDATE, PLL_UPDATE); + ret = regmap_update_bits(pll->clkr.regmap, PLL_MODE(pll), latch_bit, latch_bit); if (ret) return ret; /* Wait for 2 reference cycles before checking the ACK bit. */ udelay(1); - regmap_read(pll->clkr.regmap, PLL_MODE(pll), ®val); - if (!(regval & ALPHA_PLL_ACK_LATCH)) { + regmap_read(pll->clkr.regmap, PLL_MODE(pll), &val); + if (!(val & latch_ack)) { pr_err("Lucid PLL latch failed. Output may be unstable!\n"); return -EINVAL; } /* Return the latch input to 0 */ - ret = regmap_update_bits(pll->clkr.regmap, PLL_MODE(pll), - PLL_UPDATE, 0); + ret = regmap_update_bits(pll->clkr.regmap, PLL_MODE(pll), latch_bit, 0); if (ret) return ret; @@ -1520,6 +1524,12 @@ static int alpha_pll_trion_set_rate(struct clk_hw *hw, unsigned long rate, return 0; } +static int alpha_pll_trion_set_rate(struct clk_hw *hw, unsigned long rate, + unsigned long prate) +{ + return __alpha_pll_trion_set_rate(hw, rate, prate, PLL_UPDATE, ALPHA_PLL_ACK_LATCH); +} + const struct clk_ops clk_alpha_pll_trion_ops = { .prepare = alpha_pll_trion_prepare, .enable = clk_trion_pll_enable, @@ -1600,3 +1610,170 @@ const struct clk_ops clk_alpha_pll_agera_ops = { .set_rate = clk_alpha_pll_agera_set_rate, }; EXPORT_SYMBOL_GPL(clk_alpha_pll_agera_ops); + +static int alpha_pll_lucid_5lpe_enable(struct clk_hw *hw) +{ + struct clk_alpha_pll *pll = to_clk_alpha_pll(hw); + u32 val; + int ret; + + ret = regmap_read(pll->clkr.regmap, PLL_USER_CTL(pll), &val); + if (ret) + return ret; + + /* If in FSM mode, just vote for it */ + if (val & LUCID_5LPE_ENABLE_VOTE_RUN) { + ret = clk_enable_regmap(hw); + if (ret) + return ret; + return wait_for_pll_enable_lock(pll); + } + + /* Check if PLL is already enabled, return if enabled */ + ret = trion_pll_is_enabled(pll, pll->clkr.regmap); + if (ret < 0) + return ret; + + ret = regmap_update_bits(pll->clkr.regmap, PLL_MODE(pll), PLL_RESET_N, PLL_RESET_N); + if (ret) + return ret; + + regmap_write(pll->clkr.regmap, PLL_OPMODE(pll), PLL_RUN); + + ret = wait_for_pll_enable_lock(pll); + if (ret) + return ret; + + /* Enable the PLL outputs */ + ret = regmap_update_bits(pll->clkr.regmap, PLL_USER_CTL(pll), PLL_OUT_MASK, PLL_OUT_MASK); + if (ret) + return ret; + + /* Enable the global PLL outputs */ + return regmap_update_bits(pll->clkr.regmap, PLL_MODE(pll), PLL_OUTCTRL, PLL_OUTCTRL); +} + +static void alpha_pll_lucid_5lpe_disable(struct clk_hw *hw) +{ + struct clk_alpha_pll *pll = to_clk_alpha_pll(hw); + u32 val; + int ret; + + ret = regmap_read(pll->clkr.regmap, PLL_USER_CTL(pll), &val); + if (ret) + return; + + /* If in FSM mode, just unvote it */ + if (val & LUCID_5LPE_ENABLE_VOTE_RUN) { + clk_disable_regmap(hw); + return; + } + + /* Disable the global PLL output */ + ret = regmap_update_bits(pll->clkr.regmap, PLL_MODE(pll), PLL_OUTCTRL, 0); + if (ret) + return; + + /* Disable the PLL outputs */ + ret = regmap_update_bits(pll->clkr.regmap, PLL_USER_CTL(pll), PLL_OUT_MASK, 0); + if (ret) + return; + + /* Place the PLL mode in STANDBY */ + regmap_write(pll->clkr.regmap, PLL_OPMODE(pll), PLL_STANDBY); +} + +/* + * The Lucid 5LPE PLL requires a power-on self-calibration which happens + * when the PLL comes out of reset. Calibrate in case it is not completed. + */ +static int alpha_pll_lucid_5lpe_prepare(struct clk_hw *hw) +{ + struct clk_alpha_pll *pll = to_clk_alpha_pll(hw); + struct clk_hw *p; + u32 val = 0; + int ret; + + /* Return early if calibration is not needed. */ + regmap_read(pll->clkr.regmap, PLL_MODE(pll), &val); + if (val & LUCID_5LPE_PCAL_DONE) + return 0; + + p = clk_hw_get_parent(hw); + if (!p) + return -EINVAL; + + ret = alpha_pll_lucid_5lpe_enable(hw); + if (ret) + return ret; + + alpha_pll_lucid_5lpe_disable(hw); + + return 0; +} + +static int alpha_pll_lucid_5lpe_set_rate(struct clk_hw *hw, unsigned long rate, + unsigned long prate) +{ + return __alpha_pll_trion_set_rate(hw, rate, prate, + LUCID_5LPE_PLL_LATCH_INPUT, + LUCID_5LPE_ALPHA_PLL_ACK_LATCH); +} + +static int clk_lucid_5lpe_pll_postdiv_set_rate(struct clk_hw *hw, unsigned long rate, + unsigned long parent_rate) +{ + struct clk_alpha_pll_postdiv *pll = to_clk_alpha_pll_postdiv(hw); + int i, val = 0, div, ret; + u32 mask; + + /* + * If the PLL is in FSM mode, then treat set_rate callback as a + * no-operation. + */ + ret = regmap_read(pll->clkr.regmap, PLL_USER_CTL(pll), &val); + if (ret) + return ret; + + if (val & LUCID_5LPE_ENABLE_VOTE_RUN) + return 0; + + div = DIV_ROUND_UP_ULL((u64)parent_rate, rate); + for (i = 0; i < pll->num_post_div; i++) { + if (pll->post_div_table[i].div == div) { + val = pll->post_div_table[i].val; + break; + } + } + + mask = GENMASK(pll->width + pll->post_div_shift - 1, pll->post_div_shift); + return regmap_update_bits(pll->clkr.regmap, PLL_USER_CTL(pll), + mask, val << pll->post_div_shift); +} + +const struct clk_ops clk_alpha_pll_lucid_5lpe_ops = { + .prepare = alpha_pll_lucid_5lpe_prepare, + .enable = alpha_pll_lucid_5lpe_enable, + .disable = alpha_pll_lucid_5lpe_disable, + .is_enabled = clk_trion_pll_is_enabled, + .recalc_rate = clk_trion_pll_recalc_rate, + .round_rate = clk_alpha_pll_round_rate, + .set_rate = alpha_pll_lucid_5lpe_set_rate, +}; +EXPORT_SYMBOL(clk_alpha_pll_lucid_5lpe_ops); + +const struct clk_ops clk_alpha_pll_fixed_lucid_5lpe_ops = { + .enable = alpha_pll_lucid_5lpe_enable, + .disable = alpha_pll_lucid_5lpe_disable, + .is_enabled = clk_trion_pll_is_enabled, + .recalc_rate = clk_trion_pll_recalc_rate, + .round_rate = clk_alpha_pll_round_rate, +}; +EXPORT_SYMBOL(clk_alpha_pll_fixed_lucid_5lpe_ops); + +const struct clk_ops clk_alpha_pll_postdiv_lucid_5lpe_ops = { + .recalc_rate = clk_alpha_pll_postdiv_fabia_recalc_rate, + .round_rate = clk_alpha_pll_postdiv_fabia_round_rate, + .set_rate = clk_lucid_5lpe_pll_postdiv_set_rate, +}; +EXPORT_SYMBOL(clk_alpha_pll_postdiv_lucid_5lpe_ops); diff --git a/drivers/clk/qcom/clk-alpha-pll.h b/drivers/clk/qcom/clk-alpha-pll.h index 0ea30d2f3da1cb53b3f56182d7522b244aeed783..6943e933be0f731882ed2bbc0beb759fb3298969 100644 --- a/drivers/clk/qcom/clk-alpha-pll.h +++ b/drivers/clk/qcom/clk-alpha-pll.h @@ -144,6 +144,10 @@ extern const struct clk_ops clk_alpha_pll_lucid_ops; extern const struct clk_ops clk_alpha_pll_postdiv_lucid_ops; extern const struct clk_ops clk_alpha_pll_agera_ops; +extern const struct clk_ops clk_alpha_pll_lucid_5lpe_ops; +extern const struct clk_ops clk_alpha_pll_fixed_lucid_5lpe_ops; +extern const struct clk_ops clk_alpha_pll_postdiv_lucid_5lpe_ops; + void clk_alpha_pll_configure(struct clk_alpha_pll *pll, struct regmap *regmap, const struct alpha_pll_config *config); void clk_fabia_pll_configure(struct clk_alpha_pll *pll, struct regmap *regmap, diff --git a/drivers/clk/qcom/clk-rcg.h b/drivers/clk/qcom/clk-rcg.h index 86d2b8b90173ab40510f6e1243d268788b564d95..99efcc7f8d886140b5857076547602289d337aba 100644 --- a/drivers/clk/qcom/clk-rcg.h +++ b/drivers/clk/qcom/clk-rcg.h @@ -153,6 +153,15 @@ struct clk_rcg2 { #define to_clk_rcg2(_hw) container_of(to_clk_regmap(_hw), struct clk_rcg2, clkr) +struct clk_rcg2_gfx3d { + u8 div; + struct clk_rcg2 rcg; + struct clk_hw **hws; +}; + +#define to_clk_rcg2_gfx3d(_hw) \ + container_of(to_clk_rcg2(_hw), struct clk_rcg2_gfx3d, rcg) + extern const struct clk_ops clk_rcg2_ops; extern const struct clk_ops clk_rcg2_floor_ops; extern const struct clk_ops clk_edp_pixel_ops; diff --git a/drivers/clk/qcom/clk-rcg2.c b/drivers/clk/qcom/clk-rcg2.c index 59a5a0f261f336098748b02453a4b59a9910e7f1..42f13a2d1cc16d96a5a5cd896058400078418485 100644 --- a/drivers/clk/qcom/clk-rcg2.c +++ b/drivers/clk/qcom/clk-rcg2.c @@ -728,40 +728,51 @@ static int clk_gfx3d_determine_rate(struct clk_hw *hw, struct clk_rate_request *req) { struct clk_rate_request parent_req = { }; - struct clk_hw *p2, *p8, *p9, *xo; - unsigned long p9_rate; + struct clk_rcg2_gfx3d *cgfx = to_clk_rcg2_gfx3d(hw); + struct clk_hw *xo, *p0, *p1, *p2; + unsigned long request, p0_rate; int ret; + p0 = cgfx->hws[0]; + p1 = cgfx->hws[1]; + p2 = cgfx->hws[2]; + /* + * This function does ping-pong the RCG between PLLs: if we don't + * have at least one fixed PLL and two variable ones, + * then it's not going to work correctly. + */ + if (WARN_ON(!p0 || !p1 || !p2)) + return -EINVAL; + xo = clk_hw_get_parent_by_index(hw, 0); if (req->rate == clk_hw_get_rate(xo)) { req->best_parent_hw = xo; return 0; } - p9 = clk_hw_get_parent_by_index(hw, 2); - p2 = clk_hw_get_parent_by_index(hw, 3); - p8 = clk_hw_get_parent_by_index(hw, 4); + request = req->rate; + if (cgfx->div > 1) + parent_req.rate = request = request * cgfx->div; - /* PLL9 is a fixed rate PLL */ - p9_rate = clk_hw_get_rate(p9); + /* This has to be a fixed rate PLL */ + p0_rate = clk_hw_get_rate(p0); - parent_req.rate = req->rate = min(req->rate, p9_rate); - if (req->rate == p9_rate) { - req->rate = req->best_parent_rate = p9_rate; - req->best_parent_hw = p9; + if (request == p0_rate) { + req->rate = req->best_parent_rate = p0_rate; + req->best_parent_hw = p0; return 0; } - if (req->best_parent_hw == p9) { + if (req->best_parent_hw == p0) { /* Are we going back to a previously used rate? */ - if (clk_hw_get_rate(p8) == req->rate) - req->best_parent_hw = p8; - else + if (clk_hw_get_rate(p2) == request) req->best_parent_hw = p2; - } else if (req->best_parent_hw == p8) { - req->best_parent_hw = p2; + else + req->best_parent_hw = p1; + } else if (req->best_parent_hw == p2) { + req->best_parent_hw = p1; } else { - req->best_parent_hw = p8; + req->best_parent_hw = p2; } ret = __clk_determine_rate(req->best_parent_hw, &parent_req); @@ -769,6 +780,8 @@ static int clk_gfx3d_determine_rate(struct clk_hw *hw, return ret; req->rate = req->best_parent_rate = parent_req.rate; + if (cgfx->div > 1) + req->rate /= cgfx->div; return 0; } @@ -776,12 +789,16 @@ static int clk_gfx3d_determine_rate(struct clk_hw *hw, static int clk_gfx3d_set_rate_and_parent(struct clk_hw *hw, unsigned long rate, unsigned long parent_rate, u8 index) { - struct clk_rcg2 *rcg = to_clk_rcg2(hw); + struct clk_rcg2_gfx3d *cgfx = to_clk_rcg2_gfx3d(hw); + struct clk_rcg2 *rcg = &cgfx->rcg; u32 cfg; int ret; - /* Just mux it, we don't use the division or m/n hardware */ cfg = rcg->parent_map[index].cfg << CFG_SRC_SEL_SHIFT; + /* On some targets, the GFX3D RCG may need to divide PLL frequency */ + if (cgfx->div > 1) + cfg |= ((2 * cgfx->div) - 1) << CFG_SRC_DIV_SHIFT; + ret = regmap_write(rcg->clkr.regmap, rcg->cmd_rcgr + CFG_REG, cfg); if (ret) return ret; diff --git a/drivers/clk/qcom/clk-regmap.c b/drivers/clk/qcom/clk-regmap.c index ce80db27ccf2af5f66ada93c02850d66af460994..92ac4e0d7dbe26efa836ce210cf30208daea8c2f 100644 --- a/drivers/clk/qcom/clk-regmap.c +++ b/drivers/clk/qcom/clk-regmap.c @@ -87,6 +87,7 @@ EXPORT_SYMBOL_GPL(clk_disable_regmap); /** * devm_clk_register_regmap - register a clk_regmap clock * + * @dev: reference to the caller's device * @rclk: clk to operate on * * Clocks that use regmap for their register I/O should register their diff --git a/drivers/clk/qcom/clk-rpm.c b/drivers/clk/qcom/clk-rpm.c index f71d228fd6bd5940c02302147eb928dacf96ae65..a18811c38018794e2bcf75566c9d5f3df53c9dbc 100644 --- a/drivers/clk/qcom/clk-rpm.c +++ b/drivers/clk/qcom/clk-rpm.c @@ -73,62 +73,6 @@ }, \ } -#define DEFINE_CLK_RPM_PXO_BRANCH(_platform, _name, _active, r_id, r) \ - static struct clk_rpm _platform##_##_active; \ - static struct clk_rpm _platform##_##_name = { \ - .rpm_clk_id = (r_id), \ - .active_only = true, \ - .peer = &_platform##_##_active, \ - .rate = (r), \ - .branch = true, \ - .hw.init = &(struct clk_init_data){ \ - .ops = &clk_rpm_branch_ops, \ - .name = #_name, \ - .parent_names = (const char *[]){ "pxo_board" }, \ - .num_parents = 1, \ - }, \ - }; \ - static struct clk_rpm _platform##_##_active = { \ - .rpm_clk_id = (r_id), \ - .peer = &_platform##_##_name, \ - .rate = (r), \ - .branch = true, \ - .hw.init = &(struct clk_init_data){ \ - .ops = &clk_rpm_branch_ops, \ - .name = #_active, \ - .parent_names = (const char *[]){ "pxo_board" }, \ - .num_parents = 1, \ - }, \ - } - -#define DEFINE_CLK_RPM_CXO_BRANCH(_platform, _name, _active, r_id, r) \ - static struct clk_rpm _platform##_##_active; \ - static struct clk_rpm _platform##_##_name = { \ - .rpm_clk_id = (r_id), \ - .peer = &_platform##_##_active, \ - .rate = (r), \ - .branch = true, \ - .hw.init = &(struct clk_init_data){ \ - .ops = &clk_rpm_branch_ops, \ - .name = #_name, \ - .parent_names = (const char *[]){ "cxo_board" }, \ - .num_parents = 1, \ - }, \ - }; \ - static struct clk_rpm _platform##_##_active = { \ - .rpm_clk_id = (r_id), \ - .active_only = true, \ - .peer = &_platform##_##_name, \ - .rate = (r), \ - .branch = true, \ - .hw.init = &(struct clk_init_data){ \ - .ops = &clk_rpm_branch_ops, \ - .name = #_active, \ - .parent_names = (const char *[]){ "cxo_board" }, \ - .num_parents = 1, \ - }, \ - } - #define to_clk_rpm(_hw) container_of(_hw, struct clk_rpm, hw) struct rpm_cc; @@ -450,13 +394,6 @@ static const struct clk_ops clk_rpm_ops = { .recalc_rate = clk_rpm_recalc_rate, }; -static const struct clk_ops clk_rpm_branch_ops = { - .prepare = clk_rpm_prepare, - .unprepare = clk_rpm_unprepare, - .round_rate = clk_rpm_round_rate, - .recalc_rate = clk_rpm_recalc_rate, -}; - /* MSM8660/APQ8060 */ DEFINE_CLK_RPM(msm8660, afab_clk, afab_a_clk, QCOM_RPM_APPS_FABRIC_CLK); DEFINE_CLK_RPM(msm8660, sfab_clk, sfab_a_clk, QCOM_RPM_SYS_FABRIC_CLK); diff --git a/drivers/clk/qcom/clk-rpmh.c b/drivers/clk/qcom/clk-rpmh.c index 6a2a13c5058e39c269c3f8384dc2ee38502d75f0..91dc390a583b51c8c8712ccb880e2cdc2842d1c1 100644 --- a/drivers/clk/qcom/clk-rpmh.c +++ b/drivers/clk/qcom/clk-rpmh.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 /* - * Copyright (c) 2018-2020, The Linux Foundation. All rights reserved. + * Copyright (c) 2018-2021, The Linux Foundation. All rights reserved. */ #include @@ -348,6 +348,10 @@ DEFINE_CLK_RPMH_VRM(sdm845, rf_clk1, rf_clk1_ao, "rfclka1", 1); DEFINE_CLK_RPMH_VRM(sdm845, rf_clk2, rf_clk2_ao, "rfclka2", 1); DEFINE_CLK_RPMH_VRM(sdm845, rf_clk3, rf_clk3_ao, "rfclka3", 1); DEFINE_CLK_RPMH_VRM(sm8150, rf_clk3, rf_clk3_ao, "rfclka3", 1); +DEFINE_CLK_RPMH_VRM(sc8180x, rf_clk1, rf_clk1_ao, "rfclkd1", 1); +DEFINE_CLK_RPMH_VRM(sc8180x, rf_clk2, rf_clk2_ao, "rfclkd2", 1); +DEFINE_CLK_RPMH_VRM(sc8180x, rf_clk3, rf_clk3_ao, "rfclkd3", 1); +DEFINE_CLK_RPMH_VRM(sc8180x, rf_clk4, rf_clk4_ao, "rfclkd4", 1); DEFINE_CLK_RPMH_BCM(sdm845, ipa, "IP0"); DEFINE_CLK_RPMH_BCM(sdm845, ce, "CE0"); @@ -431,6 +435,26 @@ static const struct clk_rpmh_desc clk_rpmh_sc7180 = { .num_clks = ARRAY_SIZE(sc7180_rpmh_clocks), }; +static struct clk_hw *sc8180x_rpmh_clocks[] = { + [RPMH_CXO_CLK] = &sdm845_bi_tcxo.hw, + [RPMH_CXO_CLK_A] = &sdm845_bi_tcxo_ao.hw, + [RPMH_LN_BB_CLK2] = &sdm845_ln_bb_clk2.hw, + [RPMH_LN_BB_CLK2_A] = &sdm845_ln_bb_clk2_ao.hw, + [RPMH_LN_BB_CLK3] = &sdm845_ln_bb_clk3.hw, + [RPMH_LN_BB_CLK3_A] = &sdm845_ln_bb_clk3_ao.hw, + [RPMH_RF_CLK1] = &sc8180x_rf_clk1.hw, + [RPMH_RF_CLK1_A] = &sc8180x_rf_clk1_ao.hw, + [RPMH_RF_CLK2] = &sc8180x_rf_clk2.hw, + [RPMH_RF_CLK2_A] = &sc8180x_rf_clk2_ao.hw, + [RPMH_RF_CLK3] = &sc8180x_rf_clk3.hw, + [RPMH_RF_CLK3_A] = &sc8180x_rf_clk3_ao.hw, +}; + +static const struct clk_rpmh_desc clk_rpmh_sc8180x = { + .clks = sc8180x_rpmh_clocks, + .num_clks = ARRAY_SIZE(sc8180x_rpmh_clocks), +}; + DEFINE_CLK_RPMH_VRM(sm8250, ln_bb_clk1, ln_bb_clk1_ao, "lnbclka1", 2); static struct clk_hw *sm8250_rpmh_clocks[] = { @@ -486,6 +510,27 @@ static const struct clk_rpmh_desc clk_rpmh_sm8350 = { .num_clks = ARRAY_SIZE(sm8350_rpmh_clocks), }; +static struct clk_hw *sc7280_rpmh_clocks[] = { + [RPMH_CXO_CLK] = &sdm845_bi_tcxo.hw, + [RPMH_CXO_CLK_A] = &sdm845_bi_tcxo_ao.hw, + [RPMH_LN_BB_CLK2] = &sdm845_ln_bb_clk2.hw, + [RPMH_LN_BB_CLK2_A] = &sdm845_ln_bb_clk2_ao.hw, + [RPMH_RF_CLK1] = &sdm845_rf_clk1.hw, + [RPMH_RF_CLK1_A] = &sdm845_rf_clk1_ao.hw, + [RPMH_RF_CLK3] = &sdm845_rf_clk3.hw, + [RPMH_RF_CLK3_A] = &sdm845_rf_clk3_ao.hw, + [RPMH_RF_CLK4] = &sm8350_rf_clk4.hw, + [RPMH_RF_CLK4_A] = &sm8350_rf_clk4_ao.hw, + [RPMH_IPA_CLK] = &sdm845_ipa.hw, + [RPMH_PKA_CLK] = &sm8350_pka.hw, + [RPMH_HWKM_CLK] = &sm8350_hwkm.hw, +}; + +static const struct clk_rpmh_desc clk_rpmh_sc7280 = { + .clks = sc7280_rpmh_clocks, + .num_clks = ARRAY_SIZE(sc7280_rpmh_clocks), +}; + static struct clk_hw *of_clk_rpmh_hw_get(struct of_phandle_args *clkspec, void *data) { @@ -570,11 +615,13 @@ static int clk_rpmh_probe(struct platform_device *pdev) static const struct of_device_id clk_rpmh_match_table[] = { { .compatible = "qcom,sc7180-rpmh-clk", .data = &clk_rpmh_sc7180}, + { .compatible = "qcom,sc8180x-rpmh-clk", .data = &clk_rpmh_sc8180x}, { .compatible = "qcom,sdm845-rpmh-clk", .data = &clk_rpmh_sdm845}, { .compatible = "qcom,sdx55-rpmh-clk", .data = &clk_rpmh_sdx55}, { .compatible = "qcom,sm8150-rpmh-clk", .data = &clk_rpmh_sm8150}, { .compatible = "qcom,sm8250-rpmh-clk", .data = &clk_rpmh_sm8250}, { .compatible = "qcom,sm8350-rpmh-clk", .data = &clk_rpmh_sm8350}, + { .compatible = "qcom,sc7280-rpmh-clk", .data = &clk_rpmh_sc7280}, { } }; MODULE_DEVICE_TABLE(of, clk_rpmh_match_table); diff --git a/drivers/clk/qcom/gcc-ipq4019.c b/drivers/clk/qcom/gcc-ipq4019.c index ef5137fd50f3f25c650c70825c9dfbdb38b808a4..8abad4032de713750edd0a9f74ca73cae284b0ce 100644 --- a/drivers/clk/qcom/gcc-ipq4019.c +++ b/drivers/clk/qcom/gcc-ipq4019.c @@ -1276,16 +1276,15 @@ static int clk_cpu_div_set_rate(struct clk_hw *hw, unsigned long rate, struct clk_fepll *pll = to_clk_fepll(hw); const struct freq_tbl *f; u32 mask; - int ret; f = qcom_find_freq(pll->freq_tbl, rate); if (!f) return -EINVAL; mask = (BIT(pll->cdiv.width) - 1) << pll->cdiv.shift; - ret = regmap_update_bits(pll->cdiv.clkr.regmap, - pll->cdiv.reg, mask, - f->pre_div << pll->cdiv.shift); + regmap_update_bits(pll->cdiv.clkr.regmap, + pll->cdiv.reg, mask, + f->pre_div << pll->cdiv.shift); /* * There is no status bit which can be checked for successful CPU * divider update operation so using delay for the same. diff --git a/drivers/clk/qcom/gcc-msm8998.c b/drivers/clk/qcom/gcc-msm8998.c index 9d7016bcd6800e695bf7ad2b00448cddd212da0a..050c91af888e37bd539a3d6f324a1115b578b769 100644 --- a/drivers/clk/qcom/gcc-msm8998.c +++ b/drivers/clk/qcom/gcc-msm8998.c @@ -135,7 +135,7 @@ static struct pll_vco fabia_vco[] = { static struct clk_alpha_pll gpll0 = { .offset = 0x0, - .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_FABIA], .vco_table = fabia_vco, .num_vco = ARRAY_SIZE(fabia_vco), .clkr = { @@ -145,58 +145,58 @@ static struct clk_alpha_pll gpll0 = { .name = "gpll0", .parent_names = (const char *[]){ "xo" }, .num_parents = 1, - .ops = &clk_alpha_pll_ops, + .ops = &clk_alpha_pll_fixed_fabia_ops, } }, }; static struct clk_alpha_pll_postdiv gpll0_out_even = { .offset = 0x0, - .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_FABIA], .clkr.hw.init = &(struct clk_init_data){ .name = "gpll0_out_even", .parent_names = (const char *[]){ "gpll0" }, .num_parents = 1, - .ops = &clk_alpha_pll_postdiv_ops, + .ops = &clk_alpha_pll_postdiv_fabia_ops, }, }; static struct clk_alpha_pll_postdiv gpll0_out_main = { .offset = 0x0, - .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_FABIA], .clkr.hw.init = &(struct clk_init_data){ .name = "gpll0_out_main", .parent_names = (const char *[]){ "gpll0" }, .num_parents = 1, - .ops = &clk_alpha_pll_postdiv_ops, + .ops = &clk_alpha_pll_postdiv_fabia_ops, }, }; static struct clk_alpha_pll_postdiv gpll0_out_odd = { .offset = 0x0, - .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_FABIA], .clkr.hw.init = &(struct clk_init_data){ .name = "gpll0_out_odd", .parent_names = (const char *[]){ "gpll0" }, .num_parents = 1, - .ops = &clk_alpha_pll_postdiv_ops, + .ops = &clk_alpha_pll_postdiv_fabia_ops, }, }; static struct clk_alpha_pll_postdiv gpll0_out_test = { .offset = 0x0, - .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_FABIA], .clkr.hw.init = &(struct clk_init_data){ .name = "gpll0_out_test", .parent_names = (const char *[]){ "gpll0" }, .num_parents = 1, - .ops = &clk_alpha_pll_postdiv_ops, + .ops = &clk_alpha_pll_postdiv_fabia_ops, }, }; static struct clk_alpha_pll gpll1 = { .offset = 0x1000, - .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_FABIA], .vco_table = fabia_vco, .num_vco = ARRAY_SIZE(fabia_vco), .clkr = { @@ -206,58 +206,58 @@ static struct clk_alpha_pll gpll1 = { .name = "gpll1", .parent_names = (const char *[]){ "xo" }, .num_parents = 1, - .ops = &clk_alpha_pll_ops, + .ops = &clk_alpha_pll_fixed_fabia_ops, } }, }; static struct clk_alpha_pll_postdiv gpll1_out_even = { .offset = 0x1000, - .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_FABIA], .clkr.hw.init = &(struct clk_init_data){ .name = "gpll1_out_even", .parent_names = (const char *[]){ "gpll1" }, .num_parents = 1, - .ops = &clk_alpha_pll_postdiv_ops, + .ops = &clk_alpha_pll_postdiv_fabia_ops, }, }; static struct clk_alpha_pll_postdiv gpll1_out_main = { .offset = 0x1000, - .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_FABIA], .clkr.hw.init = &(struct clk_init_data){ .name = "gpll1_out_main", .parent_names = (const char *[]){ "gpll1" }, .num_parents = 1, - .ops = &clk_alpha_pll_postdiv_ops, + .ops = &clk_alpha_pll_postdiv_fabia_ops, }, }; static struct clk_alpha_pll_postdiv gpll1_out_odd = { .offset = 0x1000, - .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_FABIA], .clkr.hw.init = &(struct clk_init_data){ .name = "gpll1_out_odd", .parent_names = (const char *[]){ "gpll1" }, .num_parents = 1, - .ops = &clk_alpha_pll_postdiv_ops, + .ops = &clk_alpha_pll_postdiv_fabia_ops, }, }; static struct clk_alpha_pll_postdiv gpll1_out_test = { .offset = 0x1000, - .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_FABIA], .clkr.hw.init = &(struct clk_init_data){ .name = "gpll1_out_test", .parent_names = (const char *[]){ "gpll1" }, .num_parents = 1, - .ops = &clk_alpha_pll_postdiv_ops, + .ops = &clk_alpha_pll_postdiv_fabia_ops, }, }; static struct clk_alpha_pll gpll2 = { .offset = 0x2000, - .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_FABIA], .vco_table = fabia_vco, .num_vco = ARRAY_SIZE(fabia_vco), .clkr = { @@ -267,58 +267,58 @@ static struct clk_alpha_pll gpll2 = { .name = "gpll2", .parent_names = (const char *[]){ "xo" }, .num_parents = 1, - .ops = &clk_alpha_pll_ops, + .ops = &clk_alpha_pll_fixed_fabia_ops, } }, }; static struct clk_alpha_pll_postdiv gpll2_out_even = { .offset = 0x2000, - .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_FABIA], .clkr.hw.init = &(struct clk_init_data){ .name = "gpll2_out_even", .parent_names = (const char *[]){ "gpll2" }, .num_parents = 1, - .ops = &clk_alpha_pll_postdiv_ops, + .ops = &clk_alpha_pll_postdiv_fabia_ops, }, }; static struct clk_alpha_pll_postdiv gpll2_out_main = { .offset = 0x2000, - .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_FABIA], .clkr.hw.init = &(struct clk_init_data){ .name = "gpll2_out_main", .parent_names = (const char *[]){ "gpll2" }, .num_parents = 1, - .ops = &clk_alpha_pll_postdiv_ops, + .ops = &clk_alpha_pll_postdiv_fabia_ops, }, }; static struct clk_alpha_pll_postdiv gpll2_out_odd = { .offset = 0x2000, - .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_FABIA], .clkr.hw.init = &(struct clk_init_data){ .name = "gpll2_out_odd", .parent_names = (const char *[]){ "gpll2" }, .num_parents = 1, - .ops = &clk_alpha_pll_postdiv_ops, + .ops = &clk_alpha_pll_postdiv_fabia_ops, }, }; static struct clk_alpha_pll_postdiv gpll2_out_test = { .offset = 0x2000, - .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_FABIA], .clkr.hw.init = &(struct clk_init_data){ .name = "gpll2_out_test", .parent_names = (const char *[]){ "gpll2" }, .num_parents = 1, - .ops = &clk_alpha_pll_postdiv_ops, + .ops = &clk_alpha_pll_postdiv_fabia_ops, }, }; static struct clk_alpha_pll gpll3 = { .offset = 0x3000, - .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_FABIA], .vco_table = fabia_vco, .num_vco = ARRAY_SIZE(fabia_vco), .clkr = { @@ -328,58 +328,58 @@ static struct clk_alpha_pll gpll3 = { .name = "gpll3", .parent_names = (const char *[]){ "xo" }, .num_parents = 1, - .ops = &clk_alpha_pll_ops, + .ops = &clk_alpha_pll_fixed_fabia_ops, } }, }; static struct clk_alpha_pll_postdiv gpll3_out_even = { .offset = 0x3000, - .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_FABIA], .clkr.hw.init = &(struct clk_init_data){ .name = "gpll3_out_even", .parent_names = (const char *[]){ "gpll3" }, .num_parents = 1, - .ops = &clk_alpha_pll_postdiv_ops, + .ops = &clk_alpha_pll_postdiv_fabia_ops, }, }; static struct clk_alpha_pll_postdiv gpll3_out_main = { .offset = 0x3000, - .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_FABIA], .clkr.hw.init = &(struct clk_init_data){ .name = "gpll3_out_main", .parent_names = (const char *[]){ "gpll3" }, .num_parents = 1, - .ops = &clk_alpha_pll_postdiv_ops, + .ops = &clk_alpha_pll_postdiv_fabia_ops, }, }; static struct clk_alpha_pll_postdiv gpll3_out_odd = { .offset = 0x3000, - .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_FABIA], .clkr.hw.init = &(struct clk_init_data){ .name = "gpll3_out_odd", .parent_names = (const char *[]){ "gpll3" }, .num_parents = 1, - .ops = &clk_alpha_pll_postdiv_ops, + .ops = &clk_alpha_pll_postdiv_fabia_ops, }, }; static struct clk_alpha_pll_postdiv gpll3_out_test = { .offset = 0x3000, - .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_FABIA], .clkr.hw.init = &(struct clk_init_data){ .name = "gpll3_out_test", .parent_names = (const char *[]){ "gpll3" }, .num_parents = 1, - .ops = &clk_alpha_pll_postdiv_ops, + .ops = &clk_alpha_pll_postdiv_fabia_ops, }, }; static struct clk_alpha_pll gpll4 = { .offset = 0x77000, - .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_FABIA], .vco_table = fabia_vco, .num_vco = ARRAY_SIZE(fabia_vco), .clkr = { @@ -389,52 +389,52 @@ static struct clk_alpha_pll gpll4 = { .name = "gpll4", .parent_names = (const char *[]){ "xo" }, .num_parents = 1, - .ops = &clk_alpha_pll_ops, + .ops = &clk_alpha_pll_fixed_fabia_ops, } }, }; static struct clk_alpha_pll_postdiv gpll4_out_even = { .offset = 0x77000, - .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_FABIA], .clkr.hw.init = &(struct clk_init_data){ .name = "gpll4_out_even", .parent_names = (const char *[]){ "gpll4" }, .num_parents = 1, - .ops = &clk_alpha_pll_postdiv_ops, + .ops = &clk_alpha_pll_postdiv_fabia_ops, }, }; static struct clk_alpha_pll_postdiv gpll4_out_main = { .offset = 0x77000, - .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_FABIA], .clkr.hw.init = &(struct clk_init_data){ .name = "gpll4_out_main", .parent_names = (const char *[]){ "gpll4" }, .num_parents = 1, - .ops = &clk_alpha_pll_postdiv_ops, + .ops = &clk_alpha_pll_postdiv_fabia_ops, }, }; static struct clk_alpha_pll_postdiv gpll4_out_odd = { .offset = 0x77000, - .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_FABIA], .clkr.hw.init = &(struct clk_init_data){ .name = "gpll4_out_odd", .parent_names = (const char *[]){ "gpll4" }, .num_parents = 1, - .ops = &clk_alpha_pll_postdiv_ops, + .ops = &clk_alpha_pll_postdiv_fabia_ops, }, }; static struct clk_alpha_pll_postdiv gpll4_out_test = { .offset = 0x77000, - .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_FABIA], .clkr.hw.init = &(struct clk_init_data){ .name = "gpll4_out_test", .parent_names = (const char *[]){ "gpll4" }, .num_parents = 1, - .ops = &clk_alpha_pll_postdiv_ops, + .ops = &clk_alpha_pll_postdiv_fabia_ops, }, }; @@ -1341,6 +1341,22 @@ static struct clk_branch gcc_boot_rom_ahb_clk = { }, }; +static struct clk_branch gcc_mmss_gpll0_clk = { + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x5200c, + .enable_mask = BIT(1), + .hw.init = &(struct clk_init_data){ + .name = "gcc_mmss_gpll0_clk", + .parent_names = (const char *[]){ + "gpll0_out_main", + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + static struct clk_branch gcc_mss_gpll0_div_clk_src = { .halt_check = BRANCH_HALT_DELAY, .clkr = { @@ -2065,6 +2081,12 @@ static struct clk_branch gcc_gpu_cfg_ahb_clk = { .hw.init = &(struct clk_init_data){ .name = "gcc_gpu_cfg_ahb_clk", .ops = &clk_branch2_ops, + /* + * The GPU IOMMU depends on this clock and hypervisor + * will crash the SoC if this clock goes down, due to + * secure contexts protection. + */ + .flags = CLK_IS_CRITICAL, }, }, }; @@ -2144,6 +2166,25 @@ static struct clk_branch gcc_hmss_trig_clk = { }, }; +static struct freq_tbl ftbl_hmss_gpll0_clk_src[] = { + F( 300000000, P_GPLL0_OUT_MAIN, 2, 0, 0), + F( 600000000, P_GPLL0_OUT_MAIN, 1, 0, 0), + { } +}; + +static struct clk_rcg2 hmss_gpll0_clk_src = { + .cmd_rcgr = 0x4805c, + .hid_width = 5, + .parent_map = gcc_parent_map_1, + .freq_tbl = ftbl_hmss_gpll0_clk_src, + .clkr.hw.init = &(struct clk_init_data) { + .name = "hmss_gpll0_clk_src", + .parent_names = gcc_parent_names_1, + .num_parents = ARRAY_SIZE(gcc_parent_names_1), + .ops = &clk_rcg2_ops, + }, +}; + static struct clk_branch gcc_mmss_noc_cfg_ahb_clk = { .halt_reg = 0x9004, .halt_check = BRANCH_HALT, @@ -2944,6 +2985,8 @@ static struct clk_regmap *gcc_msm8998_clocks[] = { [GCC_MSS_GPLL0_DIV_CLK_SRC] = &gcc_mss_gpll0_div_clk_src.clkr, [GCC_MSS_SNOC_AXI_CLK] = &gcc_mss_snoc_axi_clk.clkr, [GCC_MSS_MNOC_BIMC_AXI_CLK] = &gcc_mss_mnoc_bimc_axi_clk.clkr, + [GCC_MMSS_GPLL0_CLK] = &gcc_mmss_gpll0_clk.clkr, + [HMSS_GPLL0_CLK_SRC] = &hmss_gpll0_clk_src.clkr, }; static struct gdsc *gcc_msm8998_gdscs[] = { diff --git a/drivers/clk/qcom/gcc-sc7180.c b/drivers/clk/qcom/gcc-sc7180.c index b05901b249172221fb464f74272c31042e12f4c2..88e896abb6631c3c3b5ba0ef4f28839baa044b6d 100644 --- a/drivers/clk/qcom/gcc-sc7180.c +++ b/drivers/clk/qcom/gcc-sc7180.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * Copyright (c) 2019-2020, The Linux Foundation. All rights reserved. + * Copyright (c) 2019-2021, The Linux Foundation. All rights reserved. */ #include @@ -919,19 +919,6 @@ static struct clk_branch gcc_camera_throttle_hf_axi_clk = { }, }; -static struct clk_branch gcc_camera_xo_clk = { - .halt_reg = 0xb02c, - .halt_check = BRANCH_HALT, - .clkr = { - .enable_reg = 0xb02c, - .enable_mask = BIT(0), - .hw.init = &(struct clk_init_data){ - .name = "gcc_camera_xo_clk", - .ops = &clk_branch2_ops, - }, - }, -}; - static struct clk_branch gcc_ce1_ahb_clk = { .halt_reg = 0x4100c, .halt_check = BRANCH_HALT_VOTED, @@ -1096,19 +1083,6 @@ static struct clk_branch gcc_disp_throttle_hf_axi_clk = { }, }; -static struct clk_branch gcc_disp_xo_clk = { - .halt_reg = 0xb030, - .halt_check = BRANCH_HALT, - .clkr = { - .enable_reg = 0xb030, - .enable_mask = BIT(0), - .hw.init = &(struct clk_init_data){ - .name = "gcc_disp_xo_clk", - .ops = &clk_branch2_ops, - }, - }, -}; - static struct clk_branch gcc_gp1_clk = { .halt_reg = 0x64000, .halt_check = BRANCH_HALT, @@ -2159,19 +2133,6 @@ static struct clk_branch gcc_video_throttle_axi_clk = { }, }; -static struct clk_branch gcc_video_xo_clk = { - .halt_reg = 0xb028, - .halt_check = BRANCH_HALT, - .clkr = { - .enable_reg = 0xb028, - .enable_mask = BIT(0), - .hw.init = &(struct clk_init_data){ - .name = "gcc_video_xo_clk", - .ops = &clk_branch2_ops, - }, - }, -}; - static struct clk_branch gcc_mss_cfg_ahb_clk = { .halt_reg = 0x8a000, .halt_check = BRANCH_HALT, @@ -2304,7 +2265,6 @@ static struct clk_regmap *gcc_sc7180_clocks[] = { [GCC_BOOT_ROM_AHB_CLK] = &gcc_boot_rom_ahb_clk.clkr, [GCC_CAMERA_HF_AXI_CLK] = &gcc_camera_hf_axi_clk.clkr, [GCC_CAMERA_THROTTLE_HF_AXI_CLK] = &gcc_camera_throttle_hf_axi_clk.clkr, - [GCC_CAMERA_XO_CLK] = &gcc_camera_xo_clk.clkr, [GCC_CE1_AHB_CLK] = &gcc_ce1_ahb_clk.clkr, [GCC_CE1_AXI_CLK] = &gcc_ce1_axi_clk.clkr, [GCC_CE1_CLK] = &gcc_ce1_clk.clkr, @@ -2317,7 +2277,6 @@ static struct clk_regmap *gcc_sc7180_clocks[] = { [GCC_DISP_GPLL0_DIV_CLK_SRC] = &gcc_disp_gpll0_div_clk_src.clkr, [GCC_DISP_HF_AXI_CLK] = &gcc_disp_hf_axi_clk.clkr, [GCC_DISP_THROTTLE_HF_AXI_CLK] = &gcc_disp_throttle_hf_axi_clk.clkr, - [GCC_DISP_XO_CLK] = &gcc_disp_xo_clk.clkr, [GCC_GP1_CLK] = &gcc_gp1_clk.clkr, [GCC_GP1_CLK_SRC] = &gcc_gp1_clk_src.clkr, [GCC_GP2_CLK] = &gcc_gp2_clk.clkr, @@ -2413,7 +2372,6 @@ static struct clk_regmap *gcc_sc7180_clocks[] = { [GCC_VIDEO_AXI_CLK] = &gcc_video_axi_clk.clkr, [GCC_VIDEO_GPLL0_DIV_CLK_SRC] = &gcc_video_gpll0_div_clk_src.clkr, [GCC_VIDEO_THROTTLE_AXI_CLK] = &gcc_video_throttle_axi_clk.clkr, - [GCC_VIDEO_XO_CLK] = &gcc_video_xo_clk.clkr, [GPLL0] = &gpll0.clkr, [GPLL0_OUT_EVEN] = &gpll0_out_even.clkr, [GPLL6] = &gpll6.clkr, @@ -2510,6 +2468,9 @@ static int gcc_sc7180_probe(struct platform_device *pdev) regmap_update_bits(regmap, 0x0b004, BIT(0), BIT(0)); regmap_update_bits(regmap, 0x0b008, BIT(0), BIT(0)); regmap_update_bits(regmap, 0x0b00c, BIT(0), BIT(0)); + regmap_update_bits(regmap, 0x0b02c, BIT(0), BIT(0)); + regmap_update_bits(regmap, 0x0b028, BIT(0), BIT(0)); + regmap_update_bits(regmap, 0x0b030, BIT(0), BIT(0)); regmap_update_bits(regmap, 0x71004, BIT(0), BIT(0)); ret = qcom_cc_register_rcg_dfs(regmap, gcc_dfs_clocks, diff --git a/drivers/clk/qcom/gcc-sc7280.c b/drivers/clk/qcom/gcc-sc7280.c new file mode 100644 index 0000000000000000000000000000000000000000..22736c16ed1609b47a9bb18a2d5bb07912a199f2 --- /dev/null +++ b/drivers/clk/qcom/gcc-sc7280.c @@ -0,0 +1,3603 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include + +#include + +#include "clk-alpha-pll.h" +#include "clk-branch.h" +#include "clk-rcg.h" +#include "clk-regmap-divider.h" +#include "clk-regmap-mux.h" +#include "common.h" +#include "gdsc.h" +#include "reset.h" + +enum { + P_BI_TCXO, + P_GCC_GPLL0_OUT_EVEN, + P_GCC_GPLL0_OUT_MAIN, + P_GCC_GPLL0_OUT_ODD, + P_GCC_GPLL10_OUT_MAIN, + P_GCC_GPLL4_OUT_MAIN, + P_GCC_GPLL9_OUT_MAIN, + P_PCIE_0_PIPE_CLK, + P_PCIE_1_PIPE_CLK, + P_SLEEP_CLK, + P_UFS_PHY_RX_SYMBOL_0_CLK, + P_UFS_PHY_RX_SYMBOL_1_CLK, + P_UFS_PHY_TX_SYMBOL_0_CLK, + P_USB3_PHY_WRAPPER_GCC_USB30_PIPE_CLK, + P_GCC_MSS_GPLL0_MAIN_DIV_CLK, +}; + +static struct clk_alpha_pll gcc_gpll0 = { + .offset = 0x0, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID], + .clkr = { + .enable_reg = 0x52010, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gpll0", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "bi_tcxo", + }, + .num_parents = 1, + .ops = &clk_alpha_pll_fixed_lucid_ops, + }, + }, +}; + +static const struct clk_div_table post_div_table_gcc_gpll0_out_even[] = { + { 0x1, 2 }, + { } +}; + +static struct clk_alpha_pll_postdiv gcc_gpll0_out_even = { + .offset = 0x0, + .post_div_shift = 8, + .post_div_table = post_div_table_gcc_gpll0_out_even, + .num_post_div = ARRAY_SIZE(post_div_table_gcc_gpll0_out_even), + .width = 4, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID], + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_gpll0_out_even", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_gpll0.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_alpha_pll_postdiv_lucid_ops, + }, +}; + +static const struct clk_div_table post_div_table_gcc_gpll0_out_odd[] = { + { 0x3, 3 }, + { } +}; + +static struct clk_alpha_pll_postdiv gcc_gpll0_out_odd = { + .offset = 0x0, + .post_div_shift = 12, + .post_div_table = post_div_table_gcc_gpll0_out_odd, + .num_post_div = ARRAY_SIZE(post_div_table_gcc_gpll0_out_odd), + .width = 4, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID], + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_gpll0_out_odd", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_gpll0.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_alpha_pll_postdiv_lucid_ops, + }, +}; + +static struct clk_alpha_pll gcc_gpll1 = { + .offset = 0x1000, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID], + .clkr = { + .enable_reg = 0x52010, + .enable_mask = BIT(1), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gpll1", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "bi_tcxo", + }, + .num_parents = 1, + .ops = &clk_alpha_pll_fixed_lucid_ops, + }, + }, +}; + +static struct clk_alpha_pll gcc_gpll10 = { + .offset = 0x1e000, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID], + .clkr = { + .enable_reg = 0x52010, + .enable_mask = BIT(9), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gpll10", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "bi_tcxo", + }, + .num_parents = 1, + .ops = &clk_alpha_pll_fixed_lucid_ops, + }, + }, +}; + +static struct clk_alpha_pll gcc_gpll4 = { + .offset = 0x76000, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID], + .clkr = { + .enable_reg = 0x52010, + .enable_mask = BIT(4), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gpll4", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "bi_tcxo", + }, + .num_parents = 1, + .ops = &clk_alpha_pll_fixed_lucid_ops, + }, + }, +}; + +static struct clk_alpha_pll gcc_gpll9 = { + .offset = 0x1c000, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID], + .clkr = { + .enable_reg = 0x52010, + .enable_mask = BIT(8), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gpll9", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "bi_tcxo", + }, + .num_parents = 1, + .ops = &clk_alpha_pll_fixed_lucid_ops, + }, + }, +}; + +static struct clk_branch gcc_mss_gpll0_main_div_clk_src = { + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(17), + .hw.init = &(struct clk_init_data){ + .name = "gcc_mss_gpll0_main_div_clk_src", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_gpll0_out_even.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static const struct parent_map gcc_parent_map_0[] = { + { P_BI_TCXO, 0 }, + { P_GCC_GPLL0_OUT_MAIN, 1 }, + { P_GCC_GPLL0_OUT_EVEN, 6 }, +}; + +static const struct clk_parent_data gcc_parent_data_0[] = { + { .fw_name = "bi_tcxo" }, + { .hw = &gcc_gpll0.clkr.hw }, + { .hw = &gcc_gpll0_out_even.clkr.hw }, +}; + +static const struct clk_parent_data gcc_parent_data_0_ao[] = { + { .fw_name = "bi_tcxo_ao" }, + { .hw = &gcc_gpll0.clkr.hw }, + { .hw = &gcc_gpll0_out_even.clkr.hw }, +}; + +static const struct parent_map gcc_parent_map_1[] = { + { P_BI_TCXO, 0 }, + { P_GCC_GPLL0_OUT_MAIN, 1 }, + { P_GCC_GPLL0_OUT_ODD, 3 }, + { P_GCC_GPLL0_OUT_EVEN, 6 }, +}; + +static const struct clk_parent_data gcc_parent_data_1[] = { + { .fw_name = "bi_tcxo" }, + { .hw = &gcc_gpll0.clkr.hw }, + { .hw = &gcc_gpll0_out_odd.clkr.hw }, + { .hw = &gcc_gpll0_out_even.clkr.hw }, +}; + +static const struct parent_map gcc_parent_map_2[] = { + { P_BI_TCXO, 0 }, + { P_SLEEP_CLK, 5 }, +}; + +static const struct clk_parent_data gcc_parent_data_2[] = { + { .fw_name = "bi_tcxo" }, + { .fw_name = "sleep_clk" }, +}; + +static const struct parent_map gcc_parent_map_3[] = { + { P_BI_TCXO, 0 }, +}; + +static const struct clk_parent_data gcc_parent_data_3[] = { + { .fw_name = "bi_tcxo" }, +}; + +static const struct parent_map gcc_parent_map_4[] = { + { P_BI_TCXO, 0 }, + { P_GCC_GPLL0_OUT_MAIN, 1 }, + { P_GCC_GPLL0_OUT_ODD, 3 }, + { P_SLEEP_CLK, 5 }, + { P_GCC_GPLL0_OUT_EVEN, 6 }, +}; + +static const struct clk_parent_data gcc_parent_data_4[] = { + { .fw_name = "bi_tcxo" }, + { .hw = &gcc_gpll0.clkr.hw }, + { .hw = &gcc_gpll0_out_odd.clkr.hw }, + { .fw_name = "sleep_clk" }, + { .hw = &gcc_gpll0_out_even.clkr.hw }, +}; + +static const struct parent_map gcc_parent_map_5[] = { + { P_BI_TCXO, 0 }, + { P_GCC_GPLL0_OUT_EVEN, 6 }, +}; + +static const struct clk_parent_data gcc_parent_data_5[] = { + { .fw_name = "bi_tcxo" }, + { .hw = &gcc_gpll0_out_even.clkr.hw }, +}; + +static const struct parent_map gcc_parent_map_6[] = { + { P_PCIE_0_PIPE_CLK, 0 }, + { P_BI_TCXO, 2 }, +}; + +static const struct clk_parent_data gcc_parent_data_6[] = { + { .fw_name = "pcie_0_pipe_clk", .name = "pcie_0_pipe_clk" }, + { .fw_name = "bi_tcxo" }, +}; + +static const struct parent_map gcc_parent_map_7[] = { + { P_PCIE_1_PIPE_CLK, 0 }, + { P_BI_TCXO, 2 }, +}; + +static const struct clk_parent_data gcc_parent_data_7[] = { + { .fw_name = "pcie_1_pipe_clk", .name = "pcie_1_pipe_clk" }, + { .fw_name = "bi_tcxo" }, +}; + +static const struct parent_map gcc_parent_map_8[] = { + { P_BI_TCXO, 0 }, + { P_GCC_GPLL0_OUT_MAIN, 1 }, + { P_GCC_GPLL0_OUT_ODD, 3 }, + { P_GCC_GPLL10_OUT_MAIN, 5 }, + { P_GCC_GPLL0_OUT_EVEN, 6 }, +}; + +static const struct clk_parent_data gcc_parent_data_8[] = { + { .fw_name = "bi_tcxo" }, + { .hw = &gcc_gpll0.clkr.hw }, + { .hw = &gcc_gpll0_out_odd.clkr.hw }, + { .hw = &gcc_gpll10.clkr.hw }, + { .hw = &gcc_gpll0_out_even.clkr.hw }, +}; + +static const struct parent_map gcc_parent_map_9[] = { + { P_BI_TCXO, 0 }, + { P_GCC_GPLL0_OUT_MAIN, 1 }, + { P_GCC_GPLL9_OUT_MAIN, 2 }, + { P_GCC_GPLL0_OUT_ODD, 3 }, + { P_GCC_GPLL4_OUT_MAIN, 5 }, + { P_GCC_GPLL0_OUT_EVEN, 6 }, +}; + +static const struct clk_parent_data gcc_parent_data_9[] = { + { .fw_name = "bi_tcxo" }, + { .hw = &gcc_gpll0.clkr.hw }, + { .hw = &gcc_gpll9.clkr.hw }, + { .hw = &gcc_gpll0_out_odd.clkr.hw }, + { .hw = &gcc_gpll4.clkr.hw }, + { .hw = &gcc_gpll0_out_even.clkr.hw }, +}; + +static const struct parent_map gcc_parent_map_10[] = { + { P_UFS_PHY_RX_SYMBOL_0_CLK, 0 }, + { P_BI_TCXO, 2 }, +}; + +static const struct clk_parent_data gcc_parent_data_10[] = { + { .fw_name = "ufs_phy_rx_symbol_0_clk" }, + { .fw_name = "bi_tcxo" }, +}; + +static const struct parent_map gcc_parent_map_11[] = { + { P_UFS_PHY_RX_SYMBOL_1_CLK, 0 }, + { P_BI_TCXO, 2 }, +}; + +static const struct clk_parent_data gcc_parent_data_11[] = { + { .fw_name = "ufs_phy_rx_symbol_1_clk" }, + { .fw_name = "bi_tcxo" }, +}; + +static const struct parent_map gcc_parent_map_12[] = { + { P_UFS_PHY_TX_SYMBOL_0_CLK, 0 }, + { P_BI_TCXO, 2 }, +}; + +static const struct clk_parent_data gcc_parent_data_12[] = { + { .fw_name = "ufs_phy_tx_symbol_0_clk" }, + { .fw_name = "bi_tcxo" }, +}; + +static const struct parent_map gcc_parent_map_13[] = { + { P_USB3_PHY_WRAPPER_GCC_USB30_PIPE_CLK, 0 }, + { P_BI_TCXO, 2 }, +}; + +static const struct clk_parent_data gcc_parent_data_13[] = { + { .fw_name = "usb3_phy_wrapper_gcc_usb30_pipe_clk" }, + { .fw_name = "bi_tcxo" }, +}; + +static const struct parent_map gcc_parent_map_14[] = { + { P_USB3_PHY_WRAPPER_GCC_USB30_PIPE_CLK, 0 }, + { P_BI_TCXO, 2 }, +}; + +static const struct clk_parent_data gcc_parent_data_14[] = { + { .fw_name = "usb3_phy_wrapper_gcc_usb30_pipe_clk" }, + { .fw_name = "bi_tcxo" }, +}; + +static const struct parent_map gcc_parent_map_15[] = { + { P_BI_TCXO, 0 }, + { P_GCC_MSS_GPLL0_MAIN_DIV_CLK, 1 }, +}; + +static const struct clk_parent_data gcc_parent_data_15[] = { + { .fw_name = "bi_tcxo" }, + { .hw = &gcc_mss_gpll0_main_div_clk_src.clkr.hw }, +}; + +static struct clk_regmap_mux gcc_pcie_0_pipe_clk_src = { + .reg = 0x6b054, + .shift = 0, + .width = 2, + .parent_map = gcc_parent_map_6, + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_0_pipe_clk_src", + .parent_data = gcc_parent_data_6, + .num_parents = ARRAY_SIZE(gcc_parent_data_6), + .ops = &clk_regmap_mux_closest_ops, + }, + }, +}; + +static struct clk_regmap_mux gcc_pcie_1_pipe_clk_src = { + .reg = 0x8d054, + .shift = 0, + .width = 2, + .parent_map = gcc_parent_map_7, + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_1_pipe_clk_src", + .parent_data = gcc_parent_data_7, + .num_parents = ARRAY_SIZE(gcc_parent_data_7), + .ops = &clk_regmap_mux_closest_ops, + }, + }, +}; + +static struct clk_regmap_mux gcc_ufs_phy_rx_symbol_0_clk_src = { + .reg = 0x77058, + .shift = 0, + .width = 2, + .parent_map = gcc_parent_map_10, + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_rx_symbol_0_clk_src", + .parent_data = gcc_parent_data_10, + .num_parents = ARRAY_SIZE(gcc_parent_data_10), + .ops = &clk_regmap_mux_closest_ops, + }, + }, +}; + +static struct clk_regmap_mux gcc_ufs_phy_rx_symbol_1_clk_src = { + .reg = 0x770c8, + .shift = 0, + .width = 2, + .parent_map = gcc_parent_map_11, + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_rx_symbol_1_clk_src", + .parent_data = gcc_parent_data_11, + .num_parents = ARRAY_SIZE(gcc_parent_data_11), + .ops = &clk_regmap_mux_closest_ops, + }, + }, +}; + +static struct clk_regmap_mux gcc_ufs_phy_tx_symbol_0_clk_src = { + .reg = 0x77048, + .shift = 0, + .width = 2, + .parent_map = gcc_parent_map_12, + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_tx_symbol_0_clk_src", + .parent_data = gcc_parent_data_12, + .num_parents = ARRAY_SIZE(gcc_parent_data_12), + .ops = &clk_regmap_mux_closest_ops, + }, + }, +}; + +static struct clk_regmap_mux gcc_usb3_prim_phy_pipe_clk_src = { + .reg = 0xf060, + .shift = 0, + .width = 2, + .parent_map = gcc_parent_map_13, + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_prim_phy_pipe_clk_src", + .parent_data = gcc_parent_data_13, + .num_parents = ARRAY_SIZE(gcc_parent_data_13), + .ops = &clk_regmap_mux_closest_ops, + }, + }, +}; + +static struct clk_regmap_mux gcc_usb3_sec_phy_pipe_clk_src = { + .reg = 0x9e060, + .shift = 0, + .width = 2, + .parent_map = gcc_parent_map_14, + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_sec_phy_pipe_clk_src", + .parent_data = gcc_parent_data_14, + .num_parents = ARRAY_SIZE(gcc_parent_data_14), + .ops = &clk_regmap_mux_closest_ops, + }, + }, +}; +static const struct freq_tbl ftbl_gcc_cpuss_ahb_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_cpuss_ahb_clk_src = { + .cmd_rcgr = 0x4800c, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_cpuss_ahb_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_cpuss_ahb_clk_src", + .parent_data = gcc_parent_data_0_ao, + .num_parents = ARRAY_SIZE(gcc_parent_data_0_ao), + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_gp1_clk_src[] = { + F(50000000, P_GCC_GPLL0_OUT_EVEN, 6, 0, 0), + F(100000000, P_GCC_GPLL0_OUT_EVEN, 3, 0, 0), + F(200000000, P_GCC_GPLL0_OUT_ODD, 1, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_gp1_clk_src = { + .cmd_rcgr = 0x64004, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_4, + .freq_tbl = ftbl_gcc_gp1_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_gp1_clk_src", + .parent_data = gcc_parent_data_4, + .num_parents = ARRAY_SIZE(gcc_parent_data_4), + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_gp2_clk_src = { + .cmd_rcgr = 0x65004, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_4, + .freq_tbl = ftbl_gcc_gp1_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_gp2_clk_src", + .parent_data = gcc_parent_data_4, + .num_parents = ARRAY_SIZE(gcc_parent_data_4), + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_gp3_clk_src = { + .cmd_rcgr = 0x66004, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_4, + .freq_tbl = ftbl_gcc_gp1_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_gp3_clk_src", + .parent_data = gcc_parent_data_4, + .num_parents = ARRAY_SIZE(gcc_parent_data_4), + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_pcie_0_aux_clk_src[] = { + F(9600000, P_BI_TCXO, 2, 0, 0), + F(19200000, P_BI_TCXO, 1, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_pcie_0_aux_clk_src = { + .cmd_rcgr = 0x6b058, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_2, + .freq_tbl = ftbl_gcc_pcie_0_aux_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_0_aux_clk_src", + .parent_data = gcc_parent_data_2, + .num_parents = ARRAY_SIZE(gcc_parent_data_2), + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_pcie_0_phy_rchng_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + F(100000000, P_GCC_GPLL0_OUT_EVEN, 3, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_pcie_0_phy_rchng_clk_src = { + .cmd_rcgr = 0x6b03c, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_pcie_0_phy_rchng_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_0_phy_rchng_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_pcie_1_aux_clk_src = { + .cmd_rcgr = 0x8d058, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_2, + .freq_tbl = ftbl_gcc_pcie_0_aux_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_1_aux_clk_src", + .parent_data = gcc_parent_data_2, + .num_parents = ARRAY_SIZE(gcc_parent_data_2), + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_pcie_1_phy_rchng_clk_src = { + .cmd_rcgr = 0x8d03c, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_pcie_0_phy_rchng_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_1_phy_rchng_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_pdm2_clk_src[] = { + F(60000000, P_GCC_GPLL0_OUT_EVEN, 5, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_pdm2_clk_src = { + .cmd_rcgr = 0x33010, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_pdm2_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_pdm2_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_qspi_core_clk_src[] = { + F(100000000, P_GCC_GPLL0_OUT_MAIN, 6, 0, 0), + F(150000000, P_GCC_GPLL0_OUT_MAIN, 4, 0, 0), + F(200000000, P_GCC_GPLL0_OUT_MAIN, 3, 0, 0), + F(300000000, P_GCC_GPLL0_OUT_MAIN, 2, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_qspi_core_clk_src = { + .cmd_rcgr = 0x4b00c, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qspi_core_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_qspi_core_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .ops = &clk_rcg2_floor_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_qupv3_wrap0_s0_clk_src[] = { + F(7372800, P_GCC_GPLL0_OUT_EVEN, 1, 384, 15625), + F(14745600, P_GCC_GPLL0_OUT_EVEN, 1, 768, 15625), + F(19200000, P_BI_TCXO, 1, 0, 0), + F(29491200, P_GCC_GPLL0_OUT_EVEN, 1, 1536, 15625), + F(32000000, P_GCC_GPLL0_OUT_EVEN, 1, 8, 75), + F(48000000, P_GCC_GPLL0_OUT_EVEN, 1, 4, 25), + F(64000000, P_GCC_GPLL0_OUT_EVEN, 1, 16, 75), + F(75000000, P_GCC_GPLL0_OUT_EVEN, 4, 0, 0), + F(80000000, P_GCC_GPLL0_OUT_EVEN, 1, 4, 15), + F(96000000, P_GCC_GPLL0_OUT_EVEN, 1, 8, 25), + F(100000000, P_GCC_GPLL0_OUT_MAIN, 6, 0, 0), + F(102400000, P_GCC_GPLL0_OUT_EVEN, 1, 128, 375), + F(112000000, P_GCC_GPLL0_OUT_EVEN, 1, 28, 75), + F(117964800, P_GCC_GPLL0_OUT_EVEN, 1, 6144, 15625), + F(120000000, P_GCC_GPLL0_OUT_EVEN, 2.5, 0, 0), + { } +}; + +static struct clk_init_data gcc_qupv3_wrap0_s0_clk_src_init = { + .name = "gcc_qupv3_wrap0_s0_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap0_s0_clk_src = { + .cmd_rcgr = 0x17010, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap0_s0_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap0_s1_clk_src_init = { + .name = "gcc_qupv3_wrap0_s1_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap0_s1_clk_src = { + .cmd_rcgr = 0x17140, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap0_s1_clk_src_init, +}; + +static const struct freq_tbl ftbl_gcc_qupv3_wrap0_s2_clk_src[] = { + F(7372800, P_GCC_GPLL0_OUT_EVEN, 1, 384, 15625), + F(14745600, P_GCC_GPLL0_OUT_EVEN, 1, 768, 15625), + F(19200000, P_BI_TCXO, 1, 0, 0), + F(29491200, P_GCC_GPLL0_OUT_EVEN, 1, 1536, 15625), + F(32000000, P_GCC_GPLL0_OUT_EVEN, 1, 8, 75), + F(48000000, P_GCC_GPLL0_OUT_EVEN, 1, 4, 25), + F(64000000, P_GCC_GPLL0_OUT_EVEN, 1, 16, 75), + F(75000000, P_GCC_GPLL0_OUT_EVEN, 4, 0, 0), + F(80000000, P_GCC_GPLL0_OUT_EVEN, 1, 4, 15), + F(96000000, P_GCC_GPLL0_OUT_EVEN, 1, 8, 25), + F(100000000, P_GCC_GPLL0_OUT_MAIN, 6, 0, 0), + { } +}; + +static struct clk_init_data gcc_qupv3_wrap0_s2_clk_src_init = { + .name = "gcc_qupv3_wrap0_s2_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap0_s2_clk_src = { + .cmd_rcgr = 0x17270, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s2_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap0_s2_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap0_s3_clk_src_init = { + .name = "gcc_qupv3_wrap0_s3_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap0_s3_clk_src = { + .cmd_rcgr = 0x173a0, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s2_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap0_s3_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap0_s4_clk_src_init = { + .name = "gcc_qupv3_wrap0_s4_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap0_s4_clk_src = { + .cmd_rcgr = 0x174d0, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s2_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap0_s4_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap0_s5_clk_src_init = { + .name = "gcc_qupv3_wrap0_s5_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap0_s5_clk_src = { + .cmd_rcgr = 0x17600, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s2_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap0_s5_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap0_s6_clk_src_init = { + .name = "gcc_qupv3_wrap0_s6_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap0_s6_clk_src = { + .cmd_rcgr = 0x17730, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s2_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap0_s6_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap0_s7_clk_src_init = { + .name = "gcc_qupv3_wrap0_s7_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap0_s7_clk_src = { + .cmd_rcgr = 0x17860, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s2_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap0_s7_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap1_s0_clk_src_init = { + .name = "gcc_qupv3_wrap1_s0_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap1_s0_clk_src = { + .cmd_rcgr = 0x18010, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap1_s0_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap1_s1_clk_src_init = { + .name = "gcc_qupv3_wrap1_s1_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap1_s1_clk_src = { + .cmd_rcgr = 0x18140, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap1_s1_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap1_s2_clk_src_init = { + .name = "gcc_qupv3_wrap1_s2_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap1_s2_clk_src = { + .cmd_rcgr = 0x18270, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s2_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap1_s2_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap1_s3_clk_src_init = { + .name = "gcc_qupv3_wrap1_s3_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap1_s3_clk_src = { + .cmd_rcgr = 0x183a0, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s2_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap1_s3_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap1_s4_clk_src_init = { + .name = "gcc_qupv3_wrap1_s4_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap1_s4_clk_src = { + .cmd_rcgr = 0x184d0, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s2_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap1_s4_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap1_s5_clk_src_init = { + .name = "gcc_qupv3_wrap1_s5_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap1_s5_clk_src = { + .cmd_rcgr = 0x18600, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s2_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap1_s5_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap1_s6_clk_src_init = { + .name = "gcc_qupv3_wrap1_s6_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap1_s6_clk_src = { + .cmd_rcgr = 0x18730, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s2_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap1_s6_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap1_s7_clk_src_init = { + .name = "gcc_qupv3_wrap1_s7_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap1_s7_clk_src = { + .cmd_rcgr = 0x18860, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s2_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap1_s7_clk_src_init, +}; + +static const struct freq_tbl ftbl_gcc_sdcc1_apps_clk_src[] = { + F(144000, P_BI_TCXO, 16, 3, 25), + F(400000, P_BI_TCXO, 12, 1, 4), + F(19200000, P_BI_TCXO, 1, 0, 0), + F(20000000, P_GCC_GPLL0_OUT_EVEN, 5, 1, 3), + F(25000000, P_GCC_GPLL0_OUT_EVEN, 12, 0, 0), + F(50000000, P_GCC_GPLL0_OUT_EVEN, 6, 0, 0), + F(100000000, P_GCC_GPLL0_OUT_EVEN, 3, 0, 0), + F(192000000, P_GCC_GPLL10_OUT_MAIN, 2, 0, 0), + F(384000000, P_GCC_GPLL10_OUT_MAIN, 1, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_sdcc1_apps_clk_src = { + .cmd_rcgr = 0x7500c, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_8, + .freq_tbl = ftbl_gcc_sdcc1_apps_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc1_apps_clk_src", + .parent_data = gcc_parent_data_8, + .num_parents = ARRAY_SIZE(gcc_parent_data_8), + .ops = &clk_rcg2_floor_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_sdcc1_ice_core_clk_src[] = { + F(100000000, P_GCC_GPLL0_OUT_EVEN, 3, 0, 0), + F(150000000, P_GCC_GPLL0_OUT_EVEN, 2, 0, 0), + F(300000000, P_GCC_GPLL0_OUT_EVEN, 1, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_sdcc1_ice_core_clk_src = { + .cmd_rcgr = 0x7502c, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_1, + .freq_tbl = ftbl_gcc_sdcc1_ice_core_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc1_ice_core_clk_src", + .parent_data = gcc_parent_data_1, + .num_parents = ARRAY_SIZE(gcc_parent_data_1), + .ops = &clk_rcg2_floor_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_sdcc2_apps_clk_src[] = { + F(400000, P_BI_TCXO, 12, 1, 4), + F(19200000, P_BI_TCXO, 1, 0, 0), + F(25000000, P_GCC_GPLL0_OUT_EVEN, 12, 0, 0), + F(50000000, P_GCC_GPLL0_OUT_EVEN, 6, 0, 0), + F(100000000, P_GCC_GPLL0_OUT_EVEN, 3, 0, 0), + F(202000000, P_GCC_GPLL9_OUT_MAIN, 4, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_sdcc2_apps_clk_src = { + .cmd_rcgr = 0x1400c, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_9, + .freq_tbl = ftbl_gcc_sdcc2_apps_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc2_apps_clk_src", + .parent_data = gcc_parent_data_9, + .num_parents = ARRAY_SIZE(gcc_parent_data_9), + .flags = CLK_OPS_PARENT_ENABLE, + .ops = &clk_rcg2_floor_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_sdcc4_apps_clk_src[] = { + F(400000, P_BI_TCXO, 12, 1, 4), + F(19200000, P_BI_TCXO, 1, 0, 0), + F(25000000, P_GCC_GPLL0_OUT_EVEN, 12, 0, 0), + F(50000000, P_GCC_GPLL0_OUT_EVEN, 6, 0, 0), + F(100000000, P_GCC_GPLL0_OUT_EVEN, 3, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_sdcc4_apps_clk_src = { + .cmd_rcgr = 0x1600c, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_1, + .freq_tbl = ftbl_gcc_sdcc4_apps_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc4_apps_clk_src", + .parent_data = gcc_parent_data_1, + .num_parents = ARRAY_SIZE(gcc_parent_data_1), + .ops = &clk_rcg2_floor_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_ufs_phy_axi_clk_src[] = { + F(25000000, P_GCC_GPLL0_OUT_EVEN, 12, 0, 0), + F(75000000, P_GCC_GPLL0_OUT_EVEN, 4, 0, 0), + F(150000000, P_GCC_GPLL0_OUT_EVEN, 2, 0, 0), + F(300000000, P_GCC_GPLL0_OUT_EVEN, 1, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_ufs_phy_axi_clk_src = { + .cmd_rcgr = 0x77024, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_ufs_phy_axi_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_axi_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_ufs_phy_ice_core_clk_src[] = { + F(75000000, P_GCC_GPLL0_OUT_EVEN, 4, 0, 0), + F(150000000, P_GCC_GPLL0_OUT_EVEN, 2, 0, 0), + F(300000000, P_GCC_GPLL0_OUT_EVEN, 1, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_ufs_phy_ice_core_clk_src = { + .cmd_rcgr = 0x7706c, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_ufs_phy_ice_core_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_ice_core_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_ufs_phy_phy_aux_clk_src = { + .cmd_rcgr = 0x770a0, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_3, + .freq_tbl = ftbl_gcc_pcie_0_aux_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_phy_aux_clk_src", + .parent_data = gcc_parent_data_3, + .num_parents = ARRAY_SIZE(gcc_parent_data_3), + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_ufs_phy_unipro_core_clk_src = { + .cmd_rcgr = 0x77084, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_ufs_phy_ice_core_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_unipro_core_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = ARRAY_SIZE(gcc_parent_data_0), + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_usb30_prim_master_clk_src[] = { + F(66666667, P_GCC_GPLL0_OUT_EVEN, 4.5, 0, 0), + F(133333333, P_GCC_GPLL0_OUT_MAIN, 4.5, 0, 0), + F(200000000, P_GCC_GPLL0_OUT_ODD, 1, 0, 0), + F(240000000, P_GCC_GPLL0_OUT_MAIN, 2.5, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_usb30_prim_master_clk_src = { + .cmd_rcgr = 0xf020, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_1, + .freq_tbl = ftbl_gcc_usb30_prim_master_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_prim_master_clk_src", + .parent_data = gcc_parent_data_1, + .num_parents = ARRAY_SIZE(gcc_parent_data_1), + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_usb30_prim_mock_utmi_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_usb30_prim_mock_utmi_clk_src = { + .cmd_rcgr = 0xf038, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_3, + .freq_tbl = ftbl_gcc_usb30_prim_mock_utmi_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_prim_mock_utmi_clk_src", + .parent_data = gcc_parent_data_3, + .num_parents = ARRAY_SIZE(gcc_parent_data_3), + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_usb30_sec_master_clk_src[] = { + F(60000000, P_GCC_GPLL0_OUT_EVEN, 5, 0, 0), + F(120000000, P_GCC_GPLL0_OUT_EVEN, 2.5, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_usb30_sec_master_clk_src = { + .cmd_rcgr = 0x9e020, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_5, + .freq_tbl = ftbl_gcc_usb30_sec_master_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_sec_master_clk_src", + .parent_data = gcc_parent_data_5, + .num_parents = ARRAY_SIZE(gcc_parent_data_5), + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_usb30_sec_mock_utmi_clk_src = { + .cmd_rcgr = 0x9e038, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_3, + .freq_tbl = ftbl_gcc_usb30_prim_mock_utmi_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_sec_mock_utmi_clk_src", + .parent_data = gcc_parent_data_3, + .num_parents = ARRAY_SIZE(gcc_parent_data_3), + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_usb3_prim_phy_aux_clk_src = { + .cmd_rcgr = 0xf064, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_2, + .freq_tbl = ftbl_gcc_usb30_prim_mock_utmi_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_prim_phy_aux_clk_src", + .parent_data = gcc_parent_data_2, + .num_parents = ARRAY_SIZE(gcc_parent_data_2), + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_usb3_sec_phy_aux_clk_src = { + .cmd_rcgr = 0x9e064, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_2, + .freq_tbl = ftbl_gcc_usb30_prim_mock_utmi_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_sec_phy_aux_clk_src", + .parent_data = gcc_parent_data_2, + .num_parents = ARRAY_SIZE(gcc_parent_data_2), + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_sec_ctrl_clk_src[] = { + F(4800000, P_BI_TCXO, 4, 0, 0), + F(19200000, P_BI_TCXO, 1, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_sec_ctrl_clk_src = { + .cmd_rcgr = 0x3d02c, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_3, + .freq_tbl = ftbl_gcc_sec_ctrl_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_sec_ctrl_clk_src", + .parent_data = gcc_parent_data_3, + .num_parents = ARRAY_SIZE(gcc_parent_data_3), + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_regmap_div gcc_cpuss_ahb_postdiv_clk_src = { + .reg = 0x48024, + .shift = 0, + .width = 4, + .clkr.hw.init = &(struct clk_init_data) { + .name = "gcc_cpuss_ahb_postdiv_clk_src", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_cpuss_ahb_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_regmap_div_ro_ops, + }, +}; + +static struct clk_regmap_div gcc_usb30_prim_mock_utmi_postdiv_clk_src = { + .reg = 0xf050, + .shift = 0, + .width = 4, + .clkr.hw.init = &(struct clk_init_data) { + .name = "gcc_usb30_prim_mock_utmi_postdiv_clk_src", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_usb30_prim_mock_utmi_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_regmap_div_ro_ops, + }, +}; + +static struct clk_regmap_div gcc_usb30_sec_mock_utmi_postdiv_clk_src = { + .reg = 0x9e050, + .shift = 0, + .width = 4, + .clkr.hw.init = &(struct clk_init_data) { + .name = "gcc_usb30_sec_mock_utmi_postdiv_clk_src", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_usb30_sec_mock_utmi_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_regmap_div_ro_ops, + }, +}; + +static struct clk_branch gcc_pcie_clkref_en = { + .halt_reg = 0x8c004, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x8c004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_clkref_en", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_edp_clkref_en = { + .halt_reg = 0x8c008, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x8c008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_edp_clkref_en", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_aggre_noc_pcie_0_axi_clk = { + .halt_reg = 0x6b080, + .halt_check = BRANCH_HALT_SKIP, + .hwcg_reg = 0x6b080, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(12), + .hw.init = &(struct clk_init_data){ + .name = "gcc_aggre_noc_pcie_0_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_aggre_noc_pcie_1_axi_clk = { + .halt_reg = 0x8d084, + .halt_check = BRANCH_HALT_SKIP, + .hwcg_reg = 0x8d084, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(11), + .hw.init = &(struct clk_init_data){ + .name = "gcc_aggre_noc_pcie_1_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_aggre_noc_pcie_tbu_clk = { + .halt_reg = 0x90010, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x90010, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(18), + .hw.init = &(struct clk_init_data){ + .name = "gcc_aggre_noc_pcie_tbu_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_aggre_noc_pcie_center_sf_axi_clk = { + .halt_reg = 0x8d088, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x8d088, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(28), + .hw.init = &(struct clk_init_data){ + .name = "gcc_aggre_noc_pcie_center_sf_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_aggre_ufs_phy_axi_clk = { + .halt_reg = 0x770cc, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x770cc, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x770cc, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_aggre_ufs_phy_axi_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_phy_axi_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_aggre_usb3_prim_axi_clk = { + .halt_reg = 0xf080, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0xf080, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0xf080, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_aggre_usb3_prim_axi_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_usb30_prim_master_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_aggre_usb3_sec_axi_clk = { + .halt_reg = 0x9e080, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x9e080, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x9e080, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_aggre_usb3_sec_axi_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_usb30_sec_master_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camera_hf_axi_clk = { + .halt_reg = 0x26010, + .halt_check = BRANCH_HALT_SKIP, + .hwcg_reg = 0x26010, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x26010, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_camera_hf_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camera_sf_axi_clk = { + .halt_reg = 0x2601c, + .halt_check = BRANCH_HALT_SKIP, + .hwcg_reg = 0x2601c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x2601c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_camera_sf_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_cfg_noc_usb3_prim_axi_clk = { + .halt_reg = 0xf07c, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0xf07c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0xf07c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_cfg_noc_usb3_prim_axi_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_usb30_prim_master_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_cfg_noc_usb3_sec_axi_clk = { + .halt_reg = 0x9e07c, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x9e07c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x9e07c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_cfg_noc_usb3_sec_axi_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_usb30_sec_master_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +/* For CPUSS functionality the AHB clock needs to be left enabled */ +static struct clk_branch gcc_cpuss_ahb_clk = { + .halt_reg = 0x48000, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x48000, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(21), + .hw.init = &(struct clk_init_data){ + .name = "gcc_cpuss_ahb_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_cpuss_ahb_postdiv_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_IS_CRITICAL | CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ddrss_gpu_axi_clk = { + .halt_reg = 0x71154, + .halt_check = BRANCH_HALT_SKIP, + .hwcg_reg = 0x71154, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x71154, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ddrss_gpu_axi_clk", + .ops = &clk_branch2_aon_ops, + }, + }, +}; + +static struct clk_branch gcc_ddrss_pcie_sf_clk = { + .halt_reg = 0x8d080, + .halt_check = BRANCH_HALT_SKIP, + .hwcg_reg = 0x8d080, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(19), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ddrss_pcie_sf_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_disp_gpll0_clk_src = { + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(7), + .hw.init = &(struct clk_init_data){ + .name = "gcc_disp_gpll0_clk_src", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_gpll0.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_disp_hf_axi_clk = { + .halt_reg = 0x2700c, + .halt_check = BRANCH_HALT_SKIP, + .hwcg_reg = 0x2700c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x2700c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_disp_hf_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_disp_sf_axi_clk = { + .halt_reg = 0x27014, + .halt_check = BRANCH_HALT_SKIP, + .hwcg_reg = 0x27014, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x27014, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_disp_sf_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_gp1_clk = { + .halt_reg = 0x64000, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x64000, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gp1_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_gp1_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_gp2_clk = { + .halt_reg = 0x65000, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x65000, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gp2_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_gp2_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_gp3_clk = { + .halt_reg = 0x66000, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x66000, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gp3_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_gp3_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_gpu_gpll0_clk_src = { + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(15), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gpu_gpll0_clk_src", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_gpll0.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_gpu_gpll0_div_clk_src = { + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(16), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gpu_gpll0_div_clk_src", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_gpll0_out_even.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_gpu_iref_en = { + .halt_reg = 0x8c014, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x8c014, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gpu_iref_en", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_gpu_memnoc_gfx_clk = { + .halt_reg = 0x7100c, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x7100c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x7100c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gpu_memnoc_gfx_clk", + .ops = &clk_branch2_aon_ops, + }, + }, +}; + +static struct clk_branch gcc_gpu_snoc_dvm_gfx_clk = { + .halt_reg = 0x71018, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x71018, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gpu_snoc_dvm_gfx_clk", + .ops = &clk_branch2_aon_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie0_phy_rchng_clk = { + .halt_reg = 0x6b038, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(22), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie0_phy_rchng_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_pcie_0_phy_rchng_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie1_phy_rchng_clk = { + .halt_reg = 0x8d038, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(23), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie1_phy_rchng_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_pcie_1_phy_rchng_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_0_aux_clk = { + .halt_reg = 0x6b028, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(3), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_0_aux_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_pcie_0_aux_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_0_cfg_ahb_clk = { + .halt_reg = 0x6b024, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x6b024, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(2), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_0_cfg_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_0_mstr_axi_clk = { + .halt_reg = 0x6b01c, + .halt_check = BRANCH_HALT_SKIP, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(1), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_0_mstr_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_0_pipe_clk = { + .halt_reg = 0x6b030, + .halt_check = BRANCH_HALT_SKIP, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(4), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_0_pipe_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_pcie_0_pipe_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_0_slv_axi_clk = { + .halt_reg = 0x6b014, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_0_slv_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_0_slv_q2a_axi_clk = { + .halt_reg = 0x6b010, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(5), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_0_slv_q2a_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_1_aux_clk = { + .halt_reg = 0x8d028, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(29), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_1_aux_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_pcie_1_aux_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_1_cfg_ahb_clk = { + .halt_reg = 0x8d024, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x8d024, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(28), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_1_cfg_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_1_mstr_axi_clk = { + .halt_reg = 0x8d01c, + .halt_check = BRANCH_HALT_SKIP, + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(27), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_1_mstr_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_1_pipe_clk = { + .halt_reg = 0x8d030, + .halt_check = BRANCH_HALT_SKIP, + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(30), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_1_pipe_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_pcie_1_pipe_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_1_slv_axi_clk = { + .halt_reg = 0x8d014, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(26), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_1_slv_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_1_slv_q2a_axi_clk = { + .halt_reg = 0x8d010, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(25), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_1_slv_q2a_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_throttle_core_clk = { + .halt_reg = 0x90018, + .halt_check = BRANCH_HALT_SKIP, + .hwcg_reg = 0x90018, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(20), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_throttle_core_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pdm2_clk = { + .halt_reg = 0x3300c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x3300c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pdm2_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_pdm2_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pdm_ahb_clk = { + .halt_reg = 0x33004, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x33004, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x33004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pdm_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pdm_xo4_clk = { + .halt_reg = 0x33008, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x33008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pdm_xo4_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qmip_camera_nrt_ahb_clk = { + .halt_reg = 0x26008, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x26008, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x26008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qmip_camera_nrt_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qmip_camera_rt_ahb_clk = { + .halt_reg = 0x2600c, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x2600c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x2600c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qmip_camera_rt_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qmip_disp_ahb_clk = { + .halt_reg = 0x27008, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x27008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qmip_disp_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qmip_video_vcodec_ahb_clk = { + .halt_reg = 0x28008, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x28008, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x28008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qmip_video_vcodec_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qspi_cnoc_periph_ahb_clk = { + .halt_reg = 0x4b004, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0x4b004, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x4b004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qspi_cnoc_periph_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qspi_core_clk = { + .halt_reg = 0x4b008, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x4b008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qspi_core_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qspi_core_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap0_core_2x_clk = { + .halt_reg = 0x23008, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(9), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_core_2x_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap0_core_clk = { + .halt_reg = 0x23000, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(8), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_core_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap0_s0_clk = { + .halt_reg = 0x1700c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(10), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_s0_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap0_s0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap0_s1_clk = { + .halt_reg = 0x1713c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(11), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_s1_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap0_s1_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap0_s2_clk = { + .halt_reg = 0x1726c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(12), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_s2_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap0_s2_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap0_s3_clk = { + .halt_reg = 0x1739c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(13), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_s3_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap0_s3_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap0_s4_clk = { + .halt_reg = 0x174cc, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(14), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_s4_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap0_s4_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap0_s5_clk = { + .halt_reg = 0x175fc, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(15), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_s5_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap0_s5_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap0_s6_clk = { + .halt_reg = 0x1772c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(16), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_s6_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap0_s6_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap0_s7_clk = { + .halt_reg = 0x1785c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(17), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_s7_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap0_s7_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap1_core_2x_clk = { + .halt_reg = 0x23140, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(18), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap1_core_2x_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap1_core_clk = { + .halt_reg = 0x23138, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(19), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap1_core_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap1_s0_clk = { + .halt_reg = 0x1800c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(22), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap1_s0_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap1_s0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap1_s1_clk = { + .halt_reg = 0x1813c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(23), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap1_s1_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap1_s1_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap1_s2_clk = { + .halt_reg = 0x1826c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(24), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap1_s2_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap1_s2_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap1_s3_clk = { + .halt_reg = 0x1839c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(25), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap1_s3_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap1_s3_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap1_s4_clk = { + .halt_reg = 0x184cc, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(26), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap1_s4_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap1_s4_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap1_s5_clk = { + .halt_reg = 0x185fc, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(27), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap1_s5_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap1_s5_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap1_s6_clk = { + .halt_reg = 0x1872c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(13), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap1_s6_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap1_s6_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap1_s7_clk = { + .halt_reg = 0x1885c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(14), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap1_s7_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap1_s7_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap_0_m_ahb_clk = { + .halt_reg = 0x17004, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x17004, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(6), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap_0_m_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap_0_s_ahb_clk = { + .halt_reg = 0x17008, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x17008, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(7), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap_0_s_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap_1_m_ahb_clk = { + .halt_reg = 0x18004, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x18004, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(20), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap_1_m_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap_1_s_ahb_clk = { + .halt_reg = 0x18008, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x18008, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(21), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap_1_s_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sdcc1_ahb_clk = { + .halt_reg = 0x75004, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x75004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc1_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sdcc1_apps_clk = { + .halt_reg = 0x75008, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x75008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc1_apps_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_sdcc1_apps_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sdcc1_ice_core_clk = { + .halt_reg = 0x75024, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x75024, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x75024, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc1_ice_core_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_sdcc1_ice_core_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sdcc2_ahb_clk = { + .halt_reg = 0x14008, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x14008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc2_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sdcc2_apps_clk = { + .halt_reg = 0x14004, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x14004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc2_apps_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_sdcc2_apps_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sdcc4_ahb_clk = { + .halt_reg = 0x16008, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x16008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc4_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sdcc4_apps_clk = { + .halt_reg = 0x16004, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x16004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc4_apps_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_sdcc4_apps_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +/* For CPUSS functionality the AHB clock needs to be left enabled */ +static struct clk_branch gcc_sys_noc_cpuss_ahb_clk = { + .halt_reg = 0x48178, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x48178, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_sys_noc_cpuss_ahb_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_cpuss_ahb_postdiv_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_IS_CRITICAL | CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_throttle_pcie_ahb_clk = { + .halt_reg = 0x9001c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x9001c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_throttle_pcie_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_titan_nrt_throttle_core_clk = { + .halt_reg = 0x26024, + .halt_check = BRANCH_HALT_SKIP, + .hwcg_reg = 0x26024, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x26024, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_titan_nrt_throttle_core_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_titan_rt_throttle_core_clk = { + .halt_reg = 0x26018, + .halt_check = BRANCH_HALT_SKIP, + .hwcg_reg = 0x26018, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x26018, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_titan_rt_throttle_core_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_1_clkref_en = { + .halt_reg = 0x8c000, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x8c000, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_1_clkref_en", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_ahb_clk = { + .halt_reg = 0x77018, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x77018, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x77018, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_axi_clk = { + .halt_reg = 0x77010, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x77010, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x77010, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_axi_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_phy_axi_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_ice_core_clk = { + .halt_reg = 0x77064, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x77064, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x77064, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_ice_core_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_phy_ice_core_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_phy_aux_clk = { + .halt_reg = 0x7709c, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x7709c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x7709c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_phy_aux_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_phy_phy_aux_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_rx_symbol_0_clk = { + .halt_reg = 0x77020, + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x77020, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_rx_symbol_0_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_phy_rx_symbol_0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_rx_symbol_1_clk = { + .halt_reg = 0x770b8, + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x770b8, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_rx_symbol_1_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_phy_rx_symbol_1_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_tx_symbol_0_clk = { + .halt_reg = 0x7701c, + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x7701c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_tx_symbol_0_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_phy_tx_symbol_0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_unipro_core_clk = { + .halt_reg = 0x7705c, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x7705c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x7705c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_unipro_core_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_phy_unipro_core_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb30_prim_master_clk = { + .halt_reg = 0xf010, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xf010, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_prim_master_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_usb30_prim_master_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb30_prim_mock_utmi_clk = { + .halt_reg = 0xf01c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xf01c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_prim_mock_utmi_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = + &gcc_usb30_prim_mock_utmi_postdiv_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb30_prim_sleep_clk = { + .halt_reg = 0xf018, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xf018, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_prim_sleep_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb30_sec_master_clk = { + .halt_reg = 0x9e010, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x9e010, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_sec_master_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_usb30_sec_master_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb30_sec_mock_utmi_clk = { + .halt_reg = 0x9e01c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x9e01c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_sec_mock_utmi_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = + &gcc_usb30_sec_mock_utmi_postdiv_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb30_sec_sleep_clk = { + .halt_reg = 0x9e018, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x9e018, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_sec_sleep_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb3_prim_phy_aux_clk = { + .halt_reg = 0xf054, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xf054, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_prim_phy_aux_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_usb3_prim_phy_aux_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb3_prim_phy_com_aux_clk = { + .halt_reg = 0xf058, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xf058, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_prim_phy_com_aux_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_usb3_prim_phy_aux_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb3_prim_phy_pipe_clk = { + .halt_reg = 0xf05c, + .halt_check = BRANCH_HALT_DELAY, + .hwcg_reg = 0xf05c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0xf05c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_prim_phy_pipe_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_usb3_prim_phy_pipe_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_cfg_noc_lpass_clk = { + .halt_reg = 0x47020, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x47020, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_cfg_noc_lpass_clk", + .ops = &clk_branch2_ops, + }, + }, +}; +static struct clk_branch gcc_mss_cfg_ahb_clk = { + .halt_reg = 0x8a000, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x8a000, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_mss_cfg_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_mss_offline_axi_clk = { + .halt_reg = 0x8a004, + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x8a004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_mss_offline_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_mss_snoc_axi_clk = { + .halt_reg = 0x8a154, + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x8a154, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_mss_snoc_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_mss_q6_memnoc_axi_clk = { + .halt_reg = 0x8a158, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x8a158, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_mss_q6_memnoc_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_regmap_mux gcc_mss_q6ss_boot_clk_src = { + .reg = 0x8a2a4, + .shift = 0, + .width = 1, + .parent_map = gcc_parent_map_15, + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "gcc_mss_q6ss_boot_clk_src", + .parent_data = gcc_parent_data_15, + .num_parents = ARRAY_SIZE(gcc_parent_data_15), + .ops = &clk_regmap_mux_closest_ops, + }, + }, +}; + +static struct clk_branch gcc_usb3_sec_phy_aux_clk = { + .halt_reg = 0x9e054, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x9e054, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_sec_phy_aux_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_usb3_sec_phy_aux_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb3_sec_phy_com_aux_clk = { + .halt_reg = 0x9e058, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x9e058, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_sec_phy_com_aux_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_usb3_sec_phy_aux_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb3_sec_phy_pipe_clk = { + .halt_reg = 0x9e05c, + .halt_check = BRANCH_HALT_SKIP, + .hwcg_reg = 0x9e05c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x9e05c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_sec_phy_pipe_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_usb3_sec_phy_pipe_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_video_axi0_clk = { + .halt_reg = 0x2800c, + .halt_check = BRANCH_HALT_SKIP, + .hwcg_reg = 0x2800c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x2800c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_video_axi0_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_video_mvp_throttle_core_clk = { + .halt_reg = 0x28010, + .halt_check = BRANCH_HALT_SKIP, + .hwcg_reg = 0x28010, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x28010, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_video_mvp_throttle_core_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_wpss_ahb_clk = { + .halt_reg = 0x9d154, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x9d154, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_wpss_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_wpss_ahb_bdg_mst_clk = { + .halt_reg = 0x9d158, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x9d158, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_wpss_ahb_bdg_mst_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_wpss_rscp_clk = { + .halt_reg = 0x9d16c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x9d16c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_wpss_rscp_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct gdsc gcc_pcie_0_gdsc = { + .gdscr = 0x6b004, + .pd = { + .name = "gcc_pcie_0_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .flags = VOTABLE, +}; + +static struct gdsc gcc_pcie_1_gdsc = { + .gdscr = 0x8d004, + .pd = { + .name = "gcc_pcie_1_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .flags = VOTABLE, +}; + +static struct gdsc gcc_ufs_phy_gdsc = { + .gdscr = 0x77004, + .pd = { + .name = "gcc_ufs_phy_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .flags = VOTABLE, +}; + +static struct gdsc gcc_usb30_prim_gdsc = { + .gdscr = 0xf004, + .pd = { + .name = "gcc_usb30_prim_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .flags = VOTABLE, +}; + +static struct gdsc gcc_usb30_sec_gdsc = { + .gdscr = 0x9e004, + .pd = { + .name = "gcc_usb30_sec_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .flags = VOTABLE, +}; + +static struct gdsc hlos1_vote_mmnoc_mmu_tbu_hf0_gdsc = { + .gdscr = 0x7d050, + .pd = { + .name = "hlos1_vote_mmnoc_mmu_tbu_hf0_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .flags = VOTABLE, +}; + +static struct gdsc hlos1_vote_mmnoc_mmu_tbu_hf1_gdsc = { + .gdscr = 0x7d058, + .pd = { + .name = "hlos1_vote_mmnoc_mmu_tbu_hf1_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .flags = VOTABLE, +}; + +static struct gdsc hlos1_vote_mmnoc_mmu_tbu_sf0_gdsc = { + .gdscr = 0x7d054, + .pd = { + .name = "hlos1_vote_mmnoc_mmu_tbu_sf0_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .flags = VOTABLE, +}; + +static struct gdsc hlos1_vote_turing_mmu_tbu0_gdsc = { + .gdscr = 0x7d05c, + .pd = { + .name = "hlos1_vote_turing_mmu_tbu0_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .flags = VOTABLE, +}; + +static struct gdsc hlos1_vote_turing_mmu_tbu1_gdsc = { + .gdscr = 0x7d060, + .pd = { + .name = "hlos1_vote_turing_mmu_tbu1_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .flags = VOTABLE, +}; + +static struct clk_regmap *gcc_sc7280_clocks[] = { + [GCC_AGGRE_NOC_PCIE_0_AXI_CLK] = &gcc_aggre_noc_pcie_0_axi_clk.clkr, + [GCC_AGGRE_NOC_PCIE_1_AXI_CLK] = &gcc_aggre_noc_pcie_1_axi_clk.clkr, + [GCC_AGGRE_UFS_PHY_AXI_CLK] = &gcc_aggre_ufs_phy_axi_clk.clkr, + [GCC_AGGRE_USB3_PRIM_AXI_CLK] = &gcc_aggre_usb3_prim_axi_clk.clkr, + [GCC_AGGRE_USB3_SEC_AXI_CLK] = &gcc_aggre_usb3_sec_axi_clk.clkr, + [GCC_CAMERA_HF_AXI_CLK] = &gcc_camera_hf_axi_clk.clkr, + [GCC_CAMERA_SF_AXI_CLK] = &gcc_camera_sf_axi_clk.clkr, + [GCC_CFG_NOC_USB3_PRIM_AXI_CLK] = &gcc_cfg_noc_usb3_prim_axi_clk.clkr, + [GCC_CFG_NOC_USB3_SEC_AXI_CLK] = &gcc_cfg_noc_usb3_sec_axi_clk.clkr, + [GCC_CPUSS_AHB_CLK] = &gcc_cpuss_ahb_clk.clkr, + [GCC_CPUSS_AHB_CLK_SRC] = &gcc_cpuss_ahb_clk_src.clkr, + [GCC_CPUSS_AHB_POSTDIV_CLK_SRC] = &gcc_cpuss_ahb_postdiv_clk_src.clkr, + [GCC_DDRSS_GPU_AXI_CLK] = &gcc_ddrss_gpu_axi_clk.clkr, + [GCC_DDRSS_PCIE_SF_CLK] = &gcc_ddrss_pcie_sf_clk.clkr, + [GCC_DISP_GPLL0_CLK_SRC] = &gcc_disp_gpll0_clk_src.clkr, + [GCC_DISP_HF_AXI_CLK] = &gcc_disp_hf_axi_clk.clkr, + [GCC_DISP_SF_AXI_CLK] = &gcc_disp_sf_axi_clk.clkr, + [GCC_GP1_CLK] = &gcc_gp1_clk.clkr, + [GCC_GP1_CLK_SRC] = &gcc_gp1_clk_src.clkr, + [GCC_GP2_CLK] = &gcc_gp2_clk.clkr, + [GCC_GP2_CLK_SRC] = &gcc_gp2_clk_src.clkr, + [GCC_GP3_CLK] = &gcc_gp3_clk.clkr, + [GCC_GP3_CLK_SRC] = &gcc_gp3_clk_src.clkr, + [GCC_GPLL0] = &gcc_gpll0.clkr, + [GCC_GPLL0_OUT_EVEN] = &gcc_gpll0_out_even.clkr, + [GCC_GPLL0_OUT_ODD] = &gcc_gpll0_out_odd.clkr, + [GCC_GPLL1] = &gcc_gpll1.clkr, + [GCC_GPLL10] = &gcc_gpll10.clkr, + [GCC_GPLL4] = &gcc_gpll4.clkr, + [GCC_GPLL9] = &gcc_gpll9.clkr, + [GCC_GPU_GPLL0_CLK_SRC] = &gcc_gpu_gpll0_clk_src.clkr, + [GCC_GPU_GPLL0_DIV_CLK_SRC] = &gcc_gpu_gpll0_div_clk_src.clkr, + [GCC_GPU_IREF_EN] = &gcc_gpu_iref_en.clkr, + [GCC_GPU_MEMNOC_GFX_CLK] = &gcc_gpu_memnoc_gfx_clk.clkr, + [GCC_GPU_SNOC_DVM_GFX_CLK] = &gcc_gpu_snoc_dvm_gfx_clk.clkr, + [GCC_PCIE0_PHY_RCHNG_CLK] = &gcc_pcie0_phy_rchng_clk.clkr, + [GCC_PCIE1_PHY_RCHNG_CLK] = &gcc_pcie1_phy_rchng_clk.clkr, + [GCC_PCIE_0_AUX_CLK] = &gcc_pcie_0_aux_clk.clkr, + [GCC_PCIE_0_AUX_CLK_SRC] = &gcc_pcie_0_aux_clk_src.clkr, + [GCC_PCIE_0_CFG_AHB_CLK] = &gcc_pcie_0_cfg_ahb_clk.clkr, + [GCC_PCIE_0_MSTR_AXI_CLK] = &gcc_pcie_0_mstr_axi_clk.clkr, + [GCC_PCIE_0_PHY_RCHNG_CLK_SRC] = &gcc_pcie_0_phy_rchng_clk_src.clkr, + [GCC_PCIE_0_PIPE_CLK] = &gcc_pcie_0_pipe_clk.clkr, + [GCC_PCIE_0_PIPE_CLK_SRC] = &gcc_pcie_0_pipe_clk_src.clkr, + [GCC_PCIE_0_SLV_AXI_CLK] = &gcc_pcie_0_slv_axi_clk.clkr, + [GCC_PCIE_0_SLV_Q2A_AXI_CLK] = &gcc_pcie_0_slv_q2a_axi_clk.clkr, + [GCC_PCIE_1_AUX_CLK] = &gcc_pcie_1_aux_clk.clkr, + [GCC_PCIE_1_AUX_CLK_SRC] = &gcc_pcie_1_aux_clk_src.clkr, + [GCC_PCIE_1_CFG_AHB_CLK] = &gcc_pcie_1_cfg_ahb_clk.clkr, + [GCC_PCIE_1_MSTR_AXI_CLK] = &gcc_pcie_1_mstr_axi_clk.clkr, + [GCC_PCIE_1_PHY_RCHNG_CLK_SRC] = &gcc_pcie_1_phy_rchng_clk_src.clkr, + [GCC_PCIE_1_PIPE_CLK] = &gcc_pcie_1_pipe_clk.clkr, + [GCC_PCIE_1_PIPE_CLK_SRC] = &gcc_pcie_1_pipe_clk_src.clkr, + [GCC_PCIE_1_SLV_AXI_CLK] = &gcc_pcie_1_slv_axi_clk.clkr, + [GCC_PCIE_1_SLV_Q2A_AXI_CLK] = &gcc_pcie_1_slv_q2a_axi_clk.clkr, + [GCC_PCIE_THROTTLE_CORE_CLK] = &gcc_pcie_throttle_core_clk.clkr, + [GCC_PDM2_CLK] = &gcc_pdm2_clk.clkr, + [GCC_PDM2_CLK_SRC] = &gcc_pdm2_clk_src.clkr, + [GCC_PDM_AHB_CLK] = &gcc_pdm_ahb_clk.clkr, + [GCC_PDM_XO4_CLK] = &gcc_pdm_xo4_clk.clkr, + [GCC_QMIP_CAMERA_NRT_AHB_CLK] = &gcc_qmip_camera_nrt_ahb_clk.clkr, + [GCC_QMIP_CAMERA_RT_AHB_CLK] = &gcc_qmip_camera_rt_ahb_clk.clkr, + [GCC_QMIP_DISP_AHB_CLK] = &gcc_qmip_disp_ahb_clk.clkr, + [GCC_QMIP_VIDEO_VCODEC_AHB_CLK] = &gcc_qmip_video_vcodec_ahb_clk.clkr, + [GCC_QSPI_CNOC_PERIPH_AHB_CLK] = &gcc_qspi_cnoc_periph_ahb_clk.clkr, + [GCC_QSPI_CORE_CLK] = &gcc_qspi_core_clk.clkr, + [GCC_QSPI_CORE_CLK_SRC] = &gcc_qspi_core_clk_src.clkr, + [GCC_QUPV3_WRAP0_CORE_2X_CLK] = &gcc_qupv3_wrap0_core_2x_clk.clkr, + [GCC_QUPV3_WRAP0_CORE_CLK] = &gcc_qupv3_wrap0_core_clk.clkr, + [GCC_QUPV3_WRAP0_S0_CLK] = &gcc_qupv3_wrap0_s0_clk.clkr, + [GCC_QUPV3_WRAP0_S0_CLK_SRC] = &gcc_qupv3_wrap0_s0_clk_src.clkr, + [GCC_QUPV3_WRAP0_S1_CLK] = &gcc_qupv3_wrap0_s1_clk.clkr, + [GCC_QUPV3_WRAP0_S1_CLK_SRC] = &gcc_qupv3_wrap0_s1_clk_src.clkr, + [GCC_QUPV3_WRAP0_S2_CLK] = &gcc_qupv3_wrap0_s2_clk.clkr, + [GCC_QUPV3_WRAP0_S2_CLK_SRC] = &gcc_qupv3_wrap0_s2_clk_src.clkr, + [GCC_QUPV3_WRAP0_S3_CLK] = &gcc_qupv3_wrap0_s3_clk.clkr, + [GCC_QUPV3_WRAP0_S3_CLK_SRC] = &gcc_qupv3_wrap0_s3_clk_src.clkr, + [GCC_QUPV3_WRAP0_S4_CLK] = &gcc_qupv3_wrap0_s4_clk.clkr, + [GCC_QUPV3_WRAP0_S4_CLK_SRC] = &gcc_qupv3_wrap0_s4_clk_src.clkr, + [GCC_QUPV3_WRAP0_S5_CLK] = &gcc_qupv3_wrap0_s5_clk.clkr, + [GCC_QUPV3_WRAP0_S5_CLK_SRC] = &gcc_qupv3_wrap0_s5_clk_src.clkr, + [GCC_QUPV3_WRAP0_S6_CLK] = &gcc_qupv3_wrap0_s6_clk.clkr, + [GCC_QUPV3_WRAP0_S6_CLK_SRC] = &gcc_qupv3_wrap0_s6_clk_src.clkr, + [GCC_QUPV3_WRAP0_S7_CLK] = &gcc_qupv3_wrap0_s7_clk.clkr, + [GCC_QUPV3_WRAP0_S7_CLK_SRC] = &gcc_qupv3_wrap0_s7_clk_src.clkr, + [GCC_QUPV3_WRAP1_CORE_2X_CLK] = &gcc_qupv3_wrap1_core_2x_clk.clkr, + [GCC_QUPV3_WRAP1_CORE_CLK] = &gcc_qupv3_wrap1_core_clk.clkr, + [GCC_QUPV3_WRAP1_S0_CLK] = &gcc_qupv3_wrap1_s0_clk.clkr, + [GCC_QUPV3_WRAP1_S0_CLK_SRC] = &gcc_qupv3_wrap1_s0_clk_src.clkr, + [GCC_QUPV3_WRAP1_S1_CLK] = &gcc_qupv3_wrap1_s1_clk.clkr, + [GCC_QUPV3_WRAP1_S1_CLK_SRC] = &gcc_qupv3_wrap1_s1_clk_src.clkr, + [GCC_QUPV3_WRAP1_S2_CLK] = &gcc_qupv3_wrap1_s2_clk.clkr, + [GCC_QUPV3_WRAP1_S2_CLK_SRC] = &gcc_qupv3_wrap1_s2_clk_src.clkr, + [GCC_QUPV3_WRAP1_S3_CLK] = &gcc_qupv3_wrap1_s3_clk.clkr, + [GCC_QUPV3_WRAP1_S3_CLK_SRC] = &gcc_qupv3_wrap1_s3_clk_src.clkr, + [GCC_QUPV3_WRAP1_S4_CLK] = &gcc_qupv3_wrap1_s4_clk.clkr, + [GCC_QUPV3_WRAP1_S4_CLK_SRC] = &gcc_qupv3_wrap1_s4_clk_src.clkr, + [GCC_QUPV3_WRAP1_S5_CLK] = &gcc_qupv3_wrap1_s5_clk.clkr, + [GCC_QUPV3_WRAP1_S5_CLK_SRC] = &gcc_qupv3_wrap1_s5_clk_src.clkr, + [GCC_QUPV3_WRAP1_S6_CLK] = &gcc_qupv3_wrap1_s6_clk.clkr, + [GCC_QUPV3_WRAP1_S6_CLK_SRC] = &gcc_qupv3_wrap1_s6_clk_src.clkr, + [GCC_QUPV3_WRAP1_S7_CLK] = &gcc_qupv3_wrap1_s7_clk.clkr, + [GCC_QUPV3_WRAP1_S7_CLK_SRC] = &gcc_qupv3_wrap1_s7_clk_src.clkr, + [GCC_QUPV3_WRAP_0_M_AHB_CLK] = &gcc_qupv3_wrap_0_m_ahb_clk.clkr, + [GCC_QUPV3_WRAP_0_S_AHB_CLK] = &gcc_qupv3_wrap_0_s_ahb_clk.clkr, + [GCC_QUPV3_WRAP_1_M_AHB_CLK] = &gcc_qupv3_wrap_1_m_ahb_clk.clkr, + [GCC_QUPV3_WRAP_1_S_AHB_CLK] = &gcc_qupv3_wrap_1_s_ahb_clk.clkr, + [GCC_SDCC1_AHB_CLK] = &gcc_sdcc1_ahb_clk.clkr, + [GCC_SDCC1_APPS_CLK] = &gcc_sdcc1_apps_clk.clkr, + [GCC_SDCC1_APPS_CLK_SRC] = &gcc_sdcc1_apps_clk_src.clkr, + [GCC_SDCC1_ICE_CORE_CLK] = &gcc_sdcc1_ice_core_clk.clkr, + [GCC_SDCC1_ICE_CORE_CLK_SRC] = &gcc_sdcc1_ice_core_clk_src.clkr, + [GCC_SDCC2_AHB_CLK] = &gcc_sdcc2_ahb_clk.clkr, + [GCC_SDCC2_APPS_CLK] = &gcc_sdcc2_apps_clk.clkr, + [GCC_SDCC2_APPS_CLK_SRC] = &gcc_sdcc2_apps_clk_src.clkr, + [GCC_SDCC4_AHB_CLK] = &gcc_sdcc4_ahb_clk.clkr, + [GCC_SDCC4_APPS_CLK] = &gcc_sdcc4_apps_clk.clkr, + [GCC_SDCC4_APPS_CLK_SRC] = &gcc_sdcc4_apps_clk_src.clkr, + [GCC_SYS_NOC_CPUSS_AHB_CLK] = &gcc_sys_noc_cpuss_ahb_clk.clkr, + [GCC_THROTTLE_PCIE_AHB_CLK] = &gcc_throttle_pcie_ahb_clk.clkr, + [GCC_TITAN_NRT_THROTTLE_CORE_CLK] = + &gcc_titan_nrt_throttle_core_clk.clkr, + [GCC_TITAN_RT_THROTTLE_CORE_CLK] = &gcc_titan_rt_throttle_core_clk.clkr, + [GCC_UFS_1_CLKREF_EN] = &gcc_ufs_1_clkref_en.clkr, + [GCC_UFS_PHY_AHB_CLK] = &gcc_ufs_phy_ahb_clk.clkr, + [GCC_UFS_PHY_AXI_CLK] = &gcc_ufs_phy_axi_clk.clkr, + [GCC_UFS_PHY_AXI_CLK_SRC] = &gcc_ufs_phy_axi_clk_src.clkr, + [GCC_UFS_PHY_ICE_CORE_CLK] = &gcc_ufs_phy_ice_core_clk.clkr, + [GCC_UFS_PHY_ICE_CORE_CLK_SRC] = &gcc_ufs_phy_ice_core_clk_src.clkr, + [GCC_UFS_PHY_PHY_AUX_CLK] = &gcc_ufs_phy_phy_aux_clk.clkr, + [GCC_UFS_PHY_PHY_AUX_CLK_SRC] = &gcc_ufs_phy_phy_aux_clk_src.clkr, + [GCC_UFS_PHY_RX_SYMBOL_0_CLK] = &gcc_ufs_phy_rx_symbol_0_clk.clkr, + [GCC_UFS_PHY_RX_SYMBOL_0_CLK_SRC] = + &gcc_ufs_phy_rx_symbol_0_clk_src.clkr, + [GCC_UFS_PHY_RX_SYMBOL_1_CLK] = &gcc_ufs_phy_rx_symbol_1_clk.clkr, + [GCC_UFS_PHY_RX_SYMBOL_1_CLK_SRC] = + &gcc_ufs_phy_rx_symbol_1_clk_src.clkr, + [GCC_UFS_PHY_TX_SYMBOL_0_CLK] = &gcc_ufs_phy_tx_symbol_0_clk.clkr, + [GCC_UFS_PHY_TX_SYMBOL_0_CLK_SRC] = + &gcc_ufs_phy_tx_symbol_0_clk_src.clkr, + [GCC_UFS_PHY_UNIPRO_CORE_CLK] = &gcc_ufs_phy_unipro_core_clk.clkr, + [GCC_UFS_PHY_UNIPRO_CORE_CLK_SRC] = + &gcc_ufs_phy_unipro_core_clk_src.clkr, + [GCC_USB30_PRIM_MASTER_CLK] = &gcc_usb30_prim_master_clk.clkr, + [GCC_USB30_PRIM_MASTER_CLK_SRC] = &gcc_usb30_prim_master_clk_src.clkr, + [GCC_USB30_PRIM_MOCK_UTMI_CLK] = &gcc_usb30_prim_mock_utmi_clk.clkr, + [GCC_USB30_PRIM_MOCK_UTMI_CLK_SRC] = + &gcc_usb30_prim_mock_utmi_clk_src.clkr, + [GCC_USB30_PRIM_MOCK_UTMI_POSTDIV_CLK_SRC] = + &gcc_usb30_prim_mock_utmi_postdiv_clk_src.clkr, + [GCC_USB30_PRIM_SLEEP_CLK] = &gcc_usb30_prim_sleep_clk.clkr, + [GCC_USB30_SEC_MASTER_CLK] = &gcc_usb30_sec_master_clk.clkr, + [GCC_USB30_SEC_MASTER_CLK_SRC] = &gcc_usb30_sec_master_clk_src.clkr, + [GCC_USB30_SEC_MOCK_UTMI_CLK] = &gcc_usb30_sec_mock_utmi_clk.clkr, + [GCC_USB30_SEC_MOCK_UTMI_CLK_SRC] = + &gcc_usb30_sec_mock_utmi_clk_src.clkr, + [GCC_USB30_SEC_MOCK_UTMI_POSTDIV_CLK_SRC] = + &gcc_usb30_sec_mock_utmi_postdiv_clk_src.clkr, + [GCC_USB30_SEC_SLEEP_CLK] = &gcc_usb30_sec_sleep_clk.clkr, + [GCC_USB3_PRIM_PHY_AUX_CLK] = &gcc_usb3_prim_phy_aux_clk.clkr, + [GCC_USB3_PRIM_PHY_AUX_CLK_SRC] = &gcc_usb3_prim_phy_aux_clk_src.clkr, + [GCC_USB3_PRIM_PHY_COM_AUX_CLK] = &gcc_usb3_prim_phy_com_aux_clk.clkr, + [GCC_USB3_PRIM_PHY_PIPE_CLK] = &gcc_usb3_prim_phy_pipe_clk.clkr, + [GCC_USB3_PRIM_PHY_PIPE_CLK_SRC] = &gcc_usb3_prim_phy_pipe_clk_src.clkr, + [GCC_USB3_SEC_PHY_AUX_CLK] = &gcc_usb3_sec_phy_aux_clk.clkr, + [GCC_USB3_SEC_PHY_AUX_CLK_SRC] = &gcc_usb3_sec_phy_aux_clk_src.clkr, + [GCC_USB3_SEC_PHY_COM_AUX_CLK] = &gcc_usb3_sec_phy_com_aux_clk.clkr, + [GCC_USB3_SEC_PHY_PIPE_CLK] = &gcc_usb3_sec_phy_pipe_clk.clkr, + [GCC_USB3_SEC_PHY_PIPE_CLK_SRC] = &gcc_usb3_sec_phy_pipe_clk_src.clkr, + [GCC_VIDEO_AXI0_CLK] = &gcc_video_axi0_clk.clkr, + [GCC_VIDEO_MVP_THROTTLE_CORE_CLK] = + &gcc_video_mvp_throttle_core_clk.clkr, + [GCC_CFG_NOC_LPASS_CLK] = &gcc_cfg_noc_lpass_clk.clkr, + [GCC_MSS_GPLL0_MAIN_DIV_CLK_SRC] = &gcc_mss_gpll0_main_div_clk_src.clkr, + [GCC_MSS_CFG_AHB_CLK] = &gcc_mss_cfg_ahb_clk.clkr, + [GCC_MSS_OFFLINE_AXI_CLK] = &gcc_mss_offline_axi_clk.clkr, + [GCC_MSS_SNOC_AXI_CLK] = &gcc_mss_snoc_axi_clk.clkr, + [GCC_MSS_Q6_MEMNOC_AXI_CLK] = &gcc_mss_q6_memnoc_axi_clk.clkr, + [GCC_MSS_Q6SS_BOOT_CLK_SRC] = &gcc_mss_q6ss_boot_clk_src.clkr, + [GCC_AGGRE_NOC_PCIE_TBU_CLK] = &gcc_aggre_noc_pcie_tbu_clk.clkr, + [GCC_AGGRE_NOC_PCIE_CENTER_SF_AXI_CLK] = + &gcc_aggre_noc_pcie_center_sf_axi_clk.clkr, + [GCC_PCIE_CLKREF_EN] = &gcc_pcie_clkref_en.clkr, + [GCC_EDP_CLKREF_EN] = &gcc_edp_clkref_en.clkr, + [GCC_SEC_CTRL_CLK_SRC] = &gcc_sec_ctrl_clk_src.clkr, + [GCC_WPSS_AHB_CLK] = &gcc_wpss_ahb_clk.clkr, + [GCC_WPSS_AHB_BDG_MST_CLK] = &gcc_wpss_ahb_bdg_mst_clk.clkr, + [GCC_WPSS_RSCP_CLK] = &gcc_wpss_rscp_clk.clkr, +}; + +static struct gdsc *gcc_sc7280_gdscs[] = { + [GCC_PCIE_0_GDSC] = &gcc_pcie_0_gdsc, + [GCC_PCIE_1_GDSC] = &gcc_pcie_1_gdsc, + [GCC_UFS_PHY_GDSC] = &gcc_ufs_phy_gdsc, + [GCC_USB30_PRIM_GDSC] = &gcc_usb30_prim_gdsc, + [GCC_USB30_SEC_GDSC] = &gcc_usb30_sec_gdsc, + [HLOS1_VOTE_MMNOC_MMU_TBU_HF0_GDSC] = &hlos1_vote_mmnoc_mmu_tbu_hf0_gdsc, + [HLOS1_VOTE_MMNOC_MMU_TBU_HF1_GDSC] = &hlos1_vote_mmnoc_mmu_tbu_hf1_gdsc, + [HLOS1_VOTE_MMNOC_MMU_TBU_SF0_GDSC] = &hlos1_vote_mmnoc_mmu_tbu_sf0_gdsc, + [HLOS1_VOTE_TURING_MMU_TBU0_GDSC] = &hlos1_vote_turing_mmu_tbu0_gdsc, + [HLOS1_VOTE_TURING_MMU_TBU1_GDSC] = &hlos1_vote_turing_mmu_tbu1_gdsc, +}; + +static const struct qcom_reset_map gcc_sc7280_resets[] = { + [GCC_PCIE_0_BCR] = { 0x6b000 }, + [GCC_PCIE_0_PHY_BCR] = { 0x6c01c }, + [GCC_PCIE_1_BCR] = { 0x8d000 }, + [GCC_PCIE_1_PHY_BCR] = { 0x8e01c }, + [GCC_QUSB2PHY_PRIM_BCR] = { 0x12000 }, + [GCC_QUSB2PHY_SEC_BCR] = { 0x12004 }, + [GCC_SDCC1_BCR] = { 0x75000 }, + [GCC_SDCC2_BCR] = { 0x14000 }, + [GCC_SDCC4_BCR] = { 0x16000 }, + [GCC_UFS_PHY_BCR] = { 0x77000 }, + [GCC_USB30_PRIM_BCR] = { 0xf000 }, + [GCC_USB30_SEC_BCR] = { 0x9e000 }, + [GCC_USB3_DP_PHY_PRIM_BCR] = { 0x50008 }, + [GCC_USB3_PHY_PRIM_BCR] = { 0x50000 }, + [GCC_USB3PHY_PHY_PRIM_BCR] = { 0x50004 }, + [GCC_USB_PHY_CFG_AHB2PHY_BCR] = { 0x6a000 }, +}; + +static const struct clk_rcg_dfs_data gcc_dfs_clocks[] = { + DEFINE_RCG_DFS(gcc_qupv3_wrap0_s0_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap0_s1_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap0_s2_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap0_s3_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap0_s4_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap0_s5_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap0_s6_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap0_s7_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap1_s0_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap1_s1_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap1_s2_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap1_s3_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap1_s4_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap1_s5_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap1_s6_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap1_s7_clk_src), +}; + +static const struct regmap_config gcc_sc7280_regmap_config = { + .reg_bits = 32, + .reg_stride = 4, + .val_bits = 32, + .max_register = 0x9f128, + .fast_io = true, +}; + +static const struct qcom_cc_desc gcc_sc7280_desc = { + .config = &gcc_sc7280_regmap_config, + .clks = gcc_sc7280_clocks, + .num_clks = ARRAY_SIZE(gcc_sc7280_clocks), + .resets = gcc_sc7280_resets, + .num_resets = ARRAY_SIZE(gcc_sc7280_resets), + .gdscs = gcc_sc7280_gdscs, + .num_gdscs = ARRAY_SIZE(gcc_sc7280_gdscs), +}; + +static const struct of_device_id gcc_sc7280_match_table[] = { + { .compatible = "qcom,gcc-sc7280" }, + { } +}; +MODULE_DEVICE_TABLE(of, gcc_sc7280_match_table); + +static int gcc_sc7280_probe(struct platform_device *pdev) +{ + struct regmap *regmap; + int ret; + + regmap = qcom_cc_map(pdev, &gcc_sc7280_desc); + if (IS_ERR(regmap)) + return PTR_ERR(regmap); + + /* + * Keep the clocks always-ON + * GCC_CAMERA_AHB_CLK/XO_CLK, GCC_DISP_AHB_CLK/XO_CLK + * GCC_VIDEO_AHB_CLK/XO_CLK, GCC_GPU_CFG_AHB_CLK + */ + regmap_update_bits(regmap, 0x26004, BIT(0), BIT(0)); + regmap_update_bits(regmap, 0x26028, BIT(0), BIT(0)); + regmap_update_bits(regmap, 0x27004, BIT(0), BIT(0)); + regmap_update_bits(regmap, 0x2701C, BIT(0), BIT(0)); + regmap_update_bits(regmap, 0x28004, BIT(0), BIT(0)); + regmap_update_bits(regmap, 0x28014, BIT(0), BIT(0)); + regmap_update_bits(regmap, 0x71004, BIT(0), BIT(0)); + + ret = qcom_cc_register_rcg_dfs(regmap, gcc_dfs_clocks, + ARRAY_SIZE(gcc_dfs_clocks)); + if (ret) + return ret; + + return qcom_cc_really_probe(pdev, &gcc_sc7280_desc, regmap); +} + +static struct platform_driver gcc_sc7280_driver = { + .probe = gcc_sc7280_probe, + .driver = { + .name = "gcc-sc7280", + .of_match_table = gcc_sc7280_match_table, + }, +}; + +static int __init gcc_sc7280_init(void) +{ + return platform_driver_register(&gcc_sc7280_driver); +} +subsys_initcall(gcc_sc7280_init); + +static void __exit gcc_sc7280_exit(void) +{ + platform_driver_unregister(&gcc_sc7280_driver); +} +module_exit(gcc_sc7280_exit); + +MODULE_DESCRIPTION("QTI GCC SC7280 Driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/clk/qcom/gcc-sc8180x.c b/drivers/clk/qcom/gcc-sc8180x.c new file mode 100644 index 0000000000000000000000000000000000000000..90525ae1bb3a2168d13dae6208181a9eaf30a968 --- /dev/null +++ b/drivers/clk/qcom/gcc-sc8180x.c @@ -0,0 +1,4629 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2018-2019, The Linux Foundation. All rights reserved. + * Copyright (c) 2020-2021, Linaro Ltd. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "common.h" +#include "clk-alpha-pll.h" +#include "clk-branch.h" +#include "clk-pll.h" +#include "clk-rcg.h" +#include "clk-regmap.h" +#include "gdsc.h" +#include "reset.h" + +enum { + P_AUD_REF_CLK, + P_BI_TCXO, + P_GPLL0_OUT_EVEN, + P_GPLL0_OUT_MAIN, + P_GPLL1_OUT_MAIN, + P_GPLL2_OUT_MAIN, + P_GPLL4_OUT_MAIN, + P_GPLL5_OUT_MAIN, + P_GPLL7_OUT_MAIN, + P_GPLL9_OUT_MAIN, + P_SLEEP_CLK, +}; + +static struct pll_vco trion_vco[] = { + { 249600000, 2000000000, 0 }, +}; + +static struct clk_alpha_pll gpll0 = { + .offset = 0x0, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_TRION], + .vco_table = trion_vco, + .num_vco = ARRAY_SIZE(trion_vco), + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gpll0", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "bi_tcxo", + }, + .num_parents = 1, + .ops = &clk_alpha_pll_fixed_trion_ops, + }, + }, +}; + +static const struct clk_div_table post_div_table_trion_even[] = { + { 0x0, 1 }, + { 0x1, 2 }, + { 0x3, 4 }, + { 0x7, 8 }, + { } +}; + +static struct clk_alpha_pll_postdiv gpll0_out_even = { + .offset = 0x0, + .post_div_shift = 8, + .post_div_table = post_div_table_trion_even, + .num_post_div = ARRAY_SIZE(post_div_table_trion_even), + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_TRION], + .width = 4, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gpll0_out_even", + .parent_hws = (const struct clk_hw *[]){ &gpll0.clkr.hw }, + .num_parents = 1, + .ops = &clk_alpha_pll_postdiv_trion_ops, + }, +}; + +static struct clk_alpha_pll gpll1 = { + .offset = 0x1000, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_TRION], + .vco_table = trion_vco, + .num_vco = ARRAY_SIZE(trion_vco), + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(1), + .hw.init = &(struct clk_init_data){ + .name = "gpll1", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "bi_tcxo", + }, + .num_parents = 1, + .ops = &clk_alpha_pll_fixed_trion_ops, + }, + }, +}; + +static struct clk_alpha_pll gpll4 = { + .offset = 0x76000, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_TRION], + .vco_table = trion_vco, + .num_vco = ARRAY_SIZE(trion_vco), + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(4), + .hw.init = &(struct clk_init_data){ + .name = "gpll4", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "bi_tcxo", + }, + .num_parents = 1, + .ops = &clk_alpha_pll_fixed_trion_ops, + }, + }, +}; + +static struct clk_alpha_pll gpll7 = { + .offset = 0x1a000, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_TRION], + .vco_table = trion_vco, + .num_vco = ARRAY_SIZE(trion_vco), + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(7), + .hw.init = &(struct clk_init_data){ + .name = "gpll7", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "bi_tcxo", + }, + .num_parents = 1, + .ops = &clk_alpha_pll_fixed_trion_ops, + }, + }, +}; + +static const struct parent_map gcc_parent_map_0[] = { + { P_BI_TCXO, 0 }, + { P_GPLL0_OUT_MAIN, 1 }, + { P_GPLL0_OUT_EVEN, 6 }, +}; + +static const struct clk_parent_data gcc_parents_0[] = { + { .fw_name = "bi_tcxo" }, + { .hw = &gpll0.clkr.hw }, + { .hw = &gpll0_out_even.clkr.hw }, +}; + +static const struct parent_map gcc_parent_map_1[] = { + { P_BI_TCXO, 0 }, + { P_GPLL0_OUT_MAIN, 1 }, + { P_SLEEP_CLK, 5 }, + { P_GPLL0_OUT_EVEN, 6 }, +}; + +static const struct clk_parent_data gcc_parents_1[] = { + { .fw_name = "bi_tcxo", }, + { .hw = &gpll0.clkr.hw }, + { .fw_name = "sleep_clk", }, + { .hw = &gpll0_out_even.clkr.hw }, +}; + +static const struct parent_map gcc_parent_map_2[] = { + { P_BI_TCXO, 0 }, + { P_SLEEP_CLK, 5 }, +}; + +static const struct clk_parent_data gcc_parents_2[] = { + { .fw_name = "bi_tcxo", }, + { .fw_name = "sleep_clk", }, +}; + +static const struct parent_map gcc_parent_map_3[] = { + { P_BI_TCXO, 0 }, + { P_GPLL0_OUT_MAIN, 1 }, + { P_GPLL2_OUT_MAIN, 2 }, + { P_GPLL5_OUT_MAIN, 3 }, + { P_GPLL1_OUT_MAIN, 4 }, + { P_GPLL4_OUT_MAIN, 5 }, + { P_GPLL0_OUT_EVEN, 6 }, +}; + +static const struct clk_parent_data gcc_parents_3[] = { + { .fw_name = "bi_tcxo", }, + { .hw = &gpll0.clkr.hw }, + { .name = "gpll2" }, + { .name = "gpll5" }, + { .hw = &gpll1.clkr.hw }, + { .hw = &gpll4.clkr.hw }, + { .hw = &gpll0_out_even.clkr.hw }, +}; + +static const struct parent_map gcc_parent_map_4[] = { + { P_BI_TCXO, 0 }, +}; + +static const struct clk_parent_data gcc_parents_4[] = { + { .fw_name = "bi_tcxo", }, +}; + +static const struct parent_map gcc_parent_map_5[] = { + { P_BI_TCXO, 0 }, + { P_GPLL0_OUT_MAIN, 1 }, +}; + +static const struct clk_parent_data gcc_parents_5[] = { + { .fw_name = "bi_tcxo", }, + { .hw = &gpll0.clkr.hw }, +}; + +static const struct parent_map gcc_parent_map_6[] = { + { P_BI_TCXO, 0 }, + { P_GPLL0_OUT_MAIN, 1 }, + { P_GPLL7_OUT_MAIN, 3 }, + { P_GPLL0_OUT_EVEN, 6 }, +}; + +static const struct clk_parent_data gcc_parents_6[] = { + { .fw_name = "bi_tcxo", }, + { .hw = &gpll0.clkr.hw }, + { .hw = &gpll7.clkr.hw }, + { .hw = &gpll0_out_even.clkr.hw }, +}; + +static const struct parent_map gcc_parent_map_7[] = { + { P_BI_TCXO, 0 }, + { P_GPLL0_OUT_MAIN, 1 }, + { P_GPLL9_OUT_MAIN, 2 }, + { P_GPLL4_OUT_MAIN, 5 }, + { P_GPLL0_OUT_EVEN, 6 }, +}; + +static const struct clk_parent_data gcc_parents_7[] = { + { .fw_name = "bi_tcxo", }, + { .hw = &gpll0.clkr.hw }, + { .name = "gppl9" }, + { .hw = &gpll4.clkr.hw }, + { .hw = &gpll0_out_even.clkr.hw }, +}; + +static const struct parent_map gcc_parent_map_8[] = { + { P_BI_TCXO, 0 }, + { P_GPLL0_OUT_MAIN, 1 }, + { P_AUD_REF_CLK, 2 }, + { P_GPLL0_OUT_EVEN, 6 }, +}; + +static const struct clk_parent_data gcc_parents_8[] = { + { .fw_name = "bi_tcxo", }, + { .hw = &gpll0.clkr.hw }, + { .name = "aud_ref_clk" }, + { .hw = &gpll0_out_even.clkr.hw }, +}; + +static const struct freq_tbl ftbl_gcc_cpuss_ahb_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + F(50000000, P_GPLL0_OUT_MAIN, 12, 0, 0), + F(100000000, P_GPLL0_OUT_MAIN, 6, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_cpuss_ahb_clk_src = { + .cmd_rcgr = 0x48014, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_cpuss_ahb_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_cpuss_ahb_clk_src", + .parent_data = gcc_parents_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_emac_ptp_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + F(50000000, P_GPLL0_OUT_EVEN, 6, 0, 0), + F(125000000, P_GPLL7_OUT_MAIN, 4, 0, 0), + F(250000000, P_GPLL7_OUT_MAIN, 2, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_emac_ptp_clk_src = { + .cmd_rcgr = 0x6038, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_6, + .freq_tbl = ftbl_gcc_emac_ptp_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_emac_ptp_clk_src", + .parent_data = gcc_parents_6, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_emac_rgmii_clk_src[] = { + F(2500000, P_BI_TCXO, 1, 25, 192), + F(5000000, P_BI_TCXO, 1, 25, 96), + F(19200000, P_BI_TCXO, 1, 0, 0), + F(25000000, P_GPLL0_OUT_EVEN, 12, 0, 0), + F(50000000, P_GPLL0_OUT_EVEN, 6, 0, 0), + F(125000000, P_GPLL7_OUT_MAIN, 4, 0, 0), + F(250000000, P_GPLL7_OUT_MAIN, 2, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_emac_rgmii_clk_src = { + .cmd_rcgr = 0x601c, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_6, + .freq_tbl = ftbl_gcc_emac_rgmii_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_emac_rgmii_clk_src", + .parent_data = gcc_parents_6, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_gp1_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + F(25000000, P_GPLL0_OUT_EVEN, 12, 0, 0), + F(50000000, P_GPLL0_OUT_EVEN, 6, 0, 0), + F(100000000, P_GPLL0_OUT_MAIN, 6, 0, 0), + F(200000000, P_GPLL0_OUT_MAIN, 3, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_gp1_clk_src = { + .cmd_rcgr = 0x64004, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_1, + .freq_tbl = ftbl_gcc_gp1_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_gp1_clk_src", + .parent_data = gcc_parents_1, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_gp2_clk_src = { + .cmd_rcgr = 0x65004, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_1, + .freq_tbl = ftbl_gcc_gp1_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_gp2_clk_src", + .parent_data = gcc_parents_1, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_gp3_clk_src = { + .cmd_rcgr = 0x66004, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_1, + .freq_tbl = ftbl_gcc_gp1_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_gp3_clk_src", + .parent_data = gcc_parents_1, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_gp4_clk_src = { + .cmd_rcgr = 0xbe004, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_1, + .freq_tbl = ftbl_gcc_gp1_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_gp4_clk_src", + .parent_data = gcc_parents_1, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_gp5_clk_src = { + .cmd_rcgr = 0xbf004, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_1, + .freq_tbl = ftbl_gcc_gp1_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_gp5_clk_src", + .parent_data = gcc_parents_1, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_npu_axi_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + F(60000000, P_GPLL0_OUT_EVEN, 5, 0, 0), + F(150000000, P_GPLL0_OUT_EVEN, 2, 0, 0), + F(200000000, P_GPLL0_OUT_MAIN, 3, 0, 0), + F(300000000, P_GPLL0_OUT_MAIN, 2, 0, 0), + F(403000000, P_GPLL4_OUT_MAIN, 2, 0, 0), + F(533000000, P_GPLL1_OUT_MAIN, 2, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_npu_axi_clk_src = { + .cmd_rcgr = 0x4d014, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_3, + .freq_tbl = ftbl_gcc_npu_axi_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_npu_axi_clk_src", + .parent_data = gcc_parents_3, + .num_parents = 7, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_pcie_0_aux_clk_src[] = { + F(9600000, P_BI_TCXO, 2, 0, 0), + F(19200000, P_BI_TCXO, 1, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_pcie_0_aux_clk_src = { + .cmd_rcgr = 0x6b02c, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_2, + .freq_tbl = ftbl_gcc_pcie_0_aux_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_0_aux_clk_src", + .parent_data = gcc_parents_2, + .num_parents = 2, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_pcie_1_aux_clk_src = { + .cmd_rcgr = 0x8d02c, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_2, + .freq_tbl = ftbl_gcc_pcie_0_aux_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_1_aux_clk_src", + .parent_data = gcc_parents_2, + .num_parents = 2, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_pcie_2_aux_clk_src = { + .cmd_rcgr = 0x9d02c, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_2, + .freq_tbl = ftbl_gcc_pcie_0_aux_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_2_aux_clk_src", + .parent_data = gcc_parents_2, + .num_parents = 2, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_pcie_3_aux_clk_src = { + .cmd_rcgr = 0xa302c, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_2, + .freq_tbl = ftbl_gcc_pcie_0_aux_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_3_aux_clk_src", + .parent_data = gcc_parents_2, + .num_parents = 2, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_pcie_phy_refgen_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + F(100000000, P_GPLL0_OUT_MAIN, 6, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_pcie_phy_refgen_clk_src = { + .cmd_rcgr = 0x6f014, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_pcie_phy_refgen_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_phy_refgen_clk_src", + .parent_data = gcc_parents_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_pdm2_clk_src[] = { + F(9600000, P_BI_TCXO, 2, 0, 0), + F(19200000, P_BI_TCXO, 1, 0, 0), + F(60000000, P_GPLL0_OUT_MAIN, 10, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_pdm2_clk_src = { + .cmd_rcgr = 0x33010, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_pdm2_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_pdm2_clk_src", + .parent_data = gcc_parents_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_qspi_1_core_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + F(75000000, P_GPLL0_OUT_EVEN, 4, 0, 0), + F(150000000, P_GPLL0_OUT_MAIN, 4, 0, 0), + F(300000000, P_GPLL0_OUT_MAIN, 2, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_qspi_1_core_clk_src = { + .cmd_rcgr = 0x4a00c, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qspi_1_core_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_qspi_1_core_clk_src", + .parent_data = gcc_parents_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_qspi_core_clk_src = { + .cmd_rcgr = 0x4b008, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qspi_1_core_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_qspi_core_clk_src", + .parent_data = gcc_parents_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_qupv3_wrap0_s0_clk_src[] = { + F(7372800, P_GPLL0_OUT_EVEN, 1, 384, 15625), + F(14745600, P_GPLL0_OUT_EVEN, 1, 768, 15625), + F(19200000, P_BI_TCXO, 1, 0, 0), + F(29491200, P_GPLL0_OUT_EVEN, 1, 1536, 15625), + F(32000000, P_GPLL0_OUT_EVEN, 1, 8, 75), + F(48000000, P_GPLL0_OUT_EVEN, 1, 4, 25), + F(50000000, P_GPLL0_OUT_EVEN, 6, 0, 0), + F(64000000, P_GPLL0_OUT_EVEN, 1, 16, 75), + F(75000000, P_GPLL0_OUT_EVEN, 4, 0, 0), + F(80000000, P_GPLL0_OUT_EVEN, 1, 4, 15), + F(96000000, P_GPLL0_OUT_EVEN, 1, 8, 25), + F(100000000, P_GPLL0_OUT_MAIN, 6, 0, 0), + F(102400000, P_GPLL0_OUT_EVEN, 1, 128, 375), + F(112000000, P_GPLL0_OUT_EVEN, 1, 28, 75), + F(117964800, P_GPLL0_OUT_EVEN, 1, 6144, 15625), + F(120000000, P_GPLL0_OUT_EVEN, 2.5, 0, 0), + F(128000000, P_GPLL0_OUT_MAIN, 1, 16, 75), + { } +}; + +static struct clk_rcg2 gcc_qupv3_wrap0_s0_clk_src = { + .cmd_rcgr = 0x17148, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_s0_clk_src", + .parent_data = gcc_parents_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_qupv3_wrap0_s1_clk_src = { + .cmd_rcgr = 0x17278, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_s1_clk_src", + .parent_data = gcc_parents_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_qupv3_wrap0_s2_clk_src = { + .cmd_rcgr = 0x173a8, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_s2_clk_src", + .parent_data = gcc_parents_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_qupv3_wrap0_s3_clk_src = { + .cmd_rcgr = 0x174d8, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_s3_clk_src", + .parent_data = gcc_parents_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_qupv3_wrap0_s4_clk_src = { + .cmd_rcgr = 0x17608, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_s4_clk_src", + .parent_data = gcc_parents_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_qupv3_wrap0_s5_clk_src = { + .cmd_rcgr = 0x17738, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_s5_clk_src", + .parent_data = gcc_parents_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_qupv3_wrap0_s6_clk_src = { + .cmd_rcgr = 0x17868, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_s6_clk_src", + .parent_data = gcc_parents_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_qupv3_wrap0_s7_clk_src = { + .cmd_rcgr = 0x17998, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_s7_clk_src", + .parent_data = gcc_parents_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_qupv3_wrap1_s0_clk_src = { + .cmd_rcgr = 0x18148, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap1_s0_clk_src", + .parent_data = gcc_parents_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_qupv3_wrap1_s1_clk_src = { + .cmd_rcgr = 0x18278, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap1_s1_clk_src", + .parent_data = gcc_parents_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_qupv3_wrap1_s2_clk_src = { + .cmd_rcgr = 0x183a8, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap1_s2_clk_src", + .parent_data = gcc_parents_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_qupv3_wrap1_s3_clk_src = { + .cmd_rcgr = 0x184d8, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap1_s3_clk_src", + .parent_data = gcc_parents_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_qupv3_wrap1_s4_clk_src = { + .cmd_rcgr = 0x18608, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap1_s4_clk_src", + .parent_data = gcc_parents_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_qupv3_wrap1_s5_clk_src = { + .cmd_rcgr = 0x18738, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap1_s5_clk_src", + .parent_data = gcc_parents_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_qupv3_wrap2_s0_clk_src = { + .cmd_rcgr = 0x1e148, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap2_s0_clk_src", + .parent_data = gcc_parents_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_qupv3_wrap2_s1_clk_src = { + .cmd_rcgr = 0x1e278, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap2_s1_clk_src", + .parent_data = gcc_parents_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_qupv3_wrap2_s2_clk_src = { + .cmd_rcgr = 0x1e3a8, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap2_s2_clk_src", + .parent_data = gcc_parents_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_qupv3_wrap2_s3_clk_src = { + .cmd_rcgr = 0x1e4d8, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap2_s3_clk_src", + .parent_data = gcc_parents_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_qupv3_wrap2_s4_clk_src = { + .cmd_rcgr = 0x1e608, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap2_s4_clk_src", + .parent_data = gcc_parents_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_qupv3_wrap2_s5_clk_src = { + .cmd_rcgr = 0x1e738, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap2_s5_clk_src", + .parent_data = gcc_parents_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_sdcc2_apps_clk_src[] = { + F(400000, P_BI_TCXO, 12, 1, 4), + F(9600000, P_BI_TCXO, 2, 0, 0), + F(19200000, P_BI_TCXO, 1, 0, 0), + F(25000000, P_GPLL0_OUT_MAIN, 12, 1, 2), + F(50000000, P_GPLL0_OUT_MAIN, 12, 0, 0), + F(100000000, P_GPLL0_OUT_MAIN, 6, 0, 0), + F(200000000, P_GPLL0_OUT_MAIN, 3, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_sdcc2_apps_clk_src = { + .cmd_rcgr = 0x1400c, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_7, + .freq_tbl = ftbl_gcc_sdcc2_apps_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc2_apps_clk_src", + .parent_data = gcc_parents_7, + .num_parents = 5, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_floor_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_sdcc4_apps_clk_src[] = { + F(400000, P_BI_TCXO, 12, 1, 4), + F(9600000, P_BI_TCXO, 2, 0, 0), + F(19200000, P_BI_TCXO, 1, 0, 0), + F(37500000, P_GPLL0_OUT_MAIN, 16, 0, 0), + F(50000000, P_GPLL0_OUT_MAIN, 12, 0, 0), + F(75000000, P_GPLL0_OUT_MAIN, 8, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_sdcc4_apps_clk_src = { + .cmd_rcgr = 0x1600c, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_5, + .freq_tbl = ftbl_gcc_sdcc4_apps_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc4_apps_clk_src", + .parent_data = gcc_parents_5, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_floor_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_tsif_ref_clk_src[] = { + F(105495, P_BI_TCXO, 2, 1, 91), + { } +}; + +static struct clk_rcg2 gcc_tsif_ref_clk_src = { + .cmd_rcgr = 0x36010, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_8, + .freq_tbl = ftbl_gcc_tsif_ref_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_tsif_ref_clk_src", + .parent_data = gcc_parents_8, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_ufs_card_2_axi_clk_src[] = { + F(37500000, P_GPLL0_OUT_EVEN, 8, 0, 0), + F(75000000, P_GPLL0_OUT_EVEN, 4, 0, 0), + F(150000000, P_GPLL0_OUT_MAIN, 4, 0, 0), + F(300000000, P_GPLL0_OUT_MAIN, 2, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_ufs_card_2_axi_clk_src = { + .cmd_rcgr = 0xa2020, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_ufs_card_2_axi_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_2_axi_clk_src", + .parent_data = gcc_parents_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_ufs_card_2_ice_core_clk_src = { + .cmd_rcgr = 0xa2060, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_ufs_card_2_axi_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_2_ice_core_clk_src", + .parent_data = gcc_parents_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_ufs_card_2_phy_aux_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_ufs_card_2_phy_aux_clk_src = { + .cmd_rcgr = 0xa2094, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_4, + .freq_tbl = ftbl_gcc_ufs_card_2_phy_aux_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_2_phy_aux_clk_src", + .parent_data = gcc_parents_4, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_ufs_card_2_unipro_core_clk_src = { + .cmd_rcgr = 0xa2078, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_ufs_card_2_axi_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_2_unipro_core_clk_src", + .parent_data = gcc_parents_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_ufs_card_axi_clk_src[] = { + F(25000000, P_GPLL0_OUT_EVEN, 12, 0, 0), + F(50000000, P_GPLL0_OUT_EVEN, 6, 0, 0), + F(100000000, P_GPLL0_OUT_MAIN, 6, 0, 0), + F(200000000, P_GPLL0_OUT_MAIN, 3, 0, 0), + F(240000000, P_GPLL0_OUT_MAIN, 2.5, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_ufs_card_axi_clk_src = { + .cmd_rcgr = 0x75020, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_ufs_card_axi_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_axi_clk_src", + .parent_data = gcc_parents_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_ufs_card_ice_core_clk_src[] = { + F(75000000, P_GPLL0_OUT_EVEN, 4, 0, 0), + F(150000000, P_GPLL0_OUT_MAIN, 4, 0, 0), + F(300000000, P_GPLL0_OUT_MAIN, 2, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_ufs_card_ice_core_clk_src = { + .cmd_rcgr = 0x75060, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_ufs_card_ice_core_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_ice_core_clk_src", + .parent_data = gcc_parents_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_ufs_card_phy_aux_clk_src = { + .cmd_rcgr = 0x75094, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_4, + .freq_tbl = ftbl_gcc_ufs_card_2_phy_aux_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_phy_aux_clk_src", + .parent_data = gcc_parents_4, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_ufs_card_unipro_core_clk_src[] = { + F(37500000, P_GPLL0_OUT_EVEN, 8, 0, 0), + F(75000000, P_GPLL0_OUT_MAIN, 8, 0, 0), + F(150000000, P_GPLL0_OUT_MAIN, 4, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_ufs_card_unipro_core_clk_src = { + .cmd_rcgr = 0x75078, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_ufs_card_unipro_core_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_unipro_core_clk_src", + .parent_data = gcc_parents_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_ufs_phy_axi_clk_src[] = { + F(25000000, P_GPLL0_OUT_EVEN, 12, 0, 0), + F(37500000, P_GPLL0_OUT_EVEN, 8, 0, 0), + F(75000000, P_GPLL0_OUT_EVEN, 4, 0, 0), + F(150000000, P_GPLL0_OUT_MAIN, 4, 0, 0), + F(300000000, P_GPLL0_OUT_MAIN, 2, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_ufs_phy_axi_clk_src = { + .cmd_rcgr = 0x77020, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_ufs_phy_axi_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_axi_clk_src", + .parent_data = gcc_parents_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_ufs_phy_ice_core_clk_src = { + .cmd_rcgr = 0x77060, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_ufs_card_2_axi_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_ice_core_clk_src", + .parent_data = gcc_parents_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_ufs_phy_phy_aux_clk_src = { + .cmd_rcgr = 0x77094, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_4, + .freq_tbl = ftbl_gcc_pcie_0_aux_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_phy_aux_clk_src", + .parent_data = gcc_parents_4, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_ufs_phy_unipro_core_clk_src = { + .cmd_rcgr = 0x77078, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_ufs_card_2_axi_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_unipro_core_clk_src", + .parent_data = gcc_parents_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_usb30_mp_master_clk_src[] = { + F(33333333, P_GPLL0_OUT_EVEN, 9, 0, 0), + F(66666667, P_GPLL0_OUT_EVEN, 4.5, 0, 0), + F(133333333, P_GPLL0_OUT_MAIN, 4.5, 0, 0), + F(200000000, P_GPLL0_OUT_MAIN, 3, 0, 0), + F(240000000, P_GPLL0_OUT_MAIN, 2.5, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_usb30_mp_master_clk_src = { + .cmd_rcgr = 0xa601c, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_usb30_mp_master_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_mp_master_clk_src", + .parent_data = gcc_parents_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_usb30_mp_mock_utmi_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + F(20000000, P_GPLL0_OUT_EVEN, 15, 0, 0), + F(40000000, P_GPLL0_OUT_EVEN, 7.5, 0, 0), + F(60000000, P_GPLL0_OUT_MAIN, 10, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_usb30_mp_mock_utmi_clk_src = { + .cmd_rcgr = 0xa6034, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_usb30_mp_mock_utmi_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_mp_mock_utmi_clk_src", + .parent_data = gcc_parents_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_usb30_prim_master_clk_src = { + .cmd_rcgr = 0xf01c, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_usb30_mp_master_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_prim_master_clk_src", + .parent_data = gcc_parents_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_usb30_prim_mock_utmi_clk_src = { + .cmd_rcgr = 0xf034, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_usb30_mp_mock_utmi_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_prim_mock_utmi_clk_src", + .parent_data = gcc_parents_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_usb30_sec_master_clk_src = { + .cmd_rcgr = 0x1001c, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_usb30_mp_master_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_sec_master_clk_src", + .parent_data = gcc_parents_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_usb30_sec_mock_utmi_clk_src = { + .cmd_rcgr = 0x10034, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_usb30_mp_mock_utmi_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_sec_mock_utmi_clk_src", + .parent_data = gcc_parents_0, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_usb3_mp_phy_aux_clk_src = { + .cmd_rcgr = 0xa6068, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_2, + .freq_tbl = ftbl_gcc_ufs_card_2_phy_aux_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_mp_phy_aux_clk_src", + .parent_data = gcc_parents_2, + .num_parents = 2, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_usb3_prim_phy_aux_clk_src = { + .cmd_rcgr = 0xf060, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_2, + .freq_tbl = ftbl_gcc_ufs_card_2_phy_aux_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_prim_phy_aux_clk_src", + .parent_data = gcc_parents_2, + .num_parents = 2, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_usb3_sec_phy_aux_clk_src = { + .cmd_rcgr = 0x10060, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_2, + .freq_tbl = ftbl_gcc_ufs_card_2_phy_aux_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_sec_phy_aux_clk_src", + .parent_data = gcc_parents_2, + .num_parents = 2, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_branch gcc_aggre_noc_pcie_tbu_clk = { + .halt_reg = 0x90018, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x90018, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_aggre_noc_pcie_tbu_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_aggre_ufs_card_axi_clk = { + .halt_reg = 0x750c0, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0x750c0, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x750c0, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_aggre_ufs_card_axi_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_ufs_card_axi_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_aggre_ufs_card_axi_hw_ctl_clk = { + .halt_reg = 0x750c0, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0x750c0, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x750c0, + .enable_mask = BIT(1), + .hw.init = &(struct clk_init_data){ + .name = "gcc_aggre_ufs_card_axi_hw_ctl_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_aggre_ufs_card_axi_clk.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch_simple_ops, + }, + }, +}; + +static struct clk_branch gcc_aggre_ufs_phy_axi_clk = { + .halt_reg = 0x770c0, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0x770c0, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x770c0, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_aggre_ufs_phy_axi_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_ufs_phy_axi_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_aggre_ufs_phy_axi_hw_ctl_clk = { + .halt_reg = 0x770c0, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0x770c0, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x770c0, + .enable_mask = BIT(1), + .hw.init = &(struct clk_init_data){ + .name = "gcc_aggre_ufs_phy_axi_hw_ctl_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_aggre_ufs_phy_axi_clk.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch_simple_ops, + }, + }, +}; + +static struct clk_branch gcc_aggre_usb3_mp_axi_clk = { + .halt_reg = 0xa6084, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xa6084, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_aggre_usb3_mp_axi_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_usb30_mp_master_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_aggre_usb3_prim_axi_clk = { + .halt_reg = 0xf07c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xf07c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_aggre_usb3_prim_axi_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_usb30_prim_master_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_aggre_usb3_sec_axi_clk = { + .halt_reg = 0x1007c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x1007c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_aggre_usb3_sec_axi_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_usb30_sec_master_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_boot_rom_ahb_clk = { + .halt_reg = 0x38004, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x38004, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52004, + .enable_mask = BIT(10), + .hw.init = &(struct clk_init_data){ + .name = "gcc_boot_rom_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camera_hf_axi_clk = { + .halt_reg = 0xb030, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xb030, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_camera_hf_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_camera_sf_axi_clk = { + .halt_reg = 0xb034, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xb034, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_camera_sf_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_cfg_noc_usb3_mp_axi_clk = { + .halt_reg = 0xa609c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xa609c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_cfg_noc_usb3_mp_axi_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_usb30_mp_master_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_cfg_noc_usb3_prim_axi_clk = { + .halt_reg = 0xf078, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xf078, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_cfg_noc_usb3_prim_axi_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_usb30_prim_master_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_cfg_noc_usb3_sec_axi_clk = { + .halt_reg = 0x10078, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x10078, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_cfg_noc_usb3_sec_axi_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_usb30_sec_master_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +/* For CPUSS functionality the AHB clock needs to be left enabled */ +static struct clk_branch gcc_cpuss_ahb_clk = { + .halt_reg = 0x48000, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52004, + .enable_mask = BIT(21), + .hw.init = &(struct clk_init_data){ + .name = "gcc_cpuss_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_cpuss_ahb_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_IS_CRITICAL | CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_cpuss_rbcpr_clk = { + .halt_reg = 0x48008, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x48008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_cpuss_rbcpr_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ddrss_gpu_axi_clk = { + .halt_reg = 0x71154, + .halt_check = BRANCH_VOTED, + .clkr = { + .enable_reg = 0x71154, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ddrss_gpu_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_disp_hf_axi_clk = { + .halt_reg = 0xb038, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xb038, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_disp_hf_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_disp_sf_axi_clk = { + .halt_reg = 0xb03c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xb03c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_disp_sf_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_emac_axi_clk = { + .halt_reg = 0x6010, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x6010, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_emac_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_emac_ptp_clk = { + .halt_reg = 0x6034, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x6034, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_emac_ptp_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_emac_ptp_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_emac_rgmii_clk = { + .halt_reg = 0x6018, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x6018, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_emac_rgmii_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_emac_rgmii_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_emac_slv_ahb_clk = { + .halt_reg = 0x6014, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0x6014, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x6014, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_emac_slv_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_gp1_clk = { + .halt_reg = 0x64000, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x64000, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gp1_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_gp1_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_gp2_clk = { + .halt_reg = 0x65000, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x65000, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gp2_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_gp2_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_gp3_clk = { + .halt_reg = 0x66000, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x66000, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gp3_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_gp3_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_gp4_clk = { + .halt_reg = 0xbe000, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xbe000, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gp4_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_gp4_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_gp5_clk = { + .halt_reg = 0xbf000, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xbf000, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gp5_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_gp5_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_gpu_gpll0_clk_src = { + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x52004, + .enable_mask = BIT(15), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gpu_gpll0_clk_src", + .parent_hws = (const struct clk_hw *[]){ &gpll0.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_gpu_gpll0_div_clk_src = { + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x52004, + .enable_mask = BIT(16), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gpu_gpll0_div_clk_src", + .parent_hws = (const struct clk_hw *[]){ + &gpll0_out_even.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_gpu_memnoc_gfx_clk = { + .halt_reg = 0x7100c, + .halt_check = BRANCH_VOTED, + .clkr = { + .enable_reg = 0x7100c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gpu_memnoc_gfx_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_gpu_snoc_dvm_gfx_clk = { + .halt_reg = 0x71018, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x71018, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gpu_snoc_dvm_gfx_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_npu_at_clk = { + .halt_reg = 0x4d010, + .halt_check = BRANCH_VOTED, + .clkr = { + .enable_reg = 0x4d010, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_npu_at_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_npu_axi_clk = { + .halt_reg = 0x4d008, + .halt_check = BRANCH_VOTED, + .clkr = { + .enable_reg = 0x4d008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_npu_axi_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_npu_axi_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_npu_gpll0_clk_src = { + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x52004, + .enable_mask = BIT(18), + .hw.init = &(struct clk_init_data){ + .name = "gcc_npu_gpll0_clk_src", + .parent_hws = (const struct clk_hw *[]){ &gpll0.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_npu_gpll0_div_clk_src = { + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x52004, + .enable_mask = BIT(19), + .hw.init = &(struct clk_init_data){ + .name = "gcc_npu_gpll0_div_clk_src", + .parent_hws = (const struct clk_hw *[]){ + &gpll0_out_even.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_npu_trig_clk = { + .halt_reg = 0x4d00c, + .halt_check = BRANCH_VOTED, + .clkr = { + .enable_reg = 0x4d00c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_npu_trig_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie0_phy_refgen_clk = { + .halt_reg = 0x6f02c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x6f02c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie0_phy_refgen_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_pcie_phy_refgen_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie1_phy_refgen_clk = { + .halt_reg = 0x6f030, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x6f030, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie1_phy_refgen_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_pcie_phy_refgen_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie2_phy_refgen_clk = { + .halt_reg = 0x6f034, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x6f034, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie2_phy_refgen_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_pcie_phy_refgen_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie3_phy_refgen_clk = { + .halt_reg = 0x6f038, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x6f038, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie3_phy_refgen_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_pcie_phy_refgen_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_0_aux_clk = { + .halt_reg = 0x6b020, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x5200c, + .enable_mask = BIT(3), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_0_aux_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_pcie_0_aux_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_0_cfg_ahb_clk = { + .halt_reg = 0x6b01c, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x6b01c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x5200c, + .enable_mask = BIT(2), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_0_cfg_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_0_clkref_clk = { + .halt_reg = 0x8c00c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x8c00c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_0_clkref_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_0_mstr_axi_clk = { + .halt_reg = 0x6b018, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x5200c, + .enable_mask = BIT(1), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_0_mstr_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_0_pipe_clk = { + .halt_reg = 0x6b024, + .halt_check = BRANCH_HALT_SKIP, + .clkr = { + .enable_reg = 0x5200c, + .enable_mask = BIT(4), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_0_pipe_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_0_slv_axi_clk = { + .halt_reg = 0x6b014, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x6b014, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x5200c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_0_slv_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_0_slv_q2a_axi_clk = { + .halt_reg = 0x6b010, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x5200c, + .enable_mask = BIT(5), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_0_slv_q2a_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_1_aux_clk = { + .halt_reg = 0x8d020, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52004, + .enable_mask = BIT(29), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_1_aux_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_pcie_1_aux_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_1_cfg_ahb_clk = { + .halt_reg = 0x8d01c, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x8d01c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52004, + .enable_mask = BIT(28), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_1_cfg_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_1_clkref_clk = { + .halt_reg = 0x8c02c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x8c02c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_1_clkref_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_1_mstr_axi_clk = { + .halt_reg = 0x8d018, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52004, + .enable_mask = BIT(27), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_1_mstr_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_1_pipe_clk = { + .halt_reg = 0x8d024, + .halt_check = BRANCH_HALT_SKIP, + .clkr = { + .enable_reg = 0x52004, + .enable_mask = BIT(30), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_1_pipe_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_1_slv_axi_clk = { + .halt_reg = 0x8d014, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x8d014, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52004, + .enable_mask = BIT(26), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_1_slv_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_1_slv_q2a_axi_clk = { + .halt_reg = 0x8d010, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52004, + .enable_mask = BIT(25), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_1_slv_q2a_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_2_aux_clk = { + .halt_reg = 0x9d020, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52014, + .enable_mask = BIT(14), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_2_aux_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_pcie_2_aux_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_2_cfg_ahb_clk = { + .halt_reg = 0x9d01c, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x9d01c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52014, + .enable_mask = BIT(13), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_2_cfg_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_2_clkref_clk = { + .halt_reg = 0x8c014, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x8c014, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_2_clkref_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_2_mstr_axi_clk = { + .halt_reg = 0x9d018, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52014, + .enable_mask = BIT(12), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_2_mstr_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_2_pipe_clk = { + .halt_reg = 0x9d024, + .halt_check = BRANCH_HALT_SKIP, + .clkr = { + .enable_reg = 0x52014, + .enable_mask = BIT(15), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_2_pipe_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_2_slv_axi_clk = { + .halt_reg = 0x9d014, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x9d014, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52014, + .enable_mask = BIT(11), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_2_slv_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_2_slv_q2a_axi_clk = { + .halt_reg = 0x9d010, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52014, + .enable_mask = BIT(10), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_2_slv_q2a_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_3_aux_clk = { + .halt_reg = 0xa3020, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52014, + .enable_mask = BIT(20), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_3_aux_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_pcie_3_aux_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_3_cfg_ahb_clk = { + .halt_reg = 0xa301c, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0xa301c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52014, + .enable_mask = BIT(19), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_3_cfg_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_3_clkref_clk = { + .halt_reg = 0x8c018, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x8c018, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_3_clkref_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_3_mstr_axi_clk = { + .halt_reg = 0xa3018, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52014, + .enable_mask = BIT(18), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_3_mstr_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_3_pipe_clk = { + .halt_reg = 0xa3024, + .halt_check = BRANCH_HALT_SKIP, + .clkr = { + .enable_reg = 0x52014, + .enable_mask = BIT(21), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_3_pipe_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_3_slv_axi_clk = { + .halt_reg = 0xa3014, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0xa3014, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52014, + .enable_mask = BIT(17), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_3_slv_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_3_slv_q2a_axi_clk = { + .halt_reg = 0xa3010, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52014, + .enable_mask = BIT(16), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_3_slv_q2a_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_phy_aux_clk = { + .halt_reg = 0x6f004, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x6f004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_phy_aux_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_pcie_0_aux_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pdm2_clk = { + .halt_reg = 0x3300c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x3300c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pdm2_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_pdm2_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pdm_ahb_clk = { + .halt_reg = 0x33004, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0x33004, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x33004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pdm_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pdm_xo4_clk = { + .halt_reg = 0x33008, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x33008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pdm_xo4_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_prng_ahb_clk = { + .halt_reg = 0x34004, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52004, + .enable_mask = BIT(13), + .hw.init = &(struct clk_init_data){ + .name = "gcc_prng_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qmip_camera_nrt_ahb_clk = { + .halt_reg = 0xb018, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0xb018, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0xb018, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qmip_camera_nrt_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qmip_camera_rt_ahb_clk = { + .halt_reg = 0xb01c, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0xb01c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0xb01c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qmip_camera_rt_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qmip_disp_ahb_clk = { + .halt_reg = 0xb020, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0xb020, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0xb020, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qmip_disp_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qmip_video_cvp_ahb_clk = { + .halt_reg = 0xb010, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0xb010, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0xb010, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qmip_video_cvp_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qmip_video_vcodec_ahb_clk = { + .halt_reg = 0xb014, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0xb014, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0xb014, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qmip_video_vcodec_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qspi_1_cnoc_periph_ahb_clk = { + .halt_reg = 0x4a004, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x4a004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qspi_1_cnoc_periph_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qspi_1_core_clk = { + .halt_reg = 0x4a008, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x4a008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qspi_1_core_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_qspi_1_core_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qspi_cnoc_periph_ahb_clk = { + .halt_reg = 0x4b000, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x4b000, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qspi_cnoc_periph_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qspi_core_clk = { + .halt_reg = 0x4b004, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x4b004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qspi_core_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_qspi_core_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap0_s0_clk = { + .halt_reg = 0x17144, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x5200c, + .enable_mask = BIT(10), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_s0_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_qupv3_wrap0_s0_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap0_s1_clk = { + .halt_reg = 0x17274, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x5200c, + .enable_mask = BIT(11), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_s1_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_qupv3_wrap0_s1_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap0_s2_clk = { + .halt_reg = 0x173a4, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x5200c, + .enable_mask = BIT(12), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_s2_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_qupv3_wrap0_s2_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap0_s3_clk = { + .halt_reg = 0x174d4, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x5200c, + .enable_mask = BIT(13), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_s3_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_qupv3_wrap0_s3_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap0_s4_clk = { + .halt_reg = 0x17604, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x5200c, + .enable_mask = BIT(14), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_s4_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_qupv3_wrap0_s4_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap0_s5_clk = { + .halt_reg = 0x17734, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x5200c, + .enable_mask = BIT(15), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_s5_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_qupv3_wrap0_s5_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap0_s6_clk = { + .halt_reg = 0x17864, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x5200c, + .enable_mask = BIT(16), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_s6_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_qupv3_wrap0_s6_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap0_s7_clk = { + .halt_reg = 0x17994, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x5200c, + .enable_mask = BIT(17), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_s7_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_qupv3_wrap0_s7_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap1_s0_clk = { + .halt_reg = 0x18144, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x5200c, + .enable_mask = BIT(22), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap1_s0_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_qupv3_wrap1_s0_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap1_s1_clk = { + .halt_reg = 0x18274, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x5200c, + .enable_mask = BIT(23), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap1_s1_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_qupv3_wrap1_s1_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap1_s2_clk = { + .halt_reg = 0x183a4, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x5200c, + .enable_mask = BIT(24), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap1_s2_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_qupv3_wrap1_s2_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap1_s3_clk = { + .halt_reg = 0x184d4, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x5200c, + .enable_mask = BIT(25), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap1_s3_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_qupv3_wrap1_s3_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap1_s4_clk = { + .halt_reg = 0x18604, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x5200c, + .enable_mask = BIT(26), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap1_s4_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_qupv3_wrap1_s4_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap1_s5_clk = { + .halt_reg = 0x18734, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x5200c, + .enable_mask = BIT(27), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap1_s5_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_qupv3_wrap1_s5_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap2_s0_clk = { + .halt_reg = 0x1e144, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52014, + .enable_mask = BIT(4), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap2_s0_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_qupv3_wrap2_s0_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap2_s1_clk = { + .halt_reg = 0x1e274, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52014, + .enable_mask = BIT(5), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap2_s1_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_qupv3_wrap2_s1_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap2_s2_clk = { + .halt_reg = 0x1e3a4, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52014, + .enable_mask = BIT(6), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap2_s2_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_qupv3_wrap2_s2_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap2_s3_clk = { + .halt_reg = 0x1e4d4, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52014, + .enable_mask = BIT(7), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap2_s3_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_qupv3_wrap2_s3_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap2_s4_clk = { + .halt_reg = 0x1e604, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52014, + .enable_mask = BIT(8), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap2_s4_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_qupv3_wrap2_s4_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap2_s5_clk = { + .halt_reg = 0x1e734, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52014, + .enable_mask = BIT(9), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap2_s5_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_qupv3_wrap2_s5_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap_0_m_ahb_clk = { + .halt_reg = 0x17004, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x5200c, + .enable_mask = BIT(6), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap_0_m_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap_0_s_ahb_clk = { + .halt_reg = 0x17008, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x17008, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x5200c, + .enable_mask = BIT(7), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap_0_s_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap_1_m_ahb_clk = { + .halt_reg = 0x18004, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x5200c, + .enable_mask = BIT(20), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap_1_m_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap_1_s_ahb_clk = { + .halt_reg = 0x18008, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x18008, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x5200c, + .enable_mask = BIT(21), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap_1_s_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap_2_m_ahb_clk = { + .halt_reg = 0x1e004, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52014, + .enable_mask = BIT(2), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap_2_m_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap_2_s_ahb_clk = { + .halt_reg = 0x1e008, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x1e008, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52014, + .enable_mask = BIT(1), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap_2_s_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sdcc2_ahb_clk = { + .halt_reg = 0x14008, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x14008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc2_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sdcc2_apps_clk = { + .halt_reg = 0x14004, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x14004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc2_apps_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_sdcc2_apps_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sdcc4_ahb_clk = { + .halt_reg = 0x16008, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x16008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc4_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sdcc4_apps_clk = { + .halt_reg = 0x16004, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x16004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc4_apps_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_sdcc4_apps_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +/* For CPUSS functionality the SYS NOC clock needs to be left enabled */ +static struct clk_branch gcc_sys_noc_cpuss_ahb_clk = { + .halt_reg = 0x4819c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_sys_noc_cpuss_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_cpuss_ahb_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_IS_CRITICAL | CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_tsif_ahb_clk = { + .halt_reg = 0x36004, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x36004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_tsif_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_tsif_inactivity_timers_clk = { + .halt_reg = 0x3600c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x3600c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_tsif_inactivity_timers_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_tsif_ref_clk = { + .halt_reg = 0x36008, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x36008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_tsif_ref_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_tsif_ref_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_card_2_ahb_clk = { + .halt_reg = 0xa2014, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0xa2014, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0xa2014, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_2_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_card_2_axi_clk = { + .halt_reg = 0xa2010, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0xa2010, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0xa2010, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_2_axi_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_ufs_card_2_axi_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_card_2_ice_core_clk = { + .halt_reg = 0xa205c, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0xa205c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0xa205c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_2_ice_core_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_ufs_card_2_ice_core_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_card_2_phy_aux_clk = { + .halt_reg = 0xa2090, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0xa2090, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0xa2090, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_2_phy_aux_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_ufs_card_2_phy_aux_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_card_2_rx_symbol_0_clk = { + .halt_reg = 0xa201c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xa201c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_2_rx_symbol_0_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_card_2_rx_symbol_1_clk = { + .halt_reg = 0xa20ac, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xa20ac, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_2_rx_symbol_1_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_card_2_tx_symbol_0_clk = { + .halt_reg = 0xa2018, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xa2018, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_2_tx_symbol_0_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_card_2_unipro_core_clk = { + .halt_reg = 0xa2058, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0xa2058, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0xa2058, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_2_unipro_core_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_ufs_card_2_unipro_core_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_card_ahb_clk = { + .halt_reg = 0x75014, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0x75014, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x75014, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_card_axi_clk = { + .halt_reg = 0x75010, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0x75010, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x75010, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_axi_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_ufs_card_axi_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_card_axi_hw_ctl_clk = { + .halt_reg = 0x75010, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0x75010, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x75010, + .enable_mask = BIT(1), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_axi_hw_ctl_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_ufs_card_axi_clk.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch_simple_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_card_ice_core_clk = { + .halt_reg = 0x7505c, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0x7505c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x7505c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_ice_core_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_ufs_card_ice_core_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_card_ice_core_hw_ctl_clk = { + .halt_reg = 0x7505c, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0x7505c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x7505c, + .enable_mask = BIT(1), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_ice_core_hw_ctl_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_ufs_card_ice_core_clk.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch_simple_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_card_phy_aux_clk = { + .halt_reg = 0x75090, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0x75090, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x75090, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_phy_aux_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_ufs_card_phy_aux_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_card_phy_aux_hw_ctl_clk = { + .halt_reg = 0x75090, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0x75090, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x75090, + .enable_mask = BIT(1), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_phy_aux_hw_ctl_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_ufs_card_phy_aux_clk.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch_simple_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_card_rx_symbol_0_clk = { + .halt_reg = 0x7501c, + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x7501c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_rx_symbol_0_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_card_rx_symbol_1_clk = { + .halt_reg = 0x750ac, + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x750ac, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_rx_symbol_1_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_card_tx_symbol_0_clk = { + .halt_reg = 0x75018, + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x75018, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_tx_symbol_0_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_card_unipro_core_clk = { + .halt_reg = 0x75058, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0x75058, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x75058, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_unipro_core_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_ufs_card_unipro_core_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_card_unipro_core_hw_ctl_clk = { + .halt_reg = 0x75058, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0x75058, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x75058, + .enable_mask = BIT(1), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_unipro_core_hw_ctl_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_ufs_card_unipro_core_clk.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch_simple_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_ahb_clk = { + .halt_reg = 0x77014, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0x77014, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x77014, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_axi_clk = { + .halt_reg = 0x77010, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0x77010, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x77010, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_axi_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_ufs_phy_axi_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_axi_hw_ctl_clk = { + .halt_reg = 0x77010, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0x77010, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x77010, + .enable_mask = BIT(1), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_axi_hw_ctl_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_ufs_phy_axi_clk.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch_simple_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_ice_core_clk = { + .halt_reg = 0x7705c, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0x7705c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x7705c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_ice_core_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_ufs_phy_ice_core_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_ice_core_hw_ctl_clk = { + .halt_reg = 0x7705c, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0x7705c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x7705c, + .enable_mask = BIT(1), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_ice_core_hw_ctl_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_ufs_phy_ice_core_clk.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch_simple_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_phy_aux_clk = { + .halt_reg = 0x77090, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0x77090, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x77090, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_phy_aux_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_ufs_phy_phy_aux_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_phy_aux_hw_ctl_clk = { + .halt_reg = 0x77090, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0x77090, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x77090, + .enable_mask = BIT(1), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_phy_aux_hw_ctl_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_ufs_phy_phy_aux_clk.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch_simple_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_rx_symbol_0_clk = { + .halt_reg = 0x7701c, + .halt_check = BRANCH_HALT_SKIP, + .clkr = { + .enable_reg = 0x7701c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_rx_symbol_0_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_rx_symbol_1_clk = { + .halt_reg = 0x770ac, + .halt_check = BRANCH_HALT_SKIP, + .clkr = { + .enable_reg = 0x770ac, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_rx_symbol_1_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_tx_symbol_0_clk = { + .halt_reg = 0x77018, + .halt_check = BRANCH_HALT_SKIP, + .clkr = { + .enable_reg = 0x77018, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_tx_symbol_0_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_unipro_core_clk = { + .halt_reg = 0x77058, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0x77058, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x77058, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_unipro_core_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_ufs_phy_unipro_core_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_unipro_core_hw_ctl_clk = { + .halt_reg = 0x77058, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0x77058, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x77058, + .enable_mask = BIT(1), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_unipro_core_hw_ctl_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_ufs_phy_unipro_core_clk.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch_simple_ops, + }, + }, +}; + +static struct clk_branch gcc_usb30_mp_master_clk = { + .halt_reg = 0xa6010, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xa6010, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_mp_master_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_usb30_mp_master_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb30_mp_mock_utmi_clk = { + .halt_reg = 0xa6018, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xa6018, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_mp_mock_utmi_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_usb30_mp_mock_utmi_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb30_mp_sleep_clk = { + .halt_reg = 0xa6014, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xa6014, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_mp_sleep_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb30_prim_master_clk = { + .halt_reg = 0xf010, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xf010, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_prim_master_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_usb30_prim_master_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb30_prim_mock_utmi_clk = { + .halt_reg = 0xf018, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xf018, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_prim_mock_utmi_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_usb30_prim_mock_utmi_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb30_prim_sleep_clk = { + .halt_reg = 0xf014, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xf014, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_prim_sleep_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb30_sec_master_clk = { + .halt_reg = 0x10010, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x10010, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_sec_master_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_usb30_sec_master_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb30_sec_mock_utmi_clk = { + .halt_reg = 0x10018, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x10018, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_sec_mock_utmi_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_usb30_sec_mock_utmi_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb30_sec_sleep_clk = { + .halt_reg = 0x10014, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x10014, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_sec_sleep_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb3_mp_phy_aux_clk = { + .halt_reg = 0xa6050, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xa6050, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_mp_phy_aux_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_usb3_mp_phy_aux_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb3_mp_phy_com_aux_clk = { + .halt_reg = 0xa6054, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xa6054, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_mp_phy_com_aux_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_usb3_mp_phy_aux_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb3_mp_phy_pipe_0_clk = { + .halt_reg = 0xa6058, + .halt_check = BRANCH_HALT_SKIP, + .clkr = { + .enable_reg = 0xa6058, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_mp_phy_pipe_0_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb3_mp_phy_pipe_1_clk = { + .halt_reg = 0xa605c, + .halt_check = BRANCH_HALT_SKIP, + .clkr = { + .enable_reg = 0xa605c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_mp_phy_pipe_1_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb3_prim_clkref_clk = { + .halt_reg = 0x8c008, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x8c008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_prim_clkref_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb3_prim_phy_aux_clk = { + .halt_reg = 0xf050, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xf050, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_prim_phy_aux_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_usb3_prim_phy_aux_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb3_prim_phy_com_aux_clk = { + .halt_reg = 0xf054, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xf054, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_prim_phy_com_aux_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_usb3_prim_phy_aux_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb3_prim_phy_pipe_clk = { + .halt_reg = 0xf058, + .halt_check = BRANCH_HALT_SKIP, + .clkr = { + .enable_reg = 0xf058, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_prim_phy_pipe_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb3_sec_clkref_clk = { + .halt_reg = 0x8c028, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x8c028, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_sec_clkref_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb3_sec_phy_aux_clk = { + .halt_reg = 0x10050, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x10050, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_sec_phy_aux_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_usb3_sec_phy_aux_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb3_sec_phy_com_aux_clk = { + .halt_reg = 0x10054, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x10054, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_sec_phy_com_aux_clk", + .parent_hws = (const struct clk_hw *[]){ + &gcc_usb3_sec_phy_aux_clk_src.clkr.hw + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb3_sec_phy_pipe_clk = { + .halt_reg = 0x10058, + .halt_check = BRANCH_HALT_SKIP, + .clkr = { + .enable_reg = 0x10058, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_sec_phy_pipe_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_video_axi0_clk = { + .halt_reg = 0xb024, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xb024, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_video_axi0_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_video_axi1_clk = { + .halt_reg = 0xb028, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xb028, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_video_axi1_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_video_axic_clk = { + .halt_reg = 0xb02c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xb02c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_video_axic_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct gdsc usb30_sec_gdsc = { + .gdscr = 0x10004, + .pd = { + .name = "usb30_sec_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .flags = POLL_CFG_GDSCR, +}; + +static struct gdsc emac_gdsc = { + .gdscr = 0x6004, + .pd = { + .name = "emac_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .flags = POLL_CFG_GDSCR, +}; + +static struct gdsc usb30_prim_gdsc = { + .gdscr = 0xf004, + .pd = { + .name = "usb30_prim_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .flags = POLL_CFG_GDSCR, +}; + +static struct gdsc pcie_0_gdsc = { + .gdscr = 0x6b004, + .pd = { + .name = "pcie_0_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .flags = POLL_CFG_GDSCR, +}; + +static struct gdsc ufs_card_gdsc = { + .gdscr = 0x75004, + .pd = { + .name = "ufs_card_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .flags = POLL_CFG_GDSCR, +}; + +static struct gdsc ufs_phy_gdsc = { + .gdscr = 0x77004, + .pd = { + .name = "ufs_phy_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .flags = POLL_CFG_GDSCR, +}; + +static struct gdsc pcie_1_gdsc = { + .gdscr = 0x8d004, + .pd = { + .name = "pcie_1_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .flags = POLL_CFG_GDSCR, +}; + +static struct gdsc pcie_2_gdsc = { + .gdscr = 0x9d004, + .pd = { + .name = "pcie_2_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .flags = POLL_CFG_GDSCR, +}; + +static struct gdsc ufs_card_2_gdsc = { + .gdscr = 0xa2004, + .pd = { + .name = "ufs_card_2_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .flags = POLL_CFG_GDSCR, +}; + +static struct gdsc pcie_3_gdsc = { + .gdscr = 0xa3004, + .pd = { + .name = "pcie_3_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .flags = POLL_CFG_GDSCR, +}; + +static struct gdsc usb30_mp_gdsc = { + .gdscr = 0xa6004, + .pd = { + .name = "usb30_mp_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .flags = POLL_CFG_GDSCR, +}; + +static struct clk_regmap *gcc_sc8180x_clocks[] = { + [GCC_AGGRE_NOC_PCIE_TBU_CLK] = &gcc_aggre_noc_pcie_tbu_clk.clkr, + [GCC_AGGRE_UFS_CARD_AXI_CLK] = &gcc_aggre_ufs_card_axi_clk.clkr, + [GCC_AGGRE_UFS_CARD_AXI_HW_CTL_CLK] = &gcc_aggre_ufs_card_axi_hw_ctl_clk.clkr, + [GCC_AGGRE_UFS_PHY_AXI_CLK] = &gcc_aggre_ufs_phy_axi_clk.clkr, + [GCC_AGGRE_UFS_PHY_AXI_HW_CTL_CLK] = &gcc_aggre_ufs_phy_axi_hw_ctl_clk.clkr, + [GCC_AGGRE_USB3_MP_AXI_CLK] = &gcc_aggre_usb3_mp_axi_clk.clkr, + [GCC_AGGRE_USB3_PRIM_AXI_CLK] = &gcc_aggre_usb3_prim_axi_clk.clkr, + [GCC_AGGRE_USB3_SEC_AXI_CLK] = &gcc_aggre_usb3_sec_axi_clk.clkr, + [GCC_BOOT_ROM_AHB_CLK] = &gcc_boot_rom_ahb_clk.clkr, + [GCC_CAMERA_HF_AXI_CLK] = &gcc_camera_hf_axi_clk.clkr, + [GCC_CAMERA_SF_AXI_CLK] = &gcc_camera_sf_axi_clk.clkr, + [GCC_CFG_NOC_USB3_MP_AXI_CLK] = &gcc_cfg_noc_usb3_mp_axi_clk.clkr, + [GCC_CFG_NOC_USB3_PRIM_AXI_CLK] = &gcc_cfg_noc_usb3_prim_axi_clk.clkr, + [GCC_CFG_NOC_USB3_SEC_AXI_CLK] = &gcc_cfg_noc_usb3_sec_axi_clk.clkr, + [GCC_CPUSS_AHB_CLK] = &gcc_cpuss_ahb_clk.clkr, + [GCC_CPUSS_AHB_CLK_SRC] = &gcc_cpuss_ahb_clk_src.clkr, + [GCC_CPUSS_RBCPR_CLK] = &gcc_cpuss_rbcpr_clk.clkr, + [GCC_DDRSS_GPU_AXI_CLK] = &gcc_ddrss_gpu_axi_clk.clkr, + [GCC_DISP_HF_AXI_CLK] = &gcc_disp_hf_axi_clk.clkr, + [GCC_DISP_SF_AXI_CLK] = &gcc_disp_sf_axi_clk.clkr, + [GCC_EMAC_AXI_CLK] = &gcc_emac_axi_clk.clkr, + [GCC_EMAC_PTP_CLK] = &gcc_emac_ptp_clk.clkr, + [GCC_EMAC_PTP_CLK_SRC] = &gcc_emac_ptp_clk_src.clkr, + [GCC_EMAC_RGMII_CLK] = &gcc_emac_rgmii_clk.clkr, + [GCC_EMAC_RGMII_CLK_SRC] = &gcc_emac_rgmii_clk_src.clkr, + [GCC_EMAC_SLV_AHB_CLK] = &gcc_emac_slv_ahb_clk.clkr, + [GCC_GP1_CLK] = &gcc_gp1_clk.clkr, + [GCC_GP1_CLK_SRC] = &gcc_gp1_clk_src.clkr, + [GCC_GP2_CLK] = &gcc_gp2_clk.clkr, + [GCC_GP2_CLK_SRC] = &gcc_gp2_clk_src.clkr, + [GCC_GP3_CLK] = &gcc_gp3_clk.clkr, + [GCC_GP3_CLK_SRC] = &gcc_gp3_clk_src.clkr, + [GCC_GP4_CLK] = &gcc_gp4_clk.clkr, + [GCC_GP4_CLK_SRC] = &gcc_gp4_clk_src.clkr, + [GCC_GP5_CLK] = &gcc_gp5_clk.clkr, + [GCC_GP5_CLK_SRC] = &gcc_gp5_clk_src.clkr, + [GCC_GPU_GPLL0_CLK_SRC] = &gcc_gpu_gpll0_clk_src.clkr, + [GCC_GPU_GPLL0_DIV_CLK_SRC] = &gcc_gpu_gpll0_div_clk_src.clkr, + [GCC_GPU_MEMNOC_GFX_CLK] = &gcc_gpu_memnoc_gfx_clk.clkr, + [GCC_GPU_SNOC_DVM_GFX_CLK] = &gcc_gpu_snoc_dvm_gfx_clk.clkr, + [GCC_NPU_AT_CLK] = &gcc_npu_at_clk.clkr, + [GCC_NPU_AXI_CLK] = &gcc_npu_axi_clk.clkr, + [GCC_NPU_AXI_CLK_SRC] = &gcc_npu_axi_clk_src.clkr, + [GCC_NPU_GPLL0_CLK_SRC] = &gcc_npu_gpll0_clk_src.clkr, + [GCC_NPU_GPLL0_DIV_CLK_SRC] = &gcc_npu_gpll0_div_clk_src.clkr, + [GCC_NPU_TRIG_CLK] = &gcc_npu_trig_clk.clkr, + [GCC_PCIE0_PHY_REFGEN_CLK] = &gcc_pcie0_phy_refgen_clk.clkr, + [GCC_PCIE1_PHY_REFGEN_CLK] = &gcc_pcie1_phy_refgen_clk.clkr, + [GCC_PCIE2_PHY_REFGEN_CLK] = &gcc_pcie2_phy_refgen_clk.clkr, + [GCC_PCIE3_PHY_REFGEN_CLK] = &gcc_pcie3_phy_refgen_clk.clkr, + [GCC_PCIE_0_AUX_CLK] = &gcc_pcie_0_aux_clk.clkr, + [GCC_PCIE_0_AUX_CLK_SRC] = &gcc_pcie_0_aux_clk_src.clkr, + [GCC_PCIE_0_CFG_AHB_CLK] = &gcc_pcie_0_cfg_ahb_clk.clkr, + [GCC_PCIE_0_CLKREF_CLK] = &gcc_pcie_0_clkref_clk.clkr, + [GCC_PCIE_0_MSTR_AXI_CLK] = &gcc_pcie_0_mstr_axi_clk.clkr, + [GCC_PCIE_0_PIPE_CLK] = &gcc_pcie_0_pipe_clk.clkr, + [GCC_PCIE_0_SLV_AXI_CLK] = &gcc_pcie_0_slv_axi_clk.clkr, + [GCC_PCIE_0_SLV_Q2A_AXI_CLK] = &gcc_pcie_0_slv_q2a_axi_clk.clkr, + [GCC_PCIE_1_AUX_CLK] = &gcc_pcie_1_aux_clk.clkr, + [GCC_PCIE_1_AUX_CLK_SRC] = &gcc_pcie_1_aux_clk_src.clkr, + [GCC_PCIE_1_CFG_AHB_CLK] = &gcc_pcie_1_cfg_ahb_clk.clkr, + [GCC_PCIE_1_CLKREF_CLK] = &gcc_pcie_1_clkref_clk.clkr, + [GCC_PCIE_1_MSTR_AXI_CLK] = &gcc_pcie_1_mstr_axi_clk.clkr, + [GCC_PCIE_1_PIPE_CLK] = &gcc_pcie_1_pipe_clk.clkr, + [GCC_PCIE_1_SLV_AXI_CLK] = &gcc_pcie_1_slv_axi_clk.clkr, + [GCC_PCIE_1_SLV_Q2A_AXI_CLK] = &gcc_pcie_1_slv_q2a_axi_clk.clkr, + [GCC_PCIE_2_AUX_CLK] = &gcc_pcie_2_aux_clk.clkr, + [GCC_PCIE_2_AUX_CLK_SRC] = &gcc_pcie_2_aux_clk_src.clkr, + [GCC_PCIE_2_CFG_AHB_CLK] = &gcc_pcie_2_cfg_ahb_clk.clkr, + [GCC_PCIE_2_CLKREF_CLK] = &gcc_pcie_2_clkref_clk.clkr, + [GCC_PCIE_2_MSTR_AXI_CLK] = &gcc_pcie_2_mstr_axi_clk.clkr, + [GCC_PCIE_2_PIPE_CLK] = &gcc_pcie_2_pipe_clk.clkr, + [GCC_PCIE_2_SLV_AXI_CLK] = &gcc_pcie_2_slv_axi_clk.clkr, + [GCC_PCIE_2_SLV_Q2A_AXI_CLK] = &gcc_pcie_2_slv_q2a_axi_clk.clkr, + [GCC_PCIE_3_AUX_CLK] = &gcc_pcie_3_aux_clk.clkr, + [GCC_PCIE_3_AUX_CLK_SRC] = &gcc_pcie_3_aux_clk_src.clkr, + [GCC_PCIE_3_CFG_AHB_CLK] = &gcc_pcie_3_cfg_ahb_clk.clkr, + [GCC_PCIE_3_CLKREF_CLK] = &gcc_pcie_3_clkref_clk.clkr, + [GCC_PCIE_3_MSTR_AXI_CLK] = &gcc_pcie_3_mstr_axi_clk.clkr, + [GCC_PCIE_3_PIPE_CLK] = &gcc_pcie_3_pipe_clk.clkr, + [GCC_PCIE_3_SLV_AXI_CLK] = &gcc_pcie_3_slv_axi_clk.clkr, + [GCC_PCIE_3_SLV_Q2A_AXI_CLK] = &gcc_pcie_3_slv_q2a_axi_clk.clkr, + [GCC_PCIE_PHY_AUX_CLK] = &gcc_pcie_phy_aux_clk.clkr, + [GCC_PCIE_PHY_REFGEN_CLK_SRC] = &gcc_pcie_phy_refgen_clk_src.clkr, + [GCC_PDM2_CLK] = &gcc_pdm2_clk.clkr, + [GCC_PDM2_CLK_SRC] = &gcc_pdm2_clk_src.clkr, + [GCC_PDM_AHB_CLK] = &gcc_pdm_ahb_clk.clkr, + [GCC_PDM_XO4_CLK] = &gcc_pdm_xo4_clk.clkr, + [GCC_PRNG_AHB_CLK] = &gcc_prng_ahb_clk.clkr, + [GCC_QMIP_CAMERA_NRT_AHB_CLK] = &gcc_qmip_camera_nrt_ahb_clk.clkr, + [GCC_QMIP_CAMERA_RT_AHB_CLK] = &gcc_qmip_camera_rt_ahb_clk.clkr, + [GCC_QMIP_DISP_AHB_CLK] = &gcc_qmip_disp_ahb_clk.clkr, + [GCC_QMIP_VIDEO_CVP_AHB_CLK] = &gcc_qmip_video_cvp_ahb_clk.clkr, + [GCC_QMIP_VIDEO_VCODEC_AHB_CLK] = &gcc_qmip_video_vcodec_ahb_clk.clkr, + [GCC_QSPI_1_CNOC_PERIPH_AHB_CLK] = &gcc_qspi_1_cnoc_periph_ahb_clk.clkr, + [GCC_QSPI_1_CORE_CLK] = &gcc_qspi_1_core_clk.clkr, + [GCC_QSPI_1_CORE_CLK_SRC] = &gcc_qspi_1_core_clk_src.clkr, + [GCC_QSPI_CNOC_PERIPH_AHB_CLK] = &gcc_qspi_cnoc_periph_ahb_clk.clkr, + [GCC_QSPI_CORE_CLK] = &gcc_qspi_core_clk.clkr, + [GCC_QSPI_CORE_CLK_SRC] = &gcc_qspi_core_clk_src.clkr, + [GCC_QUPV3_WRAP0_S0_CLK] = &gcc_qupv3_wrap0_s0_clk.clkr, + [GCC_QUPV3_WRAP0_S0_CLK_SRC] = &gcc_qupv3_wrap0_s0_clk_src.clkr, + [GCC_QUPV3_WRAP0_S1_CLK] = &gcc_qupv3_wrap0_s1_clk.clkr, + [GCC_QUPV3_WRAP0_S1_CLK_SRC] = &gcc_qupv3_wrap0_s1_clk_src.clkr, + [GCC_QUPV3_WRAP0_S2_CLK] = &gcc_qupv3_wrap0_s2_clk.clkr, + [GCC_QUPV3_WRAP0_S2_CLK_SRC] = &gcc_qupv3_wrap0_s2_clk_src.clkr, + [GCC_QUPV3_WRAP0_S3_CLK] = &gcc_qupv3_wrap0_s3_clk.clkr, + [GCC_QUPV3_WRAP0_S3_CLK_SRC] = &gcc_qupv3_wrap0_s3_clk_src.clkr, + [GCC_QUPV3_WRAP0_S4_CLK] = &gcc_qupv3_wrap0_s4_clk.clkr, + [GCC_QUPV3_WRAP0_S4_CLK_SRC] = &gcc_qupv3_wrap0_s4_clk_src.clkr, + [GCC_QUPV3_WRAP0_S5_CLK] = &gcc_qupv3_wrap0_s5_clk.clkr, + [GCC_QUPV3_WRAP0_S5_CLK_SRC] = &gcc_qupv3_wrap0_s5_clk_src.clkr, + [GCC_QUPV3_WRAP0_S6_CLK] = &gcc_qupv3_wrap0_s6_clk.clkr, + [GCC_QUPV3_WRAP0_S6_CLK_SRC] = &gcc_qupv3_wrap0_s6_clk_src.clkr, + [GCC_QUPV3_WRAP0_S7_CLK] = &gcc_qupv3_wrap0_s7_clk.clkr, + [GCC_QUPV3_WRAP0_S7_CLK_SRC] = &gcc_qupv3_wrap0_s7_clk_src.clkr, + [GCC_QUPV3_WRAP1_S0_CLK] = &gcc_qupv3_wrap1_s0_clk.clkr, + [GCC_QUPV3_WRAP1_S0_CLK_SRC] = &gcc_qupv3_wrap1_s0_clk_src.clkr, + [GCC_QUPV3_WRAP1_S1_CLK] = &gcc_qupv3_wrap1_s1_clk.clkr, + [GCC_QUPV3_WRAP1_S1_CLK_SRC] = &gcc_qupv3_wrap1_s1_clk_src.clkr, + [GCC_QUPV3_WRAP1_S2_CLK] = &gcc_qupv3_wrap1_s2_clk.clkr, + [GCC_QUPV3_WRAP1_S2_CLK_SRC] = &gcc_qupv3_wrap1_s2_clk_src.clkr, + [GCC_QUPV3_WRAP1_S3_CLK] = &gcc_qupv3_wrap1_s3_clk.clkr, + [GCC_QUPV3_WRAP1_S3_CLK_SRC] = &gcc_qupv3_wrap1_s3_clk_src.clkr, + [GCC_QUPV3_WRAP1_S4_CLK] = &gcc_qupv3_wrap1_s4_clk.clkr, + [GCC_QUPV3_WRAP1_S4_CLK_SRC] = &gcc_qupv3_wrap1_s4_clk_src.clkr, + [GCC_QUPV3_WRAP1_S5_CLK] = &gcc_qupv3_wrap1_s5_clk.clkr, + [GCC_QUPV3_WRAP1_S5_CLK_SRC] = &gcc_qupv3_wrap1_s5_clk_src.clkr, + [GCC_QUPV3_WRAP2_S0_CLK] = &gcc_qupv3_wrap2_s0_clk.clkr, + [GCC_QUPV3_WRAP2_S0_CLK_SRC] = &gcc_qupv3_wrap2_s0_clk_src.clkr, + [GCC_QUPV3_WRAP2_S1_CLK] = &gcc_qupv3_wrap2_s1_clk.clkr, + [GCC_QUPV3_WRAP2_S1_CLK_SRC] = &gcc_qupv3_wrap2_s1_clk_src.clkr, + [GCC_QUPV3_WRAP2_S2_CLK] = &gcc_qupv3_wrap2_s2_clk.clkr, + [GCC_QUPV3_WRAP2_S2_CLK_SRC] = &gcc_qupv3_wrap2_s2_clk_src.clkr, + [GCC_QUPV3_WRAP2_S3_CLK] = &gcc_qupv3_wrap2_s3_clk.clkr, + [GCC_QUPV3_WRAP2_S3_CLK_SRC] = &gcc_qupv3_wrap2_s3_clk_src.clkr, + [GCC_QUPV3_WRAP2_S4_CLK] = &gcc_qupv3_wrap2_s4_clk.clkr, + [GCC_QUPV3_WRAP2_S4_CLK_SRC] = &gcc_qupv3_wrap2_s4_clk_src.clkr, + [GCC_QUPV3_WRAP2_S5_CLK] = &gcc_qupv3_wrap2_s5_clk.clkr, + [GCC_QUPV3_WRAP2_S5_CLK_SRC] = &gcc_qupv3_wrap2_s5_clk_src.clkr, + [GCC_QUPV3_WRAP_0_M_AHB_CLK] = &gcc_qupv3_wrap_0_m_ahb_clk.clkr, + [GCC_QUPV3_WRAP_0_S_AHB_CLK] = &gcc_qupv3_wrap_0_s_ahb_clk.clkr, + [GCC_QUPV3_WRAP_1_M_AHB_CLK] = &gcc_qupv3_wrap_1_m_ahb_clk.clkr, + [GCC_QUPV3_WRAP_1_S_AHB_CLK] = &gcc_qupv3_wrap_1_s_ahb_clk.clkr, + [GCC_QUPV3_WRAP_2_M_AHB_CLK] = &gcc_qupv3_wrap_2_m_ahb_clk.clkr, + [GCC_QUPV3_WRAP_2_S_AHB_CLK] = &gcc_qupv3_wrap_2_s_ahb_clk.clkr, + [GCC_SDCC2_AHB_CLK] = &gcc_sdcc2_ahb_clk.clkr, + [GCC_SDCC2_APPS_CLK] = &gcc_sdcc2_apps_clk.clkr, + [GCC_SDCC2_APPS_CLK_SRC] = &gcc_sdcc2_apps_clk_src.clkr, + [GCC_SDCC4_AHB_CLK] = &gcc_sdcc4_ahb_clk.clkr, + [GCC_SDCC4_APPS_CLK] = &gcc_sdcc4_apps_clk.clkr, + [GCC_SDCC4_APPS_CLK_SRC] = &gcc_sdcc4_apps_clk_src.clkr, + [GCC_SYS_NOC_CPUSS_AHB_CLK] = &gcc_sys_noc_cpuss_ahb_clk.clkr, + [GCC_TSIF_AHB_CLK] = &gcc_tsif_ahb_clk.clkr, + [GCC_TSIF_INACTIVITY_TIMERS_CLK] = &gcc_tsif_inactivity_timers_clk.clkr, + [GCC_TSIF_REF_CLK] = &gcc_tsif_ref_clk.clkr, + [GCC_TSIF_REF_CLK_SRC] = &gcc_tsif_ref_clk_src.clkr, + [GCC_UFS_CARD_2_AHB_CLK] = &gcc_ufs_card_2_ahb_clk.clkr, + [GCC_UFS_CARD_2_AXI_CLK] = &gcc_ufs_card_2_axi_clk.clkr, + [GCC_UFS_CARD_2_AXI_CLK_SRC] = &gcc_ufs_card_2_axi_clk_src.clkr, + [GCC_UFS_CARD_2_ICE_CORE_CLK] = &gcc_ufs_card_2_ice_core_clk.clkr, + [GCC_UFS_CARD_2_ICE_CORE_CLK_SRC] = &gcc_ufs_card_2_ice_core_clk_src.clkr, + [GCC_UFS_CARD_2_PHY_AUX_CLK] = &gcc_ufs_card_2_phy_aux_clk.clkr, + [GCC_UFS_CARD_2_PHY_AUX_CLK_SRC] = &gcc_ufs_card_2_phy_aux_clk_src.clkr, + [GCC_UFS_CARD_2_RX_SYMBOL_0_CLK] = &gcc_ufs_card_2_rx_symbol_0_clk.clkr, + [GCC_UFS_CARD_2_RX_SYMBOL_1_CLK] = &gcc_ufs_card_2_rx_symbol_1_clk.clkr, + [GCC_UFS_CARD_2_TX_SYMBOL_0_CLK] = &gcc_ufs_card_2_tx_symbol_0_clk.clkr, + [GCC_UFS_CARD_2_UNIPRO_CORE_CLK] = &gcc_ufs_card_2_unipro_core_clk.clkr, + [GCC_UFS_CARD_2_UNIPRO_CORE_CLK_SRC] = &gcc_ufs_card_2_unipro_core_clk_src.clkr, + [GCC_UFS_CARD_AHB_CLK] = &gcc_ufs_card_ahb_clk.clkr, + [GCC_UFS_CARD_AXI_CLK] = &gcc_ufs_card_axi_clk.clkr, + [GCC_UFS_CARD_AXI_CLK_SRC] = &gcc_ufs_card_axi_clk_src.clkr, + [GCC_UFS_CARD_AXI_HW_CTL_CLK] = &gcc_ufs_card_axi_hw_ctl_clk.clkr, + [GCC_UFS_CARD_ICE_CORE_CLK] = &gcc_ufs_card_ice_core_clk.clkr, + [GCC_UFS_CARD_ICE_CORE_CLK_SRC] = &gcc_ufs_card_ice_core_clk_src.clkr, + [GCC_UFS_CARD_ICE_CORE_HW_CTL_CLK] = &gcc_ufs_card_ice_core_hw_ctl_clk.clkr, + [GCC_UFS_CARD_PHY_AUX_CLK] = &gcc_ufs_card_phy_aux_clk.clkr, + [GCC_UFS_CARD_PHY_AUX_CLK_SRC] = &gcc_ufs_card_phy_aux_clk_src.clkr, + [GCC_UFS_CARD_PHY_AUX_HW_CTL_CLK] = &gcc_ufs_card_phy_aux_hw_ctl_clk.clkr, + [GCC_UFS_CARD_RX_SYMBOL_0_CLK] = &gcc_ufs_card_rx_symbol_0_clk.clkr, + [GCC_UFS_CARD_RX_SYMBOL_1_CLK] = &gcc_ufs_card_rx_symbol_1_clk.clkr, + [GCC_UFS_CARD_TX_SYMBOL_0_CLK] = &gcc_ufs_card_tx_symbol_0_clk.clkr, + [GCC_UFS_CARD_UNIPRO_CORE_CLK] = &gcc_ufs_card_unipro_core_clk.clkr, + [GCC_UFS_CARD_UNIPRO_CORE_CLK_SRC] = &gcc_ufs_card_unipro_core_clk_src.clkr, + [GCC_UFS_CARD_UNIPRO_CORE_HW_CTL_CLK] = &gcc_ufs_card_unipro_core_hw_ctl_clk.clkr, + [GCC_UFS_PHY_AHB_CLK] = &gcc_ufs_phy_ahb_clk.clkr, + [GCC_UFS_PHY_AXI_CLK] = &gcc_ufs_phy_axi_clk.clkr, + [GCC_UFS_PHY_AXI_CLK_SRC] = &gcc_ufs_phy_axi_clk_src.clkr, + [GCC_UFS_PHY_AXI_HW_CTL_CLK] = &gcc_ufs_phy_axi_hw_ctl_clk.clkr, + [GCC_UFS_PHY_ICE_CORE_CLK] = &gcc_ufs_phy_ice_core_clk.clkr, + [GCC_UFS_PHY_ICE_CORE_CLK_SRC] = &gcc_ufs_phy_ice_core_clk_src.clkr, + [GCC_UFS_PHY_ICE_CORE_HW_CTL_CLK] = &gcc_ufs_phy_ice_core_hw_ctl_clk.clkr, + [GCC_UFS_PHY_PHY_AUX_CLK] = &gcc_ufs_phy_phy_aux_clk.clkr, + [GCC_UFS_PHY_PHY_AUX_CLK_SRC] = &gcc_ufs_phy_phy_aux_clk_src.clkr, + [GCC_UFS_PHY_PHY_AUX_HW_CTL_CLK] = &gcc_ufs_phy_phy_aux_hw_ctl_clk.clkr, + [GCC_UFS_PHY_RX_SYMBOL_0_CLK] = &gcc_ufs_phy_rx_symbol_0_clk.clkr, + [GCC_UFS_PHY_RX_SYMBOL_1_CLK] = &gcc_ufs_phy_rx_symbol_1_clk.clkr, + [GCC_UFS_PHY_TX_SYMBOL_0_CLK] = &gcc_ufs_phy_tx_symbol_0_clk.clkr, + [GCC_UFS_PHY_UNIPRO_CORE_CLK] = &gcc_ufs_phy_unipro_core_clk.clkr, + [GCC_UFS_PHY_UNIPRO_CORE_CLK_SRC] = &gcc_ufs_phy_unipro_core_clk_src.clkr, + [GCC_UFS_PHY_UNIPRO_CORE_HW_CTL_CLK] = &gcc_ufs_phy_unipro_core_hw_ctl_clk.clkr, + [GCC_USB30_MP_MASTER_CLK] = &gcc_usb30_mp_master_clk.clkr, + [GCC_USB30_MP_MASTER_CLK_SRC] = &gcc_usb30_mp_master_clk_src.clkr, + [GCC_USB30_MP_MOCK_UTMI_CLK] = &gcc_usb30_mp_mock_utmi_clk.clkr, + [GCC_USB30_MP_MOCK_UTMI_CLK_SRC] = &gcc_usb30_mp_mock_utmi_clk_src.clkr, + [GCC_USB30_MP_SLEEP_CLK] = &gcc_usb30_mp_sleep_clk.clkr, + [GCC_USB30_PRIM_MASTER_CLK] = &gcc_usb30_prim_master_clk.clkr, + [GCC_USB30_PRIM_MASTER_CLK_SRC] = &gcc_usb30_prim_master_clk_src.clkr, + [GCC_USB30_PRIM_MOCK_UTMI_CLK] = &gcc_usb30_prim_mock_utmi_clk.clkr, + [GCC_USB30_PRIM_MOCK_UTMI_CLK_SRC] = &gcc_usb30_prim_mock_utmi_clk_src.clkr, + [GCC_USB30_PRIM_SLEEP_CLK] = &gcc_usb30_prim_sleep_clk.clkr, + [GCC_USB30_SEC_MASTER_CLK] = &gcc_usb30_sec_master_clk.clkr, + [GCC_USB30_SEC_MASTER_CLK_SRC] = &gcc_usb30_sec_master_clk_src.clkr, + [GCC_USB30_SEC_MOCK_UTMI_CLK] = &gcc_usb30_sec_mock_utmi_clk.clkr, + [GCC_USB30_SEC_MOCK_UTMI_CLK_SRC] = &gcc_usb30_sec_mock_utmi_clk_src.clkr, + [GCC_USB30_SEC_SLEEP_CLK] = &gcc_usb30_sec_sleep_clk.clkr, + [GCC_USB3_MP_PHY_AUX_CLK] = &gcc_usb3_mp_phy_aux_clk.clkr, + [GCC_USB3_MP_PHY_AUX_CLK_SRC] = &gcc_usb3_mp_phy_aux_clk_src.clkr, + [GCC_USB3_MP_PHY_COM_AUX_CLK] = &gcc_usb3_mp_phy_com_aux_clk.clkr, + [GCC_USB3_MP_PHY_PIPE_0_CLK] = &gcc_usb3_mp_phy_pipe_0_clk.clkr, + [GCC_USB3_MP_PHY_PIPE_1_CLK] = &gcc_usb3_mp_phy_pipe_1_clk.clkr, + [GCC_USB3_PRIM_CLKREF_CLK] = &gcc_usb3_prim_clkref_clk.clkr, + [GCC_USB3_PRIM_PHY_AUX_CLK] = &gcc_usb3_prim_phy_aux_clk.clkr, + [GCC_USB3_PRIM_PHY_AUX_CLK_SRC] = &gcc_usb3_prim_phy_aux_clk_src.clkr, + [GCC_USB3_PRIM_PHY_COM_AUX_CLK] = &gcc_usb3_prim_phy_com_aux_clk.clkr, + [GCC_USB3_PRIM_PHY_PIPE_CLK] = &gcc_usb3_prim_phy_pipe_clk.clkr, + [GCC_USB3_SEC_CLKREF_CLK] = &gcc_usb3_sec_clkref_clk.clkr, + [GCC_USB3_SEC_PHY_AUX_CLK] = &gcc_usb3_sec_phy_aux_clk.clkr, + [GCC_USB3_SEC_PHY_AUX_CLK_SRC] = &gcc_usb3_sec_phy_aux_clk_src.clkr, + [GCC_USB3_SEC_PHY_COM_AUX_CLK] = &gcc_usb3_sec_phy_com_aux_clk.clkr, + [GCC_USB3_SEC_PHY_PIPE_CLK] = &gcc_usb3_sec_phy_pipe_clk.clkr, + [GCC_VIDEO_AXI0_CLK] = &gcc_video_axi0_clk.clkr, + [GCC_VIDEO_AXI1_CLK] = &gcc_video_axi1_clk.clkr, + [GCC_VIDEO_AXIC_CLK] = &gcc_video_axic_clk.clkr, + [GPLL0] = &gpll0.clkr, + [GPLL0_OUT_EVEN] = &gpll0_out_even.clkr, + [GPLL1] = &gpll1.clkr, + [GPLL4] = &gpll4.clkr, + [GPLL7] = &gpll7.clkr, +}; + +static const struct qcom_reset_map gcc_sc8180x_resets[] = { + [GCC_EMAC_BCR] = { 0x6000 }, + [GCC_GPU_BCR] = { 0x71000 }, + [GCC_MMSS_BCR] = { 0xb000 }, + [GCC_NPU_BCR] = { 0x4d000 }, + [GCC_PCIE_0_BCR] = { 0x6b000 }, + [GCC_PCIE_0_PHY_BCR] = { 0x6c01c }, + [GCC_PCIE_1_BCR] = { 0x8d000 }, + [GCC_PCIE_1_PHY_BCR] = { 0x8e01c }, + [GCC_PCIE_2_BCR] = { 0x9d000 }, + [GCC_PCIE_2_PHY_BCR] = { 0xa701c }, + [GCC_PCIE_3_BCR] = { 0xa3000 }, + [GCC_PCIE_3_PHY_BCR] = { 0xa801c }, + [GCC_PCIE_PHY_BCR] = { 0x6f000 }, + [GCC_PDM_BCR] = { 0x33000 }, + [GCC_PRNG_BCR] = { 0x34000 }, + [GCC_QSPI_1_BCR] = { 0x4a000 }, + [GCC_QSPI_BCR] = { 0x24008 }, + [GCC_QUPV3_WRAPPER_0_BCR] = { 0x17000 }, + [GCC_QUPV3_WRAPPER_1_BCR] = { 0x18000 }, + [GCC_QUPV3_WRAPPER_2_BCR] = { 0x1e000 }, + [GCC_QUSB2PHY_5_BCR] = { 0x12010 }, + [GCC_QUSB2PHY_MP0_BCR] = { 0x12008 }, + [GCC_QUSB2PHY_MP1_BCR] = { 0x1200c }, + [GCC_QUSB2PHY_PRIM_BCR] = { 0x12000 }, + [GCC_QUSB2PHY_SEC_BCR] = { 0x12004 }, + [GCC_USB3_PHY_PRIM_SP0_BCR] = { 0x50000 }, + [GCC_USB3_PHY_PRIM_SP1_BCR] = { 0x50004 }, + [GCC_USB3_DP_PHY_PRIM_SP0_BCR] = { 0x50010 }, + [GCC_USB3_DP_PHY_PRIM_SP1_BCR] = { 0x50014 }, + [GCC_USB3_PHY_SEC_BCR] = { 0x50018 }, + [GCC_USB3PHY_PHY_SEC_BCR] = { 0x5001c }, + [GCC_USB3_DP_PHY_SEC_BCR] = { 0x50020 }, + [GCC_SDCC2_BCR] = { 0x14000 }, + [GCC_SDCC4_BCR] = { 0x16000 }, + [GCC_TSIF_BCR] = { 0x36000 }, + [GCC_UFS_CARD_2_BCR] = { 0xa2000 }, + [GCC_UFS_CARD_BCR] = { 0x75000 }, + [GCC_UFS_PHY_BCR] = { 0x77000 }, + [GCC_USB30_MP_BCR] = { 0xa6000 }, + [GCC_USB30_PRIM_BCR] = { 0xf000 }, + [GCC_USB30_SEC_BCR] = { 0x10000 }, + [GCC_USB_PHY_CFG_AHB2PHY_BCR] = { 0x6a000 }, + [GCC_VIDEO_AXIC_CLK_BCR] = { 0xb02c, 2 }, + [GCC_VIDEO_AXI0_CLK_BCR] = { 0xb024, 2 }, + [GCC_VIDEO_AXI1_CLK_BCR] = { 0xb028, 2 }, +}; + +static struct gdsc *gcc_sc8180x_gdscs[] = { + [EMAC_GDSC] = &emac_gdsc, + [PCIE_0_GDSC] = &pcie_0_gdsc, + [PCIE_1_GDSC] = &pcie_1_gdsc, + [PCIE_2_GDSC] = &pcie_2_gdsc, + [PCIE_3_GDSC] = &pcie_3_gdsc, + [UFS_CARD_GDSC] = &ufs_card_gdsc, + [UFS_CARD_2_GDSC] = &ufs_card_2_gdsc, + [UFS_PHY_GDSC] = &ufs_phy_gdsc, + [USB30_MP_GDSC] = &usb30_mp_gdsc, + [USB30_PRIM_GDSC] = &usb30_prim_gdsc, + [USB30_SEC_GDSC] = &usb30_sec_gdsc, +}; + +static const struct regmap_config gcc_sc8180x_regmap_config = { + .reg_bits = 32, + .reg_stride = 4, + .val_bits = 32, + .max_register = 0xc0004, + .fast_io = true, +}; + +static const struct qcom_cc_desc gcc_sc8180x_desc = { + .config = &gcc_sc8180x_regmap_config, + .clks = gcc_sc8180x_clocks, + .num_clks = ARRAY_SIZE(gcc_sc8180x_clocks), + .resets = gcc_sc8180x_resets, + .num_resets = ARRAY_SIZE(gcc_sc8180x_resets), + .gdscs = gcc_sc8180x_gdscs, + .num_gdscs = ARRAY_SIZE(gcc_sc8180x_gdscs), +}; + +static const struct of_device_id gcc_sc8180x_match_table[] = { + { .compatible = "qcom,gcc-sc8180x" }, + { } +}; +MODULE_DEVICE_TABLE(of, gcc_sc8180x_match_table); + +static int gcc_sc8180x_probe(struct platform_device *pdev) +{ + struct regmap *regmap; + + regmap = qcom_cc_map(pdev, &gcc_sc8180x_desc); + if (IS_ERR(regmap)) + return PTR_ERR(regmap); + + /* + * Enable the following always-on clocks: + * GCC_VIDEO_AHB_CLK, GCC_CAMERA_AHB_CLK, GCC_DISP_AHB_CLK, + * GCC_VIDEO_XO_CLK, GCC_CAMERA_XO_CLK, GCC_DISP_XO_CLK, + * GCC_CPUSS_GNOC_CLK, GCC_CPUSS_DVM_BUS_CLK, GCC_NPU_CFG_AHB_CLK and + * GCC_GPU_CFG_AHB_CLK + */ + regmap_update_bits(regmap, 0xb004, BIT(0), BIT(0)); + regmap_update_bits(regmap, 0xb008, BIT(0), BIT(0)); + regmap_update_bits(regmap, 0xb00c, BIT(0), BIT(0)); + regmap_update_bits(regmap, 0xb040, BIT(0), BIT(0)); + regmap_update_bits(regmap, 0xb044, BIT(0), BIT(0)); + regmap_update_bits(regmap, 0xb048, BIT(0), BIT(0)); + regmap_update_bits(regmap, 0x48004, BIT(0), BIT(0)); + regmap_update_bits(regmap, 0x48190, BIT(0), BIT(0)); + regmap_update_bits(regmap, 0x4d004, BIT(0), BIT(0)); + regmap_update_bits(regmap, 0x71004, BIT(0), BIT(0)); + + /* Disable the GPLL0 active input to NPU and GPU via MISC registers */ + regmap_update_bits(regmap, 0x4d110, 0x3, 0x3); + regmap_update_bits(regmap, 0x71028, 0x3, 0x3); + + return qcom_cc_really_probe(pdev, &gcc_sc8180x_desc, regmap); +} + +static struct platform_driver gcc_sc8180x_driver = { + .probe = gcc_sc8180x_probe, + .driver = { + .name = "gcc-sc8180x", + .of_match_table = gcc_sc8180x_match_table, + }, +}; + +static int __init gcc_sc8180x_init(void) +{ + return platform_driver_register(&gcc_sc8180x_driver); +} +core_initcall(gcc_sc8180x_init); + +static void __exit gcc_sc8180x_exit(void) +{ + platform_driver_unregister(&gcc_sc8180x_driver); +} +module_exit(gcc_sc8180x_exit); + +MODULE_DESCRIPTION("QTI GCC SC8180x driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/clk/qcom/gcc-sdm660.c b/drivers/clk/qcom/gcc-sdm660.c index 31258795e7b86bbe18d755c6a224269a7cec8ec4..6394257ca8c0ea12f5051157d8e4d8a3d6fd3958 100644 --- a/drivers/clk/qcom/gcc-sdm660.c +++ b/drivers/clk/qcom/gcc-sdm660.c @@ -1571,6 +1571,7 @@ static struct clk_branch gcc_gpu_cfg_ahb_clk = { .hw.init = &(struct clk_init_data){ .name = "gcc_gpu_cfg_ahb_clk", .ops = &clk_branch2_ops, + .flags = CLK_IS_CRITICAL, }, }, }; @@ -1684,6 +1685,12 @@ static struct clk_branch gcc_mmss_noc_cfg_ahb_clk = { .hw.init = &(struct clk_init_data){ .name = "gcc_mmss_noc_cfg_ahb_clk", .ops = &clk_branch2_ops, + /* + * Any access to mmss depends on this clock. + * Gating this clock has been shown to crash the system + * when mmssnoc_axi_rpm_clk is inited in rpmcc. + */ + .flags = CLK_IS_CRITICAL, }, }, }; diff --git a/drivers/clk/qcom/gcc-sm8350.c b/drivers/clk/qcom/gcc-sm8350.c new file mode 100644 index 0000000000000000000000000000000000000000..1c23b9f849009a003e415d0080bf00b6ac385264 --- /dev/null +++ b/drivers/clk/qcom/gcc-sm8350.c @@ -0,0 +1,3890 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2019-2020, The Linux Foundation. All rights reserved. + * Copyright (c) 2020-2021, Linaro Limited + */ + +#include +#include +#include + +#include + +#include "clk-alpha-pll.h" +#include "clk-branch.h" +#include "clk-rcg.h" +#include "clk-regmap.h" +#include "clk-regmap-divider.h" +#include "clk-regmap-mux.h" +#include "gdsc.h" +#include "reset.h" + +enum { + P_BI_TCXO, + P_CORE_BI_PLL_TEST_SE, + P_GCC_GPLL0_OUT_EVEN, + P_GCC_GPLL0_OUT_MAIN, + P_GCC_GPLL4_OUT_MAIN, + P_GCC_GPLL9_OUT_MAIN, + P_PCIE_0_PIPE_CLK, + P_PCIE_1_PIPE_CLK, + P_SLEEP_CLK, + P_UFS_CARD_RX_SYMBOL_0_CLK, + P_UFS_CARD_RX_SYMBOL_1_CLK, + P_UFS_CARD_TX_SYMBOL_0_CLK, + P_UFS_PHY_RX_SYMBOL_0_CLK, + P_UFS_PHY_RX_SYMBOL_1_CLK, + P_UFS_PHY_TX_SYMBOL_0_CLK, + P_USB3_PHY_WRAPPER_GCC_USB30_PIPE_CLK, + P_USB3_UNI_PHY_SEC_GCC_USB30_PIPE_CLK, +}; + +static struct clk_alpha_pll gcc_gpll0 = { + .offset = 0x0, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID], + .clkr = { + .enable_reg = 0x52018, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gpll0", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "bi_tcxo", + }, + .num_parents = 1, + .ops = &clk_alpha_pll_fixed_lucid_5lpe_ops, + }, + }, +}; + +static const struct clk_div_table post_div_table_gcc_gpll0_out_even[] = { + { 0x1, 2 }, + { } +}; + +static struct clk_alpha_pll_postdiv gcc_gpll0_out_even = { + .offset = 0x0, + .post_div_shift = 8, + .post_div_table = post_div_table_gcc_gpll0_out_even, + .num_post_div = ARRAY_SIZE(post_div_table_gcc_gpll0_out_even), + .width = 4, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID], + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_gpll0_out_even", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_gpll0.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_alpha_pll_postdiv_lucid_5lpe_ops, + }, +}; + +static struct clk_alpha_pll gcc_gpll4 = { + .offset = 0x76000, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID], + .clkr = { + .enable_reg = 0x52018, + .enable_mask = BIT(4), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gpll4", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "bi_tcxo", + .name = "bi_tcxo", + }, + .num_parents = 1, + .ops = &clk_alpha_pll_fixed_lucid_5lpe_ops, + }, + }, +}; + +static struct clk_alpha_pll gcc_gpll9 = { + .offset = 0x1c000, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID], + .clkr = { + .enable_reg = 0x52018, + .enable_mask = BIT(9), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gpll9", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "bi_tcxo", + .name = "bi_tcxo", + }, + .num_parents = 1, + .ops = &clk_alpha_pll_fixed_lucid_5lpe_ops, + }, + }, +}; + +static const struct parent_map gcc_parent_map_0[] = { + { P_BI_TCXO, 0 }, + { P_GCC_GPLL0_OUT_MAIN, 1 }, + { P_GCC_GPLL0_OUT_EVEN, 6 }, + { P_CORE_BI_PLL_TEST_SE, 7 }, +}; + +static const struct clk_parent_data gcc_parent_data_0[] = { + { .fw_name = "bi_tcxo" }, + { .hw = &gcc_gpll0.clkr.hw }, + { .hw = &gcc_gpll0_out_even.clkr.hw }, + { .fw_name = "core_bi_pll_test_se" }, +}; + +static const struct parent_map gcc_parent_map_1[] = { + { P_BI_TCXO, 0 }, + { P_GCC_GPLL0_OUT_MAIN, 1 }, + { P_SLEEP_CLK, 5 }, + { P_GCC_GPLL0_OUT_EVEN, 6 }, + { P_CORE_BI_PLL_TEST_SE, 7 }, +}; + +static const struct clk_parent_data gcc_parent_data_1[] = { + { .fw_name = "bi_tcxo" }, + { .hw = &gcc_gpll0.clkr.hw }, + { .fw_name = "sleep_clk" }, + { .hw = &gcc_gpll0_out_even.clkr.hw }, + { .fw_name = "core_bi_pll_test_se" }, +}; + +static const struct parent_map gcc_parent_map_2[] = { + { P_BI_TCXO, 0 }, + { P_SLEEP_CLK, 5 }, + { P_CORE_BI_PLL_TEST_SE, 7 }, +}; + +static const struct clk_parent_data gcc_parent_data_2[] = { + { .fw_name = "bi_tcxo" }, + { .fw_name = "sleep_clk" }, + { .fw_name = "core_bi_pll_test_se" }, +}; + +static const struct parent_map gcc_parent_map_3[] = { + { P_BI_TCXO, 0 }, + { P_CORE_BI_PLL_TEST_SE, 7 }, +}; + +static const struct clk_parent_data gcc_parent_data_3[] = { + { .fw_name = "bi_tcxo" }, + { .fw_name = "core_bi_pll_test_se" }, +}; + +static const struct parent_map gcc_parent_map_4[] = { + { P_PCIE_0_PIPE_CLK, 0 }, + { P_BI_TCXO, 2 }, +}; + +static const struct clk_parent_data gcc_parent_data_4[] = { + { .fw_name = "pcie_0_pipe_clk", }, + { .fw_name = "bi_tcxo" }, +}; + +static const struct parent_map gcc_parent_map_5[] = { + { P_PCIE_1_PIPE_CLK, 0 }, + { P_BI_TCXO, 2 }, +}; + +static const struct clk_parent_data gcc_parent_data_5[] = { + { .fw_name = "pcie_1_pipe_clk" }, + { .fw_name = "bi_tcxo" }, +}; + +static const struct parent_map gcc_parent_map_6[] = { + { P_BI_TCXO, 0 }, + { P_GCC_GPLL0_OUT_MAIN, 1 }, + { P_GCC_GPLL9_OUT_MAIN, 2 }, + { P_GCC_GPLL4_OUT_MAIN, 5 }, + { P_GCC_GPLL0_OUT_EVEN, 6 }, + { P_CORE_BI_PLL_TEST_SE, 7 }, +}; + +static const struct clk_parent_data gcc_parent_data_6[] = { + { .fw_name = "bi_tcxo" }, + { .hw = &gcc_gpll0.clkr.hw }, + { .hw = &gcc_gpll9.clkr.hw }, + { .hw = &gcc_gpll4.clkr.hw }, + { .hw = &gcc_gpll0_out_even.clkr.hw }, + { .fw_name = "core_bi_pll_test_se" }, +}; + +static const struct parent_map gcc_parent_map_7[] = { + { P_UFS_CARD_RX_SYMBOL_0_CLK, 0 }, + { P_BI_TCXO, 2 }, +}; + +static const struct clk_parent_data gcc_parent_data_7[] = { + { .fw_name = "ufs_card_rx_symbol_0_clk" }, + { .fw_name = "bi_tcxo" }, +}; + +static const struct parent_map gcc_parent_map_8[] = { + { P_UFS_CARD_RX_SYMBOL_1_CLK, 0 }, + { P_BI_TCXO, 2 }, +}; + +static const struct clk_parent_data gcc_parent_data_8[] = { + { .fw_name = "ufs_card_rx_symbol_1_clk" }, + { .fw_name = "bi_tcxo" }, +}; + +static const struct parent_map gcc_parent_map_9[] = { + { P_UFS_CARD_TX_SYMBOL_0_CLK, 0 }, + { P_BI_TCXO, 2 }, +}; + +static const struct clk_parent_data gcc_parent_data_9[] = { + { .fw_name = "ufs_card_tx_symbol_0_clk" }, + { .fw_name = "bi_tcxo" }, +}; + +static const struct parent_map gcc_parent_map_10[] = { + { P_UFS_PHY_RX_SYMBOL_0_CLK, 0 }, + { P_BI_TCXO, 2 }, +}; + +static const struct clk_parent_data gcc_parent_data_10[] = { + { .fw_name = "ufs_phy_rx_symbol_0_clk" }, + { .fw_name = "bi_tcxo" }, +}; + +static const struct parent_map gcc_parent_map_11[] = { + { P_UFS_PHY_RX_SYMBOL_1_CLK, 0 }, + { P_BI_TCXO, 2 }, +}; + +static const struct clk_parent_data gcc_parent_data_11[] = { + { .fw_name = "ufs_phy_rx_symbol_1_clk" }, + { .fw_name = "bi_tcxo" }, +}; + +static const struct parent_map gcc_parent_map_12[] = { + { P_UFS_PHY_TX_SYMBOL_0_CLK, 0 }, + { P_BI_TCXO, 2 }, +}; + +static const struct clk_parent_data gcc_parent_data_12[] = { + { .fw_name = "ufs_phy_tx_symbol_0_clk" }, + { .fw_name = "bi_tcxo" }, +}; + +static const struct parent_map gcc_parent_map_13[] = { + { P_USB3_PHY_WRAPPER_GCC_USB30_PIPE_CLK, 0 }, + { P_CORE_BI_PLL_TEST_SE, 1 }, + { P_BI_TCXO, 2 }, +}; + +static const struct clk_parent_data gcc_parent_data_13[] = { + { .fw_name = "usb3_phy_wrapper_gcc_usb30_pipe_clk" }, + { .fw_name = "core_bi_pll_test_se" }, + { .fw_name = "bi_tcxo" }, +}; + +static const struct parent_map gcc_parent_map_14[] = { + { P_USB3_UNI_PHY_SEC_GCC_USB30_PIPE_CLK, 0 }, + { P_CORE_BI_PLL_TEST_SE, 1 }, + { P_BI_TCXO, 2 }, +}; + +static const struct clk_parent_data gcc_parent_data_14[] = { + { .fw_name = "usb3_uni_phy_sec_gcc_usb30_pipe_clk" }, + { .fw_name = "core_bi_pll_test_se" }, + { .fw_name = "bi_tcxo" }, +}; + +static struct clk_regmap_mux gcc_pcie_0_pipe_clk_src = { + .reg = 0x6b054, + .shift = 0, + .width = 2, + .parent_map = gcc_parent_map_4, + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_0_pipe_clk_src", + .parent_data = gcc_parent_data_4, + .num_parents = 2, + .ops = &clk_regmap_mux_closest_ops, + }, + }, +}; + +static struct clk_regmap_mux gcc_pcie_1_pipe_clk_src = { + .reg = 0x8d054, + .shift = 0, + .width = 2, + .parent_map = gcc_parent_map_5, + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_1_pipe_clk_src", + .parent_data = gcc_parent_data_5, + .num_parents = 2, + .ops = &clk_regmap_mux_closest_ops, + }, + }, +}; + +static struct clk_regmap_mux gcc_ufs_card_rx_symbol_0_clk_src = { + .reg = 0x75058, + .shift = 0, + .width = 2, + .parent_map = gcc_parent_map_7, + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_rx_symbol_0_clk_src", + .parent_data = gcc_parent_data_7, + .num_parents = 2, + .ops = &clk_regmap_mux_closest_ops, + }, + }, +}; + +static struct clk_regmap_mux gcc_ufs_card_rx_symbol_1_clk_src = { + .reg = 0x750c8, + .shift = 0, + .width = 2, + .parent_map = gcc_parent_map_8, + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_rx_symbol_1_clk_src", + .parent_data = gcc_parent_data_8, + .num_parents = 2, + .ops = &clk_regmap_mux_closest_ops, + }, + }, +}; + +static struct clk_regmap_mux gcc_ufs_card_tx_symbol_0_clk_src = { + .reg = 0x75048, + .shift = 0, + .width = 2, + .parent_map = gcc_parent_map_9, + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_tx_symbol_0_clk_src", + .parent_data = gcc_parent_data_9, + .num_parents = 2, + .ops = &clk_regmap_mux_closest_ops, + }, + }, +}; + +static struct clk_regmap_mux gcc_ufs_phy_rx_symbol_0_clk_src = { + .reg = 0x77058, + .shift = 0, + .width = 2, + .parent_map = gcc_parent_map_10, + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_rx_symbol_0_clk_src", + .parent_data = gcc_parent_data_10, + .num_parents = 2, + .ops = &clk_regmap_mux_closest_ops, + }, + }, +}; + +static struct clk_regmap_mux gcc_ufs_phy_rx_symbol_1_clk_src = { + .reg = 0x770c8, + .shift = 0, + .width = 2, + .parent_map = gcc_parent_map_11, + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_rx_symbol_1_clk_src", + .parent_data = gcc_parent_data_11, + .num_parents = 2, + .ops = &clk_regmap_mux_closest_ops, + }, + }, +}; + +static struct clk_regmap_mux gcc_ufs_phy_tx_symbol_0_clk_src = { + .reg = 0x77048, + .shift = 0, + .width = 2, + .parent_map = gcc_parent_map_12, + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_tx_symbol_0_clk_src", + .parent_data = gcc_parent_data_12, + .num_parents = 2, + .ops = &clk_regmap_mux_closest_ops, + }, + }, +}; + +static struct clk_regmap_mux gcc_usb3_prim_phy_pipe_clk_src = { + .reg = 0xf060, + .shift = 0, + .width = 2, + .parent_map = gcc_parent_map_13, + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_prim_phy_pipe_clk_src", + .parent_data = gcc_parent_data_13, + .num_parents = 3, + .ops = &clk_regmap_mux_closest_ops, + }, + }, +}; + +static struct clk_regmap_mux gcc_usb3_sec_phy_pipe_clk_src = { + .reg = 0x10060, + .shift = 0, + .width = 2, + .parent_map = gcc_parent_map_14, + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_sec_phy_pipe_clk_src", + .parent_data = gcc_parent_data_14, + .num_parents = 3, + .ops = &clk_regmap_mux_closest_ops, + }, + }, +}; + +static const struct freq_tbl ftbl_gcc_gp1_clk_src[] = { + F(50000000, P_GCC_GPLL0_OUT_EVEN, 6, 0, 0), + F(100000000, P_GCC_GPLL0_OUT_MAIN, 6, 0, 0), + F(200000000, P_GCC_GPLL0_OUT_MAIN, 3, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_gp1_clk_src = { + .cmd_rcgr = 0x64004, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_1, + .freq_tbl = ftbl_gcc_gp1_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_gp1_clk_src", + .parent_data = gcc_parent_data_1, + .num_parents = 5, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_gp2_clk_src = { + .cmd_rcgr = 0x65004, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_1, + .freq_tbl = ftbl_gcc_gp1_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_gp2_clk_src", + .parent_data = gcc_parent_data_1, + .num_parents = 5, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_gp3_clk_src = { + .cmd_rcgr = 0x66004, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_1, + .freq_tbl = ftbl_gcc_gp1_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_gp3_clk_src", + .parent_data = gcc_parent_data_1, + .num_parents = 5, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_pcie_0_aux_clk_src[] = { + F(9600000, P_BI_TCXO, 2, 0, 0), + F(19200000, P_BI_TCXO, 1, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_pcie_0_aux_clk_src = { + .cmd_rcgr = 0x6b058, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_2, + .freq_tbl = ftbl_gcc_pcie_0_aux_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_0_aux_clk_src", + .parent_data = gcc_parent_data_2, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_pcie_0_phy_rchng_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + F(100000000, P_GCC_GPLL0_OUT_MAIN, 6, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_pcie_0_phy_rchng_clk_src = { + .cmd_rcgr = 0x6b03c, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_pcie_0_phy_rchng_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_0_phy_rchng_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_pcie_1_aux_clk_src = { + .cmd_rcgr = 0x8d058, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_2, + .freq_tbl = ftbl_gcc_pcie_0_aux_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_1_aux_clk_src", + .parent_data = gcc_parent_data_2, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_pcie_1_phy_rchng_clk_src = { + .cmd_rcgr = 0x8d03c, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_pcie_0_phy_rchng_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_1_phy_rchng_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_pdm2_clk_src[] = { + F(60000000, P_GCC_GPLL0_OUT_MAIN, 10, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_pdm2_clk_src = { + .cmd_rcgr = 0x33010, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_pdm2_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_pdm2_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_qupv3_wrap0_s0_clk_src[] = { + F(7372800, P_GCC_GPLL0_OUT_EVEN, 1, 384, 15625), + F(14745600, P_GCC_GPLL0_OUT_EVEN, 1, 768, 15625), + F(19200000, P_BI_TCXO, 1, 0, 0), + F(29491200, P_GCC_GPLL0_OUT_EVEN, 1, 1536, 15625), + F(32000000, P_GCC_GPLL0_OUT_EVEN, 1, 8, 75), + F(48000000, P_GCC_GPLL0_OUT_EVEN, 1, 4, 25), + F(64000000, P_GCC_GPLL0_OUT_EVEN, 1, 16, 75), + F(75000000, P_GCC_GPLL0_OUT_EVEN, 4, 0, 0), + F(80000000, P_GCC_GPLL0_OUT_EVEN, 1, 4, 15), + F(96000000, P_GCC_GPLL0_OUT_EVEN, 1, 8, 25), + F(100000000, P_GCC_GPLL0_OUT_MAIN, 6, 0, 0), + { } +}; + +static struct clk_init_data gcc_qupv3_wrap0_s0_clk_src_init = { + .name = "gcc_qupv3_wrap0_s0_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap0_s0_clk_src = { + .cmd_rcgr = 0x17010, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap0_s0_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap0_s1_clk_src_init = { + .name = "gcc_qupv3_wrap0_s1_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap0_s1_clk_src = { + .cmd_rcgr = 0x17140, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap0_s1_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap0_s2_clk_src_init = { + .name = "gcc_qupv3_wrap0_s2_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap0_s2_clk_src = { + .cmd_rcgr = 0x17270, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap0_s2_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap0_s3_clk_src_init = { + .name = "gcc_qupv3_wrap0_s3_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap0_s3_clk_src = { + .cmd_rcgr = 0x173a0, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap0_s3_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap0_s4_clk_src_init = { + .name = "gcc_qupv3_wrap0_s4_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap0_s4_clk_src = { + .cmd_rcgr = 0x174d0, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap0_s4_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap0_s5_clk_src_init = { + .name = "gcc_qupv3_wrap0_s5_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap0_s5_clk_src = { + .cmd_rcgr = 0x17600, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap0_s5_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap0_s6_clk_src_init = { + .name = "gcc_qupv3_wrap0_s6_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap0_s6_clk_src = { + .cmd_rcgr = 0x17730, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap0_s6_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap0_s7_clk_src_init = { + .name = "gcc_qupv3_wrap0_s7_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap0_s7_clk_src = { + .cmd_rcgr = 0x17860, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap0_s7_clk_src_init, +}; + +static const struct freq_tbl ftbl_gcc_qupv3_wrap1_s0_clk_src[] = { + F(7372800, P_GCC_GPLL0_OUT_EVEN, 1, 384, 15625), + F(14745600, P_GCC_GPLL0_OUT_EVEN, 1, 768, 15625), + F(19200000, P_BI_TCXO, 1, 0, 0), + F(29491200, P_GCC_GPLL0_OUT_EVEN, 1, 1536, 15625), + F(32000000, P_GCC_GPLL0_OUT_EVEN, 1, 8, 75), + F(48000000, P_GCC_GPLL0_OUT_EVEN, 1, 4, 25), + F(64000000, P_GCC_GPLL0_OUT_EVEN, 1, 16, 75), + F(75000000, P_GCC_GPLL0_OUT_EVEN, 4, 0, 0), + F(80000000, P_GCC_GPLL0_OUT_EVEN, 1, 4, 15), + F(96000000, P_GCC_GPLL0_OUT_EVEN, 1, 8, 25), + F(100000000, P_GCC_GPLL0_OUT_MAIN, 6, 0, 0), + F(102400000, P_GCC_GPLL0_OUT_EVEN, 1, 128, 375), + F(112000000, P_GCC_GPLL0_OUT_EVEN, 1, 28, 75), + F(117964800, P_GCC_GPLL0_OUT_EVEN, 1, 6144, 15625), + F(120000000, P_GCC_GPLL0_OUT_MAIN, 5, 0, 0), + { } +}; + +static struct clk_init_data gcc_qupv3_wrap1_s0_clk_src_init = { + .name = "gcc_qupv3_wrap1_s0_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap1_s0_clk_src = { + .cmd_rcgr = 0x18010, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap1_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap1_s0_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap1_s1_clk_src_init = { + .name = "gcc_qupv3_wrap1_s1_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap1_s1_clk_src = { + .cmd_rcgr = 0x18140, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap1_s1_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap1_s2_clk_src_init = { + .name = "gcc_qupv3_wrap1_s2_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap1_s2_clk_src = { + .cmd_rcgr = 0x18270, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap1_s2_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap1_s3_clk_src_init = { + .name = "gcc_qupv3_wrap1_s3_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap1_s3_clk_src = { + .cmd_rcgr = 0x183a0, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap1_s3_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap1_s4_clk_src_init = { + .name = "gcc_qupv3_wrap1_s4_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap1_s4_clk_src = { + .cmd_rcgr = 0x184d0, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap1_s4_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap1_s5_clk_src_init = { + .name = "gcc_qupv3_wrap1_s5_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap1_s5_clk_src = { + .cmd_rcgr = 0x18600, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap1_s5_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap2_s0_clk_src_init = { + .name = "gcc_qupv3_wrap2_s0_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap2_s0_clk_src = { + .cmd_rcgr = 0x1e010, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap1_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap2_s0_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap2_s1_clk_src_init = { + .name = "gcc_qupv3_wrap2_s1_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap2_s1_clk_src = { + .cmd_rcgr = 0x1e140, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap1_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap2_s1_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap2_s2_clk_src_init = { + .name = "gcc_qupv3_wrap2_s2_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap2_s2_clk_src = { + .cmd_rcgr = 0x1e270, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap2_s2_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap2_s3_clk_src_init = { + .name = "gcc_qupv3_wrap2_s3_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap2_s3_clk_src = { + .cmd_rcgr = 0x1e3a0, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap2_s3_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap2_s4_clk_src_init = { + .name = "gcc_qupv3_wrap2_s4_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap2_s4_clk_src = { + .cmd_rcgr = 0x1e4d0, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap2_s4_clk_src_init, +}; + +static struct clk_init_data gcc_qupv3_wrap2_s5_clk_src_init = { + .name = "gcc_qupv3_wrap2_s5_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, +}; + +static struct clk_rcg2 gcc_qupv3_wrap2_s5_clk_src = { + .cmd_rcgr = 0x1e600, + .mnd_width = 16, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_qupv3_wrap0_s0_clk_src, + .clkr.hw.init = &gcc_qupv3_wrap2_s5_clk_src_init, +}; + +static const struct freq_tbl ftbl_gcc_sdcc2_apps_clk_src[] = { + F(400000, P_BI_TCXO, 12, 1, 4), + F(25000000, P_GCC_GPLL0_OUT_EVEN, 12, 0, 0), + F(50000000, P_GCC_GPLL0_OUT_EVEN, 6, 0, 0), + F(100000000, P_GCC_GPLL0_OUT_EVEN, 3, 0, 0), + F(202000000, P_GCC_GPLL9_OUT_MAIN, 4, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_sdcc2_apps_clk_src = { + .cmd_rcgr = 0x1400c, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_6, + .freq_tbl = ftbl_gcc_sdcc2_apps_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc2_apps_clk_src", + .parent_data = gcc_parent_data_6, + .num_parents = 6, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_floor_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_sdcc4_apps_clk_src[] = { + F(400000, P_BI_TCXO, 12, 1, 4), + F(25000000, P_GCC_GPLL0_OUT_EVEN, 12, 0, 0), + F(100000000, P_GCC_GPLL0_OUT_EVEN, 3, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_sdcc4_apps_clk_src = { + .cmd_rcgr = 0x1600c, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_sdcc4_apps_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc4_apps_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_floor_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_ufs_card_axi_clk_src[] = { + F(25000000, P_GCC_GPLL0_OUT_EVEN, 12, 0, 0), + F(75000000, P_GCC_GPLL0_OUT_EVEN, 4, 0, 0), + F(150000000, P_GCC_GPLL0_OUT_MAIN, 4, 0, 0), + F(300000000, P_GCC_GPLL0_OUT_MAIN, 2, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_ufs_card_axi_clk_src = { + .cmd_rcgr = 0x75024, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_ufs_card_axi_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_axi_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_ufs_card_ice_core_clk_src[] = { + F(75000000, P_GCC_GPLL0_OUT_EVEN, 4, 0, 0), + F(150000000, P_GCC_GPLL0_OUT_MAIN, 4, 0, 0), + F(300000000, P_GCC_GPLL0_OUT_MAIN, 2, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_ufs_card_ice_core_clk_src = { + .cmd_rcgr = 0x7506c, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_ufs_card_ice_core_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_ice_core_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_ufs_card_phy_aux_clk_src[] = { + F(19200000, P_BI_TCXO, 1, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_ufs_card_phy_aux_clk_src = { + .cmd_rcgr = 0x750a0, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_3, + .freq_tbl = ftbl_gcc_ufs_card_phy_aux_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_phy_aux_clk_src", + .parent_data = gcc_parent_data_3, + .num_parents = 2, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_ufs_card_unipro_core_clk_src = { + .cmd_rcgr = 0x75084, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_ufs_card_ice_core_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_unipro_core_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_ufs_phy_axi_clk_src = { + .cmd_rcgr = 0x77024, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_ufs_card_axi_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_axi_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_ufs_phy_ice_core_clk_src = { + .cmd_rcgr = 0x7706c, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_ufs_card_ice_core_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_ice_core_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_ufs_phy_phy_aux_clk_src = { + .cmd_rcgr = 0x770a0, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_3, + .freq_tbl = ftbl_gcc_pcie_0_aux_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_phy_aux_clk_src", + .parent_data = gcc_parent_data_3, + .num_parents = 2, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_ufs_phy_unipro_core_clk_src = { + .cmd_rcgr = 0x77084, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_ufs_card_ice_core_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_unipro_core_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_gcc_usb30_prim_master_clk_src[] = { + F(66666667, P_GCC_GPLL0_OUT_EVEN, 4.5, 0, 0), + F(133333333, P_GCC_GPLL0_OUT_MAIN, 4.5, 0, 0), + F(200000000, P_GCC_GPLL0_OUT_MAIN, 3, 0, 0), + F(240000000, P_GCC_GPLL0_OUT_MAIN, 2.5, 0, 0), + { } +}; + +static struct clk_rcg2 gcc_usb30_prim_master_clk_src = { + .cmd_rcgr = 0xf020, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_usb30_prim_master_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_prim_master_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_usb30_prim_mock_utmi_clk_src = { + .cmd_rcgr = 0xf038, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_ufs_card_phy_aux_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_prim_mock_utmi_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_usb30_sec_master_clk_src = { + .cmd_rcgr = 0x10020, + .mnd_width = 8, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_usb30_prim_master_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_sec_master_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_usb30_sec_mock_utmi_clk_src = { + .cmd_rcgr = 0x10038, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_0, + .freq_tbl = ftbl_gcc_ufs_card_phy_aux_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_sec_mock_utmi_clk_src", + .parent_data = gcc_parent_data_0, + .num_parents = 4, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_usb3_prim_phy_aux_clk_src = { + .cmd_rcgr = 0xf064, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_2, + .freq_tbl = ftbl_gcc_ufs_card_phy_aux_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_prim_phy_aux_clk_src", + .parent_data = gcc_parent_data_2, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 gcc_usb3_sec_phy_aux_clk_src = { + .cmd_rcgr = 0x10064, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gcc_parent_map_2, + .freq_tbl = ftbl_gcc_ufs_card_phy_aux_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_sec_phy_aux_clk_src", + .parent_data = gcc_parent_data_2, + .num_parents = 3, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_regmap_div gcc_usb30_prim_mock_utmi_postdiv_clk_src = { + .reg = 0xf050, + .shift = 0, + .width = 4, + .clkr.hw.init = &(struct clk_init_data) { + .name = "gcc_usb30_prim_mock_utmi_postdiv_clk_src", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_usb30_prim_mock_utmi_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_regmap_div_ro_ops, + }, +}; + +static struct clk_regmap_div gcc_usb30_sec_mock_utmi_postdiv_clk_src = { + .reg = 0x10050, + .shift = 0, + .width = 4, + .clkr.hw.init = &(struct clk_init_data) { + .name = "gcc_usb30_sec_mock_utmi_postdiv_clk_src", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_usb30_sec_mock_utmi_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_regmap_div_ro_ops, + }, +}; + +/* external clocks so add BRANCH_HALT_SKIP */ +static struct clk_branch gcc_aggre_noc_pcie_0_axi_clk = { + .halt_reg = 0x6b080, + .halt_check = BRANCH_HALT_SKIP, + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(12), + .hw.init = &(struct clk_init_data){ + .name = "gcc_aggre_noc_pcie_0_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +/* external clocks so add BRANCH_HALT_SKIP */ +static struct clk_branch gcc_aggre_noc_pcie_1_axi_clk = { + .halt_reg = 0x8d084, + .halt_check = BRANCH_HALT_SKIP, + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(11), + .hw.init = &(struct clk_init_data){ + .name = "gcc_aggre_noc_pcie_1_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_aggre_noc_pcie_tbu_clk = { + .halt_reg = 0x9000c, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x9000c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(18), + .hw.init = &(struct clk_init_data){ + .name = "gcc_aggre_noc_pcie_tbu_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_aggre_ufs_card_axi_clk = { + .halt_reg = 0x750cc, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x750cc, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x750cc, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_aggre_ufs_card_axi_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_card_axi_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_aggre_ufs_card_axi_hw_ctl_clk = { + .halt_reg = 0x750cc, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x750cc, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x750cc, + .enable_mask = BIT(1), + .hw.init = &(struct clk_init_data){ + .name = "gcc_aggre_ufs_card_axi_hw_ctl_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_card_axi_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_aggre_ufs_phy_axi_clk = { + .halt_reg = 0x770cc, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x770cc, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x770cc, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_aggre_ufs_phy_axi_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_phy_axi_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_aggre_ufs_phy_axi_hw_ctl_clk = { + .halt_reg = 0x770cc, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x770cc, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x770cc, + .enable_mask = BIT(1), + .hw.init = &(struct clk_init_data){ + .name = "gcc_aggre_ufs_phy_axi_hw_ctl_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_phy_axi_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_aggre_usb3_prim_axi_clk = { + .halt_reg = 0xf080, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0xf080, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0xf080, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_aggre_usb3_prim_axi_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_usb30_prim_master_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_aggre_usb3_sec_axi_clk = { + .halt_reg = 0x10080, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x10080, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x10080, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_aggre_usb3_sec_axi_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_usb30_sec_master_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_boot_rom_ahb_clk = { + .halt_reg = 0x38004, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x38004, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(10), + .hw.init = &(struct clk_init_data){ + .name = "gcc_boot_rom_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +/* external clocks so add BRANCH_HALT_SKIP */ +static struct clk_branch gcc_camera_hf_axi_clk = { + .halt_reg = 0x26010, + .halt_check = BRANCH_HALT_SKIP, + .hwcg_reg = 0x26010, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x26010, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_camera_hf_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +/* external clocks so add BRANCH_HALT_SKIP */ +static struct clk_branch gcc_camera_sf_axi_clk = { + .halt_reg = 0x26014, + .halt_check = BRANCH_HALT_SKIP, + .hwcg_reg = 0x26014, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x26014, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_camera_sf_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_cfg_noc_usb3_prim_axi_clk = { + .halt_reg = 0xf07c, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0xf07c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0xf07c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_cfg_noc_usb3_prim_axi_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_usb30_prim_master_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_cfg_noc_usb3_sec_axi_clk = { + .halt_reg = 0x1007c, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x1007c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x1007c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_cfg_noc_usb3_sec_axi_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_usb30_sec_master_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +/* external clocks so add BRANCH_HALT_SKIP */ +static struct clk_branch gcc_ddrss_gpu_axi_clk = { + .halt_reg = 0x71154, + .halt_check = BRANCH_HALT_SKIP, + .hwcg_reg = 0x71154, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x71154, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ddrss_gpu_axi_clk", + .ops = &clk_branch2_aon_ops, + }, + }, +}; + +/* external clocks so add BRANCH_HALT_SKIP */ +static struct clk_branch gcc_ddrss_pcie_sf_tbu_clk = { + .halt_reg = 0x8d080, + .halt_check = BRANCH_HALT_SKIP, + .hwcg_reg = 0x8d080, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(19), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ddrss_pcie_sf_tbu_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +/* external clocks so add BRANCH_HALT_SKIP */ +static struct clk_branch gcc_disp_hf_axi_clk = { + .halt_reg = 0x2700c, + .halt_check = BRANCH_HALT_SKIP, + .hwcg_reg = 0x2700c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x2700c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_disp_hf_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +/* external clocks so add BRANCH_HALT_SKIP */ +static struct clk_branch gcc_disp_sf_axi_clk = { + .halt_reg = 0x27014, + .halt_check = BRANCH_HALT_SKIP, + .hwcg_reg = 0x27014, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x27014, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_disp_sf_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_gp1_clk = { + .halt_reg = 0x64000, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x64000, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gp1_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_gp1_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_gp2_clk = { + .halt_reg = 0x65000, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x65000, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gp2_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_gp2_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_gp3_clk = { + .halt_reg = 0x66000, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x66000, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gp3_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_gp3_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +/* Clock ON depends on external parent clock, so don't poll */ +static struct clk_branch gcc_gpu_gpll0_clk_src = { + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(15), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gpu_gpll0_clk_src", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_gpll0.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +/* Clock ON depends on external parent clock, so don't poll */ +static struct clk_branch gcc_gpu_gpll0_div_clk_src = { + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(16), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gpu_gpll0_div_clk_src", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_gpll0_out_even.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_gpu_iref_en = { + .halt_reg = 0x8c014, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x8c014, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gpu_iref_en", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_gpu_memnoc_gfx_clk = { + .halt_reg = 0x7100c, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x7100c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x7100c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gpu_memnoc_gfx_clk", + .ops = &clk_branch2_aon_ops, + }, + }, +}; + +static struct clk_branch gcc_gpu_snoc_dvm_gfx_clk = { + .halt_reg = 0x71018, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x71018, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_gpu_snoc_dvm_gfx_clk", + .ops = &clk_branch2_aon_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie0_phy_rchng_clk = { + .halt_reg = 0x6b038, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(22), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie0_phy_rchng_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_pcie_0_phy_rchng_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie1_phy_rchng_clk = { + .halt_reg = 0x8d038, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(23), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie1_phy_rchng_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_pcie_1_phy_rchng_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_0_aux_clk = { + .halt_reg = 0x6b028, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(3), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_0_aux_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_pcie_0_aux_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_0_cfg_ahb_clk = { + .halt_reg = 0x6b024, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x6b024, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(2), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_0_cfg_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_0_clkref_en = { + .halt_reg = 0x8c004, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x8c004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_0_clkref_en", + .ops = &clk_branch2_ops, + }, + }, +}; + +/* external clocks so add BRANCH_HALT_SKIP */ +static struct clk_branch gcc_pcie_0_mstr_axi_clk = { + .halt_reg = 0x6b01c, + .halt_check = BRANCH_HALT_SKIP, + .hwcg_reg = 0x6b01c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(1), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_0_mstr_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +/* external clocks so add BRANCH_HALT_SKIP */ +static struct clk_branch gcc_pcie_0_pipe_clk = { + .halt_reg = 0x6b030, + .halt_check = BRANCH_HALT_SKIP, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(4), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_0_pipe_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_pcie_0_pipe_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_0_slv_axi_clk = { + .halt_reg = 0x6b014, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x6b014, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_0_slv_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_0_slv_q2a_axi_clk = { + .halt_reg = 0x6b010, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(5), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_0_slv_q2a_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_1_aux_clk = { + .halt_reg = 0x8d028, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(29), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_1_aux_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_pcie_1_aux_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_1_cfg_ahb_clk = { + .halt_reg = 0x8d024, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x8d024, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(28), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_1_cfg_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_1_clkref_en = { + .halt_reg = 0x8c008, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x8c008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_1_clkref_en", + .ops = &clk_branch2_ops, + }, + }, +}; + +/* external clocks so add BRANCH_HALT_SKIP */ +static struct clk_branch gcc_pcie_1_mstr_axi_clk = { + .halt_reg = 0x8d01c, + .halt_check = BRANCH_HALT_SKIP, + .hwcg_reg = 0x8d01c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(27), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_1_mstr_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +/* external clocks so add BRANCH_HALT_SKIP */ +static struct clk_branch gcc_pcie_1_pipe_clk = { + .halt_reg = 0x8d030, + .halt_check = BRANCH_HALT_SKIP, + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(30), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_1_pipe_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_pcie_1_pipe_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_1_slv_axi_clk = { + .halt_reg = 0x8d014, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x8d014, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(26), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_1_slv_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pcie_1_slv_q2a_axi_clk = { + .halt_reg = 0x8d010, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52000, + .enable_mask = BIT(25), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pcie_1_slv_q2a_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pdm2_clk = { + .halt_reg = 0x3300c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x3300c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pdm2_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_pdm2_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pdm_ahb_clk = { + .halt_reg = 0x33004, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x33004, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x33004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pdm_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_pdm_xo4_clk = { + .halt_reg = 0x33008, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x33008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_pdm_xo4_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qmip_camera_nrt_ahb_clk = { + .halt_reg = 0x26008, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x26008, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x26008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qmip_camera_nrt_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qmip_camera_rt_ahb_clk = { + .halt_reg = 0x2600c, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x2600c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x2600c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qmip_camera_rt_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qmip_disp_ahb_clk = { + .halt_reg = 0x27008, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x27008, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x27008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qmip_disp_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qmip_video_cvp_ahb_clk = { + .halt_reg = 0x28008, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x28008, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x28008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qmip_video_cvp_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qmip_video_vcodec_ahb_clk = { + .halt_reg = 0x2800c, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x2800c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x2800c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qmip_video_vcodec_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap0_core_2x_clk = { + .halt_reg = 0x23008, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(9), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_core_2x_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap0_core_clk = { + .halt_reg = 0x23000, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(8), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_core_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap0_s0_clk = { + .halt_reg = 0x1700c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(10), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_s0_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap0_s0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap0_s1_clk = { + .halt_reg = 0x1713c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(11), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_s1_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap0_s1_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap0_s2_clk = { + .halt_reg = 0x1726c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(12), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_s2_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap0_s2_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap0_s3_clk = { + .halt_reg = 0x1739c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(13), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_s3_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap0_s3_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap0_s4_clk = { + .halt_reg = 0x174cc, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(14), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_s4_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap0_s4_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap0_s5_clk = { + .halt_reg = 0x175fc, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(15), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_s5_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap0_s5_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap0_s6_clk = { + .halt_reg = 0x1772c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(16), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_s6_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap0_s6_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap0_s7_clk = { + .halt_reg = 0x1785c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(17), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap0_s7_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap0_s7_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap1_core_2x_clk = { + .halt_reg = 0x23140, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(18), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap1_core_2x_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap1_core_clk = { + .halt_reg = 0x23138, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(19), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap1_core_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap_1_m_ahb_clk = { + .halt_reg = 0x18004, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x18004, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(20), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap_1_m_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap_1_s_ahb_clk = { + .halt_reg = 0x18008, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x18008, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(21), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap_1_s_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap1_s0_clk = { + .halt_reg = 0x1800c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(22), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap1_s0_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap1_s0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap1_s1_clk = { + .halt_reg = 0x1813c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(23), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap1_s1_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap1_s1_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap1_s2_clk = { + .halt_reg = 0x1826c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(24), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap1_s2_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap1_s2_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap1_s3_clk = { + .halt_reg = 0x1839c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(25), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap1_s3_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap1_s3_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap1_s4_clk = { + .halt_reg = 0x184cc, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(26), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap1_s4_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap1_s4_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap1_s5_clk = { + .halt_reg = 0x185fc, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(27), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap1_s5_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap1_s5_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap2_core_2x_clk = { + .halt_reg = 0x23278, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52010, + .enable_mask = BIT(3), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap2_core_2x_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap2_core_clk = { + .halt_reg = 0x23270, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52010, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap2_core_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap2_s0_clk = { + .halt_reg = 0x1e00c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52010, + .enable_mask = BIT(4), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap2_s0_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap2_s0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap2_s1_clk = { + .halt_reg = 0x1e13c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52010, + .enable_mask = BIT(5), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap2_s1_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap2_s1_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap2_s2_clk = { + .halt_reg = 0x1e26c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52010, + .enable_mask = BIT(6), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap2_s2_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap2_s2_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap2_s3_clk = { + .halt_reg = 0x1e39c, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52010, + .enable_mask = BIT(7), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap2_s3_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap2_s3_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap2_s4_clk = { + .halt_reg = 0x1e4cc, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52010, + .enable_mask = BIT(8), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap2_s4_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap2_s4_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap2_s5_clk = { + .halt_reg = 0x1e5fc, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0x52010, + .enable_mask = BIT(9), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap2_s5_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_qupv3_wrap2_s5_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap_0_m_ahb_clk = { + .halt_reg = 0x17004, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x17004, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(6), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap_0_m_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap_0_s_ahb_clk = { + .halt_reg = 0x17008, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x17008, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52008, + .enable_mask = BIT(7), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap_0_s_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap_2_m_ahb_clk = { + .halt_reg = 0x1e004, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x1e004, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52010, + .enable_mask = BIT(2), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap_2_m_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_qupv3_wrap_2_s_ahb_clk = { + .halt_reg = 0x1e008, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x1e008, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x52010, + .enable_mask = BIT(1), + .hw.init = &(struct clk_init_data){ + .name = "gcc_qupv3_wrap_2_s_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sdcc2_ahb_clk = { + .halt_reg = 0x14008, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x14008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc2_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sdcc2_apps_clk = { + .halt_reg = 0x14004, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x14004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc2_apps_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_sdcc2_apps_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sdcc4_ahb_clk = { + .halt_reg = 0x16008, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x16008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc4_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_sdcc4_apps_clk = { + .halt_reg = 0x16004, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x16004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_sdcc4_apps_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_sdcc4_apps_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_throttle_pcie_ahb_clk = { + .halt_reg = 0x9044, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x9044, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_throttle_pcie_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_1_clkref_en = { + .halt_reg = 0x8c000, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x8c000, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_1_clkref_en", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_card_ahb_clk = { + .halt_reg = 0x75018, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x75018, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x75018, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_card_axi_clk = { + .halt_reg = 0x75010, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x75010, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x75010, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_axi_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_card_axi_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_card_axi_hw_ctl_clk = { + .halt_reg = 0x75010, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x75010, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x75010, + .enable_mask = BIT(1), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_axi_hw_ctl_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_card_axi_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_card_ice_core_clk = { + .halt_reg = 0x75064, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x75064, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x75064, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_ice_core_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_card_ice_core_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_card_ice_core_hw_ctl_clk = { + .halt_reg = 0x75064, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x75064, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x75064, + .enable_mask = BIT(1), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_ice_core_hw_ctl_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_card_ice_core_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_card_phy_aux_clk = { + .halt_reg = 0x7509c, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x7509c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x7509c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_phy_aux_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_card_phy_aux_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_card_phy_aux_hw_ctl_clk = { + .halt_reg = 0x7509c, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x7509c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x7509c, + .enable_mask = BIT(1), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_phy_aux_hw_ctl_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_card_phy_aux_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +/* Clock ON depends on external parent clock, so don't poll */ +static struct clk_branch gcc_ufs_card_rx_symbol_0_clk = { + .halt_reg = 0x75020, + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x75020, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_rx_symbol_0_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_card_rx_symbol_0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +/* Clock ON depends on external parent clock, so don't poll */ +static struct clk_branch gcc_ufs_card_rx_symbol_1_clk = { + .halt_reg = 0x750b8, + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x750b8, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_rx_symbol_1_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_card_rx_symbol_1_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +/* Clock ON depends on external parent clock, so don't poll */ +static struct clk_branch gcc_ufs_card_tx_symbol_0_clk = { + .halt_reg = 0x7501c, + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x7501c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_tx_symbol_0_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_card_tx_symbol_0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_card_unipro_core_clk = { + .halt_reg = 0x7505c, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x7505c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x7505c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_unipro_core_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_card_unipro_core_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_card_unipro_core_hw_ctl_clk = { + .halt_reg = 0x7505c, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x7505c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x7505c, + .enable_mask = BIT(1), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_card_unipro_core_hw_ctl_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_card_unipro_core_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_ahb_clk = { + .halt_reg = 0x77018, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x77018, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x77018, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_ahb_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_axi_clk = { + .halt_reg = 0x77010, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x77010, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x77010, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_axi_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_phy_axi_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_axi_hw_ctl_clk = { + .halt_reg = 0x77010, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x77010, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x77010, + .enable_mask = BIT(1), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_axi_hw_ctl_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_phy_axi_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_ice_core_clk = { + .halt_reg = 0x77064, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x77064, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x77064, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_ice_core_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_phy_ice_core_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_ice_core_hw_ctl_clk = { + .halt_reg = 0x77064, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x77064, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x77064, + .enable_mask = BIT(1), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_ice_core_hw_ctl_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_phy_ice_core_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_phy_aux_clk = { + .halt_reg = 0x7709c, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x7709c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x7709c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_phy_aux_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_phy_phy_aux_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_phy_aux_hw_ctl_clk = { + .halt_reg = 0x7709c, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x7709c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x7709c, + .enable_mask = BIT(1), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_phy_aux_hw_ctl_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_phy_phy_aux_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +/* Clock ON depends on external parent clock, so don't poll */ +static struct clk_branch gcc_ufs_phy_rx_symbol_0_clk = { + .halt_reg = 0x77020, + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x77020, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_rx_symbol_0_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_phy_rx_symbol_0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +/* Clock ON depends on external parent clock, so don't poll */ +static struct clk_branch gcc_ufs_phy_rx_symbol_1_clk = { + .halt_reg = 0x770b8, + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x770b8, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_rx_symbol_1_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_phy_rx_symbol_1_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +/* Clock ON depends on external parent clock, so don't poll */ +static struct clk_branch gcc_ufs_phy_tx_symbol_0_clk = { + .halt_reg = 0x7701c, + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x7701c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_tx_symbol_0_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_phy_tx_symbol_0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_unipro_core_clk = { + .halt_reg = 0x7705c, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x7705c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x7705c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_unipro_core_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_phy_unipro_core_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_ufs_phy_unipro_core_hw_ctl_clk = { + .halt_reg = 0x7705c, + .halt_check = BRANCH_HALT_VOTED, + .hwcg_reg = 0x7705c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x7705c, + .enable_mask = BIT(1), + .hw.init = &(struct clk_init_data){ + .name = "gcc_ufs_phy_unipro_core_hw_ctl_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_ufs_phy_unipro_core_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb30_prim_master_clk = { + .halt_reg = 0xf010, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xf010, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_prim_master_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_usb30_prim_master_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb30_prim_master_clk__force_mem_core_on = { + .halt_reg = 0xf010, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xf010, + .enable_mask = BIT(14), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_prim_master_clk__force_mem_core_on", + .ops = &clk_branch_simple_ops, + }, + }, +}; + +static struct clk_branch gcc_usb30_prim_mock_utmi_clk = { + .halt_reg = 0xf01c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xf01c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_prim_mock_utmi_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = + &gcc_usb30_prim_mock_utmi_postdiv_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb30_prim_sleep_clk = { + .halt_reg = 0xf018, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xf018, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_prim_sleep_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb30_sec_master_clk = { + .halt_reg = 0x10010, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x10010, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_sec_master_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_usb30_sec_master_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb30_sec_master_clk__force_mem_core_on = { + .halt_reg = 0x10010, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x10010, + .enable_mask = BIT(14), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_sec_master_clk__force_mem_core_on", + .ops = &clk_branch_simple_ops, + }, + }, +}; + +static struct clk_branch gcc_usb30_sec_mock_utmi_clk = { + .halt_reg = 0x1001c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x1001c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_sec_mock_utmi_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = + &gcc_usb30_sec_mock_utmi_postdiv_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb30_sec_sleep_clk = { + .halt_reg = 0x10018, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x10018, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb30_sec_sleep_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb3_prim_phy_aux_clk = { + .halt_reg = 0xf054, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xf054, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_prim_phy_aux_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_usb3_prim_phy_aux_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb3_prim_phy_com_aux_clk = { + .halt_reg = 0xf058, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xf058, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_prim_phy_com_aux_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_usb3_prim_phy_aux_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +/* Clock ON depends on external parent clock, so don't poll */ +static struct clk_branch gcc_usb3_prim_phy_pipe_clk = { + .halt_reg = 0xf05c, + .halt_check = BRANCH_HALT_DELAY, + .hwcg_reg = 0xf05c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0xf05c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_prim_phy_pipe_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_usb3_prim_phy_pipe_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb3_sec_clkref_en = { + .halt_reg = 0x8c010, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x8c010, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_sec_clkref_en", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb3_sec_phy_aux_clk = { + .halt_reg = 0x10054, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x10054, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_sec_phy_aux_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_usb3_sec_phy_aux_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gcc_usb3_sec_phy_com_aux_clk = { + .halt_reg = 0x10058, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x10058, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_sec_phy_com_aux_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_usb3_sec_phy_aux_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +/* Clock ON depends on external parent clock, so don't poll */ +static struct clk_branch gcc_usb3_sec_phy_pipe_clk = { + .halt_reg = 0x1005c, + .halt_check = BRANCH_HALT_DELAY, + .clkr = { + .enable_reg = 0x1005c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_usb3_sec_phy_pipe_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gcc_usb3_sec_phy_pipe_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +/* external clocks so add BRANCH_HALT_SKIP */ +static struct clk_branch gcc_video_axi0_clk = { + .halt_reg = 0x28010, + .halt_check = BRANCH_HALT_SKIP, + .hwcg_reg = 0x28010, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x28010, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_video_axi0_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +/* external clocks so add BRANCH_HALT_SKIP */ +static struct clk_branch gcc_video_axi1_clk = { + .halt_reg = 0x28018, + .halt_check = BRANCH_HALT_SKIP, + .hwcg_reg = 0x28018, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x28018, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gcc_video_axi1_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct gdsc pcie_0_gdsc = { + .gdscr = 0x6b004, + .pd = { + .name = "pcie_0_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, +}; + +static struct gdsc pcie_1_gdsc = { + .gdscr = 0x8d004, + .pd = { + .name = "pcie_1_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, +}; + +static struct gdsc ufs_card_gdsc = { + .gdscr = 0x75004, + .pd = { + .name = "ufs_card_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, +}; + +static struct gdsc ufs_phy_gdsc = { + .gdscr = 0x77004, + .pd = { + .name = "ufs_phy_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, +}; + +static struct gdsc usb30_prim_gdsc = { + .gdscr = 0xf004, + .pd = { + .name = "usb30_prim_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, +}; + +static struct gdsc usb30_sec_gdsc = { + .gdscr = 0x10004, + .pd = { + .name = "usb30_sec_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, +}; + +static struct gdsc hlos1_vote_mmnoc_mmu_tbu_hf0_gdsc = { + .gdscr = 0x7d050, + .pd = { + .name = "hlos1_vote_mmnoc_mmu_tbu_hf0_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .flags = VOTABLE, +}; + +static struct gdsc hlos1_vote_mmnoc_mmu_tbu_hf1_gdsc = { + .gdscr = 0x7d058, + .pd = { + .name = "hlos1_vote_mmnoc_mmu_tbu_hf1_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .flags = VOTABLE, +}; + +static struct gdsc hlos1_vote_mmnoc_mmu_tbu_sf0_gdsc = { + .gdscr = 0x7d054, + .pd = { + .name = "hlos1_vote_mmnoc_mmu_tbu_sf0_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .flags = VOTABLE, +}; + +static struct gdsc hlos1_vote_mmnoc_mmu_tbu_sf1_gdsc = { + .gdscr = 0x7d06c, + .pd = { + .name = "hlos1_vote_mmnoc_mmu_tbu_sf1_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .flags = VOTABLE, +}; + +static struct clk_regmap *gcc_sm8350_clocks[] = { + [GCC_AGGRE_NOC_PCIE_0_AXI_CLK] = &gcc_aggre_noc_pcie_0_axi_clk.clkr, + [GCC_AGGRE_NOC_PCIE_1_AXI_CLK] = &gcc_aggre_noc_pcie_1_axi_clk.clkr, + [GCC_AGGRE_NOC_PCIE_TBU_CLK] = &gcc_aggre_noc_pcie_tbu_clk.clkr, + [GCC_AGGRE_UFS_CARD_AXI_CLK] = &gcc_aggre_ufs_card_axi_clk.clkr, + [GCC_AGGRE_UFS_CARD_AXI_HW_CTL_CLK] = &gcc_aggre_ufs_card_axi_hw_ctl_clk.clkr, + [GCC_AGGRE_UFS_PHY_AXI_CLK] = &gcc_aggre_ufs_phy_axi_clk.clkr, + [GCC_AGGRE_UFS_PHY_AXI_HW_CTL_CLK] = &gcc_aggre_ufs_phy_axi_hw_ctl_clk.clkr, + [GCC_AGGRE_USB3_PRIM_AXI_CLK] = &gcc_aggre_usb3_prim_axi_clk.clkr, + [GCC_AGGRE_USB3_SEC_AXI_CLK] = &gcc_aggre_usb3_sec_axi_clk.clkr, + [GCC_BOOT_ROM_AHB_CLK] = &gcc_boot_rom_ahb_clk.clkr, + [GCC_CAMERA_HF_AXI_CLK] = &gcc_camera_hf_axi_clk.clkr, + [GCC_CAMERA_SF_AXI_CLK] = &gcc_camera_sf_axi_clk.clkr, + [GCC_CFG_NOC_USB3_PRIM_AXI_CLK] = &gcc_cfg_noc_usb3_prim_axi_clk.clkr, + [GCC_CFG_NOC_USB3_SEC_AXI_CLK] = &gcc_cfg_noc_usb3_sec_axi_clk.clkr, + [GCC_DDRSS_GPU_AXI_CLK] = &gcc_ddrss_gpu_axi_clk.clkr, + [GCC_DDRSS_PCIE_SF_TBU_CLK] = &gcc_ddrss_pcie_sf_tbu_clk.clkr, + [GCC_DISP_HF_AXI_CLK] = &gcc_disp_hf_axi_clk.clkr, + [GCC_DISP_SF_AXI_CLK] = &gcc_disp_sf_axi_clk.clkr, + [GCC_GP1_CLK] = &gcc_gp1_clk.clkr, + [GCC_GP1_CLK_SRC] = &gcc_gp1_clk_src.clkr, + [GCC_GP2_CLK] = &gcc_gp2_clk.clkr, + [GCC_GP2_CLK_SRC] = &gcc_gp2_clk_src.clkr, + [GCC_GP3_CLK] = &gcc_gp3_clk.clkr, + [GCC_GP3_CLK_SRC] = &gcc_gp3_clk_src.clkr, + [GCC_GPLL0] = &gcc_gpll0.clkr, + [GCC_GPLL0_OUT_EVEN] = &gcc_gpll0_out_even.clkr, + [GCC_GPLL4] = &gcc_gpll4.clkr, + [GCC_GPLL9] = &gcc_gpll9.clkr, + [GCC_GPU_GPLL0_CLK_SRC] = &gcc_gpu_gpll0_clk_src.clkr, + [GCC_GPU_GPLL0_DIV_CLK_SRC] = &gcc_gpu_gpll0_div_clk_src.clkr, + [GCC_GPU_IREF_EN] = &gcc_gpu_iref_en.clkr, + [GCC_GPU_MEMNOC_GFX_CLK] = &gcc_gpu_memnoc_gfx_clk.clkr, + [GCC_GPU_SNOC_DVM_GFX_CLK] = &gcc_gpu_snoc_dvm_gfx_clk.clkr, + [GCC_PCIE0_PHY_RCHNG_CLK] = &gcc_pcie0_phy_rchng_clk.clkr, + [GCC_PCIE1_PHY_RCHNG_CLK] = &gcc_pcie1_phy_rchng_clk.clkr, + [GCC_PCIE_0_AUX_CLK] = &gcc_pcie_0_aux_clk.clkr, + [GCC_PCIE_0_AUX_CLK_SRC] = &gcc_pcie_0_aux_clk_src.clkr, + [GCC_PCIE_0_CFG_AHB_CLK] = &gcc_pcie_0_cfg_ahb_clk.clkr, + [GCC_PCIE_0_CLKREF_EN] = &gcc_pcie_0_clkref_en.clkr, + [GCC_PCIE_0_MSTR_AXI_CLK] = &gcc_pcie_0_mstr_axi_clk.clkr, + [GCC_PCIE_0_PHY_RCHNG_CLK_SRC] = &gcc_pcie_0_phy_rchng_clk_src.clkr, + [GCC_PCIE_0_PIPE_CLK] = &gcc_pcie_0_pipe_clk.clkr, + [GCC_PCIE_0_PIPE_CLK_SRC] = &gcc_pcie_0_pipe_clk_src.clkr, + [GCC_PCIE_0_SLV_AXI_CLK] = &gcc_pcie_0_slv_axi_clk.clkr, + [GCC_PCIE_0_SLV_Q2A_AXI_CLK] = &gcc_pcie_0_slv_q2a_axi_clk.clkr, + [GCC_PCIE_1_AUX_CLK] = &gcc_pcie_1_aux_clk.clkr, + [GCC_PCIE_1_AUX_CLK_SRC] = &gcc_pcie_1_aux_clk_src.clkr, + [GCC_PCIE_1_CFG_AHB_CLK] = &gcc_pcie_1_cfg_ahb_clk.clkr, + [GCC_PCIE_1_CLKREF_EN] = &gcc_pcie_1_clkref_en.clkr, + [GCC_PCIE_1_MSTR_AXI_CLK] = &gcc_pcie_1_mstr_axi_clk.clkr, + [GCC_PCIE_1_PHY_RCHNG_CLK_SRC] = &gcc_pcie_1_phy_rchng_clk_src.clkr, + [GCC_PCIE_1_PIPE_CLK] = &gcc_pcie_1_pipe_clk.clkr, + [GCC_PCIE_1_PIPE_CLK_SRC] = &gcc_pcie_1_pipe_clk_src.clkr, + [GCC_PCIE_1_SLV_AXI_CLK] = &gcc_pcie_1_slv_axi_clk.clkr, + [GCC_PCIE_1_SLV_Q2A_AXI_CLK] = &gcc_pcie_1_slv_q2a_axi_clk.clkr, + [GCC_PDM2_CLK] = &gcc_pdm2_clk.clkr, + [GCC_PDM2_CLK_SRC] = &gcc_pdm2_clk_src.clkr, + [GCC_PDM_AHB_CLK] = &gcc_pdm_ahb_clk.clkr, + [GCC_PDM_XO4_CLK] = &gcc_pdm_xo4_clk.clkr, + [GCC_QMIP_CAMERA_NRT_AHB_CLK] = &gcc_qmip_camera_nrt_ahb_clk.clkr, + [GCC_QMIP_CAMERA_RT_AHB_CLK] = &gcc_qmip_camera_rt_ahb_clk.clkr, + [GCC_QMIP_DISP_AHB_CLK] = &gcc_qmip_disp_ahb_clk.clkr, + [GCC_QMIP_VIDEO_CVP_AHB_CLK] = &gcc_qmip_video_cvp_ahb_clk.clkr, + [GCC_QMIP_VIDEO_VCODEC_AHB_CLK] = &gcc_qmip_video_vcodec_ahb_clk.clkr, + [GCC_QUPV3_WRAP0_CORE_2X_CLK] = &gcc_qupv3_wrap0_core_2x_clk.clkr, + [GCC_QUPV3_WRAP0_CORE_CLK] = &gcc_qupv3_wrap0_core_clk.clkr, + [GCC_QUPV3_WRAP0_S0_CLK] = &gcc_qupv3_wrap0_s0_clk.clkr, + [GCC_QUPV3_WRAP0_S0_CLK_SRC] = &gcc_qupv3_wrap0_s0_clk_src.clkr, + [GCC_QUPV3_WRAP0_S1_CLK] = &gcc_qupv3_wrap0_s1_clk.clkr, + [GCC_QUPV3_WRAP0_S1_CLK_SRC] = &gcc_qupv3_wrap0_s1_clk_src.clkr, + [GCC_QUPV3_WRAP0_S2_CLK] = &gcc_qupv3_wrap0_s2_clk.clkr, + [GCC_QUPV3_WRAP0_S2_CLK_SRC] = &gcc_qupv3_wrap0_s2_clk_src.clkr, + [GCC_QUPV3_WRAP0_S3_CLK] = &gcc_qupv3_wrap0_s3_clk.clkr, + [GCC_QUPV3_WRAP0_S3_CLK_SRC] = &gcc_qupv3_wrap0_s3_clk_src.clkr, + [GCC_QUPV3_WRAP0_S4_CLK] = &gcc_qupv3_wrap0_s4_clk.clkr, + [GCC_QUPV3_WRAP0_S4_CLK_SRC] = &gcc_qupv3_wrap0_s4_clk_src.clkr, + [GCC_QUPV3_WRAP0_S5_CLK] = &gcc_qupv3_wrap0_s5_clk.clkr, + [GCC_QUPV3_WRAP0_S5_CLK_SRC] = &gcc_qupv3_wrap0_s5_clk_src.clkr, + [GCC_QUPV3_WRAP0_S6_CLK] = &gcc_qupv3_wrap0_s6_clk.clkr, + [GCC_QUPV3_WRAP0_S6_CLK_SRC] = &gcc_qupv3_wrap0_s6_clk_src.clkr, + [GCC_QUPV3_WRAP0_S7_CLK] = &gcc_qupv3_wrap0_s7_clk.clkr, + [GCC_QUPV3_WRAP0_S7_CLK_SRC] = &gcc_qupv3_wrap0_s7_clk_src.clkr, + [GCC_QUPV3_WRAP1_CORE_2X_CLK] = &gcc_qupv3_wrap1_core_2x_clk.clkr, + [GCC_QUPV3_WRAP1_CORE_CLK] = &gcc_qupv3_wrap1_core_clk.clkr, + [GCC_QUPV3_WRAP1_S0_CLK] = &gcc_qupv3_wrap1_s0_clk.clkr, + [GCC_QUPV3_WRAP1_S0_CLK_SRC] = &gcc_qupv3_wrap1_s0_clk_src.clkr, + [GCC_QUPV3_WRAP1_S1_CLK] = &gcc_qupv3_wrap1_s1_clk.clkr, + [GCC_QUPV3_WRAP1_S1_CLK_SRC] = &gcc_qupv3_wrap1_s1_clk_src.clkr, + [GCC_QUPV3_WRAP1_S2_CLK] = &gcc_qupv3_wrap1_s2_clk.clkr, + [GCC_QUPV3_WRAP1_S2_CLK_SRC] = &gcc_qupv3_wrap1_s2_clk_src.clkr, + [GCC_QUPV3_WRAP1_S3_CLK] = &gcc_qupv3_wrap1_s3_clk.clkr, + [GCC_QUPV3_WRAP1_S3_CLK_SRC] = &gcc_qupv3_wrap1_s3_clk_src.clkr, + [GCC_QUPV3_WRAP1_S4_CLK] = &gcc_qupv3_wrap1_s4_clk.clkr, + [GCC_QUPV3_WRAP1_S4_CLK_SRC] = &gcc_qupv3_wrap1_s4_clk_src.clkr, + [GCC_QUPV3_WRAP1_S5_CLK] = &gcc_qupv3_wrap1_s5_clk.clkr, + [GCC_QUPV3_WRAP1_S5_CLK_SRC] = &gcc_qupv3_wrap1_s5_clk_src.clkr, + [GCC_QUPV3_WRAP2_CORE_2X_CLK] = &gcc_qupv3_wrap2_core_2x_clk.clkr, + [GCC_QUPV3_WRAP2_CORE_CLK] = &gcc_qupv3_wrap2_core_clk.clkr, + [GCC_QUPV3_WRAP2_S0_CLK] = &gcc_qupv3_wrap2_s0_clk.clkr, + [GCC_QUPV3_WRAP2_S0_CLK_SRC] = &gcc_qupv3_wrap2_s0_clk_src.clkr, + [GCC_QUPV3_WRAP2_S1_CLK] = &gcc_qupv3_wrap2_s1_clk.clkr, + [GCC_QUPV3_WRAP2_S1_CLK_SRC] = &gcc_qupv3_wrap2_s1_clk_src.clkr, + [GCC_QUPV3_WRAP2_S2_CLK] = &gcc_qupv3_wrap2_s2_clk.clkr, + [GCC_QUPV3_WRAP2_S2_CLK_SRC] = &gcc_qupv3_wrap2_s2_clk_src.clkr, + [GCC_QUPV3_WRAP2_S3_CLK] = &gcc_qupv3_wrap2_s3_clk.clkr, + [GCC_QUPV3_WRAP2_S3_CLK_SRC] = &gcc_qupv3_wrap2_s3_clk_src.clkr, + [GCC_QUPV3_WRAP2_S4_CLK] = &gcc_qupv3_wrap2_s4_clk.clkr, + [GCC_QUPV3_WRAP2_S4_CLK_SRC] = &gcc_qupv3_wrap2_s4_clk_src.clkr, + [GCC_QUPV3_WRAP2_S5_CLK] = &gcc_qupv3_wrap2_s5_clk.clkr, + [GCC_QUPV3_WRAP2_S5_CLK_SRC] = &gcc_qupv3_wrap2_s5_clk_src.clkr, + [GCC_QUPV3_WRAP_0_M_AHB_CLK] = &gcc_qupv3_wrap_0_m_ahb_clk.clkr, + [GCC_QUPV3_WRAP_0_S_AHB_CLK] = &gcc_qupv3_wrap_0_s_ahb_clk.clkr, + [GCC_QUPV3_WRAP_1_M_AHB_CLK] = &gcc_qupv3_wrap_1_m_ahb_clk.clkr, + [GCC_QUPV3_WRAP_1_S_AHB_CLK] = &gcc_qupv3_wrap_1_s_ahb_clk.clkr, + [GCC_QUPV3_WRAP_2_M_AHB_CLK] = &gcc_qupv3_wrap_2_m_ahb_clk.clkr, + [GCC_QUPV3_WRAP_2_S_AHB_CLK] = &gcc_qupv3_wrap_2_s_ahb_clk.clkr, + [GCC_SDCC2_AHB_CLK] = &gcc_sdcc2_ahb_clk.clkr, + [GCC_SDCC2_APPS_CLK] = &gcc_sdcc2_apps_clk.clkr, + [GCC_SDCC2_APPS_CLK_SRC] = &gcc_sdcc2_apps_clk_src.clkr, + [GCC_SDCC4_AHB_CLK] = &gcc_sdcc4_ahb_clk.clkr, + [GCC_SDCC4_APPS_CLK] = &gcc_sdcc4_apps_clk.clkr, + [GCC_SDCC4_APPS_CLK_SRC] = &gcc_sdcc4_apps_clk_src.clkr, + [GCC_THROTTLE_PCIE_AHB_CLK] = &gcc_throttle_pcie_ahb_clk.clkr, + [GCC_UFS_1_CLKREF_EN] = &gcc_ufs_1_clkref_en.clkr, + [GCC_UFS_CARD_AHB_CLK] = &gcc_ufs_card_ahb_clk.clkr, + [GCC_UFS_CARD_AXI_CLK] = &gcc_ufs_card_axi_clk.clkr, + [GCC_UFS_CARD_AXI_CLK_SRC] = &gcc_ufs_card_axi_clk_src.clkr, + [GCC_UFS_CARD_AXI_HW_CTL_CLK] = &gcc_ufs_card_axi_hw_ctl_clk.clkr, + [GCC_UFS_CARD_ICE_CORE_CLK] = &gcc_ufs_card_ice_core_clk.clkr, + [GCC_UFS_CARD_ICE_CORE_CLK_SRC] = &gcc_ufs_card_ice_core_clk_src.clkr, + [GCC_UFS_CARD_ICE_CORE_HW_CTL_CLK] = &gcc_ufs_card_ice_core_hw_ctl_clk.clkr, + [GCC_UFS_CARD_PHY_AUX_CLK] = &gcc_ufs_card_phy_aux_clk.clkr, + [GCC_UFS_CARD_PHY_AUX_CLK_SRC] = &gcc_ufs_card_phy_aux_clk_src.clkr, + [GCC_UFS_CARD_PHY_AUX_HW_CTL_CLK] = &gcc_ufs_card_phy_aux_hw_ctl_clk.clkr, + [GCC_UFS_CARD_RX_SYMBOL_0_CLK] = &gcc_ufs_card_rx_symbol_0_clk.clkr, + [GCC_UFS_CARD_RX_SYMBOL_0_CLK_SRC] = &gcc_ufs_card_rx_symbol_0_clk_src.clkr, + [GCC_UFS_CARD_RX_SYMBOL_1_CLK] = &gcc_ufs_card_rx_symbol_1_clk.clkr, + [GCC_UFS_CARD_RX_SYMBOL_1_CLK_SRC] = &gcc_ufs_card_rx_symbol_1_clk_src.clkr, + [GCC_UFS_CARD_TX_SYMBOL_0_CLK] = &gcc_ufs_card_tx_symbol_0_clk.clkr, + [GCC_UFS_CARD_TX_SYMBOL_0_CLK_SRC] = &gcc_ufs_card_tx_symbol_0_clk_src.clkr, + [GCC_UFS_CARD_UNIPRO_CORE_CLK] = &gcc_ufs_card_unipro_core_clk.clkr, + [GCC_UFS_CARD_UNIPRO_CORE_CLK_SRC] = &gcc_ufs_card_unipro_core_clk_src.clkr, + [GCC_UFS_CARD_UNIPRO_CORE_HW_CTL_CLK] = &gcc_ufs_card_unipro_core_hw_ctl_clk.clkr, + [GCC_UFS_PHY_AHB_CLK] = &gcc_ufs_phy_ahb_clk.clkr, + [GCC_UFS_PHY_AXI_CLK] = &gcc_ufs_phy_axi_clk.clkr, + [GCC_UFS_PHY_AXI_CLK_SRC] = &gcc_ufs_phy_axi_clk_src.clkr, + [GCC_UFS_PHY_AXI_HW_CTL_CLK] = &gcc_ufs_phy_axi_hw_ctl_clk.clkr, + [GCC_UFS_PHY_ICE_CORE_CLK] = &gcc_ufs_phy_ice_core_clk.clkr, + [GCC_UFS_PHY_ICE_CORE_CLK_SRC] = &gcc_ufs_phy_ice_core_clk_src.clkr, + [GCC_UFS_PHY_ICE_CORE_HW_CTL_CLK] = &gcc_ufs_phy_ice_core_hw_ctl_clk.clkr, + [GCC_UFS_PHY_PHY_AUX_CLK] = &gcc_ufs_phy_phy_aux_clk.clkr, + [GCC_UFS_PHY_PHY_AUX_CLK_SRC] = &gcc_ufs_phy_phy_aux_clk_src.clkr, + [GCC_UFS_PHY_PHY_AUX_HW_CTL_CLK] = &gcc_ufs_phy_phy_aux_hw_ctl_clk.clkr, + [GCC_UFS_PHY_RX_SYMBOL_0_CLK] = &gcc_ufs_phy_rx_symbol_0_clk.clkr, + [GCC_UFS_PHY_RX_SYMBOL_0_CLK_SRC] = &gcc_ufs_phy_rx_symbol_0_clk_src.clkr, + [GCC_UFS_PHY_RX_SYMBOL_1_CLK] = &gcc_ufs_phy_rx_symbol_1_clk.clkr, + [GCC_UFS_PHY_RX_SYMBOL_1_CLK_SRC] = &gcc_ufs_phy_rx_symbol_1_clk_src.clkr, + [GCC_UFS_PHY_TX_SYMBOL_0_CLK] = &gcc_ufs_phy_tx_symbol_0_clk.clkr, + [GCC_UFS_PHY_TX_SYMBOL_0_CLK_SRC] = &gcc_ufs_phy_tx_symbol_0_clk_src.clkr, + [GCC_UFS_PHY_UNIPRO_CORE_CLK] = &gcc_ufs_phy_unipro_core_clk.clkr, + [GCC_UFS_PHY_UNIPRO_CORE_CLK_SRC] = &gcc_ufs_phy_unipro_core_clk_src.clkr, + [GCC_UFS_PHY_UNIPRO_CORE_HW_CTL_CLK] = &gcc_ufs_phy_unipro_core_hw_ctl_clk.clkr, + [GCC_USB30_PRIM_MASTER_CLK] = &gcc_usb30_prim_master_clk.clkr, + [GCC_USB30_PRIM_MASTER_CLK__FORCE_MEM_CORE_ON] = + &gcc_usb30_prim_master_clk__force_mem_core_on.clkr, + [GCC_USB30_PRIM_MASTER_CLK_SRC] = &gcc_usb30_prim_master_clk_src.clkr, + [GCC_USB30_PRIM_MOCK_UTMI_CLK] = &gcc_usb30_prim_mock_utmi_clk.clkr, + [GCC_USB30_PRIM_MOCK_UTMI_CLK_SRC] = &gcc_usb30_prim_mock_utmi_clk_src.clkr, + [GCC_USB30_PRIM_MOCK_UTMI_POSTDIV_CLK_SRC] = &gcc_usb30_prim_mock_utmi_postdiv_clk_src.clkr, + [GCC_USB30_PRIM_SLEEP_CLK] = &gcc_usb30_prim_sleep_clk.clkr, + [GCC_USB30_SEC_MASTER_CLK] = &gcc_usb30_sec_master_clk.clkr, + [GCC_USB30_SEC_MASTER_CLK__FORCE_MEM_CORE_ON] = + &gcc_usb30_sec_master_clk__force_mem_core_on.clkr, + [GCC_USB30_SEC_MASTER_CLK_SRC] = &gcc_usb30_sec_master_clk_src.clkr, + [GCC_USB30_SEC_MOCK_UTMI_CLK] = &gcc_usb30_sec_mock_utmi_clk.clkr, + [GCC_USB30_SEC_MOCK_UTMI_CLK_SRC] = &gcc_usb30_sec_mock_utmi_clk_src.clkr, + [GCC_USB30_SEC_MOCK_UTMI_POSTDIV_CLK_SRC] = &gcc_usb30_sec_mock_utmi_postdiv_clk_src.clkr, + [GCC_USB30_SEC_SLEEP_CLK] = &gcc_usb30_sec_sleep_clk.clkr, + [GCC_USB3_PRIM_PHY_AUX_CLK] = &gcc_usb3_prim_phy_aux_clk.clkr, + [GCC_USB3_PRIM_PHY_AUX_CLK_SRC] = &gcc_usb3_prim_phy_aux_clk_src.clkr, + [GCC_USB3_PRIM_PHY_COM_AUX_CLK] = &gcc_usb3_prim_phy_com_aux_clk.clkr, + [GCC_USB3_PRIM_PHY_PIPE_CLK] = &gcc_usb3_prim_phy_pipe_clk.clkr, + [GCC_USB3_PRIM_PHY_PIPE_CLK_SRC] = &gcc_usb3_prim_phy_pipe_clk_src.clkr, + [GCC_USB3_SEC_CLKREF_EN] = &gcc_usb3_sec_clkref_en.clkr, + [GCC_USB3_SEC_PHY_AUX_CLK] = &gcc_usb3_sec_phy_aux_clk.clkr, + [GCC_USB3_SEC_PHY_AUX_CLK_SRC] = &gcc_usb3_sec_phy_aux_clk_src.clkr, + [GCC_USB3_SEC_PHY_COM_AUX_CLK] = &gcc_usb3_sec_phy_com_aux_clk.clkr, + [GCC_USB3_SEC_PHY_PIPE_CLK] = &gcc_usb3_sec_phy_pipe_clk.clkr, + [GCC_USB3_SEC_PHY_PIPE_CLK_SRC] = &gcc_usb3_sec_phy_pipe_clk_src.clkr, + [GCC_VIDEO_AXI0_CLK] = &gcc_video_axi0_clk.clkr, + [GCC_VIDEO_AXI1_CLK] = &gcc_video_axi1_clk.clkr, +}; + +static struct gdsc *gcc_sm8350_gdscs[] = { + [PCIE_0_GDSC] = &pcie_0_gdsc, + [PCIE_1_GDSC] = &pcie_1_gdsc, + [UFS_CARD_GDSC] = &ufs_card_gdsc, + [UFS_PHY_GDSC] = &ufs_phy_gdsc, + [USB30_PRIM_GDSC] = &usb30_prim_gdsc, + [USB30_SEC_GDSC] = &usb30_sec_gdsc, + [HLOS1_VOTE_MMNOC_MMU_TBU_HF0_GDSC] = &hlos1_vote_mmnoc_mmu_tbu_hf0_gdsc, + [HLOS1_VOTE_MMNOC_MMU_TBU_HF1_GDSC] = &hlos1_vote_mmnoc_mmu_tbu_hf1_gdsc, + [HLOS1_VOTE_MMNOC_MMU_TBU_SF0_GDSC] = &hlos1_vote_mmnoc_mmu_tbu_sf0_gdsc, + [HLOS1_VOTE_MMNOC_MMU_TBU_SF1_GDSC] = &hlos1_vote_mmnoc_mmu_tbu_sf1_gdsc, +}; + +static const struct qcom_reset_map gcc_sm8350_resets[] = { + [GCC_CAMERA_BCR] = { 0x26000 }, + [GCC_DISPLAY_BCR] = { 0x27000 }, + [GCC_GPU_BCR] = { 0x71000 }, + [GCC_MMSS_BCR] = { 0xb000 }, + [GCC_PCIE_0_BCR] = { 0x6b000 }, + [GCC_PCIE_0_LINK_DOWN_BCR] = { 0x6c014 }, + [GCC_PCIE_0_NOCSR_COM_PHY_BCR] = { 0x6c020 }, + [GCC_PCIE_0_PHY_BCR] = { 0x6c01c }, + [GCC_PCIE_0_PHY_NOCSR_COM_PHY_BCR] = { 0x6c028 }, + [GCC_PCIE_1_BCR] = { 0x8d000 }, + [GCC_PCIE_1_LINK_DOWN_BCR] = { 0x8e014 }, + [GCC_PCIE_1_NOCSR_COM_PHY_BCR] = { 0x8e020 }, + [GCC_PCIE_1_PHY_BCR] = { 0x8e01c }, + [GCC_PCIE_1_PHY_NOCSR_COM_PHY_BCR] = { 0x8e000 }, + [GCC_PCIE_PHY_CFG_AHB_BCR] = { 0x6f00c }, + [GCC_PCIE_PHY_COM_BCR] = { 0x6f010 }, + [GCC_PDM_BCR] = { 0x33000 }, + [GCC_QUPV3_WRAPPER_0_BCR] = { 0x17000 }, + [GCC_QUPV3_WRAPPER_1_BCR] = { 0x18000 }, + [GCC_QUPV3_WRAPPER_2_BCR] = { 0x1e000 }, + [GCC_QUSB2PHY_PRIM_BCR] = { 0x12000 }, + [GCC_QUSB2PHY_SEC_BCR] = { 0x12004 }, + [GCC_SDCC2_BCR] = { 0x14000 }, + [GCC_SDCC4_BCR] = { 0x16000 }, + [GCC_UFS_CARD_BCR] = { 0x75000 }, + [GCC_UFS_PHY_BCR] = { 0x77000 }, + [GCC_USB30_PRIM_BCR] = { 0xf000 }, + [GCC_USB30_SEC_BCR] = { 0x10000 }, + [GCC_USB3_DP_PHY_PRIM_BCR] = { 0x50008 }, + [GCC_USB3_DP_PHY_SEC_BCR] = { 0x50014 }, + [GCC_USB3_PHY_PRIM_BCR] = { 0x50000 }, + [GCC_USB3_PHY_SEC_BCR] = { 0x5000c }, + [GCC_USB3PHY_PHY_PRIM_BCR] = { 0x50004 }, + [GCC_USB3PHY_PHY_SEC_BCR] = { 0x50010 }, + [GCC_USB_PHY_CFG_AHB2PHY_BCR] = { 0x6a000 }, + [GCC_VIDEO_AXI0_CLK_ARES] = { 0x28010, 2 }, + [GCC_VIDEO_AXI1_CLK_ARES] = { 0x28018, 2 }, + [GCC_VIDEO_BCR] = { 0x28000 }, +}; + +static const struct clk_rcg_dfs_data gcc_dfs_clocks[] = { + DEFINE_RCG_DFS(gcc_qupv3_wrap0_s0_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap0_s1_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap0_s2_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap0_s3_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap0_s4_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap0_s5_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap0_s6_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap0_s7_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap1_s0_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap1_s1_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap1_s2_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap1_s3_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap1_s4_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap1_s5_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap2_s0_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap2_s1_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap2_s2_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap2_s3_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap2_s4_clk_src), + DEFINE_RCG_DFS(gcc_qupv3_wrap2_s5_clk_src), +}; + +static const struct regmap_config gcc_sm8350_regmap_config = { + .reg_bits = 32, + .reg_stride = 4, + .val_bits = 32, + .max_register = 0x9c100, + .fast_io = true, +}; + +static const struct qcom_cc_desc gcc_sm8350_desc = { + .config = &gcc_sm8350_regmap_config, + .clks = gcc_sm8350_clocks, + .num_clks = ARRAY_SIZE(gcc_sm8350_clocks), + .resets = gcc_sm8350_resets, + .num_resets = ARRAY_SIZE(gcc_sm8350_resets), + .gdscs = gcc_sm8350_gdscs, + .num_gdscs = ARRAY_SIZE(gcc_sm8350_gdscs), +}; + +static const struct of_device_id gcc_sm8350_match_table[] = { + { .compatible = "qcom,gcc-sm8350" }, + { } +}; +MODULE_DEVICE_TABLE(of, gcc_sm8350_match_table); + +static int gcc_sm8350_probe(struct platform_device *pdev) +{ + struct regmap *regmap; + int ret; + + regmap = qcom_cc_map(pdev, &gcc_sm8350_desc); + if (IS_ERR(regmap)) { + dev_err(&pdev->dev, "Failed to map gcc registers\n"); + return PTR_ERR(regmap); + } + + /* + * Keep the critical clock always-On + * GCC_CAMERA_AHB_CLK, GCC_CAMERA_XO_CLK, GCC_DISP_AHB_CLK, GCC_DISP_XO_CLK, + * GCC_GPU_CFG_AHB_CLK, GCC_VIDEO_AHB_CLK, GCC_VIDEO_XO_CLK + */ + regmap_update_bits(regmap, 0x26004, BIT(0), BIT(0)); + regmap_update_bits(regmap, 0x26018, BIT(0), BIT(0)); + regmap_update_bits(regmap, 0x27004, BIT(0), BIT(0)); + regmap_update_bits(regmap, 0x2701c, BIT(0), BIT(0)); + regmap_update_bits(regmap, 0x71004, BIT(0), BIT(0)); + regmap_update_bits(regmap, 0x28004, BIT(0), BIT(0)); + regmap_update_bits(regmap, 0x28020, BIT(0), BIT(0)); + + ret = qcom_cc_register_rcg_dfs(regmap, gcc_dfs_clocks, ARRAY_SIZE(gcc_dfs_clocks)); + if (ret) + return ret; + + /* FORCE_MEM_CORE_ON for ufs phy ice core clocks */ + regmap_update_bits(regmap, gcc_ufs_phy_ice_core_clk.halt_reg, BIT(14), BIT(14)); + + return qcom_cc_really_probe(pdev, &gcc_sm8350_desc, regmap); +} + +static struct platform_driver gcc_sm8350_driver = { + .probe = gcc_sm8350_probe, + .driver = { + .name = "sm8350-gcc", + .of_match_table = gcc_sm8350_match_table, + }, +}; + +static int __init gcc_sm8350_init(void) +{ + return platform_driver_register(&gcc_sm8350_driver); +} +subsys_initcall(gcc_sm8350_init); + +static void __exit gcc_sm8350_exit(void) +{ + platform_driver_unregister(&gcc_sm8350_driver); +} +module_exit(gcc_sm8350_exit); + +MODULE_DESCRIPTION("QTI GCC SM8350 Driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/clk/qcom/gdsc.c b/drivers/clk/qcom/gdsc.c index af26e0695b8661a2b145abcce0c27b8fc946579c..51ed640e527b4871eb007b6994ddfc8a725ae6dd 100644 --- a/drivers/clk/qcom/gdsc.c +++ b/drivers/clk/qcom/gdsc.c @@ -183,7 +183,10 @@ static inline int gdsc_assert_reset(struct gdsc *sc) static inline void gdsc_force_mem_on(struct gdsc *sc) { int i; - u32 mask = RETAIN_MEM | RETAIN_PERIPH; + u32 mask = RETAIN_MEM; + + if (!(sc->flags & NO_RET_PERIPH)) + mask |= RETAIN_PERIPH; for (i = 0; i < sc->cxc_count; i++) regmap_update_bits(sc->regmap, sc->cxcs[i], mask, mask); @@ -192,7 +195,10 @@ static inline void gdsc_force_mem_on(struct gdsc *sc) static inline void gdsc_clear_mem_on(struct gdsc *sc) { int i; - u32 mask = RETAIN_MEM | RETAIN_PERIPH; + u32 mask = RETAIN_MEM; + + if (!(sc->flags & NO_RET_PERIPH)) + mask |= RETAIN_PERIPH; for (i = 0; i < sc->cxc_count; i++) regmap_update_bits(sc->regmap, sc->cxcs[i], mask, 0); diff --git a/drivers/clk/qcom/gdsc.h b/drivers/clk/qcom/gdsc.h index bd537438c79321a9d11c1d3a84bed3776a4e1984..5bb396b344d16f814f23c0ce7f717eac859c7b54 100644 --- a/drivers/clk/qcom/gdsc.h +++ b/drivers/clk/qcom/gdsc.h @@ -42,7 +42,7 @@ struct gdsc { #define PWRSTS_ON BIT(2) #define PWRSTS_OFF_ON (PWRSTS_OFF | PWRSTS_ON) #define PWRSTS_RET_ON (PWRSTS_RET | PWRSTS_ON) - const u8 flags; + const u16 flags; #define VOTABLE BIT(0) #define CLAMP_IO BIT(1) #define HW_CTRL BIT(2) @@ -51,6 +51,7 @@ struct gdsc { #define POLL_CFG_GDSCR BIT(5) #define ALWAYS_ON BIT(6) #define RETAIN_FF_ENABLE BIT(7) +#define NO_RET_PERIPH BIT(8) struct reset_controller_dev *rcdev; unsigned int *resets; unsigned int reset_count; diff --git a/drivers/clk/qcom/gpucc-msm8998.c b/drivers/clk/qcom/gpucc-msm8998.c index 9b3923af02a142dcf7c821114adee5522a3c542f..fedfffaf0a8db98d6f6f5f827b3fafe8f0510686 100644 --- a/drivers/clk/qcom/gpucc-msm8998.c +++ b/drivers/clk/qcom/gpucc-msm8998.c @@ -50,6 +50,11 @@ static struct clk_branch gpucc_cxo_clk = { }, }; +static struct pll_vco fabia_vco[] = { + { 249600000, 2000000000, 0 }, + { 125000000, 1000000000, 1 }, +}; + static const struct clk_div_table post_div_table_fabia_even[] = { { 0x0, 1 }, { 0x1, 2 }, @@ -61,11 +66,13 @@ static const struct clk_div_table post_div_table_fabia_even[] = { static struct clk_alpha_pll gpupll0 = { .offset = 0x0, .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_FABIA], + .vco_table = fabia_vco, + .num_vco = ARRAY_SIZE(fabia_vco), .clkr.hw.init = &(struct clk_init_data){ .name = "gpupll0", .parent_hws = (const struct clk_hw *[]){ &gpucc_cxo_clk.clkr.hw }, .num_parents = 1, - .ops = &clk_alpha_pll_fixed_fabia_ops, + .ops = &clk_alpha_pll_fabia_ops, }, }; @@ -80,6 +87,7 @@ static struct clk_alpha_pll_postdiv gpupll0_out_even = { .name = "gpupll0_out_even", .parent_hws = (const struct clk_hw *[]){ &gpupll0.clkr.hw }, .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, .ops = &clk_alpha_pll_postdiv_fabia_ops, }, }; @@ -253,12 +261,16 @@ static struct gdsc gpu_cx_gdsc = { static struct gdsc gpu_gx_gdsc = { .gdscr = 0x1094, .clamp_io_ctrl = 0x130, + .resets = (unsigned int []){ GPU_GX_BCR }, + .reset_count = 1, + .cxcs = (unsigned int []){ 0x1098 }, + .cxc_count = 1, .pd = { .name = "gpu_gx", }, .parent = &gpu_cx_gdsc.pd, - .pwrsts = PWRSTS_OFF_ON, - .flags = CLAMP_IO | AON_RESET, + .pwrsts = PWRSTS_OFF_ON | PWRSTS_RET, + .flags = CLAMP_IO | SW_RESET | AON_RESET | NO_RET_PERIPH, }; static struct clk_regmap *gpucc_msm8998_clocks[] = { diff --git a/drivers/clk/qcom/gpucc-sdm660.c b/drivers/clk/qcom/gpucc-sdm660.c new file mode 100644 index 0000000000000000000000000000000000000000..1ebcceb3a50dd9814d99ce5120730be81c5768c1 --- /dev/null +++ b/drivers/clk/qcom/gpucc-sdm660.c @@ -0,0 +1,349 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2020, The Linux Foundation. All rights reserved. + * Copyright (c) 2020, AngeloGioacchino Del Regno + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "clk-alpha-pll.h" +#include "common.h" +#include "clk-regmap.h" +#include "clk-pll.h" +#include "clk-rcg.h" +#include "clk-branch.h" +#include "gdsc.h" +#include "reset.h" + +enum { + P_GPU_XO, + P_CORE_BI_PLL_TEST_SE, + P_GPLL0_OUT_MAIN, + P_GPLL0_OUT_MAIN_DIV, + P_GPU_PLL0_PLL_OUT_MAIN, + P_GPU_PLL1_PLL_OUT_MAIN, +}; + +static struct clk_branch gpucc_cxo_clk = { + .halt_reg = 0x1020, + .clkr = { + .enable_reg = 0x1020, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gpucc_cxo_clk", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "xo", + .name = "xo" + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_IS_CRITICAL, + }, + }, +}; + +static struct pll_vco gpu_vco[] = { + { 1000000000, 2000000000, 0 }, + { 500000000, 1000000000, 2 }, + { 250000000, 500000000, 3 }, +}; + +static struct clk_alpha_pll gpu_pll0_pll_out_main = { + .offset = 0x0, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .vco_table = gpu_vco, + .num_vco = ARRAY_SIZE(gpu_vco), + .clkr.hw.init = &(struct clk_init_data){ + .name = "gpu_pll0_pll_out_main", + .parent_data = &(const struct clk_parent_data){ + .hw = &gpucc_cxo_clk.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_alpha_pll_ops, + }, +}; + +static struct clk_alpha_pll gpu_pll1_pll_out_main = { + .offset = 0x40, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .vco_table = gpu_vco, + .num_vco = ARRAY_SIZE(gpu_vco), + .clkr.hw.init = &(struct clk_init_data){ + .name = "gpu_pll1_pll_out_main", + .parent_data = &(const struct clk_parent_data){ + .hw = &gpucc_cxo_clk.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_alpha_pll_ops, + }, +}; + +static const struct parent_map gpucc_parent_map_1[] = { + { P_GPU_XO, 0 }, + { P_GPU_PLL0_PLL_OUT_MAIN, 1 }, + { P_GPU_PLL1_PLL_OUT_MAIN, 3 }, + { P_GPLL0_OUT_MAIN, 5 }, +}; + +static const struct clk_parent_data gpucc_parent_data_1[] = { + { .hw = &gpucc_cxo_clk.clkr.hw }, + { .hw = &gpu_pll0_pll_out_main.clkr.hw }, + { .hw = &gpu_pll1_pll_out_main.clkr.hw }, + { .fw_name = "gcc_gpu_gpll0_clk", .name = "gcc_gpu_gpll0_clk" }, +}; + +static struct clk_rcg2_gfx3d gfx3d_clk_src = { + .div = 2, + .rcg = { + .cmd_rcgr = 0x1070, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gpucc_parent_map_1, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gfx3d_clk_src", + .parent_data = gpucc_parent_data_1, + .num_parents = 4, + .ops = &clk_gfx3d_ops, + .flags = CLK_SET_RATE_PARENT | CLK_OPS_PARENT_ENABLE, + }, + }, + .hws = (struct clk_hw*[]){ + &gpucc_cxo_clk.clkr.hw, + &gpu_pll0_pll_out_main.clkr.hw, + &gpu_pll1_pll_out_main.clkr.hw, + } +}; + +static struct clk_branch gpucc_gfx3d_clk = { + .halt_reg = 0x1098, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0x1098, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x1098, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gpucc_gfx3d_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &gfx3d_clk_src.rcg.clkr.hw, + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, +}; + +static const struct parent_map gpucc_parent_map_0[] = { + { P_GPU_XO, 0 }, + { P_GPLL0_OUT_MAIN, 5 }, + { P_GPLL0_OUT_MAIN_DIV, 6 }, +}; + +static const struct clk_parent_data gpucc_parent_data_0[] = { + { .hw = &gpucc_cxo_clk.clkr.hw }, + { .fw_name = "gcc_gpu_gpll0_clk", .name = "gcc_gpu_gpll0_clk" }, + { .fw_name = "gcc_gpu_gpll0_div_clk", .name = "gcc_gpu_gpll0_div_clk" }, +}; + +static const struct freq_tbl ftbl_rbbmtimer_clk_src[] = { + F(19200000, P_GPU_XO, 1, 0, 0), + { } +}; + +static struct clk_rcg2 rbbmtimer_clk_src = { + .cmd_rcgr = 0x10b0, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gpucc_parent_map_0, + .freq_tbl = ftbl_rbbmtimer_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "rbbmtimer_clk_src", + .parent_data = gpucc_parent_data_0, + .num_parents = 3, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_rbcpr_clk_src[] = { + F(19200000, P_GPU_XO, 1, 0, 0), + F(50000000, P_GPLL0_OUT_MAIN_DIV, 6, 0, 0), + { } +}; + +static struct clk_rcg2 rbcpr_clk_src = { + .cmd_rcgr = 0x1030, + .mnd_width = 0, + .hid_width = 5, + .parent_map = gpucc_parent_map_0, + .freq_tbl = ftbl_rbcpr_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "rbcpr_clk_src", + .parent_data = gpucc_parent_data_0, + .num_parents = 3, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_branch gpucc_rbbmtimer_clk = { + .halt_reg = 0x10d0, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x10d0, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gpucc_rbbmtimer_clk", + .parent_names = (const char *[]){ + "rbbmtimer_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch gpucc_rbcpr_clk = { + .halt_reg = 0x1054, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x1054, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "gpucc_rbcpr_clk", + .parent_names = (const char *[]){ + "rbcpr_clk_src", + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct gdsc gpu_cx_gdsc = { + .gdscr = 0x1004, + .gds_hw_ctrl = 0x1008, + .pd = { + .name = "gpu_cx", + }, + .pwrsts = PWRSTS_OFF_ON, + .flags = VOTABLE, +}; + +static struct gdsc gpu_gx_gdsc = { + .gdscr = 0x1094, + .clamp_io_ctrl = 0x130, + .resets = (unsigned int []){ GPU_GX_BCR }, + .reset_count = 1, + .cxcs = (unsigned int []){ 0x1098 }, + .cxc_count = 1, + .pd = { + .name = "gpu_gx", + }, + .parent = &gpu_cx_gdsc.pd, + .pwrsts = PWRSTS_OFF | PWRSTS_ON | PWRSTS_RET, + .flags = CLAMP_IO | SW_RESET | AON_RESET | NO_RET_PERIPH, +}; + +static struct gdsc *gpucc_sdm660_gdscs[] = { + [GPU_CX_GDSC] = &gpu_cx_gdsc, + [GPU_GX_GDSC] = &gpu_gx_gdsc, +}; + +static const struct qcom_reset_map gpucc_sdm660_resets[] = { + [GPU_CX_BCR] = { 0x1000 }, + [RBCPR_BCR] = { 0x1050 }, + [GPU_GX_BCR] = { 0x1090 }, + [SPDM_BCR] = { 0x10E0 }, +}; + +static struct clk_regmap *gpucc_sdm660_clocks[] = { + [GPUCC_CXO_CLK] = &gpucc_cxo_clk.clkr, + [GPU_PLL0_PLL] = &gpu_pll0_pll_out_main.clkr, + [GPU_PLL1_PLL] = &gpu_pll1_pll_out_main.clkr, + [GFX3D_CLK_SRC] = &gfx3d_clk_src.rcg.clkr, + [RBCPR_CLK_SRC] = &rbcpr_clk_src.clkr, + [RBBMTIMER_CLK_SRC] = &rbbmtimer_clk_src.clkr, + [GPUCC_RBCPR_CLK] = &gpucc_rbcpr_clk.clkr, + [GPUCC_GFX3D_CLK] = &gpucc_gfx3d_clk.clkr, + [GPUCC_RBBMTIMER_CLK] = &gpucc_rbbmtimer_clk.clkr, +}; + +static const struct regmap_config gpucc_660_regmap_config = { + .reg_bits = 32, + .reg_stride = 4, + .val_bits = 32, + .max_register = 0x9034, + .fast_io = true, +}; + +static const struct qcom_cc_desc gpucc_sdm660_desc = { + .config = &gpucc_660_regmap_config, + .clks = gpucc_sdm660_clocks, + .num_clks = ARRAY_SIZE(gpucc_sdm660_clocks), + .resets = gpucc_sdm660_resets, + .num_resets = ARRAY_SIZE(gpucc_sdm660_resets), + .gdscs = gpucc_sdm660_gdscs, + .num_gdscs = ARRAY_SIZE(gpucc_sdm660_gdscs), +}; + +static const struct of_device_id gpucc_sdm660_match_table[] = { + { .compatible = "qcom,gpucc-sdm660" }, + { .compatible = "qcom,gpucc-sdm630" }, + { } +}; +MODULE_DEVICE_TABLE(of, gpucc_sdm660_match_table); + +static int gpucc_sdm660_probe(struct platform_device *pdev) +{ + struct regmap *regmap; + struct alpha_pll_config gpu_pll_config = { + .config_ctl_val = 0x4001055b, + .alpha = 0xaaaaab00, + .alpha_en_mask = BIT(24), + .vco_val = 0x2 << 20, + .vco_mask = 0x3 << 20, + .main_output_mask = 0x1, + }; + + regmap = qcom_cc_map(pdev, &gpucc_sdm660_desc); + if (IS_ERR(regmap)) + return PTR_ERR(regmap); + + /* 800MHz configuration for GPU PLL0 */ + gpu_pll_config.l = 0x29; + gpu_pll_config.alpha_hi = 0xaa; + clk_alpha_pll_configure(&gpu_pll0_pll_out_main, regmap, &gpu_pll_config); + + /* 740MHz configuration for GPU PLL1 */ + gpu_pll_config.l = 0x26; + gpu_pll_config.alpha_hi = 0x8a; + clk_alpha_pll_configure(&gpu_pll1_pll_out_main, regmap, &gpu_pll_config); + + return qcom_cc_really_probe(pdev, &gpucc_sdm660_desc, regmap); +} + +static struct platform_driver gpucc_sdm660_driver = { + .probe = gpucc_sdm660_probe, + .driver = { + .name = "gpucc-sdm660", + .of_match_table = gpucc_sdm660_match_table, + }, +}; +module_platform_driver(gpucc_sdm660_driver); + +MODULE_DESCRIPTION("Qualcomm SDM630/SDM660 GPUCC Driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/clk/qcom/lpass-gfm-sm8250.c b/drivers/clk/qcom/lpass-gfm-sm8250.c index d366c7c2abc778d83d9df02e601043308ec11a1c..f5e31e692b9b4976d9450037b2ed3c176e2c2062 100644 --- a/drivers/clk/qcom/lpass-gfm-sm8250.c +++ b/drivers/clk/qcom/lpass-gfm-sm8250.c @@ -33,14 +33,13 @@ struct clk_gfm { void __iomem *gfm_mux; }; -#define GFM_MASK BIT(1) #define to_clk_gfm(_hw) container_of(_hw, struct clk_gfm, hw) static u8 clk_gfm_get_parent(struct clk_hw *hw) { struct clk_gfm *clk = to_clk_gfm(hw); - return readl(clk->gfm_mux) & GFM_MASK; + return readl(clk->gfm_mux) & clk->mux_mask; } static int clk_gfm_set_parent(struct clk_hw *hw, u8 index) @@ -51,9 +50,10 @@ static int clk_gfm_set_parent(struct clk_hw *hw, u8 index) val = readl(clk->gfm_mux); if (index) - val |= GFM_MASK; + val |= clk->mux_mask; else - val &= ~GFM_MASK; + val &= ~clk->mux_mask; + writel(val, clk->gfm_mux); diff --git a/drivers/clk/qcom/mmcc-msm8974.c b/drivers/clk/qcom/mmcc-msm8974.c index 015426262d0808bd513fb037dd473ff45a35c472..a1552b6771bc6d8db7c981f6680516bdf0c2af21 100644 --- a/drivers/clk/qcom/mmcc-msm8974.c +++ b/drivers/clk/qcom/mmcc-msm8974.c @@ -74,22 +74,6 @@ static const char * const mmcc_xo_mmpll0_dsi_hdmi_gpll0[] = { "dsi1pll", }; -static const struct parent_map mmcc_xo_mmpll0_1_2_gpll0_map[] = { - { P_XO, 0 }, - { P_MMPLL0, 1 }, - { P_MMPLL1, 2 }, - { P_GPLL0, 5 }, - { P_MMPLL2, 3 } -}; - -static const char * const mmcc_xo_mmpll0_1_2_gpll0[] = { - "xo", - "mmpll0_vote", - "mmpll1_vote", - "mmss_gpll0_vote", - "mmpll2", -}; - static const struct parent_map mmcc_xo_mmpll0_1_3_gpll0_map[] = { { P_XO, 0 }, { P_MMPLL0, 1 }, diff --git a/drivers/clk/qcom/mmcc-msm8996.c b/drivers/clk/qcom/mmcc-msm8996.c index 3b3aac07fb2d5cb063645175e25ec81f4863bf9b..24843e4f2599d0625a7fb804406580819944e96d 100644 --- a/drivers/clk/qcom/mmcc-msm8996.c +++ b/drivers/clk/qcom/mmcc-msm8996.c @@ -528,16 +528,23 @@ static struct clk_rcg2 maxi_clk_src = { }, }; -static struct clk_rcg2 gfx3d_clk_src = { - .cmd_rcgr = 0x4000, - .hid_width = 5, - .parent_map = mmss_xo_mmpll0_mmpll9_mmpll2_mmpll8_gpll0_map, - .clkr.hw.init = &(struct clk_init_data){ - .name = "gfx3d_clk_src", - .parent_names = mmss_xo_mmpll0_mmpll9_mmpll2_mmpll8_gpll0, - .num_parents = 6, - .ops = &clk_gfx3d_ops, - .flags = CLK_SET_RATE_PARENT, +static struct clk_rcg2_gfx3d gfx3d_clk_src = { + .rcg = { + .cmd_rcgr = 0x4000, + .hid_width = 5, + .parent_map = mmss_xo_mmpll0_mmpll9_mmpll2_mmpll8_gpll0_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "gfx3d_clk_src", + .parent_names = mmss_xo_mmpll0_mmpll9_mmpll2_mmpll8_gpll0, + .num_parents = 6, + .ops = &clk_gfx3d_ops, + .flags = CLK_SET_RATE_PARENT, + }, + }, + .hws = (struct clk_hw*[]) { + &mmpll9.clkr.hw, + &mmpll2.clkr.hw, + &mmpll8.clkr.hw }, }; @@ -3089,7 +3096,7 @@ static struct clk_regmap *mmcc_msm8996_clocks[] = { [AHB_CLK_SRC] = &ahb_clk_src.clkr, [AXI_CLK_SRC] = &axi_clk_src.clkr, [MAXI_CLK_SRC] = &maxi_clk_src.clkr, - [GFX3D_CLK_SRC] = &gfx3d_clk_src.clkr, + [GFX3D_CLK_SRC] = &gfx3d_clk_src.rcg.clkr, [RBBMTIMER_CLK_SRC] = &rbbmtimer_clk_src.clkr, [ISENSE_CLK_SRC] = &isense_clk_src.clkr, [RBCPR_CLK_SRC] = &rbcpr_clk_src.clkr, diff --git a/drivers/clk/qcom/mmcc-msm8998.c b/drivers/clk/qcom/mmcc-msm8998.c index dd68983fe22efa9357e32a718e2bd4c816892bcd..467dadccde026322ab3df7c98cd57a1513f19f68 100644 --- a/drivers/clk/qcom/mmcc-msm8998.c +++ b/drivers/clk/qcom/mmcc-msm8998.c @@ -1211,6 +1211,8 @@ static struct clk_rcg2 vfe1_clk_src = { static struct clk_branch misc_ahb_clk = { .halt_reg = 0x328, + .hwcg_reg = 0x328, + .hwcg_bit = 1, .clkr = { .enable_reg = 0x328, .enable_mask = BIT(0), @@ -1241,6 +1243,8 @@ static struct clk_branch video_core_clk = { static struct clk_branch video_ahb_clk = { .halt_reg = 0x1030, + .hwcg_reg = 0x1030, + .hwcg_bit = 1, .clkr = { .enable_reg = 0x1030, .enable_mask = BIT(0), @@ -1315,6 +1319,8 @@ static struct clk_branch video_subcore1_clk = { static struct clk_branch mdss_ahb_clk = { .halt_reg = 0x2308, + .hwcg_reg = 0x2308, + .hwcg_bit = 1, .clkr = { .enable_reg = 0x2308, .enable_mask = BIT(0), @@ -2496,6 +2502,8 @@ static struct clk_branch mnoc_ahb_clk = { static struct clk_branch bimc_smmu_ahb_clk = { .halt_reg = 0xe004, + .hwcg_reg = 0xe004, + .hwcg_bit = 1, .clkr = { .enable_reg = 0xe004, .enable_mask = BIT(0), @@ -2511,6 +2519,8 @@ static struct clk_branch bimc_smmu_ahb_clk = { static struct clk_branch bimc_smmu_axi_clk = { .halt_reg = 0xe008, + .hwcg_reg = 0xe008, + .hwcg_bit = 1, .clkr = { .enable_reg = 0xe008, .enable_mask = BIT(0), @@ -2653,7 +2663,7 @@ static struct gdsc bimc_smmu_gdsc = { .name = "bimc_smmu", }, .pwrsts = PWRSTS_OFF_ON, - .flags = HW_CTRL, + .flags = HW_CTRL | ALWAYS_ON, }; static struct clk_regmap *mmcc_msm8998_clocks[] = { diff --git a/drivers/clk/qcom/mmcc-sdm660.c b/drivers/clk/qcom/mmcc-sdm660.c new file mode 100644 index 0000000000000000000000000000000000000000..941993bc610dfe5c5013a4ef5857ab134a07b711 --- /dev/null +++ b/drivers/clk/qcom/mmcc-sdm660.c @@ -0,0 +1,2864 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2020, The Linux Foundation. All rights reserved. + * Copyright (c) 2020, Martin Botka + * Copyright (c) 2020, Konrad Dybcio + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +#include + +#include "common.h" +#include "clk-regmap.h" +#include "clk-regmap-divider.h" +#include "clk-alpha-pll.h" +#include "clk-rcg.h" +#include "clk-branch.h" +#include "reset.h" +#include "gdsc.h" + +enum { + P_XO, + P_DSI0PLL_BYTE, + P_DSI0PLL, + P_DSI1PLL_BYTE, + P_DSI1PLL, + P_GPLL0, + P_GPLL0_DIV, + P_MMPLL0, + P_MMPLL10, + P_MMPLL3, + P_MMPLL4, + P_MMPLL5, + P_MMPLL6, + P_MMPLL7, + P_MMPLL8, + P_SLEEP_CLK, + P_DP_PHY_PLL_LINK_CLK, + P_DP_PHY_PLL_VCO_DIV, +}; + +static const struct parent_map mmcc_xo_mmpll0_mmpll4_mmpll7_mmpll8_gpll0_gpll0_div_map[] = { + { P_XO, 0 }, + { P_MMPLL0, 1 }, + { P_MMPLL4, 2 }, + { P_MMPLL7, 3 }, + { P_MMPLL8, 4 }, + { P_GPLL0, 5 }, + { P_GPLL0_DIV, 6 }, +}; + +/* Voteable PLL */ +static struct clk_alpha_pll mmpll0 = { + .offset = 0xc000, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .clkr = { + .enable_reg = 0x1f0, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mmpll0", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "xo", + }, + .num_parents = 1, + .ops = &clk_alpha_pll_ops, + }, + }, +}; + +static struct clk_alpha_pll mmpll6 = { + .offset = 0xf0, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .clkr = { + .enable_reg = 0x1f0, + .enable_mask = BIT(2), + .hw.init = &(struct clk_init_data){ + .name = "mmpll6", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "xo", + }, + .num_parents = 1, + .ops = &clk_alpha_pll_ops, + }, + }, +}; + +/* APSS controlled PLLs */ +static struct pll_vco vco[] = { + { 1000000000, 2000000000, 0 }, + { 750000000, 1500000000, 1 }, + { 500000000, 1000000000, 2 }, + { 250000000, 500000000, 3 }, +}; + +static struct pll_vco mmpll3_vco[] = { + { 750000000, 1500000000, 1 }, +}; + +static const struct alpha_pll_config mmpll10_config = { + .l = 0x1e, + .config_ctl_val = 0x00004289, + .main_output_mask = 0x1, +}; + +static struct clk_alpha_pll mmpll10 = { + .offset = 0x190, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "mmpll10", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "xo", + }, + .num_parents = 1, + .ops = &clk_alpha_pll_ops, + }, + }, +}; + +static const struct alpha_pll_config mmpll3_config = { + .l = 0x2e, + .config_ctl_val = 0x4001055b, + .vco_val = 0x1 << 20, + .vco_mask = 0x3 << 20, + .main_output_mask = 0x1, +}; + +static struct clk_alpha_pll mmpll3 = { + .offset = 0x0, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .vco_table = mmpll3_vco, + .num_vco = ARRAY_SIZE(mmpll3_vco), + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "mmpll3", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "xo", + }, + .num_parents = 1, + .ops = &clk_alpha_pll_ops, + }, + }, +}; + +static const struct alpha_pll_config mmpll4_config = { + .l = 0x28, + .config_ctl_val = 0x4001055b, + .vco_val = 0x2 << 20, + .vco_mask = 0x3 << 20, + .main_output_mask = 0x1, +}; + +static struct clk_alpha_pll mmpll4 = { + .offset = 0x50, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .vco_table = vco, + .num_vco = ARRAY_SIZE(vco), + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "mmpll4", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "xo", + }, + .num_parents = 1, + .ops = &clk_alpha_pll_ops, + }, + }, +}; + +static const struct alpha_pll_config mmpll5_config = { + .l = 0x2a, + .config_ctl_val = 0x4001055b, + .alpha_hi = 0xf8, + .alpha_en_mask = BIT(24), + .vco_val = 0x2 << 20, + .vco_mask = 0x3 << 20, + .main_output_mask = 0x1, +}; + +static struct clk_alpha_pll mmpll5 = { + .offset = 0xa0, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .vco_table = vco, + .num_vco = ARRAY_SIZE(vco), + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "mmpll5", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "xo", + }, + .num_parents = 1, + .ops = &clk_alpha_pll_ops, + }, + }, +}; + +static const struct alpha_pll_config mmpll7_config = { + .l = 0x32, + .config_ctl_val = 0x4001055b, + .vco_val = 0x2 << 20, + .vco_mask = 0x3 << 20, + .main_output_mask = 0x1, +}; + +static struct clk_alpha_pll mmpll7 = { + .offset = 0x140, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .vco_table = vco, + .num_vco = ARRAY_SIZE(vco), + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "mmpll7", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "xo", + }, + .num_parents = 1, + .ops = &clk_alpha_pll_ops, + }, + }, +}; + +static const struct alpha_pll_config mmpll8_config = { + .l = 0x30, + .alpha_hi = 0x70, + .alpha_en_mask = BIT(24), + .config_ctl_val = 0x4001055b, + .vco_val = 0x2 << 20, + .vco_mask = 0x3 << 20, + .main_output_mask = 0x1, +}; + +static struct clk_alpha_pll mmpll8 = { + .offset = 0x1c0, + .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], + .vco_table = vco, + .num_vco = ARRAY_SIZE(vco), + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "mmpll8", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "xo", + }, + .num_parents = 1, + .ops = &clk_alpha_pll_ops, + }, + }, +}; + +static const struct clk_parent_data mmcc_xo_mmpll0_mmpll4_mmpll7_mmpll8_gpll0_gpll0_div[] = { + { .fw_name = "xo" }, + { .hw = &mmpll0.clkr.hw }, + { .hw = &mmpll4.clkr.hw }, + { .hw = &mmpll7.clkr.hw }, + { .hw = &mmpll8.clkr.hw }, + { .fw_name = "gpll0" }, + { .fw_name = "gpll0_div" }, +}; + +static const struct parent_map mmcc_xo_dsibyte_map[] = { + { P_XO, 0 }, + { P_DSI0PLL_BYTE, 1 }, + { P_DSI1PLL_BYTE, 2 }, +}; + +static const struct clk_parent_data mmcc_xo_dsibyte[] = { + { .fw_name = "xo" }, + { .fw_name = "dsi0pllbyte" }, + { .fw_name = "dsi1pllbyte" }, +}; + +static const struct parent_map mmcc_xo_mmpll0_mmpll4_mmpll7_mmpll10_gpll0_gpll0_div_map[] = { + { P_XO, 0 }, + { P_MMPLL0, 1 }, + { P_MMPLL4, 2 }, + { P_MMPLL7, 3 }, + { P_MMPLL10, 4 }, + { P_GPLL0, 5 }, + { P_GPLL0_DIV, 6 }, +}; + +static const struct clk_parent_data mmcc_xo_mmpll0_mmpll4_mmpll7_mmpll10_gpll0_gpll0_div[] = { + { .fw_name = "xo" }, + { .hw = &mmpll0.clkr.hw }, + { .hw = &mmpll4.clkr.hw }, + { .hw = &mmpll7.clkr.hw }, + { .hw = &mmpll10.clkr.hw }, + { .fw_name = "gpll0" }, + { .fw_name = "gpll0_div" }, +}; + +static const struct parent_map mmcc_xo_mmpll4_mmpll7_mmpll10_sleep_gpll0_gpll0_div_map[] = { + { P_XO, 0 }, + { P_MMPLL4, 1 }, + { P_MMPLL7, 2 }, + { P_MMPLL10, 3 }, + { P_SLEEP_CLK, 4 }, + { P_GPLL0, 5 }, + { P_GPLL0_DIV, 6 }, +}; + +static const struct clk_parent_data mmcc_xo_mmpll4_mmpll7_mmpll10_sleep_gpll0_gpll0_div[] = { + { .fw_name = "xo" }, + { .hw = &mmpll4.clkr.hw }, + { .hw = &mmpll7.clkr.hw }, + { .hw = &mmpll10.clkr.hw }, + { .fw_name = "sleep_clk" }, + { .fw_name = "gpll0" }, + { .fw_name = "gpll0_div" }, +}; + +static const struct parent_map mmcc_xo_mmpll0_mmpll7_mmpll10_sleep_gpll0_gpll0_div_map[] = { + { P_XO, 0 }, + { P_MMPLL0, 1 }, + { P_MMPLL7, 2 }, + { P_MMPLL10, 3 }, + { P_SLEEP_CLK, 4 }, + { P_GPLL0, 5 }, + { P_GPLL0_DIV, 6 }, +}; + +static const struct clk_parent_data mmcc_xo_mmpll0_mmpll7_mmpll10_sleep_gpll0_gpll0_div[] = { + { .fw_name = "xo" }, + { .hw = &mmpll0.clkr.hw }, + { .hw = &mmpll7.clkr.hw }, + { .hw = &mmpll10.clkr.hw }, + { .fw_name = "sleep_clk" }, + { .fw_name = "gpll0" }, + { .fw_name = "gpll0_div" }, +}; + +static const struct parent_map mmcc_xo_gpll0_gpll0_div_map[] = { + { P_XO, 0 }, + { P_GPLL0, 5 }, + { P_GPLL0_DIV, 6 }, +}; + +static const struct clk_parent_data mmcc_xo_gpll0_gpll0_div[] = { + { .fw_name = "xo" }, + { .fw_name = "gpll0" }, + { .fw_name = "gpll0_div" }, +}; + +static const struct parent_map mmcc_xo_dplink_dpvco_map[] = { + { P_XO, 0 }, + { P_DP_PHY_PLL_LINK_CLK, 1 }, + { P_DP_PHY_PLL_VCO_DIV, 2 }, +}; + +static const struct clk_parent_data mmcc_xo_dplink_dpvco[] = { + { .fw_name = "xo" }, + { .fw_name = "dp_link_2x_clk_divsel_five" }, + { .fw_name = "dp_vco_divided_clk_src_mux" }, +}; + +static const struct parent_map mmcc_xo_mmpll0_mmpll5_mmpll7_gpll0_gpll0_div_map[] = { + { P_XO, 0 }, + { P_MMPLL0, 1 }, + { P_MMPLL5, 2 }, + { P_MMPLL7, 3 }, + { P_GPLL0, 5 }, + { P_GPLL0_DIV, 6 }, +}; + +static const struct clk_parent_data mmcc_xo_mmpll0_mmpll5_mmpll7_gpll0_gpll0_div[] = { + { .fw_name = "xo" }, + { .hw = &mmpll0.clkr.hw }, + { .hw = &mmpll5.clkr.hw }, + { .hw = &mmpll7.clkr.hw }, + { .fw_name = "gpll0" }, + { .fw_name = "gpll0_div" }, +}; + +static const struct parent_map mmcc_xo_dsi0pll_dsi1pll_map[] = { + { P_XO, 0 }, + { P_DSI0PLL, 1 }, + { P_DSI1PLL, 2 }, +}; + +static const struct clk_parent_data mmcc_xo_dsi0pll_dsi1pll[] = { + { .fw_name = "xo" }, + { .fw_name = "dsi0pll" }, + { .fw_name = "dsi1pll" }, +}; + +static const struct parent_map mmcc_mmpll0_mmpll4_mmpll7_mmpll10_mmpll6_gpll0_map[] = { + { P_XO, 0 }, + { P_MMPLL0, 1 }, + { P_MMPLL4, 2 }, + { P_MMPLL7, 3 }, + { P_MMPLL10, 4 }, + { P_MMPLL6, 5 }, + { P_GPLL0, 6 }, +}; + +static const struct clk_parent_data mmcc_mmpll0_mmpll4_mmpll7_mmpll10_mmpll6_gpll0[] = { + { .fw_name = "xo" }, + { .hw = &mmpll0.clkr.hw }, + { .hw = &mmpll4.clkr.hw }, + { .hw = &mmpll7.clkr.hw }, + { .hw = &mmpll10.clkr.hw }, + { .hw = &mmpll6.clkr.hw }, + { .fw_name = "gpll0" }, +}; + +static const struct parent_map mmcc_xo_mmpll0_gpll0_gpll0_div_map[] = { + { P_XO, 0 }, + { P_MMPLL0, 1 }, + { P_GPLL0, 5 }, + { P_GPLL0_DIV, 6 }, +}; + +static const struct clk_parent_data mmcc_xo_mmpll0_gpll0_gpll0_div[] = { + { .fw_name = "xo" }, + { .hw = &mmpll0.clkr.hw }, + { .fw_name = "gpll0" }, + { .fw_name = "gpll0_div" }, +}; + +static const struct parent_map mmcc_xo_mmpll0_mmpll4_mmpll7_mmpll10_gpll0_mmpll6_map[] = { + { P_XO, 0 }, + { P_MMPLL0, 1 }, + { P_MMPLL4, 2 }, + { P_MMPLL7, 3 }, + { P_MMPLL10, 4 }, + { P_GPLL0, 5 }, + { P_MMPLL6, 6 }, +}; + +static const struct clk_parent_data mmcc_xo_mmpll0_mmpll4_mmpll7_mmpll10_gpll0_mmpll6[] = { + { .fw_name = "xo" }, + { .hw = &mmpll0.clkr.hw }, + { .hw = &mmpll4.clkr.hw }, + { .hw = &mmpll7.clkr.hw }, + { .hw = &mmpll10.clkr.hw }, + { .fw_name = "gpll0" }, + { .hw = &mmpll6.clkr.hw }, +}; + +static const struct parent_map mmcc_xo_mmpll0_mmpll8_mmpll3_mmpll6_gpll0_mmpll7_map[] = { + { P_XO, 0 }, + { P_MMPLL0, 1 }, + { P_MMPLL8, 2 }, + { P_MMPLL3, 3 }, + { P_MMPLL6, 4 }, + { P_GPLL0, 5 }, + { P_MMPLL7, 6 }, +}; + +static const struct clk_parent_data mmcc_xo_mmpll0_mmpll8_mmpll3_mmpll6_gpll0_mmpll7[] = { + { .fw_name = "xo" }, + { .hw = &mmpll0.clkr.hw }, + { .hw = &mmpll8.clkr.hw }, + { .hw = &mmpll3.clkr.hw }, + { .hw = &mmpll6.clkr.hw }, + { .fw_name = "gpll0" }, + { .hw = &mmpll7.clkr.hw }, +}; + +static const struct freq_tbl ftbl_ahb_clk_src[] = { + F(19200000, P_XO, 1, 0, 0), + F(40000000, P_GPLL0_DIV, 7.5, 0, 0), + F(80800000, P_MMPLL0, 10, 0, 0), + { } +}; + +static struct clk_rcg2 ahb_clk_src = { + .cmd_rcgr = 0x5000, + .mnd_width = 0, + .hid_width = 5, + .parent_map = mmcc_xo_mmpll0_gpll0_gpll0_div_map, + .freq_tbl = ftbl_ahb_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "ahb_clk_src", + .parent_data = mmcc_xo_mmpll0_gpll0_gpll0_div, + .num_parents = 4, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 byte0_clk_src = { + .cmd_rcgr = 0x2120, + .mnd_width = 0, + .hid_width = 5, + .parent_map = mmcc_xo_dsibyte_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "byte0_clk_src", + .parent_data = mmcc_xo_dsibyte, + .num_parents = 3, + .ops = &clk_byte2_ops, + .flags = CLK_SET_RATE_PARENT | CLK_GET_RATE_NOCACHE, + }, +}; + +static struct clk_rcg2 byte1_clk_src = { + .cmd_rcgr = 0x2140, + .mnd_width = 0, + .hid_width = 5, + .parent_map = mmcc_xo_dsibyte_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "byte1_clk_src", + .parent_data = mmcc_xo_dsibyte, + .num_parents = 3, + .ops = &clk_byte2_ops, + .flags = CLK_SET_RATE_PARENT | CLK_GET_RATE_NOCACHE, + }, +}; + +static const struct freq_tbl ftbl_camss_gp0_clk_src[] = { + F(10000, P_XO, 16, 1, 120), + F(24000, P_XO, 16, 1, 50), + F(6000000, P_GPLL0_DIV, 10, 1, 5), + F(12000000, P_GPLL0_DIV, 10, 2, 5), + F(13043478, P_GPLL0_DIV, 1, 1, 23), + F(24000000, P_GPLL0_DIV, 1, 2, 25), + F(50000000, P_GPLL0_DIV, 6, 0, 0), + F(100000000, P_GPLL0_DIV, 3, 0, 0), + F(200000000, P_GPLL0, 3, 0, 0), + { } +}; + +static struct clk_rcg2 camss_gp0_clk_src = { + .cmd_rcgr = 0x3420, + .mnd_width = 8, + .hid_width = 5, + .parent_map = mmcc_xo_mmpll0_mmpll7_mmpll10_sleep_gpll0_gpll0_div_map, + .freq_tbl = ftbl_camss_gp0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "camss_gp0_clk_src", + .parent_data = mmcc_xo_mmpll0_mmpll7_mmpll10_sleep_gpll0_gpll0_div, + .num_parents = 7, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 camss_gp1_clk_src = { + .cmd_rcgr = 0x3450, + .mnd_width = 8, + .hid_width = 5, + .parent_map = mmcc_xo_mmpll0_mmpll7_mmpll10_sleep_gpll0_gpll0_div_map, + .freq_tbl = ftbl_camss_gp0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "camss_gp1_clk_src", + .parent_data = mmcc_xo_mmpll0_mmpll7_mmpll10_sleep_gpll0_gpll0_div, + .num_parents = 7, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_cci_clk_src[] = { + F(37500000, P_GPLL0_DIV, 8, 0, 0), + F(50000000, P_GPLL0_DIV, 6, 0, 0), + F(100000000, P_GPLL0, 6, 0, 0), + { } +}; + +static struct clk_rcg2 cci_clk_src = { + .cmd_rcgr = 0x3300, + .mnd_width = 8, + .hid_width = 5, + .parent_map = mmcc_xo_mmpll0_mmpll7_mmpll10_sleep_gpll0_gpll0_div_map, + .freq_tbl = ftbl_cci_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "cci_clk_src", + .parent_data = mmcc_xo_mmpll0_mmpll7_mmpll10_sleep_gpll0_gpll0_div, + .num_parents = 7, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_cpp_clk_src[] = { + F(120000000, P_GPLL0, 5, 0, 0), + F(256000000, P_MMPLL4, 3, 0, 0), + F(384000000, P_MMPLL4, 2, 0, 0), + F(480000000, P_MMPLL7, 2, 0, 0), + F(540000000, P_MMPLL6, 2, 0, 0), + F(576000000, P_MMPLL10, 1, 0, 0), + { } +}; + +static struct clk_rcg2 cpp_clk_src = { + .cmd_rcgr = 0x3640, + .mnd_width = 0, + .hid_width = 5, + .parent_map = mmcc_xo_mmpll0_mmpll4_mmpll7_mmpll10_gpll0_mmpll6_map, + .freq_tbl = ftbl_cpp_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "cpp_clk_src", + .parent_data = mmcc_xo_mmpll0_mmpll4_mmpll7_mmpll10_gpll0_mmpll6, + .num_parents = 7, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_csi0_clk_src[] = { + F(100000000, P_GPLL0_DIV, 3, 0, 0), + F(200000000, P_GPLL0, 3, 0, 0), + F(310000000, P_MMPLL8, 3, 0, 0), + F(404000000, P_MMPLL0, 2, 0, 0), + F(465000000, P_MMPLL8, 2, 0, 0), + { } +}; + +static struct clk_rcg2 csi0_clk_src = { + .cmd_rcgr = 0x3090, + .mnd_width = 0, + .hid_width = 5, + .parent_map = mmcc_xo_mmpll0_mmpll4_mmpll7_mmpll8_gpll0_gpll0_div_map, + .freq_tbl = ftbl_csi0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "csi0_clk_src", + .parent_data = mmcc_xo_mmpll0_mmpll4_mmpll7_mmpll8_gpll0_gpll0_div, + .num_parents = 7, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_csi0phytimer_clk_src[] = { + F(100000000, P_GPLL0_DIV, 3, 0, 0), + F(200000000, P_GPLL0, 3, 0, 0), + F(269333333, P_MMPLL0, 3, 0, 0), + { } +}; + +static struct clk_rcg2 csi0phytimer_clk_src = { + .cmd_rcgr = 0x3000, + .mnd_width = 0, + .hid_width = 5, + .parent_map = mmcc_xo_mmpll0_mmpll4_mmpll7_mmpll10_gpll0_gpll0_div_map, + .freq_tbl = ftbl_csi0phytimer_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "csi0phytimer_clk_src", + .parent_data = mmcc_xo_mmpll0_mmpll4_mmpll7_mmpll10_gpll0_gpll0_div, + .num_parents = 7, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 csi1_clk_src = { + .cmd_rcgr = 0x3100, + .mnd_width = 0, + .hid_width = 5, + .parent_map = mmcc_xo_mmpll0_mmpll4_mmpll7_mmpll8_gpll0_gpll0_div_map, + .freq_tbl = ftbl_csi0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "csi1_clk_src", + .parent_data = mmcc_xo_mmpll0_mmpll4_mmpll7_mmpll8_gpll0_gpll0_div, + .num_parents = 7, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 csi1phytimer_clk_src = { + .cmd_rcgr = 0x3030, + .mnd_width = 0, + .hid_width = 5, + .parent_map = mmcc_xo_mmpll0_mmpll4_mmpll7_mmpll10_gpll0_gpll0_div_map, + .freq_tbl = ftbl_csi0phytimer_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "csi1phytimer_clk_src", + .parent_data = mmcc_xo_mmpll0_mmpll4_mmpll7_mmpll10_gpll0_gpll0_div, + .num_parents = 7, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 csi2_clk_src = { + .cmd_rcgr = 0x3160, + .mnd_width = 0, + .hid_width = 5, + .parent_map = mmcc_xo_mmpll0_mmpll4_mmpll7_mmpll8_gpll0_gpll0_div_map, + .freq_tbl = ftbl_csi0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "csi2_clk_src", + .parent_data = mmcc_xo_mmpll0_mmpll4_mmpll7_mmpll8_gpll0_gpll0_div, + .num_parents = 7, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 csi2phytimer_clk_src = { + .cmd_rcgr = 0x3060, + .mnd_width = 0, + .hid_width = 5, + .parent_map = mmcc_xo_mmpll0_mmpll4_mmpll7_mmpll10_gpll0_gpll0_div_map, + .freq_tbl = ftbl_csi0phytimer_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "csi2phytimer_clk_src", + .parent_data = mmcc_xo_mmpll0_mmpll4_mmpll7_mmpll10_gpll0_gpll0_div, + .num_parents = 7, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 csi3_clk_src = { + .cmd_rcgr = 0x31c0, + .mnd_width = 0, + .hid_width = 5, + .parent_map = mmcc_xo_mmpll0_mmpll4_mmpll7_mmpll8_gpll0_gpll0_div_map, + .freq_tbl = ftbl_csi0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "csi3_clk_src", + .parent_data = mmcc_xo_mmpll0_mmpll4_mmpll7_mmpll8_gpll0_gpll0_div, + .num_parents = 7, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_csiphy_clk_src[] = { + F(100000000, P_GPLL0_DIV, 3, 0, 0), + F(200000000, P_GPLL0, 3, 0, 0), + F(269333333, P_MMPLL0, 3, 0, 0), + F(320000000, P_MMPLL7, 3, 0, 0), + { } +}; + +static struct clk_rcg2 csiphy_clk_src = { + .cmd_rcgr = 0x3800, + .mnd_width = 0, + .hid_width = 5, + .parent_map = mmcc_xo_mmpll0_mmpll4_mmpll7_mmpll8_gpll0_gpll0_div_map, + .freq_tbl = ftbl_csiphy_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "csiphy_clk_src", + .parent_data = mmcc_xo_mmpll0_mmpll4_mmpll7_mmpll8_gpll0_gpll0_div, + .num_parents = 7, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_dp_aux_clk_src[] = { + F(19200000, P_XO, 1, 0, 0), + { } +}; + +static struct clk_rcg2 dp_aux_clk_src = { + .cmd_rcgr = 0x2260, + .mnd_width = 0, + .hid_width = 5, + .parent_map = mmcc_xo_gpll0_gpll0_div_map, + .freq_tbl = ftbl_dp_aux_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "dp_aux_clk_src", + .parent_data = mmcc_xo_gpll0_gpll0_div, + .num_parents = 3, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_dp_crypto_clk_src[] = { + F(101250000, P_DP_PHY_PLL_VCO_DIV, 4, 0, 0), + F(168750000, P_DP_PHY_PLL_VCO_DIV, 4, 0, 0), + F(337500000, P_DP_PHY_PLL_VCO_DIV, 4, 0, 0), + { } +}; + +static struct clk_rcg2 dp_crypto_clk_src = { + .cmd_rcgr = 0x2220, + .mnd_width = 8, + .hid_width = 5, + .parent_map = mmcc_xo_dplink_dpvco_map, + .freq_tbl = ftbl_dp_crypto_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "dp_crypto_clk_src", + .parent_data = mmcc_xo_dplink_dpvco, + .num_parents = 3, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_dp_gtc_clk_src[] = { + F(40000000, P_GPLL0_DIV, 7.5, 0, 0), + F(60000000, P_GPLL0, 10, 0, 0), + { } +}; + +static struct clk_rcg2 dp_gtc_clk_src = { + .cmd_rcgr = 0x2280, + .mnd_width = 0, + .hid_width = 5, + .parent_map = mmcc_xo_gpll0_gpll0_div_map, + .freq_tbl = ftbl_dp_gtc_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "dp_gtc_clk_src", + .parent_data = mmcc_xo_gpll0_gpll0_div, + .num_parents = 3, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_dp_link_clk_src[] = { + F(162000000, P_DP_PHY_PLL_LINK_CLK, 2, 0, 0), + F(270000000, P_DP_PHY_PLL_LINK_CLK, 2, 0, 0), + F(540000000, P_DP_PHY_PLL_LINK_CLK, 2, 0, 0), + { } +}; + +static struct clk_rcg2 dp_link_clk_src = { + .cmd_rcgr = 0x2200, + .mnd_width = 0, + .hid_width = 5, + .parent_map = mmcc_xo_dplink_dpvco_map, + .freq_tbl = ftbl_dp_link_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "dp_link_clk_src", + .parent_data = mmcc_xo_dplink_dpvco, + .num_parents = 3, + .ops = &clk_rcg2_ops, + .flags = CLK_GET_RATE_NOCACHE | CLK_SET_RATE_PARENT, + }, +}; + +static struct clk_rcg2 dp_pixel_clk_src = { + .cmd_rcgr = 0x2240, + .mnd_width = 16, + .hid_width = 5, + .parent_map = mmcc_xo_dplink_dpvco_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "dp_pixel_clk_src", + .parent_data = mmcc_xo_dplink_dpvco, + .num_parents = 3, + .ops = &clk_dp_ops, + .flags = CLK_GET_RATE_NOCACHE | CLK_SET_RATE_PARENT, + }, +}; + +static struct clk_rcg2 esc0_clk_src = { + .cmd_rcgr = 0x2160, + .mnd_width = 0, + .hid_width = 5, + .parent_map = mmcc_xo_dsibyte_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "esc0_clk_src", + .parent_data = mmcc_xo_dsibyte, + .num_parents = 3, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 esc1_clk_src = { + .cmd_rcgr = 0x2180, + .mnd_width = 0, + .hid_width = 5, + .parent_map = mmcc_xo_dsibyte_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "esc1_clk_src", + .parent_data = mmcc_xo_dsibyte, + .num_parents = 3, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_jpeg0_clk_src[] = { + F(66666667, P_GPLL0_DIV, 4.5, 0, 0), + F(133333333, P_GPLL0, 4.5, 0, 0), + F(219428571, P_MMPLL4, 3.5, 0, 0), + F(320000000, P_MMPLL7, 3, 0, 0), + F(480000000, P_MMPLL7, 2, 0, 0), + { } +}; + +static struct clk_rcg2 jpeg0_clk_src = { + .cmd_rcgr = 0x3500, + .mnd_width = 0, + .hid_width = 5, + .parent_map = mmcc_xo_mmpll0_mmpll4_mmpll7_mmpll10_gpll0_gpll0_div_map, + .freq_tbl = ftbl_jpeg0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "jpeg0_clk_src", + .parent_data = mmcc_xo_mmpll0_mmpll4_mmpll7_mmpll10_gpll0_gpll0_div, + .num_parents = 7, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_mclk0_clk_src[] = { + F(4800000, P_XO, 4, 0, 0), + F(6000000, P_GPLL0_DIV, 10, 1, 5), + F(8000000, P_GPLL0_DIV, 1, 2, 75), + F(9600000, P_XO, 2, 0, 0), + F(16666667, P_GPLL0_DIV, 2, 1, 9), + F(19200000, P_XO, 1, 0, 0), + F(24000000, P_MMPLL10, 1, 1, 24), + F(33333333, P_GPLL0_DIV, 1, 1, 9), + F(48000000, P_GPLL0, 1, 2, 25), + F(66666667, P_GPLL0, 1, 1, 9), + { } +}; + +static struct clk_rcg2 mclk0_clk_src = { + .cmd_rcgr = 0x3360, + .mnd_width = 8, + .hid_width = 5, + .parent_map = mmcc_xo_mmpll4_mmpll7_mmpll10_sleep_gpll0_gpll0_div_map, + .freq_tbl = ftbl_mclk0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "mclk0_clk_src", + .parent_data = mmcc_xo_mmpll4_mmpll7_mmpll10_sleep_gpll0_gpll0_div, + .num_parents = 7, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 mclk1_clk_src = { + .cmd_rcgr = 0x3390, + .mnd_width = 8, + .hid_width = 5, + .parent_map = mmcc_xo_mmpll4_mmpll7_mmpll10_sleep_gpll0_gpll0_div_map, + .freq_tbl = ftbl_mclk0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "mclk1_clk_src", + .parent_data = mmcc_xo_mmpll4_mmpll7_mmpll10_sleep_gpll0_gpll0_div, + .num_parents = 7, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 mclk2_clk_src = { + .cmd_rcgr = 0x33c0, + .mnd_width = 8, + .hid_width = 5, + .parent_map = mmcc_xo_mmpll4_mmpll7_mmpll10_sleep_gpll0_gpll0_div_map, + .freq_tbl = ftbl_mclk0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "mclk2_clk_src", + .parent_data = mmcc_xo_mmpll4_mmpll7_mmpll10_sleep_gpll0_gpll0_div, + .num_parents = 7, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 mclk3_clk_src = { + .cmd_rcgr = 0x33f0, + .mnd_width = 8, + .hid_width = 5, + .parent_map = mmcc_xo_mmpll4_mmpll7_mmpll10_sleep_gpll0_gpll0_div_map, + .freq_tbl = ftbl_mclk0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "mclk3_clk_src", + .parent_data = mmcc_xo_mmpll4_mmpll7_mmpll10_sleep_gpll0_gpll0_div, + .num_parents = 7, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_mdp_clk_src[] = { + F(100000000, P_GPLL0_DIV, 3, 0, 0), + F(150000000, P_GPLL0_DIV, 2, 0, 0), + F(171428571, P_GPLL0, 3.5, 0, 0), + F(200000000, P_GPLL0, 3, 0, 0), + F(275000000, P_MMPLL5, 3, 0, 0), + F(300000000, P_GPLL0, 2, 0, 0), + F(330000000, P_MMPLL5, 2.5, 0, 0), + F(412500000, P_MMPLL5, 2, 0, 0), + { } +}; + +static struct clk_rcg2 mdp_clk_src = { + .cmd_rcgr = 0x2040, + .mnd_width = 0, + .hid_width = 5, + .parent_map = mmcc_xo_mmpll0_mmpll5_mmpll7_gpll0_gpll0_div_map, + .freq_tbl = ftbl_mdp_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "mdp_clk_src", + .parent_data = mmcc_xo_mmpll0_mmpll5_mmpll7_gpll0_gpll0_div, + .num_parents = 6, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 pclk0_clk_src = { + .cmd_rcgr = 0x2000, + .mnd_width = 8, + .hid_width = 5, + .parent_map = mmcc_xo_dsi0pll_dsi1pll_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "pclk0_clk_src", + .parent_data = mmcc_xo_dsi0pll_dsi1pll, + .num_parents = 3, + .ops = &clk_pixel_ops, + .flags = CLK_SET_RATE_PARENT | CLK_GET_RATE_NOCACHE, + }, +}; + +static struct clk_rcg2 pclk1_clk_src = { + .cmd_rcgr = 0x2020, + .mnd_width = 8, + .hid_width = 5, + .parent_map = mmcc_xo_dsi0pll_dsi1pll_map, + .clkr.hw.init = &(struct clk_init_data){ + .name = "pclk1_clk_src", + .parent_data = mmcc_xo_dsi0pll_dsi1pll, + .num_parents = 3, + .ops = &clk_pixel_ops, + .flags = CLK_SET_RATE_PARENT | CLK_GET_RATE_NOCACHE, + }, +}; + +static const struct freq_tbl ftbl_rot_clk_src[] = { + F(171428571, P_GPLL0, 3.5, 0, 0), + F(275000000, P_MMPLL5, 3, 0, 0), + F(300000000, P_GPLL0, 2, 0, 0), + F(330000000, P_MMPLL5, 2.5, 0, 0), + F(412500000, P_MMPLL5, 2, 0, 0), + { } +}; + +static struct clk_rcg2 rot_clk_src = { + .cmd_rcgr = 0x21a0, + .mnd_width = 0, + .hid_width = 5, + .parent_map = mmcc_xo_mmpll0_mmpll5_mmpll7_gpll0_gpll0_div_map, + .freq_tbl = ftbl_rot_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "rot_clk_src", + .parent_data = mmcc_xo_mmpll0_mmpll5_mmpll7_gpll0_gpll0_div, + .num_parents = 6, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_vfe0_clk_src[] = { + F(120000000, P_GPLL0, 5, 0, 0), + F(200000000, P_GPLL0, 3, 0, 0), + F(256000000, P_MMPLL4, 3, 0, 0), + F(300000000, P_GPLL0, 2, 0, 0), + F(404000000, P_MMPLL0, 2, 0, 0), + F(480000000, P_MMPLL7, 2, 0, 0), + F(540000000, P_MMPLL6, 2, 0, 0), + F(576000000, P_MMPLL10, 1, 0, 0), + { } +}; + +static struct clk_rcg2 vfe0_clk_src = { + .cmd_rcgr = 0x3600, + .mnd_width = 0, + .hid_width = 5, + .parent_map = mmcc_mmpll0_mmpll4_mmpll7_mmpll10_mmpll6_gpll0_map, + .freq_tbl = ftbl_vfe0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "vfe0_clk_src", + .parent_data = mmcc_mmpll0_mmpll4_mmpll7_mmpll10_mmpll6_gpll0, + .num_parents = 7, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_rcg2 vfe1_clk_src = { + .cmd_rcgr = 0x3620, + .mnd_width = 0, + .hid_width = 5, + .parent_map = mmcc_mmpll0_mmpll4_mmpll7_mmpll10_mmpll6_gpll0_map, + .freq_tbl = ftbl_vfe0_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "vfe1_clk_src", + .parent_data = mmcc_mmpll0_mmpll4_mmpll7_mmpll10_mmpll6_gpll0, + .num_parents = 7, + .ops = &clk_rcg2_ops, + }, +}; + +static const struct freq_tbl ftbl_video_core_clk_src[] = { + F(133333333, P_GPLL0, 4.5, 0, 0), + F(269333333, P_MMPLL0, 3, 0, 0), + F(320000000, P_MMPLL7, 3, 0, 0), + F(404000000, P_MMPLL0, 2, 0, 0), + F(441600000, P_MMPLL3, 2, 0, 0), + F(518400000, P_MMPLL3, 2, 0, 0), + { } +}; + +static struct clk_rcg2 video_core_clk_src = { + .cmd_rcgr = 0x1000, + .mnd_width = 0, + .hid_width = 5, + .parent_map = mmcc_xo_mmpll0_mmpll8_mmpll3_mmpll6_gpll0_mmpll7_map, + .freq_tbl = ftbl_video_core_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "video_core_clk_src", + .parent_data = mmcc_xo_mmpll0_mmpll8_mmpll3_mmpll6_gpll0_mmpll7, + .num_parents = 7, + .ops = &clk_rcg2_ops, + .flags = CLK_IS_CRITICAL, + }, +}; + +static struct clk_rcg2 vsync_clk_src = { + .cmd_rcgr = 0x2080, + .mnd_width = 0, + .hid_width = 5, + .parent_map = mmcc_xo_gpll0_gpll0_div_map, + .freq_tbl = ftbl_dp_aux_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "vsync_clk_src", + .parent_data = mmcc_xo_gpll0_gpll0_div, + .num_parents = 3, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_branch bimc_smmu_ahb_clk = { + .halt_reg = 0xe004, + .halt_check = BRANCH_VOTED, + .hwcg_reg = 0xe004, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0xe004, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "bimc_smmu_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ &ahb_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch bimc_smmu_axi_clk = { + .halt_reg = 0xe008, + .halt_check = BRANCH_VOTED, + .hwcg_reg = 0xe008, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0xe008, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "bimc_smmu_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_ahb_clk = { + .halt_reg = 0x348c, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0x348c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x348c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ &ahb_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_cci_ahb_clk = { + .halt_reg = 0x3348, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x3348, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_cci_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ &ahb_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_cci_clk = { + .halt_reg = 0x3344, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x3344, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_cci_clk", + .parent_hws = (const struct clk_hw *[]){ &cci_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_cpp_ahb_clk = { + .halt_reg = 0x36b4, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x36b4, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_cpp_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ &ahb_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_cpp_axi_clk = { + .halt_reg = 0x36c4, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x36c4, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_cpp_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_cpp_clk = { + .halt_reg = 0x36b0, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x36b0, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_cpp_clk", + .parent_hws = (const struct clk_hw *[]){ &cpp_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_cpp_vbif_ahb_clk = { + .halt_reg = 0x36c8, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x36c8, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_cpp_vbif_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ &ahb_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_csi0_ahb_clk = { + .halt_reg = 0x30bc, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x30bc, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi0_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ &ahb_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_csi0_clk = { + .halt_reg = 0x30b4, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x30b4, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi0_clk", + .parent_hws = (const struct clk_hw *[]){ &csi0_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_csi0phytimer_clk = { + .halt_reg = 0x3024, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x3024, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi0phytimer_clk", + .parent_hws = (const struct clk_hw *[]){ &csi0phytimer_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_csi0pix_clk = { + .halt_reg = 0x30e4, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x30e4, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi0pix_clk", + .parent_hws = (const struct clk_hw *[]){ &csi0_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_csi0rdi_clk = { + .halt_reg = 0x30d4, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x30d4, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi0rdi_clk", + .parent_hws = (const struct clk_hw *[]){ &csi0_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_csi1_ahb_clk = { + .halt_reg = 0x3128, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x3128, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi1_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ &ahb_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_csi1_clk = { + .halt_reg = 0x3124, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x3124, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi1_clk", + .parent_hws = (const struct clk_hw *[]){ &csi1_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_csi1phytimer_clk = { + .halt_reg = 0x3054, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x3054, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi1phytimer_clk", + .parent_hws = (const struct clk_hw *[]){ &csi1phytimer_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_csi1pix_clk = { + .halt_reg = 0x3154, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x3154, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi1pix_clk", + .parent_hws = (const struct clk_hw *[]){ &csi1_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_csi1rdi_clk = { + .halt_reg = 0x3144, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x3144, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi1rdi_clk", + .parent_hws = (const struct clk_hw *[]){ &csi1_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_csi2_ahb_clk = { + .halt_reg = 0x3188, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x3188, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi2_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ &ahb_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_csi2_clk = { + .halt_reg = 0x3184, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x3184, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi2_clk", + .parent_hws = (const struct clk_hw *[]){ &csi2_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_csi2phytimer_clk = { + .halt_reg = 0x3084, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x3084, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi2phytimer_clk", + .parent_hws = (const struct clk_hw *[]){ &csi2phytimer_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_csi2pix_clk = { + .halt_reg = 0x31b4, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x31b4, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi2pix_clk", + .parent_hws = (const struct clk_hw *[]){ &csi2_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_csi2rdi_clk = { + .halt_reg = 0x31a4, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x31a4, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi2rdi_clk", + .parent_hws = (const struct clk_hw *[]){ &csi2_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_csi3_ahb_clk = { + .halt_reg = 0x31e8, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x31e8, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi3_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ &ahb_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_csi3_clk = { + .halt_reg = 0x31e4, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x31e4, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi3_clk", + .parent_hws = (const struct clk_hw *[]){ &csi3_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_csi3pix_clk = { + .halt_reg = 0x3214, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x3214, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi3pix_clk", + .parent_hws = (const struct clk_hw *[]){ &csi3_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_csi3rdi_clk = { + .halt_reg = 0x3204, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x3204, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi3rdi_clk", + .parent_hws = (const struct clk_hw *[]){ &csi3_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_csi_vfe0_clk = { + .halt_reg = 0x3704, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x3704, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi_vfe0_clk", + .parent_hws = (const struct clk_hw *[]){ &vfe0_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_csi_vfe1_clk = { + .halt_reg = 0x3714, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x3714, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csi_vfe1_clk", + .parent_hws = (const struct clk_hw *[]){ &vfe1_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_csiphy0_clk = { + .halt_reg = 0x3740, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x3740, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csiphy0_clk", + .parent_hws = (const struct clk_hw *[]){ &csiphy_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_csiphy1_clk = { + .halt_reg = 0x3744, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x3744, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csiphy1_clk", + .parent_hws = (const struct clk_hw *[]){ &csiphy_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_csiphy2_clk = { + .halt_reg = 0x3748, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x3748, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_csiphy2_clk", + .parent_hws = (const struct clk_hw *[]){ &csiphy_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + + +static struct clk_branch camss_cphy_csid0_clk = { + .halt_reg = 0x3730, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x3730, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_cphy_csid0_clk", + .parent_hws = (const struct clk_hw *[]){ &camss_csiphy0_clk.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_cphy_csid1_clk = { + .halt_reg = 0x3734, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x3734, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_cphy_csid1_clk", + .parent_hws = (const struct clk_hw *[]){ &camss_csiphy1_clk.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_cphy_csid2_clk = { + .halt_reg = 0x3738, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x3738, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_cphy_csid2_clk", + .parent_hws = (const struct clk_hw *[]){ &camss_csiphy2_clk.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_cphy_csid3_clk = { + .halt_reg = 0x373c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x373c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_cphy_csid3_clk", + .parent_hws = (const struct clk_hw *[]){ &csiphy_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_gp0_clk = { + .halt_reg = 0x3444, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x3444, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_gp0_clk", + .parent_hws = (const struct clk_hw *[]){ &camss_gp0_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_gp1_clk = { + .halt_reg = 0x3474, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x3474, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_gp1_clk", + .parent_hws = (const struct clk_hw *[]){ &camss_gp1_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_ispif_ahb_clk = { + .halt_reg = 0x3224, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x3224, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_ispif_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ &ahb_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_jpeg0_clk = { + .halt_reg = 0x35a8, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x35a8, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_jpeg0_clk", + .parent_hws = (const struct clk_hw *[]){ &jpeg0_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_jpeg_ahb_clk = { + .halt_reg = 0x35b4, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x35b4, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_jpeg_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ &ahb_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_jpeg_axi_clk = { + .halt_reg = 0x35b8, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x35b8, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_jpeg_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch throttle_camss_axi_clk = { + .halt_reg = 0x3c3c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x3c3c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "throttle_camss_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_mclk0_clk = { + .halt_reg = 0x3384, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x3384, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_mclk0_clk", + .parent_hws = (const struct clk_hw *[]){ &mclk0_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_mclk1_clk = { + .halt_reg = 0x33b4, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x33b4, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_mclk1_clk", + .parent_hws = (const struct clk_hw *[]){ &mclk1_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_mclk2_clk = { + .halt_reg = 0x33e4, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x33e4, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_mclk2_clk", + .parent_hws = (const struct clk_hw *[]){ &mclk2_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_mclk3_clk = { + .halt_reg = 0x3414, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x3414, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_mclk3_clk", + .parent_hws = (const struct clk_hw *[]){ &mclk3_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_micro_ahb_clk = { + .halt_reg = 0x3494, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x3494, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_micro_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ &ahb_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_top_ahb_clk = { + .halt_reg = 0x3484, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x3484, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_top_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ &ahb_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_vfe0_ahb_clk = { + .halt_reg = 0x3668, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x3668, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_vfe0_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ &ahb_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_vfe0_clk = { + .halt_reg = 0x36a8, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x36a8, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_vfe0_clk", + .parent_hws = (const struct clk_hw *[]){ &vfe0_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_vfe0_stream_clk = { + .halt_reg = 0x3720, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x3720, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_vfe0_stream_clk", + .parent_hws = (const struct clk_hw *[]){ &vfe0_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_vfe1_ahb_clk = { + .halt_reg = 0x3678, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x3678, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_vfe1_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ &ahb_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_vfe1_clk = { + .halt_reg = 0x36ac, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x36ac, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_vfe1_clk", + .parent_hws = (const struct clk_hw *[]){ &vfe1_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_vfe1_stream_clk = { + .halt_reg = 0x3724, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x3724, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_vfe1_stream_clk", + .parent_hws = (const struct clk_hw *[]){ &vfe1_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_vfe_vbif_ahb_clk = { + .halt_reg = 0x36b8, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x36b8, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_vfe_vbif_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ &ahb_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch camss_vfe_vbif_axi_clk = { + .halt_reg = 0x36bc, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x36bc, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "camss_vfe_vbif_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch csiphy_ahb2crif_clk = { + .halt_reg = 0x374c, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0x374c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x374c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "csiphy_ahb2crif_clk", + .parent_hws = (const struct clk_hw *[]){ &ahb_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch mdss_ahb_clk = { + .halt_reg = 0x2308, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0x8a004, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x2308, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mdss_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ &ahb_clk_src.clkr.hw }, + .flags = CLK_SET_RATE_PARENT, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static const struct freq_tbl ftbl_axi_clk_src[] = { + F(75000000, P_GPLL0, 8, 0, 0), + F(171428571, P_GPLL0, 3.5, 0, 0), + F(240000000, P_GPLL0, 2.5, 0, 0), + F(323200000, P_MMPLL0, 2.5, 0, 0), + F(406000000, P_MMPLL0, 2, 0, 0), + { } +}; + +/* RO to linux */ +static struct clk_rcg2 axi_clk_src = { + .cmd_rcgr = 0xd000, + .hid_width = 5, + .parent_map = mmcc_xo_mmpll0_mmpll4_mmpll7_mmpll10_gpll0_gpll0_div_map, + .freq_tbl = ftbl_axi_clk_src, + .clkr.hw.init = &(struct clk_init_data){ + .name = "axi_clk_src", + .parent_data = mmcc_xo_mmpll0_mmpll4_mmpll7_mmpll10_gpll0_gpll0_div, + .num_parents = 7, + .ops = &clk_rcg2_ops, + }, +}; + +static struct clk_branch mdss_axi_clk = { + .halt_reg = 0x2310, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x2310, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mdss_axi_clk", + .parent_hws = (const struct clk_hw *[]){ &axi_clk_src.clkr.hw }, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch throttle_mdss_axi_clk = { + .halt_reg = 0x246c, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0x246c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x246c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "throttle_mdss_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch mdss_byte0_clk = { + .halt_reg = 0x233c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x233c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mdss_byte0_clk", + .parent_hws = (const struct clk_hw *[]){ &byte0_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_regmap_div mdss_byte0_intf_div_clk = { + .reg = 0x237c, + .shift = 0, + .width = 2, + /* + * NOTE: Op does not work for div-3. Current assumption is that div-3 + * is not a recommended setting for this divider. + */ + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "mdss_byte0_intf_div_clk", + .parent_hws = (const struct clk_hw *[]){ &byte0_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_regmap_div_ops, + .flags = CLK_GET_RATE_NOCACHE, + }, + }, +}; + +static struct clk_branch mdss_byte0_intf_clk = { + .halt_reg = 0x2374, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x2374, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mdss_byte0_intf_clk", + .parent_hws = (const struct clk_hw *[]){ &mdss_byte0_intf_div_clk.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT | CLK_GET_RATE_NOCACHE, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch mdss_byte1_clk = { + .halt_reg = 0x2340, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x2340, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mdss_byte1_clk", + .parent_hws = (const struct clk_hw *[]){ &byte1_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_regmap_div mdss_byte1_intf_div_clk = { + .reg = 0x2380, + .shift = 0, + .width = 2, + /* + * NOTE: Op does not work for div-3. Current assumption is that div-3 + * is not a recommended setting for this divider. + */ + .clkr = { + .hw.init = &(struct clk_init_data){ + .name = "mdss_byte1_intf_div_clk", + .parent_hws = (const struct clk_hw *[]){ &byte1_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_regmap_div_ops, + .flags = CLK_GET_RATE_NOCACHE, + }, + }, +}; + +static struct clk_branch mdss_byte1_intf_clk = { + .halt_reg = 0x2378, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x2378, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mdss_byte1_intf_clk", + .parent_hws = (const struct clk_hw *[]){ &mdss_byte1_intf_div_clk.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT | CLK_GET_RATE_NOCACHE, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch mdss_dp_aux_clk = { + .halt_reg = 0x2364, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x2364, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mdss_dp_aux_clk", + .parent_hws = (const struct clk_hw *[]){ &dp_aux_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch mdss_dp_crypto_clk = { + .halt_reg = 0x235c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x235c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mdss_dp_crypto_clk", + .parent_hws = (const struct clk_hw *[]){ &dp_crypto_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch mdss_dp_gtc_clk = { + .halt_reg = 0x2368, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x2368, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mdss_dp_gtc_clk", + .parent_hws = (const struct clk_hw *[]){ &dp_gtc_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch mdss_dp_link_clk = { + .halt_reg = 0x2354, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x2354, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mdss_dp_link_clk", + .parent_hws = (const struct clk_hw *[]){ &dp_link_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_GET_RATE_NOCACHE | CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +/* Reset state of MDSS_DP_LINK_INTF_DIV is 0x3 (div-4) */ +static struct clk_branch mdss_dp_link_intf_clk = { + .halt_reg = 0x2358, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x2358, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mdss_dp_link_intf_clk", + .parent_hws = (const struct clk_hw *[]){ &dp_link_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch mdss_dp_pixel_clk = { + .halt_reg = 0x2360, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x2360, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mdss_dp_pixel_clk", + .parent_hws = (const struct clk_hw *[]){ &dp_pixel_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_GET_RATE_NOCACHE | CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch mdss_esc0_clk = { + .halt_reg = 0x2344, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x2344, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mdss_esc0_clk", + .parent_hws = (const struct clk_hw *[]){ &esc0_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch mdss_esc1_clk = { + .halt_reg = 0x2348, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x2348, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mdss_esc1_clk", + .parent_hws = (const struct clk_hw *[]){ &esc1_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch mdss_hdmi_dp_ahb_clk = { + .halt_reg = 0x230c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x230c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mdss_hdmi_dp_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ &ahb_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch mdss_mdp_clk = { + .halt_reg = 0x231c, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x231c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mdss_mdp_clk", + .parent_hws = (const struct clk_hw *[]){ &mdp_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch mdss_pclk0_clk = { + .halt_reg = 0x2314, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x2314, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mdss_pclk0_clk", + .parent_hws = (const struct clk_hw *[]){ &pclk0_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch mdss_pclk1_clk = { + .halt_reg = 0x2318, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x2318, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mdss_pclk1_clk", + .parent_hws = (const struct clk_hw *[]){ &pclk1_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch mdss_rot_clk = { + .halt_reg = 0x2350, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x2350, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mdss_rot_clk", + .parent_hws = (const struct clk_hw *[]){ &rot_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch mdss_vsync_clk = { + .halt_reg = 0x2328, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x2328, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mdss_vsync_clk", + .parent_hws = (const struct clk_hw *[]){ &vsync_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch mnoc_ahb_clk = { + .halt_reg = 0x5024, + .halt_check = BRANCH_VOTED, + .clkr = { + .enable_reg = 0x5024, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "mnoc_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ &ahb_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch misc_ahb_clk = { + .halt_reg = 0x328, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0x328, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x328, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "misc_ahb_clk", + /* + * Dependency to be enabled before the branch is + * enabled. + */ + .parent_hws = (const struct clk_hw *[]){ &mnoc_ahb_clk.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch misc_cxo_clk = { + .halt_reg = 0x324, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x324, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "misc_cxo_clk", + .parent_data = &(const struct clk_parent_data){ + .fw_name = "xo", + }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch snoc_dvm_axi_clk = { + .halt_reg = 0xe040, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0xe040, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "snoc_dvm_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch video_ahb_clk = { + .halt_reg = 0x1030, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0x1030, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x1030, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "video_ahb_clk", + .parent_hws = (const struct clk_hw *[]){ &ahb_clk_src.clkr.hw }, + .num_parents = 1, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch video_axi_clk = { + .halt_reg = 0x1034, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x1034, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "video_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch throttle_video_axi_clk = { + .halt_reg = 0x118c, + .halt_check = BRANCH_HALT, + .hwcg_reg = 0x118c, + .hwcg_bit = 1, + .clkr = { + .enable_reg = 0x118c, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "throttle_video_axi_clk", + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch video_core_clk = { + .halt_reg = 0x1028, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x1028, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "video_core_clk", + .parent_hws = (const struct clk_hw *[]){ &video_core_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct clk_branch video_subcore0_clk = { + .halt_reg = 0x1048, + .halt_check = BRANCH_HALT, + .clkr = { + .enable_reg = 0x1048, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "video_subcore0_clk", + .parent_hws = (const struct clk_hw *[]){ &video_core_clk_src.clkr.hw }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT | CLK_GET_RATE_NOCACHE, + .ops = &clk_branch2_ops, + }, + }, +}; + +static struct gdsc venus_gdsc = { + .gdscr = 0x1024, + .pd = { + .name = "venus", + }, + .pwrsts = PWRSTS_OFF_ON, +}; + +static struct gdsc venus_core0_gdsc = { + .gdscr = 0x1040, + .pd = { + .name = "venus_core0", + }, + .parent = &venus_gdsc.pd, + .pwrsts = PWRSTS_OFF_ON, +}; + +static struct gdsc mdss_gdsc = { + .gdscr = 0x2304, + .pd = { + .name = "mdss", + }, + .cxcs = (unsigned int []){ 0x2040 }, + .cxc_count = 1, + .pwrsts = PWRSTS_OFF_ON, +}; + +static struct gdsc camss_top_gdsc = { + .gdscr = 0x34a0, + .pd = { + .name = "camss_top", + }, + .pwrsts = PWRSTS_OFF_ON, +}; + +static struct gdsc camss_vfe0_gdsc = { + .gdscr = 0x3664, + .pd = { + .name = "camss_vfe0", + }, + .parent = &camss_top_gdsc.pd, + .pwrsts = PWRSTS_OFF_ON, +}; + +static struct gdsc camss_vfe1_gdsc = { + .gdscr = 0x3674, + .pd = { + .name = "camss_vfe1_gdsc", + }, + .parent = &camss_top_gdsc.pd, + .pwrsts = PWRSTS_OFF_ON, +}; + +static struct gdsc camss_cpp_gdsc = { + .gdscr = 0x36d4, + .pd = { + .name = "camss_cpp", + }, + .parent = &camss_top_gdsc.pd, + .pwrsts = PWRSTS_OFF_ON, +}; + +/* This GDSC seems to hang the whole multimedia subsystem. +static struct gdsc bimc_smmu_gdsc = { + .gdscr = 0xe020, + .gds_hw_ctrl = 0xe024, + .pd = { + .name = "bimc_smmu", + }, + .pwrsts = PWRSTS_OFF_ON, + .parent = &bimc_smmu_gdsc.pd, + .flags = HW_CTRL, +}; +*/ + +static struct clk_regmap *mmcc_660_clocks[] = { + [AHB_CLK_SRC] = &ahb_clk_src.clkr, + [BYTE0_CLK_SRC] = &byte0_clk_src.clkr, + [BYTE1_CLK_SRC] = &byte1_clk_src.clkr, + [CAMSS_GP0_CLK_SRC] = &camss_gp0_clk_src.clkr, + [CAMSS_GP1_CLK_SRC] = &camss_gp1_clk_src.clkr, + [CCI_CLK_SRC] = &cci_clk_src.clkr, + [CPP_CLK_SRC] = &cpp_clk_src.clkr, + [CSI0_CLK_SRC] = &csi0_clk_src.clkr, + [CSI0PHYTIMER_CLK_SRC] = &csi0phytimer_clk_src.clkr, + [CSI1_CLK_SRC] = &csi1_clk_src.clkr, + [CSI1PHYTIMER_CLK_SRC] = &csi1phytimer_clk_src.clkr, + [CSI2_CLK_SRC] = &csi2_clk_src.clkr, + [CSI2PHYTIMER_CLK_SRC] = &csi2phytimer_clk_src.clkr, + [CSI3_CLK_SRC] = &csi3_clk_src.clkr, + [CSIPHY_CLK_SRC] = &csiphy_clk_src.clkr, + [DP_AUX_CLK_SRC] = &dp_aux_clk_src.clkr, + [DP_CRYPTO_CLK_SRC] = &dp_crypto_clk_src.clkr, + [DP_GTC_CLK_SRC] = &dp_gtc_clk_src.clkr, + [DP_LINK_CLK_SRC] = &dp_link_clk_src.clkr, + [DP_PIXEL_CLK_SRC] = &dp_pixel_clk_src.clkr, + [ESC0_CLK_SRC] = &esc0_clk_src.clkr, + [ESC1_CLK_SRC] = &esc1_clk_src.clkr, + [JPEG0_CLK_SRC] = &jpeg0_clk_src.clkr, + [MCLK0_CLK_SRC] = &mclk0_clk_src.clkr, + [MCLK1_CLK_SRC] = &mclk1_clk_src.clkr, + [MCLK2_CLK_SRC] = &mclk2_clk_src.clkr, + [MCLK3_CLK_SRC] = &mclk3_clk_src.clkr, + [MDP_CLK_SRC] = &mdp_clk_src.clkr, + [MMPLL0_PLL] = &mmpll0.clkr, + [MMPLL10_PLL] = &mmpll10.clkr, + [MMPLL3_PLL] = &mmpll3.clkr, + [MMPLL4_PLL] = &mmpll4.clkr, + [MMPLL5_PLL] = &mmpll5.clkr, + [MMPLL6_PLL] = &mmpll6.clkr, + [MMPLL7_PLL] = &mmpll7.clkr, + [MMPLL8_PLL] = &mmpll8.clkr, + [BIMC_SMMU_AHB_CLK] = &bimc_smmu_ahb_clk.clkr, + [BIMC_SMMU_AXI_CLK] = &bimc_smmu_axi_clk.clkr, + [CAMSS_AHB_CLK] = &camss_ahb_clk.clkr, + [CAMSS_CCI_AHB_CLK] = &camss_cci_ahb_clk.clkr, + [CAMSS_CCI_CLK] = &camss_cci_clk.clkr, + [CAMSS_CPHY_CSID0_CLK] = &camss_cphy_csid0_clk.clkr, + [CAMSS_CPHY_CSID1_CLK] = &camss_cphy_csid1_clk.clkr, + [CAMSS_CPHY_CSID2_CLK] = &camss_cphy_csid2_clk.clkr, + [CAMSS_CPHY_CSID3_CLK] = &camss_cphy_csid3_clk.clkr, + [CAMSS_CPP_AHB_CLK] = &camss_cpp_ahb_clk.clkr, + [CAMSS_CPP_AXI_CLK] = &camss_cpp_axi_clk.clkr, + [CAMSS_CPP_CLK] = &camss_cpp_clk.clkr, + [CAMSS_CPP_VBIF_AHB_CLK] = &camss_cpp_vbif_ahb_clk.clkr, + [CAMSS_CSI0_AHB_CLK] = &camss_csi0_ahb_clk.clkr, + [CAMSS_CSI0_CLK] = &camss_csi0_clk.clkr, + [CAMSS_CSI0PHYTIMER_CLK] = &camss_csi0phytimer_clk.clkr, + [CAMSS_CSI0PIX_CLK] = &camss_csi0pix_clk.clkr, + [CAMSS_CSI0RDI_CLK] = &camss_csi0rdi_clk.clkr, + [CAMSS_CSI1_AHB_CLK] = &camss_csi1_ahb_clk.clkr, + [CAMSS_CSI1_CLK] = &camss_csi1_clk.clkr, + [CAMSS_CSI1PHYTIMER_CLK] = &camss_csi1phytimer_clk.clkr, + [CAMSS_CSI1PIX_CLK] = &camss_csi1pix_clk.clkr, + [CAMSS_CSI1RDI_CLK] = &camss_csi1rdi_clk.clkr, + [CAMSS_CSI2_AHB_CLK] = &camss_csi2_ahb_clk.clkr, + [CAMSS_CSI2_CLK] = &camss_csi2_clk.clkr, + [CAMSS_CSI2PHYTIMER_CLK] = &camss_csi2phytimer_clk.clkr, + [CAMSS_CSI2PIX_CLK] = &camss_csi2pix_clk.clkr, + [CAMSS_CSI2RDI_CLK] = &camss_csi2rdi_clk.clkr, + [CAMSS_CSI3_AHB_CLK] = &camss_csi3_ahb_clk.clkr, + [CAMSS_CSI3_CLK] = &camss_csi3_clk.clkr, + [CAMSS_CSI3PIX_CLK] = &camss_csi3pix_clk.clkr, + [CAMSS_CSI3RDI_CLK] = &camss_csi3rdi_clk.clkr, + [CAMSS_CSI_VFE0_CLK] = &camss_csi_vfe0_clk.clkr, + [CAMSS_CSI_VFE1_CLK] = &camss_csi_vfe1_clk.clkr, + [CAMSS_CSIPHY0_CLK] = &camss_csiphy0_clk.clkr, + [CAMSS_CSIPHY1_CLK] = &camss_csiphy1_clk.clkr, + [CAMSS_CSIPHY2_CLK] = &camss_csiphy2_clk.clkr, + [CAMSS_GP0_CLK] = &camss_gp0_clk.clkr, + [CAMSS_GP1_CLK] = &camss_gp1_clk.clkr, + [CAMSS_ISPIF_AHB_CLK] = &camss_ispif_ahb_clk.clkr, + [CAMSS_JPEG0_CLK] = &camss_jpeg0_clk.clkr, + [CAMSS_JPEG_AHB_CLK] = &camss_jpeg_ahb_clk.clkr, + [CAMSS_JPEG_AXI_CLK] = &camss_jpeg_axi_clk.clkr, + [CAMSS_MCLK0_CLK] = &camss_mclk0_clk.clkr, + [CAMSS_MCLK1_CLK] = &camss_mclk1_clk.clkr, + [CAMSS_MCLK2_CLK] = &camss_mclk2_clk.clkr, + [CAMSS_MCLK3_CLK] = &camss_mclk3_clk.clkr, + [CAMSS_MICRO_AHB_CLK] = &camss_micro_ahb_clk.clkr, + [CAMSS_TOP_AHB_CLK] = &camss_top_ahb_clk.clkr, + [CAMSS_VFE0_AHB_CLK] = &camss_vfe0_ahb_clk.clkr, + [CAMSS_VFE0_CLK] = &camss_vfe0_clk.clkr, + [CAMSS_VFE0_STREAM_CLK] = &camss_vfe0_stream_clk.clkr, + [CAMSS_VFE1_AHB_CLK] = &camss_vfe1_ahb_clk.clkr, + [CAMSS_VFE1_CLK] = &camss_vfe1_clk.clkr, + [CAMSS_VFE1_STREAM_CLK] = &camss_vfe1_stream_clk.clkr, + [CAMSS_VFE_VBIF_AHB_CLK] = &camss_vfe_vbif_ahb_clk.clkr, + [CAMSS_VFE_VBIF_AXI_CLK] = &camss_vfe_vbif_axi_clk.clkr, + [CSIPHY_AHB2CRIF_CLK] = &csiphy_ahb2crif_clk.clkr, + [MDSS_AHB_CLK] = &mdss_ahb_clk.clkr, + [MDSS_AXI_CLK] = &mdss_axi_clk.clkr, + [MDSS_BYTE0_CLK] = &mdss_byte0_clk.clkr, + [MDSS_BYTE0_INTF_CLK] = &mdss_byte0_intf_clk.clkr, + [MDSS_BYTE0_INTF_DIV_CLK] = &mdss_byte0_intf_div_clk.clkr, + [MDSS_BYTE1_CLK] = &mdss_byte1_clk.clkr, + [MDSS_BYTE1_INTF_CLK] = &mdss_byte1_intf_clk.clkr, + [MDSS_DP_AUX_CLK] = &mdss_dp_aux_clk.clkr, + [MDSS_DP_CRYPTO_CLK] = &mdss_dp_crypto_clk.clkr, + [MDSS_DP_GTC_CLK] = &mdss_dp_gtc_clk.clkr, + [MDSS_DP_LINK_CLK] = &mdss_dp_link_clk.clkr, + [MDSS_DP_LINK_INTF_CLK] = &mdss_dp_link_intf_clk.clkr, + [MDSS_DP_PIXEL_CLK] = &mdss_dp_pixel_clk.clkr, + [MDSS_ESC0_CLK] = &mdss_esc0_clk.clkr, + [MDSS_ESC1_CLK] = &mdss_esc1_clk.clkr, + [MDSS_HDMI_DP_AHB_CLK] = &mdss_hdmi_dp_ahb_clk.clkr, + [MDSS_MDP_CLK] = &mdss_mdp_clk.clkr, + [MDSS_PCLK0_CLK] = &mdss_pclk0_clk.clkr, + [MDSS_PCLK1_CLK] = &mdss_pclk1_clk.clkr, + [MDSS_ROT_CLK] = &mdss_rot_clk.clkr, + [MDSS_VSYNC_CLK] = &mdss_vsync_clk.clkr, + [MISC_AHB_CLK] = &misc_ahb_clk.clkr, + [MISC_CXO_CLK] = &misc_cxo_clk.clkr, + [MNOC_AHB_CLK] = &mnoc_ahb_clk.clkr, + [SNOC_DVM_AXI_CLK] = &snoc_dvm_axi_clk.clkr, + [THROTTLE_CAMSS_AXI_CLK] = &throttle_camss_axi_clk.clkr, + [THROTTLE_MDSS_AXI_CLK] = &throttle_mdss_axi_clk.clkr, + [THROTTLE_VIDEO_AXI_CLK] = &throttle_video_axi_clk.clkr, + [VIDEO_AHB_CLK] = &video_ahb_clk.clkr, + [VIDEO_AXI_CLK] = &video_axi_clk.clkr, + [VIDEO_CORE_CLK] = &video_core_clk.clkr, + [VIDEO_SUBCORE0_CLK] = &video_subcore0_clk.clkr, + [PCLK0_CLK_SRC] = &pclk0_clk_src.clkr, + [PCLK1_CLK_SRC] = &pclk1_clk_src.clkr, + [ROT_CLK_SRC] = &rot_clk_src.clkr, + [VFE0_CLK_SRC] = &vfe0_clk_src.clkr, + [VFE1_CLK_SRC] = &vfe1_clk_src.clkr, + [VIDEO_CORE_CLK_SRC] = &video_core_clk_src.clkr, + [VSYNC_CLK_SRC] = &vsync_clk_src.clkr, + [MDSS_BYTE1_INTF_DIV_CLK] = &mdss_byte1_intf_div_clk.clkr, + [AXI_CLK_SRC] = &axi_clk_src.clkr, +}; + +static struct gdsc *mmcc_sdm660_gdscs[] = { + [VENUS_GDSC] = &venus_gdsc, + [VENUS_CORE0_GDSC] = &venus_core0_gdsc, + [MDSS_GDSC] = &mdss_gdsc, + [CAMSS_TOP_GDSC] = &camss_top_gdsc, + [CAMSS_VFE0_GDSC] = &camss_vfe0_gdsc, + [CAMSS_VFE1_GDSC] = &camss_vfe1_gdsc, + [CAMSS_CPP_GDSC] = &camss_cpp_gdsc, +}; + +static const struct qcom_reset_map mmcc_660_resets[] = { + [CAMSS_MICRO_BCR] = { 0x3490 }, +}; + +static const struct regmap_config mmcc_660_regmap_config = { + .reg_bits = 32, + .reg_stride = 4, + .val_bits = 32, + .max_register = 0x40000, + .fast_io = true, +}; + +static const struct qcom_cc_desc mmcc_660_desc = { + .config = &mmcc_660_regmap_config, + .clks = mmcc_660_clocks, + .num_clks = ARRAY_SIZE(mmcc_660_clocks), + .resets = mmcc_660_resets, + .num_resets = ARRAY_SIZE(mmcc_660_resets), + .gdscs = mmcc_sdm660_gdscs, + .num_gdscs = ARRAY_SIZE(mmcc_sdm660_gdscs), +}; + +static const struct of_device_id mmcc_660_match_table[] = { + { .compatible = "qcom,mmcc-sdm660" }, + { .compatible = "qcom,mmcc-sdm630", .data = (void *)1UL }, + { } +}; +MODULE_DEVICE_TABLE(of, mmcc_660_match_table); + +static void sdm630_clock_override(void) +{ + /* SDM630 has only one DSI */ + mmcc_660_desc.clks[BYTE1_CLK_SRC] = NULL; + mmcc_660_desc.clks[MDSS_BYTE1_CLK] = NULL; + mmcc_660_desc.clks[MDSS_BYTE1_INTF_DIV_CLK] = NULL; + mmcc_660_desc.clks[MDSS_BYTE1_INTF_CLK] = NULL; + mmcc_660_desc.clks[ESC1_CLK_SRC] = NULL; + mmcc_660_desc.clks[MDSS_ESC1_CLK] = NULL; + mmcc_660_desc.clks[PCLK1_CLK_SRC] = NULL; + mmcc_660_desc.clks[MDSS_PCLK1_CLK] = NULL; +} + +static int mmcc_660_probe(struct platform_device *pdev) +{ + const struct of_device_id *id; + struct regmap *regmap; + bool is_sdm630; + + id = of_match_device(mmcc_660_match_table, &pdev->dev); + if (!id) + return -ENODEV; + is_sdm630 = !!(id->data); + + regmap = qcom_cc_map(pdev, &mmcc_660_desc); + if (IS_ERR(regmap)) + return PTR_ERR(regmap); + + if (is_sdm630) + sdm630_clock_override(); + + clk_alpha_pll_configure(&mmpll3, regmap, &mmpll3_config); + clk_alpha_pll_configure(&mmpll4, regmap, &mmpll4_config); + clk_alpha_pll_configure(&mmpll5, regmap, &mmpll5_config); + clk_alpha_pll_configure(&mmpll7, regmap, &mmpll7_config); + clk_alpha_pll_configure(&mmpll8, regmap, &mmpll8_config); + clk_alpha_pll_configure(&mmpll10, regmap, &mmpll10_config); + + return qcom_cc_really_probe(pdev, &mmcc_660_desc, regmap); +} + +static struct platform_driver mmcc_660_driver = { + .probe = mmcc_660_probe, + .driver = { + .name = "mmcc-sdm660", + .of_match_table = mmcc_660_match_table, + }, +}; +module_platform_driver(mmcc_660_driver); + +MODULE_DESCRIPTION("Qualcomm SDM630/SDM660 MMCC driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/clk/qcom/videocc-sm8250.c b/drivers/clk/qcom/videocc-sm8250.c index 2797c61f593843367570e24277c98680f3c3b2e0..b0efadc19634aa34c0d31787272516ccea16cd36 100644 --- a/drivers/clk/qcom/videocc-sm8250.c +++ b/drivers/clk/qcom/videocc-sm8250.c @@ -169,6 +169,21 @@ static struct clk_regmap_div video_cc_mvs0c_div2_div_clk_src = { }, }; +static struct clk_regmap_div video_cc_mvs0_div_clk_src = { + .reg = 0xd54, + .shift = 0, + .width = 2, + .clkr.hw.init = &(struct clk_init_data) { + .name = "video_cc_mvs0_div_clk_src", + .parent_data = &(const struct clk_parent_data){ + .hw = &video_cc_mvs0_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_regmap_div_ro_ops, + }, +}; + static struct clk_regmap_div video_cc_mvs1c_div2_div_clk_src = { .reg = 0xcf4, .shift = 0, @@ -202,6 +217,24 @@ static struct clk_branch video_cc_mvs0c_clk = { }, }; +static struct clk_branch video_cc_mvs0_clk = { + .halt_reg = 0xd34, + .halt_check = BRANCH_HALT_VOTED, + .clkr = { + .enable_reg = 0xd34, + .enable_mask = BIT(0), + .hw.init = &(struct clk_init_data){ + .name = "video_cc_mvs0_clk", + .parent_data = &(const struct clk_parent_data){ + .hw = &video_cc_mvs0_div_clk_src.clkr.hw, + }, + .num_parents = 1, + .flags = CLK_SET_RATE_PARENT, + .ops = &clk_branch2_ops, + }, + }, +}; + static struct clk_branch video_cc_mvs1_div2_clk = { .halt_reg = 0xdf4, .halt_check = BRANCH_HALT_VOTED, @@ -245,6 +278,7 @@ static struct gdsc mvs0c_gdsc = { }, .flags = 0, .pwrsts = PWRSTS_OFF_ON, + .supply = "mmcx", }; static struct gdsc mvs1c_gdsc = { @@ -254,6 +288,7 @@ static struct gdsc mvs1c_gdsc = { }, .flags = 0, .pwrsts = PWRSTS_OFF_ON, + .supply = "mmcx", }; static struct gdsc mvs0_gdsc = { @@ -263,6 +298,7 @@ static struct gdsc mvs0_gdsc = { }, .flags = HW_CTRL, .pwrsts = PWRSTS_OFF_ON, + .supply = "mmcx", }; static struct gdsc mvs1_gdsc = { @@ -272,10 +308,13 @@ static struct gdsc mvs1_gdsc = { }, .flags = HW_CTRL, .pwrsts = PWRSTS_OFF_ON, + .supply = "mmcx", }; static struct clk_regmap *video_cc_sm8250_clocks[] = { + [VIDEO_CC_MVS0_CLK] = &video_cc_mvs0_clk.clkr, [VIDEO_CC_MVS0_CLK_SRC] = &video_cc_mvs0_clk_src.clkr, + [VIDEO_CC_MVS0_DIV_CLK_SRC] = &video_cc_mvs0_div_clk_src.clkr, [VIDEO_CC_MVS0C_CLK] = &video_cc_mvs0c_clk.clkr, [VIDEO_CC_MVS0C_DIV2_DIV_CLK_SRC] = &video_cc_mvs0c_div2_div_clk_src.clkr, [VIDEO_CC_MVS1_CLK_SRC] = &video_cc_mvs1_clk_src.clkr, diff --git a/drivers/clk/renesas/Kconfig b/drivers/clk/renesas/Kconfig index 18915d668a30c579e02833b98efd76efce3acaa3..607e64a17d7273cad802907d783ab5380b793b94 100644 --- a/drivers/clk/renesas/Kconfig +++ b/drivers/clk/renesas/Kconfig @@ -148,6 +148,7 @@ config CLK_R8A77995 config CLK_R8A779A0 bool "R-Car V3U clock support" if COMPILE_TEST + select CLK_RCAR_CPG_LIB select CLK_RENESAS_CPG_MSSR config CLK_R9A06G032 @@ -162,12 +163,16 @@ config CLK_SH73A0 # Family +config CLK_RCAR_CPG_LIB + bool "CPG/MSSR library functions" if COMPILE_TEST + config CLK_RCAR_GEN2_CPG bool "R-Car Gen2 CPG clock support" if COMPILE_TEST select CLK_RENESAS_CPG_MSSR config CLK_RCAR_GEN3_CPG bool "R-Car Gen3 and RZ/G2 CPG clock support" if COMPILE_TEST + select CLK_RCAR_CPG_LIB select CLK_RENESAS_CPG_MSSR config CLK_RCAR_USB2_CLOCK_SEL diff --git a/drivers/clk/renesas/Makefile b/drivers/clk/renesas/Makefile index c803912ef2cebf7c5ac1dfe4bddc600ef8409daa..ef0d2bba92bf52ad88b5b468d056ba4b7ddf9eb7 100644 --- a/drivers/clk/renesas/Makefile +++ b/drivers/clk/renesas/Makefile @@ -32,6 +32,7 @@ obj-$(CONFIG_CLK_R9A06G032) += r9a06g032-clocks.o obj-$(CONFIG_CLK_SH73A0) += clk-sh73a0.o # Family +obj-$(CONFIG_CLK_RCAR_CPG_LIB) += rcar-cpg-lib.o obj-$(CONFIG_CLK_RCAR_GEN2_CPG) += rcar-gen2-cpg.o obj-$(CONFIG_CLK_RCAR_GEN3_CPG) += rcar-gen3-cpg.o obj-$(CONFIG_CLK_RCAR_USB2_CLOCK_SEL) += rcar-usb2-clock-sel.o diff --git a/drivers/clk/renesas/r8a7796-cpg-mssr.c b/drivers/clk/renesas/r8a7796-cpg-mssr.c index 2cd6e3876fbdcd0aaa3af9dd1f8876f683004151..41593c126faf33a73e7e7c277001fbd41b00ae3d 100644 --- a/drivers/clk/renesas/r8a7796-cpg-mssr.c +++ b/drivers/clk/renesas/r8a7796-cpg-mssr.c @@ -128,6 +128,11 @@ static const struct cpg_core_clk r8a7796_core_clks[] __initconst = { static struct mssr_mod_clk r8a7796_mod_clks[] __initdata = { DEF_MOD("fdp1-0", 119, R8A7796_CLK_S0D1), + DEF_MOD("tmu4", 121, R8A7796_CLK_S0D6), + DEF_MOD("tmu3", 122, R8A7796_CLK_S3D2), + DEF_MOD("tmu2", 123, R8A7796_CLK_S3D2), + DEF_MOD("tmu1", 124, R8A7796_CLK_S3D2), + DEF_MOD("tmu0", 125, R8A7796_CLK_CP), DEF_MOD("scif5", 202, R8A7796_CLK_S3D4), DEF_MOD("scif4", 203, R8A7796_CLK_S3D4), DEF_MOD("scif3", 204, R8A7796_CLK_S3D4), diff --git a/drivers/clk/renesas/r8a77965-cpg-mssr.c b/drivers/clk/renesas/r8a77965-cpg-mssr.c index 2b55a06ac5cf238f8a976789c068de00916480c5..46a157732759c394d535d420f9118991d887462d 100644 --- a/drivers/clk/renesas/r8a77965-cpg-mssr.c +++ b/drivers/clk/renesas/r8a77965-cpg-mssr.c @@ -123,6 +123,11 @@ static const struct cpg_core_clk r8a77965_core_clks[] __initconst = { static const struct mssr_mod_clk r8a77965_mod_clks[] __initconst = { DEF_MOD("fdp1-0", 119, R8A77965_CLK_S0D1), + DEF_MOD("tmu4", 121, R8A77965_CLK_S0D6), + DEF_MOD("tmu3", 122, R8A77965_CLK_S3D2), + DEF_MOD("tmu2", 123, R8A77965_CLK_S3D2), + DEF_MOD("tmu1", 124, R8A77965_CLK_S3D2), + DEF_MOD("tmu0", 125, R8A77965_CLK_CP), DEF_MOD("scif5", 202, R8A77965_CLK_S3D4), DEF_MOD("scif4", 203, R8A77965_CLK_S3D4), DEF_MOD("scif3", 204, R8A77965_CLK_S3D4), diff --git a/drivers/clk/renesas/r8a77990-cpg-mssr.c b/drivers/clk/renesas/r8a77990-cpg-mssr.c index 2b97ab61d044ca0753f33111d48aab77426d95e9..2d172f80b34c277d7a8c94fad642fea829e81fb9 100644 --- a/drivers/clk/renesas/r8a77990-cpg-mssr.c +++ b/drivers/clk/renesas/r8a77990-cpg-mssr.c @@ -124,6 +124,11 @@ static const struct cpg_core_clk r8a77990_core_clks[] __initconst = { }; static const struct mssr_mod_clk r8a77990_mod_clks[] __initconst = { + DEF_MOD("tmu4", 121, R8A77990_CLK_S0D6C), + DEF_MOD("tmu3", 122, R8A77990_CLK_S3D2C), + DEF_MOD("tmu2", 123, R8A77990_CLK_S3D2C), + DEF_MOD("tmu1", 124, R8A77990_CLK_S3D2C), + DEF_MOD("tmu0", 125, R8A77990_CLK_CP), DEF_MOD("scif5", 202, R8A77990_CLK_S3D4C), DEF_MOD("scif4", 203, R8A77990_CLK_S3D4C), DEF_MOD("scif3", 204, R8A77990_CLK_S3D4C), diff --git a/drivers/clk/renesas/r8a77995-cpg-mssr.c b/drivers/clk/renesas/r8a77995-cpg-mssr.c index 5b4691117b470556dbb71f79993dc703bf3a9747..9cfd00cf4e69aa457db4be2e842d9e679e1c980c 100644 --- a/drivers/clk/renesas/r8a77995-cpg-mssr.c +++ b/drivers/clk/renesas/r8a77995-cpg-mssr.c @@ -111,6 +111,11 @@ static const struct cpg_core_clk r8a77995_core_clks[] __initconst = { }; static const struct mssr_mod_clk r8a77995_mod_clks[] __initconst = { + DEF_MOD("tmu4", 121, R8A77995_CLK_S1D4C), + DEF_MOD("tmu3", 122, R8A77995_CLK_S3D2C), + DEF_MOD("tmu2", 123, R8A77995_CLK_S3D2C), + DEF_MOD("tmu1", 124, R8A77995_CLK_S3D2C), + DEF_MOD("tmu0", 125, R8A77995_CLK_CP), DEF_MOD("scif5", 202, R8A77995_CLK_S3D4C), DEF_MOD("scif4", 203, R8A77995_CLK_S3D4C), DEF_MOD("scif3", 204, R8A77995_CLK_S3D4C), diff --git a/drivers/clk/renesas/r8a779a0-cpg-mssr.c b/drivers/clk/renesas/r8a779a0-cpg-mssr.c index aa5389b04d74204b1f8b196963fc2392b375ff1f..f23fe9d5e5e1c7a3c444a40e91ee5d747b7ca07e 100644 --- a/drivers/clk/renesas/r8a779a0-cpg-mssr.c +++ b/drivers/clk/renesas/r8a779a0-cpg-mssr.c @@ -25,6 +25,7 @@ #include +#include "rcar-cpg-lib.h" #include "renesas-cpg-mssr.h" enum rcar_r8a779a0_clk_types { @@ -32,6 +33,7 @@ enum rcar_r8a779a0_clk_types { CLK_TYPE_R8A779A0_PLL1, CLK_TYPE_R8A779A0_PLL2X_3X, /* PLL[23][01] */ CLK_TYPE_R8A779A0_PLL5, + CLK_TYPE_R8A779A0_SD, CLK_TYPE_R8A779A0_MDSEL, /* Select parent/divider using mode pin */ CLK_TYPE_R8A779A0_OSC, /* OSC EXTAL predivider and fixed divider */ }; @@ -69,7 +71,6 @@ enum clk_ids { CLK_PLL5_DIV2, CLK_PLL5_DIV4, CLK_S1, - CLK_S2, CLK_S3, CLK_SDSRC, CLK_RPCSRC, @@ -83,6 +84,9 @@ enum clk_ids { DEF_BASE(_name, _id, CLK_TYPE_R8A779A0_PLL2X_3X, CLK_MAIN, \ .offset = _offset) +#define DEF_SD(_name, _id, _parent, _offset) \ + DEF_BASE(_name, _id, CLK_TYPE_R8A779A0_SD, _parent, .offset = _offset) + #define DEF_MDSEL(_name, _id, _md, _parent0, _div0, _parent1, _div1) \ DEF_BASE(_name, _id, CLK_TYPE_R8A779A0_MDSEL, \ (_parent0) << 16 | (_parent1), \ @@ -114,6 +118,7 @@ static const struct cpg_core_clk r8a779a0_core_clks[] __initconst = { DEF_FIXED(".pll5_div4", CLK_PLL5_DIV4, CLK_PLL5_DIV2, 2, 1), DEF_FIXED(".s1", CLK_S1, CLK_PLL1_DIV2, 2, 1), DEF_FIXED(".s3", CLK_S3, CLK_PLL1_DIV2, 4, 1), + DEF_FIXED(".sdsrc", CLK_SDSRC, CLK_PLL5_DIV4, 1, 1), DEF_RATE(".oco", CLK_OCO, 32768), /* Core Clock Outputs */ @@ -137,7 +142,10 @@ static const struct cpg_core_clk r8a779a0_core_clks[] __initconst = { DEF_FIXED("icu", R8A779A0_CLK_ICU, CLK_PLL5_DIV4, 2, 1), DEF_FIXED("icud2", R8A779A0_CLK_ICUD2, CLK_PLL5_DIV4, 4, 1), DEF_FIXED("vcbus", R8A779A0_CLK_VCBUS, CLK_PLL5_DIV4, 1, 1), - DEF_FIXED("cbfusa", R8A779A0_CLK_CBFUSA, CLK_MAIN, 2, 1), + DEF_FIXED("cbfusa", R8A779A0_CLK_CBFUSA, CLK_EXTAL, 2, 1), + DEF_FIXED("cp", R8A779A0_CLK_CP, CLK_EXTAL, 2, 1), + + DEF_SD("sd0", R8A779A0_CLK_SD0, CLK_SDSRC, 0x870), DEF_DIV6P1("mso", R8A779A0_CLK_MSO, CLK_PLL5_DIV4, 0x87c), DEF_DIV6P1("canfd", R8A779A0_CLK_CANFD, CLK_PLL5_DIV4, 0x878), @@ -148,14 +156,42 @@ static const struct cpg_core_clk r8a779a0_core_clks[] __initconst = { }; static const struct mssr_mod_clk r8a779a0_mod_clks[] __initconst = { + DEF_MOD("avb0", 211, R8A779A0_CLK_S3D2), + DEF_MOD("avb1", 212, R8A779A0_CLK_S3D2), + DEF_MOD("avb2", 213, R8A779A0_CLK_S3D2), + DEF_MOD("avb3", 214, R8A779A0_CLK_S3D2), + DEF_MOD("avb4", 215, R8A779A0_CLK_S3D2), + DEF_MOD("avb5", 216, R8A779A0_CLK_S3D2), DEF_MOD("csi40", 331, R8A779A0_CLK_CSI0), DEF_MOD("csi41", 400, R8A779A0_CLK_CSI0), DEF_MOD("csi42", 401, R8A779A0_CLK_CSI0), DEF_MOD("csi43", 402, R8A779A0_CLK_CSI0), + DEF_MOD("fcpvd0", 508, R8A779A0_CLK_S3D1), + DEF_MOD("fcpvd1", 509, R8A779A0_CLK_S3D1), + DEF_MOD("hscif0", 514, R8A779A0_CLK_S1D2), + DEF_MOD("hscif1", 515, R8A779A0_CLK_S1D2), + DEF_MOD("hscif2", 516, R8A779A0_CLK_S1D2), + DEF_MOD("hscif3", 517, R8A779A0_CLK_S1D2), + DEF_MOD("i2c0", 518, R8A779A0_CLK_S1D4), + DEF_MOD("i2c1", 519, R8A779A0_CLK_S1D4), + DEF_MOD("i2c2", 520, R8A779A0_CLK_S1D4), + DEF_MOD("i2c3", 521, R8A779A0_CLK_S1D4), + DEF_MOD("i2c4", 522, R8A779A0_CLK_S1D4), + DEF_MOD("i2c5", 523, R8A779A0_CLK_S1D4), + DEF_MOD("i2c6", 524, R8A779A0_CLK_S1D4), + DEF_MOD("msi0", 618, R8A779A0_CLK_MSO), + DEF_MOD("msi1", 619, R8A779A0_CLK_MSO), + DEF_MOD("msi2", 620, R8A779A0_CLK_MSO), + DEF_MOD("msi3", 621, R8A779A0_CLK_MSO), + DEF_MOD("msi4", 622, R8A779A0_CLK_MSO), + DEF_MOD("msi5", 623, R8A779A0_CLK_MSO), DEF_MOD("scif0", 702, R8A779A0_CLK_S1D8), DEF_MOD("scif1", 703, R8A779A0_CLK_S1D8), DEF_MOD("scif3", 704, R8A779A0_CLK_S1D8), DEF_MOD("scif4", 705, R8A779A0_CLK_S1D8), + DEF_MOD("sdhi0", 706, R8A779A0_CLK_SD0), + DEF_MOD("sydm1", 709, R8A779A0_CLK_S1D2), + DEF_MOD("sydm2", 710, R8A779A0_CLK_S1D2), DEF_MOD("vin00", 730, R8A779A0_CLK_S1D1), DEF_MOD("vin01", 731, R8A779A0_CLK_S1D1), DEF_MOD("vin02", 800, R8A779A0_CLK_S1D1), @@ -188,10 +224,19 @@ static const struct mssr_mod_clk r8a779a0_mod_clks[] __initconst = { DEF_MOD("vin35", 827, R8A779A0_CLK_S1D1), DEF_MOD("vin36", 828, R8A779A0_CLK_S1D1), DEF_MOD("vin37", 829, R8A779A0_CLK_S1D1), + DEF_MOD("vspd0", 830, R8A779A0_CLK_S3D1), + DEF_MOD("vspd1", 831, R8A779A0_CLK_S3D1), + DEF_MOD("rwdt", 907, R8A779A0_CLK_R), + DEF_MOD("pfc0", 915, R8A779A0_CLK_CP), + DEF_MOD("pfc1", 916, R8A779A0_CLK_CP), + DEF_MOD("pfc2", 917, R8A779A0_CLK_CP), + DEF_MOD("pfc3", 918, R8A779A0_CLK_CP), + DEF_MOD("vspx0", 1028, R8A779A0_CLK_S1D1), + DEF_MOD("vspx1", 1029, R8A779A0_CLK_S1D1), + DEF_MOD("vspx2", 1030, R8A779A0_CLK_S1D1), + DEF_MOD("vspx3", 1031, R8A779A0_CLK_S1D1), }; -static spinlock_t cpg_lock; - static const struct rcar_r8a779a0_cpg_pll_config *cpg_pll_config __initdata; static unsigned int cpg_clk_extalr __initdata; static u32 cpg_mode __initdata; @@ -230,6 +275,12 @@ static struct clk * __init rcar_r8a779a0_cpg_clk_register(struct device *dev, div = cpg_pll_config->pll5_div; break; + case CLK_TYPE_R8A779A0_SD: + return cpg_sd_clk_register(core->name, base, core->offset, + __clk_get_name(parent), notifiers, + false); + break; + case CLK_TYPE_R8A779A0_MDSEL: /* * Clock selectable between two parents and two fixed dividers @@ -261,6 +312,10 @@ static struct clk * __init rcar_r8a779a0_cpg_clk_register(struct device *dev, __clk_get_name(parent), 0, mult, div); } +static const unsigned int r8a779a0_crit_mod_clks[] __initconst = { + MOD_CLK_ID(907), /* RWDT */ +}; + /* * CPG Clock Data */ @@ -311,6 +366,10 @@ const struct cpg_mssr_info r8a779a0_cpg_mssr_info __initconst = { .num_mod_clks = ARRAY_SIZE(r8a779a0_mod_clks), .num_hw_mod_clks = 15 * 32, + /* Critical Module Clocks */ + .crit_mod_clks = r8a779a0_crit_mod_clks, + .num_crit_mod_clks = ARRAY_SIZE(r8a779a0_crit_mod_clks), + /* Callbacks */ .init = r8a779a0_cpg_mssr_init, .cpg_clk_register = rcar_r8a779a0_cpg_clk_register, diff --git a/drivers/clk/renesas/rcar-cpg-lib.c b/drivers/clk/renesas/rcar-cpg-lib.c new file mode 100644 index 0000000000000000000000000000000000000000..7e7e5d1341d5e80c6b02af145d129f9ff8eed3b9 --- /dev/null +++ b/drivers/clk/renesas/rcar-cpg-lib.c @@ -0,0 +1,270 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * R-Car Gen3 Clock Pulse Generator Library + * + * Copyright (C) 2015-2018 Glider bvba + * Copyright (C) 2019 Renesas Electronics Corp. + * + * Based on clk-rcar-gen3.c + * + * Copyright (C) 2015 Renesas Electronics Corp. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "rcar-cpg-lib.h" + +spinlock_t cpg_lock; + +void cpg_reg_modify(void __iomem *reg, u32 clear, u32 set) +{ + unsigned long flags; + u32 val; + + spin_lock_irqsave(&cpg_lock, flags); + val = readl(reg); + val &= ~clear; + val |= set; + writel(val, reg); + spin_unlock_irqrestore(&cpg_lock, flags); +}; + +static int cpg_simple_notifier_call(struct notifier_block *nb, + unsigned long action, void *data) +{ + struct cpg_simple_notifier *csn = + container_of(nb, struct cpg_simple_notifier, nb); + + switch (action) { + case PM_EVENT_SUSPEND: + csn->saved = readl(csn->reg); + return NOTIFY_OK; + + case PM_EVENT_RESUME: + writel(csn->saved, csn->reg); + return NOTIFY_OK; + } + return NOTIFY_DONE; +} + +void cpg_simple_notifier_register(struct raw_notifier_head *notifiers, + struct cpg_simple_notifier *csn) +{ + csn->nb.notifier_call = cpg_simple_notifier_call; + raw_notifier_chain_register(notifiers, &csn->nb); +} + +/* + * SDn Clock + */ +#define CPG_SD_STP_HCK BIT(9) +#define CPG_SD_STP_CK BIT(8) + +#define CPG_SD_STP_MASK (CPG_SD_STP_HCK | CPG_SD_STP_CK) +#define CPG_SD_FC_MASK (0x7 << 2 | 0x3 << 0) + +#define CPG_SD_DIV_TABLE_DATA(stp_hck, sd_srcfc, sd_fc, sd_div) \ +{ \ + .val = ((stp_hck) ? CPG_SD_STP_HCK : 0) | \ + ((sd_srcfc) << 2) | \ + ((sd_fc) << 0), \ + .div = (sd_div), \ +} + +struct sd_div_table { + u32 val; + unsigned int div; +}; + +struct sd_clock { + struct clk_hw hw; + const struct sd_div_table *div_table; + struct cpg_simple_notifier csn; + unsigned int div_num; + unsigned int cur_div_idx; +}; + +/* SDn divider + * sd_srcfc sd_fc div + * stp_hck (div) (div) = sd_srcfc x sd_fc + *--------------------------------------------------------- + * 0 0 (1) 1 (4) 4 : SDR104 / HS200 / HS400 (8 TAP) + * 0 1 (2) 1 (4) 8 : SDR50 + * 1 2 (4) 1 (4) 16 : HS / SDR25 + * 1 3 (8) 1 (4) 32 : NS / SDR12 + * 1 4 (16) 1 (4) 64 + * 0 0 (1) 0 (2) 2 + * 0 1 (2) 0 (2) 4 : SDR104 / HS200 / HS400 (4 TAP) + * 1 2 (4) 0 (2) 8 + * 1 3 (8) 0 (2) 16 + * 1 4 (16) 0 (2) 32 + * + * NOTE: There is a quirk option to ignore the first row of the dividers + * table when searching for suitable settings. This is because HS400 on + * early ES versions of H3 and M3-W requires a specific setting to work. + */ +static const struct sd_div_table cpg_sd_div_table[] = { +/* CPG_SD_DIV_TABLE_DATA(stp_hck, sd_srcfc, sd_fc, sd_div) */ + CPG_SD_DIV_TABLE_DATA(0, 0, 1, 4), + CPG_SD_DIV_TABLE_DATA(0, 1, 1, 8), + CPG_SD_DIV_TABLE_DATA(1, 2, 1, 16), + CPG_SD_DIV_TABLE_DATA(1, 3, 1, 32), + CPG_SD_DIV_TABLE_DATA(1, 4, 1, 64), + CPG_SD_DIV_TABLE_DATA(0, 0, 0, 2), + CPG_SD_DIV_TABLE_DATA(0, 1, 0, 4), + CPG_SD_DIV_TABLE_DATA(1, 2, 0, 8), + CPG_SD_DIV_TABLE_DATA(1, 3, 0, 16), + CPG_SD_DIV_TABLE_DATA(1, 4, 0, 32), +}; + +#define to_sd_clock(_hw) container_of(_hw, struct sd_clock, hw) + +static int cpg_sd_clock_enable(struct clk_hw *hw) +{ + struct sd_clock *clock = to_sd_clock(hw); + + cpg_reg_modify(clock->csn.reg, CPG_SD_STP_MASK, + clock->div_table[clock->cur_div_idx].val & + CPG_SD_STP_MASK); + + return 0; +} + +static void cpg_sd_clock_disable(struct clk_hw *hw) +{ + struct sd_clock *clock = to_sd_clock(hw); + + cpg_reg_modify(clock->csn.reg, 0, CPG_SD_STP_MASK); +} + +static int cpg_sd_clock_is_enabled(struct clk_hw *hw) +{ + struct sd_clock *clock = to_sd_clock(hw); + + return !(readl(clock->csn.reg) & CPG_SD_STP_MASK); +} + +static unsigned long cpg_sd_clock_recalc_rate(struct clk_hw *hw, + unsigned long parent_rate) +{ + struct sd_clock *clock = to_sd_clock(hw); + + return DIV_ROUND_CLOSEST(parent_rate, + clock->div_table[clock->cur_div_idx].div); +} + +static int cpg_sd_clock_determine_rate(struct clk_hw *hw, + struct clk_rate_request *req) +{ + unsigned long best_rate = ULONG_MAX, diff_min = ULONG_MAX; + struct sd_clock *clock = to_sd_clock(hw); + unsigned long calc_rate, diff; + unsigned int i; + + for (i = 0; i < clock->div_num; i++) { + calc_rate = DIV_ROUND_CLOSEST(req->best_parent_rate, + clock->div_table[i].div); + if (calc_rate < req->min_rate || calc_rate > req->max_rate) + continue; + + diff = calc_rate > req->rate ? calc_rate - req->rate + : req->rate - calc_rate; + if (diff < diff_min) { + best_rate = calc_rate; + diff_min = diff; + } + } + + if (best_rate == ULONG_MAX) + return -EINVAL; + + req->rate = best_rate; + return 0; +} + +static int cpg_sd_clock_set_rate(struct clk_hw *hw, unsigned long rate, + unsigned long parent_rate) +{ + struct sd_clock *clock = to_sd_clock(hw); + unsigned int i; + + for (i = 0; i < clock->div_num; i++) + if (rate == DIV_ROUND_CLOSEST(parent_rate, + clock->div_table[i].div)) + break; + + if (i >= clock->div_num) + return -EINVAL; + + clock->cur_div_idx = i; + + cpg_reg_modify(clock->csn.reg, CPG_SD_STP_MASK | CPG_SD_FC_MASK, + clock->div_table[i].val & + (CPG_SD_STP_MASK | CPG_SD_FC_MASK)); + + return 0; +} + +static const struct clk_ops cpg_sd_clock_ops = { + .enable = cpg_sd_clock_enable, + .disable = cpg_sd_clock_disable, + .is_enabled = cpg_sd_clock_is_enabled, + .recalc_rate = cpg_sd_clock_recalc_rate, + .determine_rate = cpg_sd_clock_determine_rate, + .set_rate = cpg_sd_clock_set_rate, +}; + +struct clk * __init cpg_sd_clk_register(const char *name, + void __iomem *base, unsigned int offset, const char *parent_name, + struct raw_notifier_head *notifiers, bool skip_first) +{ + struct clk_init_data init; + struct sd_clock *clock; + struct clk *clk; + u32 val; + + clock = kzalloc(sizeof(*clock), GFP_KERNEL); + if (!clock) + return ERR_PTR(-ENOMEM); + + init.name = name; + init.ops = &cpg_sd_clock_ops; + init.flags = CLK_SET_RATE_PARENT; + init.parent_names = &parent_name; + init.num_parents = 1; + + clock->csn.reg = base + offset; + clock->hw.init = &init; + clock->div_table = cpg_sd_div_table; + clock->div_num = ARRAY_SIZE(cpg_sd_div_table); + + if (skip_first) { + clock->div_table++; + clock->div_num--; + } + + val = readl(clock->csn.reg) & ~CPG_SD_FC_MASK; + val |= CPG_SD_STP_MASK | (clock->div_table[0].val & CPG_SD_FC_MASK); + writel(val, clock->csn.reg); + + clk = clk_register(NULL, &clock->hw); + if (IS_ERR(clk)) + goto free_clock; + + cpg_simple_notifier_register(notifiers, &clock->csn); + return clk; + +free_clock: + kfree(clock); + return clk; +} + + diff --git a/drivers/clk/renesas/rcar-cpg-lib.h b/drivers/clk/renesas/rcar-cpg-lib.h new file mode 100644 index 0000000000000000000000000000000000000000..d00c91b116ca0e36f743514c53b0cead5bb4c46a --- /dev/null +++ b/drivers/clk/renesas/rcar-cpg-lib.h @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * R-Car Gen3 Clock Pulse Generator Library + * + * Copyright (C) 2015-2018 Glider bvba + * Copyright (C) 2019 Renesas Electronics Corp. + * + * Based on clk-rcar-gen3.c + * + * Copyright (C) 2015 Renesas Electronics Corp. + */ + +#ifndef __CLK_RENESAS_RCAR_CPG_LIB_H__ +#define __CLK_RENESAS_RCAR_CPG_LIB_H__ + +extern spinlock_t cpg_lock; + +struct cpg_simple_notifier { + struct notifier_block nb; + void __iomem *reg; + u32 saved; +}; + +void cpg_simple_notifier_register(struct raw_notifier_head *notifiers, + struct cpg_simple_notifier *csn); + +void cpg_reg_modify(void __iomem *reg, u32 clear, u32 set); + +struct clk * __init cpg_sd_clk_register(const char *name, + void __iomem *base, unsigned int offset, const char *parent_name, + struct raw_notifier_head *notifiers, bool skip_first); + +#endif diff --git a/drivers/clk/renesas/rcar-gen3-cpg.c b/drivers/clk/renesas/rcar-gen3-cpg.c index 063b6115148869aa014216343cfa06407a30b77b..17826599e9dd0e02da6eb767414c6a1bcb79e1e6 100644 --- a/drivers/clk/renesas/rcar-gen3-cpg.c +++ b/drivers/clk/renesas/rcar-gen3-cpg.c @@ -23,6 +23,7 @@ #include #include "renesas-cpg-mssr.h" +#include "rcar-cpg-lib.h" #include "rcar-gen3-cpg.h" #define CPG_PLL0CR 0x00d8 @@ -31,52 +32,6 @@ #define CPG_RCKCR_CKSEL BIT(15) /* RCLK Clock Source Select */ -static spinlock_t cpg_lock; - -static void cpg_reg_modify(void __iomem *reg, u32 clear, u32 set) -{ - unsigned long flags; - u32 val; - - spin_lock_irqsave(&cpg_lock, flags); - val = readl(reg); - val &= ~clear; - val |= set; - writel(val, reg); - spin_unlock_irqrestore(&cpg_lock, flags); -}; - -struct cpg_simple_notifier { - struct notifier_block nb; - void __iomem *reg; - u32 saved; -}; - -static int cpg_simple_notifier_call(struct notifier_block *nb, - unsigned long action, void *data) -{ - struct cpg_simple_notifier *csn = - container_of(nb, struct cpg_simple_notifier, nb); - - switch (action) { - case PM_EVENT_SUSPEND: - csn->saved = readl(csn->reg); - return NOTIFY_OK; - - case PM_EVENT_RESUME: - writel(csn->saved, csn->reg); - return NOTIFY_OK; - } - return NOTIFY_DONE; -} - -static void cpg_simple_notifier_register(struct raw_notifier_head *notifiers, - struct cpg_simple_notifier *csn) -{ - csn->nb.notifier_call = cpg_simple_notifier_call; - raw_notifier_chain_register(notifiers, &csn->nb); -} - /* * Z Clock & Z2 Clock * @@ -215,217 +170,6 @@ static struct clk * __init cpg_z_clk_register(const char *name, return clk; } -/* - * SDn Clock - */ -#define CPG_SD_STP_HCK BIT(9) -#define CPG_SD_STP_CK BIT(8) - -#define CPG_SD_STP_MASK (CPG_SD_STP_HCK | CPG_SD_STP_CK) -#define CPG_SD_FC_MASK (0x7 << 2 | 0x3 << 0) - -#define CPG_SD_DIV_TABLE_DATA(stp_hck, sd_srcfc, sd_fc, sd_div) \ -{ \ - .val = ((stp_hck) ? CPG_SD_STP_HCK : 0) | \ - ((sd_srcfc) << 2) | \ - ((sd_fc) << 0), \ - .div = (sd_div), \ -} - -struct sd_div_table { - u32 val; - unsigned int div; -}; - -struct sd_clock { - struct clk_hw hw; - const struct sd_div_table *div_table; - struct cpg_simple_notifier csn; - unsigned int div_num; - unsigned int cur_div_idx; -}; - -/* SDn divider - * sd_srcfc sd_fc div - * stp_hck (div) (div) = sd_srcfc x sd_fc - *--------------------------------------------------------- - * 0 0 (1) 1 (4) 4 : SDR104 / HS200 / HS400 (8 TAP) - * 0 1 (2) 1 (4) 8 : SDR50 - * 1 2 (4) 1 (4) 16 : HS / SDR25 - * 1 3 (8) 1 (4) 32 : NS / SDR12 - * 1 4 (16) 1 (4) 64 - * 0 0 (1) 0 (2) 2 - * 0 1 (2) 0 (2) 4 : SDR104 / HS200 / HS400 (4 TAP) - * 1 2 (4) 0 (2) 8 - * 1 3 (8) 0 (2) 16 - * 1 4 (16) 0 (2) 32 - * - * NOTE: There is a quirk option to ignore the first row of the dividers - * table when searching for suitable settings. This is because HS400 on - * early ES versions of H3 and M3-W requires a specific setting to work. - */ -static const struct sd_div_table cpg_sd_div_table[] = { -/* CPG_SD_DIV_TABLE_DATA(stp_hck, sd_srcfc, sd_fc, sd_div) */ - CPG_SD_DIV_TABLE_DATA(0, 0, 1, 4), - CPG_SD_DIV_TABLE_DATA(0, 1, 1, 8), - CPG_SD_DIV_TABLE_DATA(1, 2, 1, 16), - CPG_SD_DIV_TABLE_DATA(1, 3, 1, 32), - CPG_SD_DIV_TABLE_DATA(1, 4, 1, 64), - CPG_SD_DIV_TABLE_DATA(0, 0, 0, 2), - CPG_SD_DIV_TABLE_DATA(0, 1, 0, 4), - CPG_SD_DIV_TABLE_DATA(1, 2, 0, 8), - CPG_SD_DIV_TABLE_DATA(1, 3, 0, 16), - CPG_SD_DIV_TABLE_DATA(1, 4, 0, 32), -}; - -#define to_sd_clock(_hw) container_of(_hw, struct sd_clock, hw) - -static int cpg_sd_clock_enable(struct clk_hw *hw) -{ - struct sd_clock *clock = to_sd_clock(hw); - - cpg_reg_modify(clock->csn.reg, CPG_SD_STP_MASK, - clock->div_table[clock->cur_div_idx].val & - CPG_SD_STP_MASK); - - return 0; -} - -static void cpg_sd_clock_disable(struct clk_hw *hw) -{ - struct sd_clock *clock = to_sd_clock(hw); - - cpg_reg_modify(clock->csn.reg, 0, CPG_SD_STP_MASK); -} - -static int cpg_sd_clock_is_enabled(struct clk_hw *hw) -{ - struct sd_clock *clock = to_sd_clock(hw); - - return !(readl(clock->csn.reg) & CPG_SD_STP_MASK); -} - -static unsigned long cpg_sd_clock_recalc_rate(struct clk_hw *hw, - unsigned long parent_rate) -{ - struct sd_clock *clock = to_sd_clock(hw); - - return DIV_ROUND_CLOSEST(parent_rate, - clock->div_table[clock->cur_div_idx].div); -} - -static int cpg_sd_clock_determine_rate(struct clk_hw *hw, - struct clk_rate_request *req) -{ - unsigned long best_rate = ULONG_MAX, diff_min = ULONG_MAX; - struct sd_clock *clock = to_sd_clock(hw); - unsigned long calc_rate, diff; - unsigned int i; - - for (i = 0; i < clock->div_num; i++) { - calc_rate = DIV_ROUND_CLOSEST(req->best_parent_rate, - clock->div_table[i].div); - if (calc_rate < req->min_rate || calc_rate > req->max_rate) - continue; - - diff = calc_rate > req->rate ? calc_rate - req->rate - : req->rate - calc_rate; - if (diff < diff_min) { - best_rate = calc_rate; - diff_min = diff; - } - } - - if (best_rate == ULONG_MAX) - return -EINVAL; - - req->rate = best_rate; - return 0; -} - -static int cpg_sd_clock_set_rate(struct clk_hw *hw, unsigned long rate, - unsigned long parent_rate) -{ - struct sd_clock *clock = to_sd_clock(hw); - unsigned int i; - - for (i = 0; i < clock->div_num; i++) - if (rate == DIV_ROUND_CLOSEST(parent_rate, - clock->div_table[i].div)) - break; - - if (i >= clock->div_num) - return -EINVAL; - - clock->cur_div_idx = i; - - cpg_reg_modify(clock->csn.reg, CPG_SD_STP_MASK | CPG_SD_FC_MASK, - clock->div_table[i].val & - (CPG_SD_STP_MASK | CPG_SD_FC_MASK)); - - return 0; -} - -static const struct clk_ops cpg_sd_clock_ops = { - .enable = cpg_sd_clock_enable, - .disable = cpg_sd_clock_disable, - .is_enabled = cpg_sd_clock_is_enabled, - .recalc_rate = cpg_sd_clock_recalc_rate, - .determine_rate = cpg_sd_clock_determine_rate, - .set_rate = cpg_sd_clock_set_rate, -}; - -static u32 cpg_quirks __initdata; - -#define PLL_ERRATA BIT(0) /* Missing PLL0/2/4 post-divider */ -#define RCKCR_CKSEL BIT(1) /* Manual RCLK parent selection */ -#define SD_SKIP_FIRST BIT(2) /* Skip first clock in SD table */ - -static struct clk * __init cpg_sd_clk_register(const char *name, - void __iomem *base, unsigned int offset, const char *parent_name, - struct raw_notifier_head *notifiers) -{ - struct clk_init_data init; - struct sd_clock *clock; - struct clk *clk; - u32 val; - - clock = kzalloc(sizeof(*clock), GFP_KERNEL); - if (!clock) - return ERR_PTR(-ENOMEM); - - init.name = name; - init.ops = &cpg_sd_clock_ops; - init.flags = CLK_SET_RATE_PARENT; - init.parent_names = &parent_name; - init.num_parents = 1; - - clock->csn.reg = base + offset; - clock->hw.init = &init; - clock->div_table = cpg_sd_div_table; - clock->div_num = ARRAY_SIZE(cpg_sd_div_table); - - if (cpg_quirks & SD_SKIP_FIRST) { - clock->div_table++; - clock->div_num--; - } - - val = readl(clock->csn.reg) & ~CPG_SD_FC_MASK; - val |= CPG_SD_STP_MASK | (clock->div_table[0].val & CPG_SD_FC_MASK); - writel(val, clock->csn.reg); - - clk = clk_register(NULL, &clock->hw); - if (IS_ERR(clk)) - goto free_clock; - - cpg_simple_notifier_register(notifiers, &clock->csn); - return clk; - -free_clock: - kfree(clock); - return clk; -} - struct rpc_clock { struct clk_divider div; struct clk_gate gate; @@ -518,6 +262,12 @@ static struct clk * __init cpg_rpcd2_clk_register(const char *name, static const struct rcar_gen3_cpg_pll_config *cpg_pll_config __initdata; static unsigned int cpg_clk_extalr __initdata; static u32 cpg_mode __initdata; +static u32 cpg_quirks __initdata; + +#define PLL_ERRATA BIT(0) /* Missing PLL0/2/4 post-divider */ +#define RCKCR_CKSEL BIT(1) /* Manual RCLK parent selection */ +#define SD_SKIP_FIRST BIT(2) /* Skip first clock in SD table */ + static const struct soc_device_attribute cpg_quirks_match[] __initconst = { { @@ -613,7 +363,8 @@ struct clk * __init rcar_gen3_cpg_clk_register(struct device *dev, case CLK_TYPE_GEN3_SD: return cpg_sd_clk_register(core->name, base, core->offset, - __clk_get_name(parent), notifiers); + __clk_get_name(parent), notifiers, + cpg_quirks & SD_SKIP_FIRST); case CLK_TYPE_GEN3_R: if (cpg_quirks & RCKCR_CKSEL) { diff --git a/drivers/clk/renesas/renesas-cpg-mssr.c b/drivers/clk/renesas/renesas-cpg-mssr.c index 1c3215dc4877c8b6cf31d544f07cadaefca9f7b7..bffbc3d2faf5fa16c8912173c54714d053e07ec8 100644 --- a/drivers/clk/renesas/renesas-cpg-mssr.c +++ b/drivers/clk/renesas/renesas-cpg-mssr.c @@ -136,8 +136,8 @@ static const u16 srstclr_for_v3u[] = { * @control_regs: Pointer to control registers array * @reset_regs: Pointer to reset registers array * @reset_clear_regs: Pointer to reset clearing registers array - * @smstpcr_saved[].mask: Mask of SMSTPCR[] bits under our control - * @smstpcr_saved[].val: Saved values of SMSTPCR[] + * @smstpcr_saved: [].mask: Mask of SMSTPCR[] bits under our control + * [].val: Saved values of SMSTPCR[] * @clks: Array containing all Core and Module Clocks */ struct cpg_mssr_priv { diff --git a/drivers/clk/rockchip/clk-cpu.c b/drivers/clk/rockchip/clk-cpu.c index 0dc478a19451b4d026e4b738dc0fe7bcdabe8128..fa9027fb1920ac260f7bcf6d14c75584daca50b8 100644 --- a/drivers/clk/rockchip/clk-cpu.c +++ b/drivers/clk/rockchip/clk-cpu.c @@ -51,10 +51,6 @@ */ struct rockchip_cpuclk { struct clk_hw hw; - - struct clk_mux cpu_mux; - const struct clk_ops *cpu_mux_ops; - struct clk *alt_parent; void __iomem *reg_base; struct notifier_block clk_nb; diff --git a/drivers/clk/rockchip/clk-half-divider.c b/drivers/clk/rockchip/clk-half-divider.c index ccd5c270c2132f04e13814280310e06b07c0b081..64f7faad2148f27099d1ace36da45207e2315a1c 100644 --- a/drivers/clk/rockchip/clk-half-divider.c +++ b/drivers/clk/rockchip/clk-half-divider.c @@ -145,7 +145,7 @@ static const struct clk_ops clk_half_divider_ops = { .set_rate = clk_half_divider_set_rate, }; -/** +/* * Register a clock branch. * Most clock branches have a form like * diff --git a/drivers/clk/rockchip/clk-pll.c b/drivers/clk/rockchip/clk-pll.c index 4c6c9167ef50944988291ac89ddf70203dff2fc7..fe937bcdb4876696869a840c57f4b70b7fc32b0d 100644 --- a/drivers/clk/rockchip/clk-pll.c +++ b/drivers/clk/rockchip/clk-pll.c @@ -97,7 +97,7 @@ static int rockchip_pll_wait_lock(struct rockchip_clk_pll *pll) return ret; } -/** +/* * PLL used in RK3036 */ @@ -358,7 +358,7 @@ static const struct clk_ops rockchip_rk3036_pll_clk_ops = { .init = rockchip_rk3036_pll_init, }; -/** +/* * PLL used in RK3066, RK3188 and RK3288 */ @@ -577,7 +577,7 @@ static const struct clk_ops rockchip_rk3066_pll_clk_ops = { .init = rockchip_rk3066_pll_init, }; -/** +/* * PLL used in RK3399 */ diff --git a/drivers/clk/rockchip/clk-rk3368.c b/drivers/clk/rockchip/clk-rk3368.c index 55443349439b414c8371a4549f3e0b187ef8c0cb..9a0dab9448db37d3d40d1b286a17cd20f9c5396b 100644 --- a/drivers/clk/rockchip/clk-rk3368.c +++ b/drivers/clk/rockchip/clk-rk3368.c @@ -474,7 +474,7 @@ static struct rockchip_clk_branch rk3368_clk_branches[] __initdata = { COMPOSITE_NODIV(0, "vip_src", mux_pll_src_cpll_gpll_p, 0, RK3368_CLKSEL_CON(21), 15, 1, MFLAGS, RK3368_CLKGATE_CON(4), 5, GFLAGS), - COMPOSITE_NOGATE(0, "sclk_vip_out", mux_vip_out_p, 0, + COMPOSITE_NOGATE(SCLK_VIP_OUT, "sclk_vip_out", mux_vip_out_p, 0, RK3368_CLKSEL_CON(21), 14, 1, MFLAGS, 8, 5, DFLAGS), COMPOSITE_NODIV(SCLK_EDP_24M, "sclk_edp_24m", mux_edp_24m_p, 0, @@ -818,8 +818,8 @@ static struct rockchip_clk_branch rk3368_clk_branches[] __initdata = { * pclk_vio gates * pclk_vio comes from the exactly same source as hclk_vio */ - GATE(0, "pclk_dphyrx", "hclk_vio", CLK_IGNORE_UNUSED, RK3368_CLKGATE_CON(14), 8, GFLAGS), - GATE(0, "pclk_dphytx", "hclk_vio", CLK_IGNORE_UNUSED, RK3368_CLKGATE_CON(14), 8, GFLAGS), + GATE(PCLK_DPHYRX, "pclk_dphyrx", "hclk_vio", CLK_IGNORE_UNUSED, RK3368_CLKGATE_CON(22), 11, GFLAGS), + GATE(PCLK_DPHYTX0, "pclk_dphytx0", "hclk_vio", CLK_IGNORE_UNUSED, RK3368_CLKGATE_CON(22), 10, GFLAGS), /* pclk_pd_pmu gates */ GATE(PCLK_PMUGRF, "pclk_pmugrf", "pclk_pd_pmu", CLK_IGNORE_UNUSED, RK3368_CLKGATE_CON(23), 5, GFLAGS), diff --git a/drivers/clk/rockchip/clk.c b/drivers/clk/rockchip/clk.c index 336481bc6cc726d182ef21543c158b1658138d37..049e5e0b64f62f64295fb23e6e1b373a09e5676d 100644 --- a/drivers/clk/rockchip/clk.c +++ b/drivers/clk/rockchip/clk.c @@ -24,7 +24,7 @@ #include #include "clk.h" -/** +/* * Register a clock branch. * Most clock branches have a form like * @@ -170,7 +170,7 @@ static int rockchip_clk_frac_notifier_cb(struct notifier_block *nb, return notifier_from_errno(ret); } -/** +/* * fractional divider must set that denominator is 20 times larger than * numerator to generate precise clock frequency. */ diff --git a/drivers/clk/sifive/fu540-prci.h b/drivers/clk/sifive/fu540-prci.h index c8271efa7bdc73587155ec8e1cf433d92a3abadd..c220677dc0108e1ef4fb292149e8a060b0d0f38f 100644 --- a/drivers/clk/sifive/fu540-prci.h +++ b/drivers/clk/sifive/fu540-prci.h @@ -13,9 +13,4 @@ extern struct __prci_clock __prci_init_clocks_fu540[NUM_CLOCK_FU540]; -static const struct prci_clk_desc prci_clk_fu540 = { - .clks = __prci_init_clocks_fu540, - .num_clks = ARRAY_SIZE(__prci_init_clocks_fu540), -}; - #endif /* __SIFIVE_CLK_FU540_PRCI_H */ diff --git a/drivers/clk/sifive/sifive-prci.c b/drivers/clk/sifive/sifive-prci.c index c78b042750e210926ae30e087837d29193ec3e58..1490b01ce629020ad727cbeead08f1ad6d7059a2 100644 --- a/drivers/clk/sifive/sifive-prci.c +++ b/drivers/clk/sifive/sifive-prci.c @@ -12,6 +12,11 @@ #include "fu540-prci.h" #include "fu740-prci.h" +static const struct prci_clk_desc prci_clk_fu540 = { + .clks = __prci_init_clocks_fu540, + .num_clks = ARRAY_SIZE(__prci_init_clocks_fu540), +}; + /* * Private functions */ diff --git a/drivers/clk/sirf/Makefile b/drivers/clk/sirf/Makefile deleted file mode 100644 index 0ff61f87cddb4504c0d8ef0bd698a624fe0dd474..0000000000000000000000000000000000000000 --- a/drivers/clk/sirf/Makefile +++ /dev/null @@ -1,6 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -# -# Makefile for sirf specific clk -# - -obj-$(CONFIG_ARCH_SIRF) += clk-prima2.o clk-atlas6.o clk-atlas7.o diff --git a/drivers/clk/sirf/atlas6.h b/drivers/clk/sirf/atlas6.h deleted file mode 100644 index cb871e30a1758da6a8ed6cfd2e5445e322b09abc..0000000000000000000000000000000000000000 --- a/drivers/clk/sirf/atlas6.h +++ /dev/null @@ -1,32 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#define SIRFSOC_CLKC_CLK_EN0 0x0000 -#define SIRFSOC_CLKC_CLK_EN1 0x0004 -#define SIRFSOC_CLKC_REF_CFG 0x0020 -#define SIRFSOC_CLKC_CPU_CFG 0x0024 -#define SIRFSOC_CLKC_MEM_CFG 0x0028 -#define SIRFSOC_CLKC_MEMDIV_CFG 0x002C -#define SIRFSOC_CLKC_SYS_CFG 0x0030 -#define SIRFSOC_CLKC_IO_CFG 0x0034 -#define SIRFSOC_CLKC_DSP_CFG 0x0038 -#define SIRFSOC_CLKC_GFX_CFG 0x003c -#define SIRFSOC_CLKC_MM_CFG 0x0040 -#define SIRFSOC_CLKC_GFX2D_CFG 0x0040 -#define SIRFSOC_CLKC_LCD_CFG 0x0044 -#define SIRFSOC_CLKC_MMC01_CFG 0x0048 -#define SIRFSOC_CLKC_MMC23_CFG 0x004C -#define SIRFSOC_CLKC_MMC45_CFG 0x0050 -#define SIRFSOC_CLKC_NAND_CFG 0x0054 -#define SIRFSOC_CLKC_NANDDIV_CFG 0x0058 -#define SIRFSOC_CLKC_PLL1_CFG0 0x0080 -#define SIRFSOC_CLKC_PLL2_CFG0 0x0084 -#define SIRFSOC_CLKC_PLL3_CFG0 0x0088 -#define SIRFSOC_CLKC_PLL1_CFG1 0x008c -#define SIRFSOC_CLKC_PLL2_CFG1 0x0090 -#define SIRFSOC_CLKC_PLL3_CFG1 0x0094 -#define SIRFSOC_CLKC_PLL1_CFG2 0x0098 -#define SIRFSOC_CLKC_PLL2_CFG2 0x009c -#define SIRFSOC_CLKC_PLL3_CFG2 0x00A0 -#define SIRFSOC_USBPHY_PLL_CTRL 0x0008 -#define SIRFSOC_USBPHY_PLL_POWERDOWN BIT(1) -#define SIRFSOC_USBPHY_PLL_BYPASS BIT(2) -#define SIRFSOC_USBPHY_PLL_LOCK BIT(3) diff --git a/drivers/clk/sirf/clk-atlas6.c b/drivers/clk/sirf/clk-atlas6.c deleted file mode 100644 index b95483bb6a5ecd7e5638d670df2f6251638ea756..0000000000000000000000000000000000000000 --- a/drivers/clk/sirf/clk-atlas6.c +++ /dev/null @@ -1,150 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Clock tree for CSR SiRFatlasVI - * - * Copyright (c) 2011 - 2014 Cambridge Silicon Radio Limited, a CSR plc group - * company. - */ - -#include -#include -#include -#include -#include -#include -#include - -#include "atlas6.h" -#include "clk-common.c" - -static struct clk_dmn clk_mmc01 = { - .regofs = SIRFSOC_CLKC_MMC01_CFG, - .enable_bit = 59, - .hw = { - .init = &clk_mmc01_init, - }, -}; - -static struct clk_dmn clk_mmc23 = { - .regofs = SIRFSOC_CLKC_MMC23_CFG, - .enable_bit = 60, - .hw = { - .init = &clk_mmc23_init, - }, -}; - -static struct clk_dmn clk_mmc45 = { - .regofs = SIRFSOC_CLKC_MMC45_CFG, - .enable_bit = 61, - .hw = { - .init = &clk_mmc45_init, - }, -}; - -static const struct clk_init_data clk_nand_init = { - .name = "nand", - .ops = &dmn_ops, - .parent_names = dmn_clk_parents, - .num_parents = ARRAY_SIZE(dmn_clk_parents), -}; - -static struct clk_dmn clk_nand = { - .regofs = SIRFSOC_CLKC_NAND_CFG, - .enable_bit = 34, - .hw = { - .init = &clk_nand_init, - }, -}; - -enum atlas6_clk_index { - /* 0 1 2 3 4 5 6 7 8 9 */ - rtc, osc, pll1, pll2, pll3, mem, sys, security, dsp, gps, - mf, io, cpu, uart0, uart1, uart2, tsc, i2c0, i2c1, spi0, - spi1, pwmc, efuse, pulse, dmac0, dmac1, nand, audio, usp0, usp1, - usp2, vip, gfx, gfx2d, lcd, vpp, mmc01, mmc23, mmc45, usbpll, - usb0, usb1, cphif, maxclk, -}; - -static __initdata struct clk_hw *atlas6_clk_hw_array[maxclk] = { - NULL, /* dummy */ - NULL, - &clk_pll1.hw, - &clk_pll2.hw, - &clk_pll3.hw, - &clk_mem.hw, - &clk_sys.hw, - &clk_security.hw, - &clk_dsp.hw, - &clk_gps.hw, - &clk_mf.hw, - &clk_io.hw, - &clk_cpu.hw, - &clk_uart0.hw, - &clk_uart1.hw, - &clk_uart2.hw, - &clk_tsc.hw, - &clk_i2c0.hw, - &clk_i2c1.hw, - &clk_spi0.hw, - &clk_spi1.hw, - &clk_pwmc.hw, - &clk_efuse.hw, - &clk_pulse.hw, - &clk_dmac0.hw, - &clk_dmac1.hw, - &clk_nand.hw, - &clk_audio.hw, - &clk_usp0.hw, - &clk_usp1.hw, - &clk_usp2.hw, - &clk_vip.hw, - &clk_gfx.hw, - &clk_gfx2d.hw, - &clk_lcd.hw, - &clk_vpp.hw, - &clk_mmc01.hw, - &clk_mmc23.hw, - &clk_mmc45.hw, - &usb_pll_clk_hw, - &clk_usb0.hw, - &clk_usb1.hw, - &clk_cphif.hw, -}; - -static struct clk *atlas6_clks[maxclk]; - -static void __init atlas6_clk_init(struct device_node *np) -{ - struct device_node *rscnp; - int i; - - rscnp = of_find_compatible_node(NULL, NULL, "sirf,prima2-rsc"); - sirfsoc_rsc_vbase = of_iomap(rscnp, 0); - if (!sirfsoc_rsc_vbase) - panic("unable to map rsc registers\n"); - of_node_put(rscnp); - - sirfsoc_clk_vbase = of_iomap(np, 0); - if (!sirfsoc_clk_vbase) - panic("unable to map clkc registers\n"); - - /* These are always available (RTC and 26MHz OSC)*/ - atlas6_clks[rtc] = clk_register_fixed_rate(NULL, "rtc", NULL, 0, 32768); - atlas6_clks[osc] = clk_register_fixed_rate(NULL, "osc", NULL, 0, - 26000000); - - for (i = pll1; i < maxclk; i++) { - atlas6_clks[i] = clk_register(NULL, atlas6_clk_hw_array[i]); - BUG_ON(IS_ERR(atlas6_clks[i])); - } - clk_register_clkdev(atlas6_clks[cpu], NULL, "cpu"); - clk_register_clkdev(atlas6_clks[io], NULL, "io"); - clk_register_clkdev(atlas6_clks[mem], NULL, "mem"); - clk_register_clkdev(atlas6_clks[mem], NULL, "osc"); - - clk_data.clks = atlas6_clks; - clk_data.clk_num = maxclk; - - of_clk_add_provider(np, of_clk_src_onecell_get, &clk_data); -} -CLK_OF_DECLARE(atlas6_clk, "sirf,atlas6-clkc", atlas6_clk_init); diff --git a/drivers/clk/sirf/clk-atlas7.c b/drivers/clk/sirf/clk-atlas7.c deleted file mode 100644 index 3f57fefd13bbb9ad0b00a7940cd9d53340b6caa0..0000000000000000000000000000000000000000 --- a/drivers/clk/sirf/clk-atlas7.c +++ /dev/null @@ -1,1682 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Clock tree for CSR SiRFAtlas7 - * - * Copyright (c) 2014 Cambridge Silicon Radio Limited, a CSR plc group company. - */ - -#include -#include -#include -#include -#include -#include -#include - -#define SIRFSOC_CLKC_MEMPLL_AB_FREQ 0x0000 -#define SIRFSOC_CLKC_MEMPLL_AB_SSC 0x0004 -#define SIRFSOC_CLKC_MEMPLL_AB_CTRL0 0x0008 -#define SIRFSOC_CLKC_MEMPLL_AB_CTRL1 0x000c -#define SIRFSOC_CLKC_MEMPLL_AB_STATUS 0x0010 -#define SIRFSOC_CLKC_MEMPLL_AB_SSRAM_ADDR 0x0014 -#define SIRFSOC_CLKC_MEMPLL_AB_SSRAM_DATA 0x0018 - -#define SIRFSOC_CLKC_CPUPLL_AB_FREQ 0x001c -#define SIRFSOC_CLKC_CPUPLL_AB_SSC 0x0020 -#define SIRFSOC_CLKC_CPUPLL_AB_CTRL0 0x0024 -#define SIRFSOC_CLKC_CPUPLL_AB_CTRL1 0x0028 -#define SIRFSOC_CLKC_CPUPLL_AB_STATUS 0x002c - -#define SIRFSOC_CLKC_SYS0PLL_AB_FREQ 0x0030 -#define SIRFSOC_CLKC_SYS0PLL_AB_SSC 0x0034 -#define SIRFSOC_CLKC_SYS0PLL_AB_CTRL0 0x0038 -#define SIRFSOC_CLKC_SYS0PLL_AB_CTRL1 0x003c -#define SIRFSOC_CLKC_SYS0PLL_AB_STATUS 0x0040 - -#define SIRFSOC_CLKC_SYS1PLL_AB_FREQ 0x0044 -#define SIRFSOC_CLKC_SYS1PLL_AB_SSC 0x0048 -#define SIRFSOC_CLKC_SYS1PLL_AB_CTRL0 0x004c -#define SIRFSOC_CLKC_SYS1PLL_AB_CTRL1 0x0050 -#define SIRFSOC_CLKC_SYS1PLL_AB_STATUS 0x0054 - -#define SIRFSOC_CLKC_SYS2PLL_AB_FREQ 0x0058 -#define SIRFSOC_CLKC_SYS2PLL_AB_SSC 0x005c -#define SIRFSOC_CLKC_SYS2PLL_AB_CTRL0 0x0060 -#define SIRFSOC_CLKC_SYS2PLL_AB_CTRL1 0x0064 -#define SIRFSOC_CLKC_SYS2PLL_AB_STATUS 0x0068 - -#define SIRFSOC_CLKC_SYS3PLL_AB_FREQ 0x006c -#define SIRFSOC_CLKC_SYS3PLL_AB_SSC 0x0070 -#define SIRFSOC_CLKC_SYS3PLL_AB_CTRL0 0x0074 -#define SIRFSOC_CLKC_SYS3PLL_AB_CTRL1 0x0078 -#define SIRFSOC_CLKC_SYS3PLL_AB_STATUS 0x007c - -#define SIRFSOC_ABPLL_CTRL0_SSEN 0x00001000 -#define SIRFSOC_ABPLL_CTRL0_BYPASS 0x00000010 -#define SIRFSOC_ABPLL_CTRL0_RESET 0x00000001 - -#define SIRFSOC_CLKC_AUDIO_DTO_INC 0x0088 -#define SIRFSOC_CLKC_DISP0_DTO_INC 0x008c -#define SIRFSOC_CLKC_DISP1_DTO_INC 0x0090 - -#define SIRFSOC_CLKC_AUDIO_DTO_SRC 0x0094 -#define SIRFSOC_CLKC_AUDIO_DTO_ENA 0x0098 -#define SIRFSOC_CLKC_AUDIO_DTO_DROFF 0x009c - -#define SIRFSOC_CLKC_DISP0_DTO_SRC 0x00a0 -#define SIRFSOC_CLKC_DISP0_DTO_ENA 0x00a4 -#define SIRFSOC_CLKC_DISP0_DTO_DROFF 0x00a8 - -#define SIRFSOC_CLKC_DISP1_DTO_SRC 0x00ac -#define SIRFSOC_CLKC_DISP1_DTO_ENA 0x00b0 -#define SIRFSOC_CLKC_DISP1_DTO_DROFF 0x00b4 - -#define SIRFSOC_CLKC_I2S_CLK_SEL 0x00b8 -#define SIRFSOC_CLKC_I2S_SEL_STAT 0x00bc - -#define SIRFSOC_CLKC_USBPHY_CLKDIV_CFG 0x00c0 -#define SIRFSOC_CLKC_USBPHY_CLKDIV_ENA 0x00c4 -#define SIRFSOC_CLKC_USBPHY_CLK_SEL 0x00c8 -#define SIRFSOC_CLKC_USBPHY_CLK_SEL_STAT 0x00cc - -#define SIRFSOC_CLKC_BTSS_CLKDIV_CFG 0x00d0 -#define SIRFSOC_CLKC_BTSS_CLKDIV_ENA 0x00d4 -#define SIRFSOC_CLKC_BTSS_CLK_SEL 0x00d8 -#define SIRFSOC_CLKC_BTSS_CLK_SEL_STAT 0x00dc - -#define SIRFSOC_CLKC_RGMII_CLKDIV_CFG 0x00e0 -#define SIRFSOC_CLKC_RGMII_CLKDIV_ENA 0x00e4 -#define SIRFSOC_CLKC_RGMII_CLK_SEL 0x00e8 -#define SIRFSOC_CLKC_RGMII_CLK_SEL_STAT 0x00ec - -#define SIRFSOC_CLKC_CPU_CLKDIV_CFG 0x00f0 -#define SIRFSOC_CLKC_CPU_CLKDIV_ENA 0x00f4 -#define SIRFSOC_CLKC_CPU_CLK_SEL 0x00f8 -#define SIRFSOC_CLKC_CPU_CLK_SEL_STAT 0x00fc - -#define SIRFSOC_CLKC_SDPHY01_CLKDIV_CFG 0x0100 -#define SIRFSOC_CLKC_SDPHY01_CLKDIV_ENA 0x0104 -#define SIRFSOC_CLKC_SDPHY01_CLK_SEL 0x0108 -#define SIRFSOC_CLKC_SDPHY01_CLK_SEL_STAT 0x010c - -#define SIRFSOC_CLKC_SDPHY23_CLKDIV_CFG 0x0110 -#define SIRFSOC_CLKC_SDPHY23_CLKDIV_ENA 0x0114 -#define SIRFSOC_CLKC_SDPHY23_CLK_SEL 0x0118 -#define SIRFSOC_CLKC_SDPHY23_CLK_SEL_STAT 0x011c - -#define SIRFSOC_CLKC_SDPHY45_CLKDIV_CFG 0x0120 -#define SIRFSOC_CLKC_SDPHY45_CLKDIV_ENA 0x0124 -#define SIRFSOC_CLKC_SDPHY45_CLK_SEL 0x0128 -#define SIRFSOC_CLKC_SDPHY45_CLK_SEL_STAT 0x012c - -#define SIRFSOC_CLKC_SDPHY67_CLKDIV_CFG 0x0130 -#define SIRFSOC_CLKC_SDPHY67_CLKDIV_ENA 0x0134 -#define SIRFSOC_CLKC_SDPHY67_CLK_SEL 0x0138 -#define SIRFSOC_CLKC_SDPHY67_CLK_SEL_STAT 0x013c - -#define SIRFSOC_CLKC_CAN_CLKDIV_CFG 0x0140 -#define SIRFSOC_CLKC_CAN_CLKDIV_ENA 0x0144 -#define SIRFSOC_CLKC_CAN_CLK_SEL 0x0148 -#define SIRFSOC_CLKC_CAN_CLK_SEL_STAT 0x014c - -#define SIRFSOC_CLKC_DEINT_CLKDIV_CFG 0x0150 -#define SIRFSOC_CLKC_DEINT_CLKDIV_ENA 0x0154 -#define SIRFSOC_CLKC_DEINT_CLK_SEL 0x0158 -#define SIRFSOC_CLKC_DEINT_CLK_SEL_STAT 0x015c - -#define SIRFSOC_CLKC_NAND_CLKDIV_CFG 0x0160 -#define SIRFSOC_CLKC_NAND_CLKDIV_ENA 0x0164 -#define SIRFSOC_CLKC_NAND_CLK_SEL 0x0168 -#define SIRFSOC_CLKC_NAND_CLK_SEL_STAT 0x016c - -#define SIRFSOC_CLKC_DISP0_CLKDIV_CFG 0x0170 -#define SIRFSOC_CLKC_DISP0_CLKDIV_ENA 0x0174 -#define SIRFSOC_CLKC_DISP0_CLK_SEL 0x0178 -#define SIRFSOC_CLKC_DISP0_CLK_SEL_STAT 0x017c - -#define SIRFSOC_CLKC_DISP1_CLKDIV_CFG 0x0180 -#define SIRFSOC_CLKC_DISP1_CLKDIV_ENA 0x0184 -#define SIRFSOC_CLKC_DISP1_CLK_SEL 0x0188 -#define SIRFSOC_CLKC_DISP1_CLK_SEL_STAT 0x018c - -#define SIRFSOC_CLKC_GPU_CLKDIV_CFG 0x0190 -#define SIRFSOC_CLKC_GPU_CLKDIV_ENA 0x0194 -#define SIRFSOC_CLKC_GPU_CLK_SEL 0x0198 -#define SIRFSOC_CLKC_GPU_CLK_SEL_STAT 0x019c - -#define SIRFSOC_CLKC_GNSS_CLKDIV_CFG 0x01a0 -#define SIRFSOC_CLKC_GNSS_CLKDIV_ENA 0x01a4 -#define SIRFSOC_CLKC_GNSS_CLK_SEL 0x01a8 -#define SIRFSOC_CLKC_GNSS_CLK_SEL_STAT 0x01ac - -#define SIRFSOC_CLKC_SHARED_DIVIDER_CFG0 0x01b0 -#define SIRFSOC_CLKC_SHARED_DIVIDER_CFG1 0x01b4 -#define SIRFSOC_CLKC_SHARED_DIVIDER_ENA 0x01b8 - -#define SIRFSOC_CLKC_SYS_CLK_SEL 0x01bc -#define SIRFSOC_CLKC_SYS_CLK_SEL_STAT 0x01c0 -#define SIRFSOC_CLKC_IO_CLK_SEL 0x01c4 -#define SIRFSOC_CLKC_IO_CLK_SEL_STAT 0x01c8 -#define SIRFSOC_CLKC_G2D_CLK_SEL 0x01cc -#define SIRFSOC_CLKC_G2D_CLK_SEL_STAT 0x01d0 -#define SIRFSOC_CLKC_JPENC_CLK_SEL 0x01d4 -#define SIRFSOC_CLKC_JPENC_CLK_SEL_STAT 0x01d8 -#define SIRFSOC_CLKC_VDEC_CLK_SEL 0x01dc -#define SIRFSOC_CLKC_VDEC_CLK_SEL_STAT 0x01e0 -#define SIRFSOC_CLKC_GMAC_CLK_SEL 0x01e4 -#define SIRFSOC_CLKC_GMAC_CLK_SEL_STAT 0x01e8 -#define SIRFSOC_CLKC_USB_CLK_SEL 0x01ec -#define SIRFSOC_CLKC_USB_CLK_SEL_STAT 0x01f0 -#define SIRFSOC_CLKC_KAS_CLK_SEL 0x01f4 -#define SIRFSOC_CLKC_KAS_CLK_SEL_STAT 0x01f8 -#define SIRFSOC_CLKC_SEC_CLK_SEL 0x01fc -#define SIRFSOC_CLKC_SEC_CLK_SEL_STAT 0x0200 -#define SIRFSOC_CLKC_SDR_CLK_SEL 0x0204 -#define SIRFSOC_CLKC_SDR_CLK_SEL_STAT 0x0208 -#define SIRFSOC_CLKC_VIP_CLK_SEL 0x020c -#define SIRFSOC_CLKC_VIP_CLK_SEL_STAT 0x0210 -#define SIRFSOC_CLKC_NOCD_CLK_SEL 0x0214 -#define SIRFSOC_CLKC_NOCD_CLK_SEL_STAT 0x0218 -#define SIRFSOC_CLKC_NOCR_CLK_SEL 0x021c -#define SIRFSOC_CLKC_NOCR_CLK_SEL_STAT 0x0220 -#define SIRFSOC_CLKC_TPIU_CLK_SEL 0x0224 -#define SIRFSOC_CLKC_TPIU_CLK_SEL_STAT 0x0228 - -#define SIRFSOC_CLKC_ROOT_CLK_EN0_SET 0x022c -#define SIRFSOC_CLKC_ROOT_CLK_EN0_CLR 0x0230 -#define SIRFSOC_CLKC_ROOT_CLK_EN0_STAT 0x0234 -#define SIRFSOC_CLKC_ROOT_CLK_EN1_SET 0x0238 -#define SIRFSOC_CLKC_ROOT_CLK_EN1_CLR 0x023c -#define SIRFSOC_CLKC_ROOT_CLK_EN1_STAT 0x0240 - -#define SIRFSOC_CLKC_LEAF_CLK_EN0_SET 0x0244 -#define SIRFSOC_CLKC_LEAF_CLK_EN0_CLR 0x0248 -#define SIRFSOC_CLKC_LEAF_CLK_EN0_STAT 0x024c - -#define SIRFSOC_CLKC_RSTC_A7_SW_RST 0x0308 - -#define SIRFSOC_CLKC_LEAF_CLK_EN1_SET 0x04a0 -#define SIRFSOC_CLKC_LEAF_CLK_EN2_SET 0x04b8 -#define SIRFSOC_CLKC_LEAF_CLK_EN3_SET 0x04d0 -#define SIRFSOC_CLKC_LEAF_CLK_EN4_SET 0x04e8 -#define SIRFSOC_CLKC_LEAF_CLK_EN5_SET 0x0500 -#define SIRFSOC_CLKC_LEAF_CLK_EN6_SET 0x0518 -#define SIRFSOC_CLKC_LEAF_CLK_EN7_SET 0x0530 -#define SIRFSOC_CLKC_LEAF_CLK_EN8_SET 0x0548 - -#define SIRFSOC_NOC_CLK_IDLEREQ_SET 0x02D0 -#define SIRFSOC_NOC_CLK_IDLEREQ_CLR 0x02D4 -#define SIRFSOC_NOC_CLK_SLVRDY_SET 0x02E8 -#define SIRFSOC_NOC_CLK_SLVRDY_CLR 0x02EC -#define SIRFSOC_NOC_CLK_IDLE_STATUS 0x02F4 - -struct clk_pll { - struct clk_hw hw; - u16 regofs; /* register offset */ -}; -#define to_pllclk(_hw) container_of(_hw, struct clk_pll, hw) - -struct clk_dto { - struct clk_hw hw; - u16 inc_offset; /* dto increment offset */ - u16 src_offset; /* dto src offset */ -}; -#define to_dtoclk(_hw) container_of(_hw, struct clk_dto, hw) - -enum clk_unit_type { - CLK_UNIT_NOC_OTHER, - CLK_UNIT_NOC_CLOCK, - CLK_UNIT_NOC_SOCKET, -}; - -struct clk_unit { - struct clk_hw hw; - u16 regofs; - u16 bit; - u32 type; - u8 idle_bit; - spinlock_t *lock; -}; -#define to_unitclk(_hw) container_of(_hw, struct clk_unit, hw) - -struct atlas7_div_init_data { - const char *div_name; - const char *parent_name; - const char *gate_name; - unsigned long flags; - u8 divider_flags; - u8 gate_flags; - u32 div_offset; - u8 shift; - u8 width; - u32 gate_offset; - u8 gate_bit; - spinlock_t *lock; -}; - -struct atlas7_mux_init_data { - const char *mux_name; - const char * const *parent_names; - u8 parent_num; - unsigned long flags; - u8 mux_flags; - u32 mux_offset; - u8 shift; - u8 width; -}; - -struct atlas7_unit_init_data { - u32 index; - const char *unit_name; - const char *parent_name; - unsigned long flags; - u32 regofs; - u8 bit; - u32 type; - u8 idle_bit; - spinlock_t *lock; -}; - -struct atlas7_reset_desc { - const char *name; - u32 clk_ofs; - u8 clk_bit; - u32 rst_ofs; - u8 rst_bit; - spinlock_t *lock; -}; - -static void __iomem *sirfsoc_clk_vbase; -static struct clk_onecell_data clk_data; - -static const struct clk_div_table pll_div_table[] = { - { .val = 0, .div = 1 }, - { .val = 1, .div = 2 }, - { .val = 2, .div = 4 }, - { .val = 3, .div = 8 }, - { .val = 4, .div = 16 }, - { .val = 5, .div = 32 }, -}; - -static DEFINE_SPINLOCK(cpupll_ctrl1_lock); -static DEFINE_SPINLOCK(mempll_ctrl1_lock); -static DEFINE_SPINLOCK(sys0pll_ctrl1_lock); -static DEFINE_SPINLOCK(sys1pll_ctrl1_lock); -static DEFINE_SPINLOCK(sys2pll_ctrl1_lock); -static DEFINE_SPINLOCK(sys3pll_ctrl1_lock); -static DEFINE_SPINLOCK(usbphy_div_lock); -static DEFINE_SPINLOCK(btss_div_lock); -static DEFINE_SPINLOCK(rgmii_div_lock); -static DEFINE_SPINLOCK(cpu_div_lock); -static DEFINE_SPINLOCK(sdphy01_div_lock); -static DEFINE_SPINLOCK(sdphy23_div_lock); -static DEFINE_SPINLOCK(sdphy45_div_lock); -static DEFINE_SPINLOCK(sdphy67_div_lock); -static DEFINE_SPINLOCK(can_div_lock); -static DEFINE_SPINLOCK(deint_div_lock); -static DEFINE_SPINLOCK(nand_div_lock); -static DEFINE_SPINLOCK(disp0_div_lock); -static DEFINE_SPINLOCK(disp1_div_lock); -static DEFINE_SPINLOCK(gpu_div_lock); -static DEFINE_SPINLOCK(gnss_div_lock); -/* gate register shared */ -static DEFINE_SPINLOCK(share_div_lock); -static DEFINE_SPINLOCK(root0_gate_lock); -static DEFINE_SPINLOCK(root1_gate_lock); -static DEFINE_SPINLOCK(leaf0_gate_lock); -static DEFINE_SPINLOCK(leaf1_gate_lock); -static DEFINE_SPINLOCK(leaf2_gate_lock); -static DEFINE_SPINLOCK(leaf3_gate_lock); -static DEFINE_SPINLOCK(leaf4_gate_lock); -static DEFINE_SPINLOCK(leaf5_gate_lock); -static DEFINE_SPINLOCK(leaf6_gate_lock); -static DEFINE_SPINLOCK(leaf7_gate_lock); -static DEFINE_SPINLOCK(leaf8_gate_lock); - -static inline unsigned long clkc_readl(unsigned reg) -{ - return readl(sirfsoc_clk_vbase + reg); -} - -static inline void clkc_writel(u32 val, unsigned reg) -{ - writel(val, sirfsoc_clk_vbase + reg); -} - -/* -* ABPLL -* integer mode: Fvco = Fin * 2 * NF / NR -* Spread Spectrum mode: Fvco = Fin * SSN / NR -* SSN = 2^24 / (256 * ((ssdiv >> ssdepth) << ssdepth) + (ssmod << ssdepth)) -*/ -static unsigned long pll_clk_recalc_rate(struct clk_hw *hw, - unsigned long parent_rate) -{ - unsigned long fin = parent_rate; - struct clk_pll *clk = to_pllclk(hw); - u64 rate; - u32 regctrl0 = clkc_readl(clk->regofs + SIRFSOC_CLKC_MEMPLL_AB_CTRL0 - - SIRFSOC_CLKC_MEMPLL_AB_FREQ); - u32 regfreq = clkc_readl(clk->regofs); - u32 regssc = clkc_readl(clk->regofs + SIRFSOC_CLKC_MEMPLL_AB_SSC - - SIRFSOC_CLKC_MEMPLL_AB_FREQ); - u32 nr = (regfreq >> 16 & (BIT(3) - 1)) + 1; - u32 nf = (regfreq & (BIT(9) - 1)) + 1; - u32 ssdiv = regssc >> 8 & (BIT(12) - 1); - u32 ssdepth = regssc >> 20 & (BIT(2) - 1); - u32 ssmod = regssc & (BIT(8) - 1); - - if (regctrl0 & SIRFSOC_ABPLL_CTRL0_BYPASS) - return fin; - - if (regctrl0 & SIRFSOC_ABPLL_CTRL0_SSEN) { - rate = fin; - rate *= 1 << 24; - do_div(rate, nr); - do_div(rate, (256 * ((ssdiv >> ssdepth) << ssdepth) - + (ssmod << ssdepth))); - } else { - rate = 2 * fin; - rate *= nf; - do_div(rate, nr); - } - return rate; -} - -static const struct clk_ops ab_pll_ops = { - .recalc_rate = pll_clk_recalc_rate, -}; - -static const char * const pll_clk_parents[] = { - "xin", -}; - -static const struct clk_init_data clk_cpupll_init = { - .name = "cpupll_vco", - .ops = &ab_pll_ops, - .parent_names = pll_clk_parents, - .num_parents = ARRAY_SIZE(pll_clk_parents), -}; - -static struct clk_pll clk_cpupll = { - .regofs = SIRFSOC_CLKC_CPUPLL_AB_FREQ, - .hw = { - .init = &clk_cpupll_init, - }, -}; - -static const struct clk_init_data clk_mempll_init = { - .name = "mempll_vco", - .ops = &ab_pll_ops, - .parent_names = pll_clk_parents, - .num_parents = ARRAY_SIZE(pll_clk_parents), -}; - -static struct clk_pll clk_mempll = { - .regofs = SIRFSOC_CLKC_MEMPLL_AB_FREQ, - .hw = { - .init = &clk_mempll_init, - }, -}; - -static const struct clk_init_data clk_sys0pll_init = { - .name = "sys0pll_vco", - .ops = &ab_pll_ops, - .parent_names = pll_clk_parents, - .num_parents = ARRAY_SIZE(pll_clk_parents), -}; - -static struct clk_pll clk_sys0pll = { - .regofs = SIRFSOC_CLKC_SYS0PLL_AB_FREQ, - .hw = { - .init = &clk_sys0pll_init, - }, -}; - -static const struct clk_init_data clk_sys1pll_init = { - .name = "sys1pll_vco", - .ops = &ab_pll_ops, - .parent_names = pll_clk_parents, - .num_parents = ARRAY_SIZE(pll_clk_parents), -}; - -static struct clk_pll clk_sys1pll = { - .regofs = SIRFSOC_CLKC_SYS1PLL_AB_FREQ, - .hw = { - .init = &clk_sys1pll_init, - }, -}; - -static const struct clk_init_data clk_sys2pll_init = { - .name = "sys2pll_vco", - .ops = &ab_pll_ops, - .parent_names = pll_clk_parents, - .num_parents = ARRAY_SIZE(pll_clk_parents), -}; - -static struct clk_pll clk_sys2pll = { - .regofs = SIRFSOC_CLKC_SYS2PLL_AB_FREQ, - .hw = { - .init = &clk_sys2pll_init, - }, -}; - -static const struct clk_init_data clk_sys3pll_init = { - .name = "sys3pll_vco", - .ops = &ab_pll_ops, - .parent_names = pll_clk_parents, - .num_parents = ARRAY_SIZE(pll_clk_parents), -}; - -static struct clk_pll clk_sys3pll = { - .regofs = SIRFSOC_CLKC_SYS3PLL_AB_FREQ, - .hw = { - .init = &clk_sys3pll_init, - }, -}; - -/* - * DTO in clkc, default enable double resolution mode - * double resolution mode:fout = fin * finc / 2^29 - * normal mode:fout = fin * finc / 2^28 - */ -#define DTO_RESL_DOUBLE (1ULL << 29) -#define DTO_RESL_NORMAL (1ULL << 28) - -static int dto_clk_is_enabled(struct clk_hw *hw) -{ - struct clk_dto *clk = to_dtoclk(hw); - int reg; - - reg = clk->src_offset + SIRFSOC_CLKC_AUDIO_DTO_ENA - SIRFSOC_CLKC_AUDIO_DTO_SRC; - - return !!(clkc_readl(reg) & BIT(0)); -} - -static int dto_clk_enable(struct clk_hw *hw) -{ - u32 val, reg; - struct clk_dto *clk = to_dtoclk(hw); - - reg = clk->src_offset + SIRFSOC_CLKC_AUDIO_DTO_ENA - SIRFSOC_CLKC_AUDIO_DTO_SRC; - - val = clkc_readl(reg) | BIT(0); - clkc_writel(val, reg); - return 0; -} - -static void dto_clk_disable(struct clk_hw *hw) -{ - u32 val, reg; - struct clk_dto *clk = to_dtoclk(hw); - - reg = clk->src_offset + SIRFSOC_CLKC_AUDIO_DTO_ENA - SIRFSOC_CLKC_AUDIO_DTO_SRC; - - val = clkc_readl(reg) & ~BIT(0); - clkc_writel(val, reg); -} - -static unsigned long dto_clk_recalc_rate(struct clk_hw *hw, - unsigned long parent_rate) -{ - u64 rate = parent_rate; - struct clk_dto *clk = to_dtoclk(hw); - u32 finc = clkc_readl(clk->inc_offset); - u32 droff = clkc_readl(clk->src_offset + SIRFSOC_CLKC_AUDIO_DTO_DROFF - SIRFSOC_CLKC_AUDIO_DTO_SRC); - - rate *= finc; - if (droff & BIT(0)) - /* Double resolution off */ - do_div(rate, DTO_RESL_NORMAL); - else - do_div(rate, DTO_RESL_DOUBLE); - - return rate; -} - -static long dto_clk_round_rate(struct clk_hw *hw, unsigned long rate, - unsigned long *parent_rate) -{ - u64 dividend = rate * DTO_RESL_DOUBLE; - - do_div(dividend, *parent_rate); - dividend *= *parent_rate; - do_div(dividend, DTO_RESL_DOUBLE); - - return dividend; -} - -static int dto_clk_set_rate(struct clk_hw *hw, unsigned long rate, - unsigned long parent_rate) -{ - u64 dividend = rate * DTO_RESL_DOUBLE; - struct clk_dto *clk = to_dtoclk(hw); - - do_div(dividend, parent_rate); - clkc_writel(0, clk->src_offset + SIRFSOC_CLKC_AUDIO_DTO_DROFF - SIRFSOC_CLKC_AUDIO_DTO_SRC); - clkc_writel(dividend, clk->inc_offset); - - return 0; -} - -static u8 dto_clk_get_parent(struct clk_hw *hw) -{ - struct clk_dto *clk = to_dtoclk(hw); - - return clkc_readl(clk->src_offset); -} - -/* - * dto need CLK_SET_PARENT_GATE - */ -static int dto_clk_set_parent(struct clk_hw *hw, u8 index) -{ - struct clk_dto *clk = to_dtoclk(hw); - - clkc_writel(index, clk->src_offset); - return 0; -} - -static const struct clk_ops dto_ops = { - .is_enabled = dto_clk_is_enabled, - .enable = dto_clk_enable, - .disable = dto_clk_disable, - .recalc_rate = dto_clk_recalc_rate, - .round_rate = dto_clk_round_rate, - .set_rate = dto_clk_set_rate, - .get_parent = dto_clk_get_parent, - .set_parent = dto_clk_set_parent, -}; - -/* dto parent clock as syspllvco/clk1 */ -static const char * const audiodto_clk_parents[] = { - "sys0pll_clk1", - "sys1pll_clk1", - "sys3pll_clk1", -}; - -static const struct clk_init_data clk_audiodto_init = { - .name = "audio_dto", - .ops = &dto_ops, - .parent_names = audiodto_clk_parents, - .num_parents = ARRAY_SIZE(audiodto_clk_parents), -}; - -static struct clk_dto clk_audio_dto = { - .inc_offset = SIRFSOC_CLKC_AUDIO_DTO_INC, - .src_offset = SIRFSOC_CLKC_AUDIO_DTO_SRC, - .hw = { - .init = &clk_audiodto_init, - }, -}; - -static const char * const disp0dto_clk_parents[] = { - "sys0pll_clk1", - "sys1pll_clk1", - "sys3pll_clk1", -}; - -static const struct clk_init_data clk_disp0dto_init = { - .name = "disp0_dto", - .ops = &dto_ops, - .parent_names = disp0dto_clk_parents, - .num_parents = ARRAY_SIZE(disp0dto_clk_parents), -}; - -static struct clk_dto clk_disp0_dto = { - .inc_offset = SIRFSOC_CLKC_DISP0_DTO_INC, - .src_offset = SIRFSOC_CLKC_DISP0_DTO_SRC, - .hw = { - .init = &clk_disp0dto_init, - }, -}; - -static const char * const disp1dto_clk_parents[] = { - "sys0pll_clk1", - "sys1pll_clk1", - "sys3pll_clk1", -}; - -static const struct clk_init_data clk_disp1dto_init = { - .name = "disp1_dto", - .ops = &dto_ops, - .parent_names = disp1dto_clk_parents, - .num_parents = ARRAY_SIZE(disp1dto_clk_parents), -}; - -static struct clk_dto clk_disp1_dto = { - .inc_offset = SIRFSOC_CLKC_DISP1_DTO_INC, - .src_offset = SIRFSOC_CLKC_DISP1_DTO_SRC, - .hw = { - .init = &clk_disp1dto_init, - }, -}; - -static struct atlas7_div_init_data divider_list[] __initdata = { - /* div_name, parent_name, gate_name, clk_flag, divider_flag, gate_flag, div_offset, shift, wdith, gate_offset, bit_enable, lock */ - { "sys0pll_qa1", "sys0pll_fixdiv", "sys0pll_a1", 0, 0, 0, SIRFSOC_CLKC_USBPHY_CLKDIV_CFG, 0, 6, SIRFSOC_CLKC_USBPHY_CLKDIV_ENA, 0, &usbphy_div_lock }, - { "sys1pll_qa1", "sys1pll_fixdiv", "sys1pll_a1", 0, 0, 0, SIRFSOC_CLKC_USBPHY_CLKDIV_CFG, 8, 6, SIRFSOC_CLKC_USBPHY_CLKDIV_ENA, 4, &usbphy_div_lock }, - { "sys2pll_qa1", "sys2pll_fixdiv", "sys2pll_a1", 0, 0, 0, SIRFSOC_CLKC_USBPHY_CLKDIV_CFG, 16, 6, SIRFSOC_CLKC_USBPHY_CLKDIV_ENA, 8, &usbphy_div_lock }, - { "sys3pll_qa1", "sys3pll_fixdiv", "sys3pll_a1", 0, 0, 0, SIRFSOC_CLKC_USBPHY_CLKDIV_CFG, 24, 6, SIRFSOC_CLKC_USBPHY_CLKDIV_ENA, 12, &usbphy_div_lock }, - { "sys0pll_qa2", "sys0pll_fixdiv", "sys0pll_a2", 0, 0, 0, SIRFSOC_CLKC_BTSS_CLKDIV_CFG, 0, 6, SIRFSOC_CLKC_BTSS_CLKDIV_ENA, 0, &btss_div_lock }, - { "sys1pll_qa2", "sys1pll_fixdiv", "sys1pll_a2", 0, 0, 0, SIRFSOC_CLKC_BTSS_CLKDIV_CFG, 8, 6, SIRFSOC_CLKC_BTSS_CLKDIV_ENA, 4, &btss_div_lock }, - { "sys2pll_qa2", "sys2pll_fixdiv", "sys2pll_a2", 0, 0, 0, SIRFSOC_CLKC_BTSS_CLKDIV_CFG, 16, 6, SIRFSOC_CLKC_BTSS_CLKDIV_ENA, 8, &btss_div_lock }, - { "sys3pll_qa2", "sys3pll_fixdiv", "sys3pll_a2", 0, 0, 0, SIRFSOC_CLKC_BTSS_CLKDIV_CFG, 24, 6, SIRFSOC_CLKC_BTSS_CLKDIV_ENA, 12, &btss_div_lock }, - { "sys0pll_qa3", "sys0pll_fixdiv", "sys0pll_a3", 0, 0, 0, SIRFSOC_CLKC_RGMII_CLKDIV_CFG, 0, 6, SIRFSOC_CLKC_RGMII_CLKDIV_ENA, 0, &rgmii_div_lock }, - { "sys1pll_qa3", "sys1pll_fixdiv", "sys1pll_a3", 0, 0, 0, SIRFSOC_CLKC_RGMII_CLKDIV_CFG, 8, 6, SIRFSOC_CLKC_RGMII_CLKDIV_ENA, 4, &rgmii_div_lock }, - { "sys2pll_qa3", "sys2pll_fixdiv", "sys2pll_a3", 0, 0, 0, SIRFSOC_CLKC_RGMII_CLKDIV_CFG, 16, 6, SIRFSOC_CLKC_RGMII_CLKDIV_ENA, 8, &rgmii_div_lock }, - { "sys3pll_qa3", "sys3pll_fixdiv", "sys3pll_a3", 0, 0, 0, SIRFSOC_CLKC_RGMII_CLKDIV_CFG, 24, 6, SIRFSOC_CLKC_RGMII_CLKDIV_ENA, 12, &rgmii_div_lock }, - { "sys0pll_qa4", "sys0pll_fixdiv", "sys0pll_a4", 0, 0, 0, SIRFSOC_CLKC_CPU_CLKDIV_CFG, 0, 6, SIRFSOC_CLKC_CPU_CLKDIV_ENA, 0, &cpu_div_lock }, - { "sys1pll_qa4", "sys1pll_fixdiv", "sys1pll_a4", 0, 0, CLK_IGNORE_UNUSED, SIRFSOC_CLKC_CPU_CLKDIV_CFG, 8, 6, SIRFSOC_CLKC_CPU_CLKDIV_ENA, 4, &cpu_div_lock }, - { "sys0pll_qa5", "sys0pll_fixdiv", "sys0pll_a5", 0, 0, 0, SIRFSOC_CLKC_SDPHY01_CLKDIV_CFG, 0, 6, SIRFSOC_CLKC_SDPHY01_CLKDIV_ENA, 0, &sdphy01_div_lock }, - { "sys1pll_qa5", "sys1pll_fixdiv", "sys1pll_a5", 0, 0, 0, SIRFSOC_CLKC_SDPHY01_CLKDIV_CFG, 8, 6, SIRFSOC_CLKC_SDPHY01_CLKDIV_ENA, 4, &sdphy01_div_lock }, - { "sys2pll_qa5", "sys2pll_fixdiv", "sys2pll_a5", 0, 0, 0, SIRFSOC_CLKC_SDPHY01_CLKDIV_CFG, 16, 6, SIRFSOC_CLKC_SDPHY01_CLKDIV_ENA, 8, &sdphy01_div_lock }, - { "sys3pll_qa5", "sys3pll_fixdiv", "sys3pll_a5", 0, 0, 0, SIRFSOC_CLKC_SDPHY01_CLKDIV_CFG, 24, 6, SIRFSOC_CLKC_SDPHY01_CLKDIV_ENA, 12, &sdphy01_div_lock }, - { "sys0pll_qa6", "sys0pll_fixdiv", "sys0pll_a6", 0, 0, 0, SIRFSOC_CLKC_SDPHY23_CLKDIV_CFG, 0, 6, SIRFSOC_CLKC_SDPHY23_CLKDIV_ENA, 0, &sdphy23_div_lock }, - { "sys1pll_qa6", "sys1pll_fixdiv", "sys1pll_a6", 0, 0, 0, SIRFSOC_CLKC_SDPHY23_CLKDIV_CFG, 8, 6, SIRFSOC_CLKC_SDPHY23_CLKDIV_ENA, 4, &sdphy23_div_lock }, - { "sys2pll_qa6", "sys2pll_fixdiv", "sys2pll_a6", 0, 0, 0, SIRFSOC_CLKC_SDPHY23_CLKDIV_CFG, 16, 6, SIRFSOC_CLKC_SDPHY23_CLKDIV_ENA, 8, &sdphy23_div_lock }, - { "sys3pll_qa6", "sys3pll_fixdiv", "sys3pll_a6", 0, 0, 0, SIRFSOC_CLKC_SDPHY23_CLKDIV_CFG, 24, 6, SIRFSOC_CLKC_SDPHY23_CLKDIV_ENA, 12, &sdphy23_div_lock }, - { "sys0pll_qa7", "sys0pll_fixdiv", "sys0pll_a7", 0, 0, 0, SIRFSOC_CLKC_SDPHY45_CLKDIV_CFG, 0, 6, SIRFSOC_CLKC_SDPHY45_CLKDIV_ENA, 0, &sdphy45_div_lock }, - { "sys1pll_qa7", "sys1pll_fixdiv", "sys1pll_a7", 0, 0, 0, SIRFSOC_CLKC_SDPHY45_CLKDIV_CFG, 8, 6, SIRFSOC_CLKC_SDPHY45_CLKDIV_ENA, 4, &sdphy45_div_lock }, - { "sys2pll_qa7", "sys2pll_fixdiv", "sys2pll_a7", 0, 0, 0, SIRFSOC_CLKC_SDPHY45_CLKDIV_CFG, 16, 6, SIRFSOC_CLKC_SDPHY45_CLKDIV_ENA, 8, &sdphy45_div_lock }, - { "sys3pll_qa7", "sys3pll_fixdiv", "sys3pll_a7", 0, 0, 0, SIRFSOC_CLKC_SDPHY45_CLKDIV_CFG, 24, 6, SIRFSOC_CLKC_SDPHY45_CLKDIV_ENA, 12, &sdphy45_div_lock }, - { "sys0pll_qa8", "sys0pll_fixdiv", "sys0pll_a8", 0, 0, 0, SIRFSOC_CLKC_SDPHY67_CLKDIV_CFG, 0, 6, SIRFSOC_CLKC_SDPHY67_CLKDIV_ENA, 0, &sdphy67_div_lock }, - { "sys1pll_qa8", "sys1pll_fixdiv", "sys1pll_a8", 0, 0, 0, SIRFSOC_CLKC_SDPHY67_CLKDIV_CFG, 8, 6, SIRFSOC_CLKC_SDPHY67_CLKDIV_ENA, 4, &sdphy67_div_lock }, - { "sys2pll_qa8", "sys2pll_fixdiv", "sys2pll_a8", 0, 0, 0, SIRFSOC_CLKC_SDPHY67_CLKDIV_CFG, 16, 6, SIRFSOC_CLKC_SDPHY67_CLKDIV_ENA, 8, &sdphy67_div_lock }, - { "sys3pll_qa8", "sys3pll_fixdiv", "sys3pll_a8", 0, 0, 0, SIRFSOC_CLKC_SDPHY67_CLKDIV_CFG, 24, 6, SIRFSOC_CLKC_SDPHY67_CLKDIV_ENA, 12, &sdphy67_div_lock }, - { "sys0pll_qa9", "sys0pll_fixdiv", "sys0pll_a9", 0, 0, 0, SIRFSOC_CLKC_CAN_CLKDIV_CFG, 0, 6, SIRFSOC_CLKC_CAN_CLKDIV_ENA, 0, &can_div_lock }, - { "sys1pll_qa9", "sys1pll_fixdiv", "sys1pll_a9", 0, 0, 0, SIRFSOC_CLKC_CAN_CLKDIV_CFG, 8, 6, SIRFSOC_CLKC_CAN_CLKDIV_ENA, 4, &can_div_lock }, - { "sys2pll_qa9", "sys2pll_fixdiv", "sys2pll_a9", 0, 0, 0, SIRFSOC_CLKC_CAN_CLKDIV_CFG, 16, 6, SIRFSOC_CLKC_CAN_CLKDIV_ENA, 8, &can_div_lock }, - { "sys3pll_qa9", "sys3pll_fixdiv", "sys3pll_a9", 0, 0, 0, SIRFSOC_CLKC_CAN_CLKDIV_CFG, 24, 6, SIRFSOC_CLKC_CAN_CLKDIV_ENA, 12, &can_div_lock }, - { "sys0pll_qa10", "sys0pll_fixdiv", "sys0pll_a10", 0, 0, 0, SIRFSOC_CLKC_DEINT_CLKDIV_CFG, 0, 6, SIRFSOC_CLKC_DEINT_CLKDIV_ENA, 0, &deint_div_lock }, - { "sys1pll_qa10", "sys1pll_fixdiv", "sys1pll_a10", 0, 0, 0, SIRFSOC_CLKC_DEINT_CLKDIV_CFG, 8, 6, SIRFSOC_CLKC_DEINT_CLKDIV_ENA, 4, &deint_div_lock }, - { "sys2pll_qa10", "sys2pll_fixdiv", "sys2pll_a10", 0, 0, 0, SIRFSOC_CLKC_DEINT_CLKDIV_CFG, 16, 6, SIRFSOC_CLKC_DEINT_CLKDIV_ENA, 8, &deint_div_lock }, - { "sys3pll_qa10", "sys3pll_fixdiv", "sys3pll_a10", 0, 0, 0, SIRFSOC_CLKC_DEINT_CLKDIV_CFG, 24, 6, SIRFSOC_CLKC_DEINT_CLKDIV_ENA, 12, &deint_div_lock }, - { "sys0pll_qa11", "sys0pll_fixdiv", "sys0pll_a11", 0, 0, 0, SIRFSOC_CLKC_NAND_CLKDIV_CFG, 0, 6, SIRFSOC_CLKC_NAND_CLKDIV_ENA, 0, &nand_div_lock }, - { "sys1pll_qa11", "sys1pll_fixdiv", "sys1pll_a11", 0, 0, 0, SIRFSOC_CLKC_NAND_CLKDIV_CFG, 8, 6, SIRFSOC_CLKC_NAND_CLKDIV_ENA, 4, &nand_div_lock }, - { "sys2pll_qa11", "sys2pll_fixdiv", "sys2pll_a11", 0, 0, 0, SIRFSOC_CLKC_NAND_CLKDIV_CFG, 16, 6, SIRFSOC_CLKC_NAND_CLKDIV_ENA, 8, &nand_div_lock }, - { "sys3pll_qa11", "sys3pll_fixdiv", "sys3pll_a11", 0, 0, 0, SIRFSOC_CLKC_NAND_CLKDIV_CFG, 24, 6, SIRFSOC_CLKC_NAND_CLKDIV_ENA, 12, &nand_div_lock }, - { "sys0pll_qa12", "sys0pll_fixdiv", "sys0pll_a12", 0, 0, 0, SIRFSOC_CLKC_DISP0_CLKDIV_CFG, 0, 6, SIRFSOC_CLKC_DISP0_CLKDIV_ENA, 0, &disp0_div_lock }, - { "sys1pll_qa12", "sys1pll_fixdiv", "sys1pll_a12", 0, 0, 0, SIRFSOC_CLKC_DISP0_CLKDIV_CFG, 8, 6, SIRFSOC_CLKC_DISP0_CLKDIV_ENA, 4, &disp0_div_lock }, - { "sys2pll_qa12", "sys2pll_fixdiv", "sys2pll_a12", 0, 0, 0, SIRFSOC_CLKC_DISP0_CLKDIV_CFG, 16, 6, SIRFSOC_CLKC_DISP0_CLKDIV_ENA, 8, &disp0_div_lock }, - { "sys3pll_qa12", "sys3pll_fixdiv", "sys3pll_a12", 0, 0, 0, SIRFSOC_CLKC_DISP0_CLKDIV_CFG, 24, 6, SIRFSOC_CLKC_DISP0_CLKDIV_ENA, 12, &disp0_div_lock }, - { "sys0pll_qa13", "sys0pll_fixdiv", "sys0pll_a13", 0, 0, 0, SIRFSOC_CLKC_DISP1_CLKDIV_CFG, 0, 6, SIRFSOC_CLKC_DISP1_CLKDIV_ENA, 0, &disp1_div_lock }, - { "sys1pll_qa13", "sys1pll_fixdiv", "sys1pll_a13", 0, 0, 0, SIRFSOC_CLKC_DISP1_CLKDIV_CFG, 8, 6, SIRFSOC_CLKC_DISP1_CLKDIV_ENA, 4, &disp1_div_lock }, - { "sys2pll_qa13", "sys2pll_fixdiv", "sys2pll_a13", 0, 0, 0, SIRFSOC_CLKC_DISP1_CLKDIV_CFG, 16, 6, SIRFSOC_CLKC_DISP1_CLKDIV_ENA, 8, &disp1_div_lock }, - { "sys3pll_qa13", "sys3pll_fixdiv", "sys3pll_a13", 0, 0, 0, SIRFSOC_CLKC_DISP1_CLKDIV_CFG, 24, 6, SIRFSOC_CLKC_DISP1_CLKDIV_ENA, 12, &disp1_div_lock }, - { "sys0pll_qa14", "sys0pll_fixdiv", "sys0pll_a14", 0, 0, 0, SIRFSOC_CLKC_GPU_CLKDIV_CFG, 0, 6, SIRFSOC_CLKC_GPU_CLKDIV_ENA, 0, &gpu_div_lock }, - { "sys1pll_qa14", "sys1pll_fixdiv", "sys1pll_a14", 0, 0, 0, SIRFSOC_CLKC_GPU_CLKDIV_CFG, 8, 6, SIRFSOC_CLKC_GPU_CLKDIV_ENA, 4, &gpu_div_lock }, - { "sys2pll_qa14", "sys2pll_fixdiv", "sys2pll_a14", 0, 0, 0, SIRFSOC_CLKC_GPU_CLKDIV_CFG, 16, 6, SIRFSOC_CLKC_GPU_CLKDIV_ENA, 8, &gpu_div_lock }, - { "sys3pll_qa14", "sys3pll_fixdiv", "sys3pll_a14", 0, 0, 0, SIRFSOC_CLKC_GPU_CLKDIV_CFG, 24, 6, SIRFSOC_CLKC_GPU_CLKDIV_ENA, 12, &gpu_div_lock }, - { "sys0pll_qa15", "sys0pll_fixdiv", "sys0pll_a15", 0, 0, 0, SIRFSOC_CLKC_GNSS_CLKDIV_CFG, 0, 6, SIRFSOC_CLKC_GNSS_CLKDIV_ENA, 0, &gnss_div_lock }, - { "sys1pll_qa15", "sys1pll_fixdiv", "sys1pll_a15", 0, 0, 0, SIRFSOC_CLKC_GNSS_CLKDIV_CFG, 8, 6, SIRFSOC_CLKC_GNSS_CLKDIV_ENA, 4, &gnss_div_lock }, - { "sys2pll_qa15", "sys2pll_fixdiv", "sys2pll_a15", 0, 0, 0, SIRFSOC_CLKC_GNSS_CLKDIV_CFG, 16, 6, SIRFSOC_CLKC_GNSS_CLKDIV_ENA, 8, &gnss_div_lock }, - { "sys3pll_qa15", "sys3pll_fixdiv", "sys3pll_a15", 0, 0, 0, SIRFSOC_CLKC_GNSS_CLKDIV_CFG, 24, 6, SIRFSOC_CLKC_GNSS_CLKDIV_ENA, 12, &gnss_div_lock }, - { "sys1pll_qa18", "sys1pll_fixdiv", "sys1pll_a18", 0, 0, 0, SIRFSOC_CLKC_SHARED_DIVIDER_CFG0, 24, 6, SIRFSOC_CLKC_SHARED_DIVIDER_ENA, 12, &share_div_lock }, - { "sys1pll_qa19", "sys1pll_fixdiv", "sys1pll_a19", 0, 0, CLK_IGNORE_UNUSED, SIRFSOC_CLKC_SHARED_DIVIDER_CFG0, 16, 6, SIRFSOC_CLKC_SHARED_DIVIDER_ENA, 8, &share_div_lock }, - { "sys1pll_qa20", "sys1pll_fixdiv", "sys1pll_a20", 0, 0, 0, SIRFSOC_CLKC_SHARED_DIVIDER_CFG0, 8, 6, SIRFSOC_CLKC_SHARED_DIVIDER_ENA, 4, &share_div_lock }, - { "sys2pll_qa20", "sys2pll_fixdiv", "sys2pll_a20", 0, 0, 0, SIRFSOC_CLKC_SHARED_DIVIDER_CFG0, 0, 6, SIRFSOC_CLKC_SHARED_DIVIDER_ENA, 0, &share_div_lock }, - { "sys1pll_qa17", "sys1pll_fixdiv", "sys1pll_a17", 0, 0, CLK_IGNORE_UNUSED, SIRFSOC_CLKC_SHARED_DIVIDER_CFG1, 8, 6, SIRFSOC_CLKC_SHARED_DIVIDER_ENA, 20, &share_div_lock }, - { "sys0pll_qa20", "sys0pll_fixdiv", "sys0pll_a20", 0, 0, 0, SIRFSOC_CLKC_SHARED_DIVIDER_CFG1, 0, 6, SIRFSOC_CLKC_SHARED_DIVIDER_ENA, 16, &share_div_lock }, -}; - -static const char * const i2s_clk_parents[] = { - "xin", - "xinw", - "audio_dto", - /* "pwm_i2s01" */ -}; - -static const char * const usbphy_clk_parents[] = { - "xin", - "xinw", - "sys0pll_a1", - "sys1pll_a1", - "sys2pll_a1", - "sys3pll_a1", -}; - -static const char * const btss_clk_parents[] = { - "xin", - "xinw", - "sys0pll_a2", - "sys1pll_a2", - "sys2pll_a2", - "sys3pll_a2", -}; - -static const char * const rgmii_clk_parents[] = { - "xin", - "xinw", - "sys0pll_a3", - "sys1pll_a3", - "sys2pll_a3", - "sys3pll_a3", -}; - -static const char * const cpu_clk_parents[] = { - "xin", - "xinw", - "sys0pll_a4", - "sys1pll_a4", - "cpupll_clk1", -}; - -static const char * const sdphy01_clk_parents[] = { - "xin", - "xinw", - "sys0pll_a5", - "sys1pll_a5", - "sys2pll_a5", - "sys3pll_a5", -}; - -static const char * const sdphy23_clk_parents[] = { - "xin", - "xinw", - "sys0pll_a6", - "sys1pll_a6", - "sys2pll_a6", - "sys3pll_a6", -}; - -static const char * const sdphy45_clk_parents[] = { - "xin", - "xinw", - "sys0pll_a7", - "sys1pll_a7", - "sys2pll_a7", - "sys3pll_a7", -}; - -static const char * const sdphy67_clk_parents[] = { - "xin", - "xinw", - "sys0pll_a8", - "sys1pll_a8", - "sys2pll_a8", - "sys3pll_a8", -}; - -static const char * const can_clk_parents[] = { - "xin", - "xinw", - "sys0pll_a9", - "sys1pll_a9", - "sys2pll_a9", - "sys3pll_a9", -}; - -static const char * const deint_clk_parents[] = { - "xin", - "xinw", - "sys0pll_a10", - "sys1pll_a10", - "sys2pll_a10", - "sys3pll_a10", -}; - -static const char * const nand_clk_parents[] = { - "xin", - "xinw", - "sys0pll_a11", - "sys1pll_a11", - "sys2pll_a11", - "sys3pll_a11", -}; - -static const char * const disp0_clk_parents[] = { - "xin", - "xinw", - "sys0pll_a12", - "sys1pll_a12", - "sys2pll_a12", - "sys3pll_a12", - "disp0_dto", -}; - -static const char * const disp1_clk_parents[] = { - "xin", - "xinw", - "sys0pll_a13", - "sys1pll_a13", - "sys2pll_a13", - "sys3pll_a13", - "disp1_dto", -}; - -static const char * const gpu_clk_parents[] = { - "xin", - "xinw", - "sys0pll_a14", - "sys1pll_a14", - "sys2pll_a14", - "sys3pll_a14", -}; - -static const char * const gnss_clk_parents[] = { - "xin", - "xinw", - "sys0pll_a15", - "sys1pll_a15", - "sys2pll_a15", - "sys3pll_a15", -}; - -static const char * const sys_clk_parents[] = { - "xin", - "xinw", - "sys2pll_a20", - "sys1pll_a20", - "sys1pll_a19", - "sys1pll_a18", - "sys0pll_a20", - "sys1pll_a17", -}; - -static const char * const io_clk_parents[] = { - "xin", - "xinw", - "sys2pll_a20", - "sys1pll_a20", - "sys1pll_a19", - "sys1pll_a18", - "sys0pll_a20", - "sys1pll_a17", -}; - -static const char * const g2d_clk_parents[] = { - "xin", - "xinw", - "sys2pll_a20", - "sys1pll_a20", - "sys1pll_a19", - "sys1pll_a18", - "sys0pll_a20", - "sys1pll_a17", -}; - -static const char * const jpenc_clk_parents[] = { - "xin", - "xinw", - "sys2pll_a20", - "sys1pll_a20", - "sys1pll_a19", - "sys1pll_a18", - "sys0pll_a20", - "sys1pll_a17", -}; - -static const char * const vdec_clk_parents[] = { - "xin", - "xinw", - "sys2pll_a20", - "sys1pll_a20", - "sys1pll_a19", - "sys1pll_a18", - "sys0pll_a20", - "sys1pll_a17", -}; - -static const char * const gmac_clk_parents[] = { - "xin", - "xinw", - "sys2pll_a20", - "sys1pll_a20", - "sys1pll_a19", - "sys1pll_a18", - "sys0pll_a20", - "sys1pll_a17", -}; - -static const char * const usb_clk_parents[] = { - "xin", - "xinw", - "sys2pll_a20", - "sys1pll_a20", - "sys1pll_a19", - "sys1pll_a18", - "sys0pll_a20", - "sys1pll_a17", -}; - -static const char * const kas_clk_parents[] = { - "xin", - "xinw", - "sys2pll_a20", - "sys1pll_a20", - "sys1pll_a19", - "sys1pll_a18", - "sys0pll_a20", - "sys1pll_a17", -}; - -static const char * const sec_clk_parents[] = { - "xin", - "xinw", - "sys2pll_a20", - "sys1pll_a20", - "sys1pll_a19", - "sys1pll_a18", - "sys0pll_a20", - "sys1pll_a17", -}; - -static const char * const sdr_clk_parents[] = { - "xin", - "xinw", - "sys2pll_a20", - "sys1pll_a20", - "sys1pll_a19", - "sys1pll_a18", - "sys0pll_a20", - "sys1pll_a17", -}; - -static const char * const vip_clk_parents[] = { - "xin", - "xinw", - "sys2pll_a20", - "sys1pll_a20", - "sys1pll_a19", - "sys1pll_a18", - "sys0pll_a20", - "sys1pll_a17", -}; - -static const char * const nocd_clk_parents[] = { - "xin", - "xinw", - "sys2pll_a20", - "sys1pll_a20", - "sys1pll_a19", - "sys1pll_a18", - "sys0pll_a20", - "sys1pll_a17", -}; - -static const char * const nocr_clk_parents[] = { - "xin", - "xinw", - "sys2pll_a20", - "sys1pll_a20", - "sys1pll_a19", - "sys1pll_a18", - "sys0pll_a20", - "sys1pll_a17", -}; - -static const char * const tpiu_clk_parents[] = { - "xin", - "xinw", - "sys2pll_a20", - "sys1pll_a20", - "sys1pll_a19", - "sys1pll_a18", - "sys0pll_a20", - "sys1pll_a17", -}; - -static struct atlas7_mux_init_data mux_list[] __initdata = { - /* mux_name, parent_names, parent_num, flags, mux_flags, mux_offset, shift, width */ - { "i2s_mux", i2s_clk_parents, ARRAY_SIZE(i2s_clk_parents), 0, 0, SIRFSOC_CLKC_I2S_CLK_SEL, 0, 2 }, - { "usbphy_mux", usbphy_clk_parents, ARRAY_SIZE(usbphy_clk_parents), 0, 0, SIRFSOC_CLKC_I2S_CLK_SEL, 0, 3 }, - { "btss_mux", btss_clk_parents, ARRAY_SIZE(btss_clk_parents), 0, 0, SIRFSOC_CLKC_BTSS_CLK_SEL, 0, 3 }, - { "rgmii_mux", rgmii_clk_parents, ARRAY_SIZE(rgmii_clk_parents), 0, 0, SIRFSOC_CLKC_RGMII_CLK_SEL, 0, 3 }, - { "cpu_mux", cpu_clk_parents, ARRAY_SIZE(cpu_clk_parents), 0, 0, SIRFSOC_CLKC_CPU_CLK_SEL, 0, 3 }, - { "sdphy01_mux", sdphy01_clk_parents, ARRAY_SIZE(sdphy01_clk_parents), 0, 0, SIRFSOC_CLKC_SDPHY01_CLK_SEL, 0, 3 }, - { "sdphy23_mux", sdphy23_clk_parents, ARRAY_SIZE(sdphy23_clk_parents), 0, 0, SIRFSOC_CLKC_SDPHY23_CLK_SEL, 0, 3 }, - { "sdphy45_mux", sdphy45_clk_parents, ARRAY_SIZE(sdphy45_clk_parents), 0, 0, SIRFSOC_CLKC_SDPHY45_CLK_SEL, 0, 3 }, - { "sdphy67_mux", sdphy67_clk_parents, ARRAY_SIZE(sdphy67_clk_parents), 0, 0, SIRFSOC_CLKC_SDPHY67_CLK_SEL, 0, 3 }, - { "can_mux", can_clk_parents, ARRAY_SIZE(can_clk_parents), 0, 0, SIRFSOC_CLKC_CAN_CLK_SEL, 0, 3 }, - { "deint_mux", deint_clk_parents, ARRAY_SIZE(deint_clk_parents), 0, 0, SIRFSOC_CLKC_DEINT_CLK_SEL, 0, 3 }, - { "nand_mux", nand_clk_parents, ARRAY_SIZE(nand_clk_parents), 0, 0, SIRFSOC_CLKC_NAND_CLK_SEL, 0, 3 }, - { "disp0_mux", disp0_clk_parents, ARRAY_SIZE(disp0_clk_parents), 0, 0, SIRFSOC_CLKC_DISP0_CLK_SEL, 0, 3 }, - { "disp1_mux", disp1_clk_parents, ARRAY_SIZE(disp1_clk_parents), 0, 0, SIRFSOC_CLKC_DISP1_CLK_SEL, 0, 3 }, - { "gpu_mux", gpu_clk_parents, ARRAY_SIZE(gpu_clk_parents), 0, 0, SIRFSOC_CLKC_GPU_CLK_SEL, 0, 3 }, - { "gnss_mux", gnss_clk_parents, ARRAY_SIZE(gnss_clk_parents), 0, 0, SIRFSOC_CLKC_GNSS_CLK_SEL, 0, 3 }, - { "sys_mux", sys_clk_parents, ARRAY_SIZE(sys_clk_parents), 0, 0, SIRFSOC_CLKC_SYS_CLK_SEL, 0, 3 }, - { "io_mux", io_clk_parents, ARRAY_SIZE(io_clk_parents), 0, 0, SIRFSOC_CLKC_IO_CLK_SEL, 0, 3 }, - { "g2d_mux", g2d_clk_parents, ARRAY_SIZE(g2d_clk_parents), 0, 0, SIRFSOC_CLKC_G2D_CLK_SEL, 0, 3 }, - { "jpenc_mux", jpenc_clk_parents, ARRAY_SIZE(jpenc_clk_parents), 0, 0, SIRFSOC_CLKC_JPENC_CLK_SEL, 0, 3 }, - { "vdec_mux", vdec_clk_parents, ARRAY_SIZE(vdec_clk_parents), 0, 0, SIRFSOC_CLKC_VDEC_CLK_SEL, 0, 3 }, - { "gmac_mux", gmac_clk_parents, ARRAY_SIZE(gmac_clk_parents), 0, 0, SIRFSOC_CLKC_GMAC_CLK_SEL, 0, 3 }, - { "usb_mux", usb_clk_parents, ARRAY_SIZE(usb_clk_parents), 0, 0, SIRFSOC_CLKC_USB_CLK_SEL, 0, 3 }, - { "kas_mux", kas_clk_parents, ARRAY_SIZE(kas_clk_parents), 0, 0, SIRFSOC_CLKC_KAS_CLK_SEL, 0, 3 }, - { "sec_mux", sec_clk_parents, ARRAY_SIZE(sec_clk_parents), 0, 0, SIRFSOC_CLKC_SEC_CLK_SEL, 0, 3 }, - { "sdr_mux", sdr_clk_parents, ARRAY_SIZE(sdr_clk_parents), 0, 0, SIRFSOC_CLKC_SDR_CLK_SEL, 0, 3 }, - { "vip_mux", vip_clk_parents, ARRAY_SIZE(vip_clk_parents), 0, 0, SIRFSOC_CLKC_VIP_CLK_SEL, 0, 3 }, - { "nocd_mux", nocd_clk_parents, ARRAY_SIZE(nocd_clk_parents), 0, 0, SIRFSOC_CLKC_NOCD_CLK_SEL, 0, 3 }, - { "nocr_mux", nocr_clk_parents, ARRAY_SIZE(nocr_clk_parents), 0, 0, SIRFSOC_CLKC_NOCR_CLK_SEL, 0, 3 }, - { "tpiu_mux", tpiu_clk_parents, ARRAY_SIZE(tpiu_clk_parents), 0, 0, SIRFSOC_CLKC_TPIU_CLK_SEL, 0, 3 }, -}; - - /* new unit should add start from the tail of list */ -static struct atlas7_unit_init_data unit_list[] __initdata = { - /* unit_name, parent_name, flags, regofs, bit, lock */ - { 0, "audmscm_kas", "kas_mux", 0, SIRFSOC_CLKC_ROOT_CLK_EN0_SET, 0, 0, 0, &root0_gate_lock }, - { 1, "gnssm_gnss", "gnss_mux", 0, SIRFSOC_CLKC_ROOT_CLK_EN0_SET, 1, 0, 0, &root0_gate_lock }, - { 2, "gpum_gpu", "gpu_mux", 0, SIRFSOC_CLKC_ROOT_CLK_EN0_SET, 2, 0, 0, &root0_gate_lock }, - { 3, "mediam_g2d", "g2d_mux", 0, SIRFSOC_CLKC_ROOT_CLK_EN0_SET, 3, 0, 0, &root0_gate_lock }, - { 4, "mediam_jpenc", "jpenc_mux", 0, SIRFSOC_CLKC_ROOT_CLK_EN0_SET, 4, 0, 0, &root0_gate_lock }, - { 5, "vdifm_disp0", "disp0_mux", 0, SIRFSOC_CLKC_ROOT_CLK_EN0_SET, 5, 0, 0, &root0_gate_lock }, - { 6, "vdifm_disp1", "disp1_mux", 0, SIRFSOC_CLKC_ROOT_CLK_EN0_SET, 6, 0, 0, &root0_gate_lock }, - { 7, "audmscm_i2s", "i2s_mux", 0, SIRFSOC_CLKC_ROOT_CLK_EN0_SET, 8, 0, 0, &root0_gate_lock }, - { 8, "audmscm_io", "io_mux", 0, SIRFSOC_CLKC_ROOT_CLK_EN0_SET, 11, 0, 0, &root0_gate_lock }, - { 9, "vdifm_io", "io_mux", 0, SIRFSOC_CLKC_ROOT_CLK_EN0_SET, 12, 0, 0, &root0_gate_lock }, - { 10, "gnssm_io", "io_mux", 0, SIRFSOC_CLKC_ROOT_CLK_EN0_SET, 13, 0, 0, &root0_gate_lock }, - { 11, "mediam_io", "io_mux", 0, SIRFSOC_CLKC_ROOT_CLK_EN0_SET, 14, 0, 0, &root0_gate_lock }, - { 12, "btm_io", "io_mux", 0, SIRFSOC_CLKC_ROOT_CLK_EN0_SET, 17, 0, 0, &root0_gate_lock }, - { 13, "mediam_sdphy01", "sdphy01_mux", 0, SIRFSOC_CLKC_ROOT_CLK_EN0_SET, 18, 0, 0, &root0_gate_lock }, - { 14, "vdifm_sdphy23", "sdphy23_mux", 0, SIRFSOC_CLKC_ROOT_CLK_EN0_SET, 19, 0, 0, &root0_gate_lock }, - { 15, "vdifm_sdphy45", "sdphy45_mux", 0, SIRFSOC_CLKC_ROOT_CLK_EN0_SET, 20, 0, 0, &root0_gate_lock }, - { 16, "vdifm_sdphy67", "sdphy67_mux", 0, SIRFSOC_CLKC_ROOT_CLK_EN0_SET, 21, 0, 0, &root0_gate_lock }, - { 17, "audmscm_xin", "xin", 0, SIRFSOC_CLKC_ROOT_CLK_EN0_SET, 22, 0, 0, &root0_gate_lock }, - { 18, "mediam_nand", "nand_mux", 0, SIRFSOC_CLKC_ROOT_CLK_EN0_SET, 27, 0, 0, &root0_gate_lock }, - { 19, "gnssm_sec", "sec_mux", 0, SIRFSOC_CLKC_ROOT_CLK_EN0_SET, 28, 0, 0, &root0_gate_lock }, - { 20, "cpum_cpu", "cpu_mux", 0, SIRFSOC_CLKC_ROOT_CLK_EN0_SET, 29, 0, 0, &root0_gate_lock }, - { 21, "gnssm_xin", "xin", 0, SIRFSOC_CLKC_ROOT_CLK_EN0_SET, 30, 0, 0, &root0_gate_lock }, - { 22, "vdifm_vip", "vip_mux", 0, SIRFSOC_CLKC_ROOT_CLK_EN0_SET, 31, 0, 0, &root0_gate_lock }, - { 23, "btm_btss", "btss_mux", 0, SIRFSOC_CLKC_ROOT_CLK_EN1_SET, 0, 0, 0, &root1_gate_lock }, - { 24, "mediam_usbphy", "usbphy_mux", 0, SIRFSOC_CLKC_ROOT_CLK_EN1_SET, 1, 0, 0, &root1_gate_lock }, - { 25, "rtcm_kas", "kas_mux", 0, SIRFSOC_CLKC_ROOT_CLK_EN1_SET, 2, 0, 0, &root1_gate_lock }, - { 26, "audmscm_nocd", "nocd_mux", 0, SIRFSOC_CLKC_ROOT_CLK_EN1_SET, 3, 0, 0, &root1_gate_lock }, - { 27, "vdifm_nocd", "nocd_mux", 0, SIRFSOC_CLKC_ROOT_CLK_EN1_SET, 4, 0, 0, &root1_gate_lock }, - { 28, "gnssm_nocd", "nocd_mux", 0, SIRFSOC_CLKC_ROOT_CLK_EN1_SET, 5, 0, 0, &root1_gate_lock }, - { 29, "mediam_nocd", "nocd_mux", 0, SIRFSOC_CLKC_ROOT_CLK_EN1_SET, 6, 0, 0, &root1_gate_lock }, - { 30, "cpum_nocd", "nocd_mux", 0, SIRFSOC_CLKC_ROOT_CLK_EN1_SET, 8, 0, 0, &root1_gate_lock }, - { 31, "gpum_nocd", "nocd_mux", 0, SIRFSOC_CLKC_ROOT_CLK_EN1_SET, 9, 0, 0, &root1_gate_lock }, - { 32, "audmscm_nocr", "nocr_mux", 0, SIRFSOC_CLKC_ROOT_CLK_EN1_SET, 11, 0, 0, &root1_gate_lock }, - { 33, "vdifm_nocr", "nocr_mux", 0, SIRFSOC_CLKC_ROOT_CLK_EN1_SET, 12, 0, 0, &root1_gate_lock }, - { 34, "gnssm_nocr", "nocr_mux", CLK_IGNORE_UNUSED, SIRFSOC_CLKC_ROOT_CLK_EN1_SET, 13, 0, 0, &root1_gate_lock }, - { 35, "mediam_nocr", "nocr_mux", CLK_IGNORE_UNUSED, SIRFSOC_CLKC_ROOT_CLK_EN1_SET, 14, 0, 0, &root1_gate_lock }, - { 36, "ddrm_nocr", "nocr_mux", CLK_IGNORE_UNUSED, SIRFSOC_CLKC_ROOT_CLK_EN1_SET, 15, 0, 0, &root1_gate_lock }, - { 37, "cpum_tpiu", "tpiu_mux", 0, SIRFSOC_CLKC_ROOT_CLK_EN1_SET, 16, 0, 0, &root1_gate_lock }, - { 38, "gpum_nocr", "nocr_mux", 0, SIRFSOC_CLKC_ROOT_CLK_EN1_SET, 17, 0, 0, &root1_gate_lock }, - { 39, "gnssm_rgmii", "rgmii_mux", 0, SIRFSOC_CLKC_ROOT_CLK_EN1_SET, 20, 0, 0, &root1_gate_lock }, - { 40, "mediam_vdec", "vdec_mux", 0, SIRFSOC_CLKC_ROOT_CLK_EN1_SET, 21, 0, 0, &root1_gate_lock }, - { 41, "gpum_sdr", "sdr_mux", 0, SIRFSOC_CLKC_ROOT_CLK_EN1_SET, 22, 0, 0, &root1_gate_lock }, - { 42, "vdifm_deint", "deint_mux", 0, SIRFSOC_CLKC_ROOT_CLK_EN1_SET, 23, 0, 0, &root1_gate_lock }, - { 43, "gnssm_can", "can_mux", 0, SIRFSOC_CLKC_ROOT_CLK_EN1_SET, 26, 0, 0, &root1_gate_lock }, - { 44, "mediam_usb", "usb_mux", 0, SIRFSOC_CLKC_ROOT_CLK_EN1_SET, 28, 0, 0, &root1_gate_lock }, - { 45, "gnssm_gmac", "gmac_mux", 0, SIRFSOC_CLKC_ROOT_CLK_EN1_SET, 29, 0, 0, &root1_gate_lock }, - { 46, "cvd_io", "audmscm_io", 0, SIRFSOC_CLKC_LEAF_CLK_EN1_SET, 0, CLK_UNIT_NOC_CLOCK, 4, &leaf1_gate_lock }, - { 47, "timer_io", "audmscm_io", 0, SIRFSOC_CLKC_LEAF_CLK_EN1_SET, 1, 0, 0, &leaf1_gate_lock }, - { 48, "pulse_io", "audmscm_io", 0, SIRFSOC_CLKC_LEAF_CLK_EN1_SET, 2, 0, 0, &leaf1_gate_lock }, - { 49, "tsc_io", "audmscm_io", 0, SIRFSOC_CLKC_LEAF_CLK_EN1_SET, 3, 0, 0, &leaf1_gate_lock }, - { 50, "tsc_xin", "audmscm_xin", 0, SIRFSOC_CLKC_LEAF_CLK_EN1_SET, 21, 0, 0, &leaf1_gate_lock }, - { 51, "ioctop_io", "audmscm_io", 0, SIRFSOC_CLKC_LEAF_CLK_EN1_SET, 4, 0, 0, &leaf1_gate_lock }, - { 52, "rsc_io", "audmscm_io", 0, SIRFSOC_CLKC_LEAF_CLK_EN1_SET, 5, 0, 0, &leaf1_gate_lock }, - { 53, "dvm_io", "audmscm_io", 0, SIRFSOC_CLKC_LEAF_CLK_EN1_SET, 6, CLK_UNIT_NOC_SOCKET, 7, &leaf1_gate_lock }, - { 54, "lvds_xin", "audmscm_xin", 0, SIRFSOC_CLKC_LEAF_CLK_EN1_SET, 7, CLK_UNIT_NOC_SOCKET, 8, &leaf1_gate_lock }, - { 55, "kas_kas", "audmscm_kas", 0, SIRFSOC_CLKC_LEAF_CLK_EN1_SET, 8, CLK_UNIT_NOC_CLOCK, 2, &leaf1_gate_lock }, - { 56, "ac97_kas", "audmscm_kas", 0, SIRFSOC_CLKC_LEAF_CLK_EN1_SET, 9, 0, 0, &leaf1_gate_lock }, - { 57, "usp0_kas", "audmscm_kas", 0, SIRFSOC_CLKC_LEAF_CLK_EN1_SET, 10, CLK_UNIT_NOC_SOCKET, 4, &leaf1_gate_lock }, - { 58, "usp1_kas", "audmscm_kas", 0, SIRFSOC_CLKC_LEAF_CLK_EN1_SET, 11, CLK_UNIT_NOC_SOCKET, 5, &leaf1_gate_lock }, - { 59, "usp2_kas", "audmscm_kas", 0, SIRFSOC_CLKC_LEAF_CLK_EN1_SET, 12, CLK_UNIT_NOC_SOCKET, 6, &leaf1_gate_lock }, - { 60, "dmac2_kas", "audmscm_kas", 0, SIRFSOC_CLKC_LEAF_CLK_EN1_SET, 13, CLK_UNIT_NOC_SOCKET, 1, &leaf1_gate_lock }, - { 61, "dmac3_kas", "audmscm_kas", 0, SIRFSOC_CLKC_LEAF_CLK_EN1_SET, 14, CLK_UNIT_NOC_SOCKET, 2, &leaf1_gate_lock }, - { 62, "audioif_kas", "audmscm_kas", 0, SIRFSOC_CLKC_LEAF_CLK_EN1_SET, 15, CLK_UNIT_NOC_SOCKET, 0, &leaf1_gate_lock }, - { 63, "i2s1_kas", "audmscm_kas", 0, SIRFSOC_CLKC_LEAF_CLK_EN1_SET, 17, CLK_UNIT_NOC_CLOCK, 2, &leaf1_gate_lock }, - { 64, "thaudmscm_io", "audmscm_io", 0, SIRFSOC_CLKC_LEAF_CLK_EN1_SET, 22, 0, 0, &leaf1_gate_lock }, - { 65, "analogtest_xin", "audmscm_xin", 0, SIRFSOC_CLKC_LEAF_CLK_EN1_SET, 23, 0, 0, &leaf1_gate_lock }, - { 66, "sys2pci_io", "vdifm_io", 0, SIRFSOC_CLKC_LEAF_CLK_EN2_SET, 0, CLK_UNIT_NOC_CLOCK, 20, &leaf2_gate_lock }, - { 67, "pciarb_io", "vdifm_io", 0, SIRFSOC_CLKC_LEAF_CLK_EN2_SET, 1, 0, 0, &leaf2_gate_lock }, - { 68, "pcicopy_io", "vdifm_io", 0, SIRFSOC_CLKC_LEAF_CLK_EN2_SET, 2, 0, 0, &leaf2_gate_lock }, - { 69, "rom_io", "vdifm_io", 0, SIRFSOC_CLKC_LEAF_CLK_EN2_SET, 3, 0, 0, &leaf2_gate_lock }, - { 70, "sdio23_io", "vdifm_io", 0, SIRFSOC_CLKC_LEAF_CLK_EN2_SET, 4, 0, 0, &leaf2_gate_lock }, - { 71, "sdio45_io", "vdifm_io", 0, SIRFSOC_CLKC_LEAF_CLK_EN2_SET, 5, 0, 0, &leaf2_gate_lock }, - { 72, "sdio67_io", "vdifm_io", 0, SIRFSOC_CLKC_LEAF_CLK_EN2_SET, 6, 0, 0, &leaf2_gate_lock }, - { 73, "vip1_io", "vdifm_io", 0, SIRFSOC_CLKC_LEAF_CLK_EN2_SET, 7, 0, 0, &leaf2_gate_lock }, - { 74, "vip1_vip", "vdifm_vip", 0, SIRFSOC_CLKC_LEAF_CLK_EN2_SET, 16, CLK_UNIT_NOC_CLOCK, 21, &leaf2_gate_lock }, - { 75, "sdio23_sdphy23", "vdifm_sdphy23", 0, SIRFSOC_CLKC_LEAF_CLK_EN2_SET, 8, 0, 0, &leaf2_gate_lock }, - { 76, "sdio45_sdphy45", "vdifm_sdphy45", 0, SIRFSOC_CLKC_LEAF_CLK_EN2_SET, 9, 0, 0, &leaf2_gate_lock }, - { 77, "sdio67_sdphy67", "vdifm_sdphy67", 0, SIRFSOC_CLKC_LEAF_CLK_EN2_SET, 10, 0, 0, &leaf2_gate_lock }, - { 78, "vpp0_disp0", "vdifm_disp0", 0, SIRFSOC_CLKC_LEAF_CLK_EN2_SET, 11, CLK_UNIT_NOC_CLOCK, 22, &leaf2_gate_lock }, - { 79, "lcd0_disp0", "vdifm_disp0", 0, SIRFSOC_CLKC_LEAF_CLK_EN2_SET, 12, CLK_UNIT_NOC_CLOCK, 18, &leaf2_gate_lock }, - { 80, "vpp1_disp1", "vdifm_disp1", 0, SIRFSOC_CLKC_LEAF_CLK_EN2_SET, 13, CLK_UNIT_NOC_CLOCK, 23, &leaf2_gate_lock }, - { 81, "lcd1_disp1", "vdifm_disp1", 0, SIRFSOC_CLKC_LEAF_CLK_EN2_SET, 14, CLK_UNIT_NOC_CLOCK, 19, &leaf2_gate_lock }, - { 82, "dcu_deint", "vdifm_deint", 0, SIRFSOC_CLKC_LEAF_CLK_EN2_SET, 15, CLK_UNIT_NOC_CLOCK, 17, &leaf2_gate_lock }, - { 83, "vdifm_dapa_r_nocr", "vdifm_nocr", 0, SIRFSOC_CLKC_LEAF_CLK_EN2_SET, 17, 0, 0, &leaf2_gate_lock }, - { 84, "gpio1_io", "vdifm_io", 0, SIRFSOC_CLKC_LEAF_CLK_EN2_SET, 18, 0, 0, &leaf2_gate_lock }, - { 85, "thvdifm_io", "vdifm_io", 0, SIRFSOC_CLKC_LEAF_CLK_EN2_SET, 19, 0, 0, &leaf2_gate_lock }, - { 86, "gmac_rgmii", "gnssm_rgmii", 0, SIRFSOC_CLKC_LEAF_CLK_EN3_SET, 0, 0, 0, &leaf3_gate_lock }, - { 87, "gmac_gmac", "gnssm_gmac", 0, SIRFSOC_CLKC_LEAF_CLK_EN3_SET, 1, CLK_UNIT_NOC_CLOCK, 10, &leaf3_gate_lock }, - { 88, "uart1_io", "gnssm_io", 0, SIRFSOC_CLKC_LEAF_CLK_EN3_SET, 2, CLK_UNIT_NOC_SOCKET, 14, &leaf3_gate_lock }, - { 89, "dmac0_io", "gnssm_io", 0, SIRFSOC_CLKC_LEAF_CLK_EN3_SET, 3, CLK_UNIT_NOC_SOCKET, 11, &leaf3_gate_lock }, - { 90, "uart0_io", "gnssm_io", 0, SIRFSOC_CLKC_LEAF_CLK_EN3_SET, 4, CLK_UNIT_NOC_SOCKET, 13, &leaf3_gate_lock }, - { 91, "uart2_io", "gnssm_io", 0, SIRFSOC_CLKC_LEAF_CLK_EN3_SET, 5, CLK_UNIT_NOC_SOCKET, 15, &leaf3_gate_lock }, - { 92, "uart3_io", "gnssm_io", 0, SIRFSOC_CLKC_LEAF_CLK_EN3_SET, 6, CLK_UNIT_NOC_SOCKET, 16, &leaf3_gate_lock }, - { 93, "uart4_io", "gnssm_io", 0, SIRFSOC_CLKC_LEAF_CLK_EN3_SET, 7, CLK_UNIT_NOC_SOCKET, 17, &leaf3_gate_lock }, - { 94, "uart5_io", "gnssm_io", 0, SIRFSOC_CLKC_LEAF_CLK_EN3_SET, 8, CLK_UNIT_NOC_SOCKET, 18, &leaf3_gate_lock }, - { 95, "spi1_io", "gnssm_io", 0, SIRFSOC_CLKC_LEAF_CLK_EN3_SET, 9, CLK_UNIT_NOC_SOCKET, 12, &leaf3_gate_lock }, - { 96, "gnss_gnss", "gnssm_gnss", 0, SIRFSOC_CLKC_LEAF_CLK_EN3_SET, 10, 0, 0, &leaf3_gate_lock }, - { 97, "canbus1_can", "gnssm_can", 0, SIRFSOC_CLKC_LEAF_CLK_EN3_SET, 12, CLK_UNIT_NOC_CLOCK, 7, &leaf3_gate_lock }, - { 98, "ccsec_sec", "gnssm_sec", 0, SIRFSOC_CLKC_LEAF_CLK_EN3_SET, 15, CLK_UNIT_NOC_CLOCK, 9, &leaf3_gate_lock }, - { 99, "ccpub_sec", "gnssm_sec", 0, SIRFSOC_CLKC_LEAF_CLK_EN3_SET, 16, CLK_UNIT_NOC_CLOCK, 8, &leaf3_gate_lock }, - { 100, "gnssm_dapa_r_nocr", "gnssm_nocr", 0, SIRFSOC_CLKC_LEAF_CLK_EN3_SET, 13, 0, 0, &leaf3_gate_lock }, - { 101, "thgnssm_io", "gnssm_io", 0, SIRFSOC_CLKC_LEAF_CLK_EN3_SET, 14, 0, 0, &leaf3_gate_lock }, - { 102, "media_vdec", "mediam_vdec", 0, SIRFSOC_CLKC_LEAF_CLK_EN4_SET, 0, CLK_UNIT_NOC_CLOCK, 3, &leaf4_gate_lock }, - { 103, "media_jpenc", "mediam_jpenc", 0, SIRFSOC_CLKC_LEAF_CLK_EN4_SET, 1, CLK_UNIT_NOC_CLOCK, 1, &leaf4_gate_lock }, - { 104, "g2d_g2d", "mediam_g2d", 0, SIRFSOC_CLKC_LEAF_CLK_EN4_SET, 2, CLK_UNIT_NOC_CLOCK, 12, &leaf4_gate_lock }, - { 105, "i2c0_io", "mediam_io", 0, SIRFSOC_CLKC_LEAF_CLK_EN4_SET, 3, CLK_UNIT_NOC_SOCKET, 21, &leaf4_gate_lock }, - { 106, "i2c1_io", "mediam_io", 0, SIRFSOC_CLKC_LEAF_CLK_EN4_SET, 4, CLK_UNIT_NOC_SOCKET, 20, &leaf4_gate_lock }, - { 107, "gpio0_io", "mediam_io", 0, SIRFSOC_CLKC_LEAF_CLK_EN4_SET, 5, CLK_UNIT_NOC_SOCKET, 19, &leaf4_gate_lock }, - { 108, "nand_io", "mediam_io", 0, SIRFSOC_CLKC_LEAF_CLK_EN4_SET, 6, 0, 0, &leaf4_gate_lock }, - { 109, "sdio01_io", "mediam_io", 0, SIRFSOC_CLKC_LEAF_CLK_EN4_SET, 7, 0, 0, &leaf4_gate_lock }, - { 110, "sys2pci2_io", "mediam_io", 0, SIRFSOC_CLKC_LEAF_CLK_EN4_SET, 8, CLK_UNIT_NOC_CLOCK, 13, &leaf4_gate_lock }, - { 111, "sdio01_sdphy01", "mediam_sdphy01", 0, SIRFSOC_CLKC_LEAF_CLK_EN4_SET, 9, 0, 0, &leaf4_gate_lock }, - { 112, "nand_nand", "mediam_nand", 0, SIRFSOC_CLKC_LEAF_CLK_EN4_SET, 10, CLK_UNIT_NOC_CLOCK, 14, &leaf4_gate_lock }, - { 113, "usb0_usb", "mediam_usb", 0, SIRFSOC_CLKC_LEAF_CLK_EN4_SET, 11, CLK_UNIT_NOC_CLOCK, 15, &leaf4_gate_lock }, - { 114, "usb1_usb", "mediam_usb", 0, SIRFSOC_CLKC_LEAF_CLK_EN4_SET, 12, CLK_UNIT_NOC_CLOCK, 16, &leaf4_gate_lock }, - { 115, "usbphy0_usbphy", "mediam_usbphy", 0, SIRFSOC_CLKC_LEAF_CLK_EN4_SET, 13, 0, 0, &leaf4_gate_lock }, - { 116, "usbphy1_usbphy", "mediam_usbphy", 0, SIRFSOC_CLKC_LEAF_CLK_EN4_SET, 14, 0, 0, &leaf4_gate_lock }, - { 117, "thmediam_io", "mediam_io", 0, SIRFSOC_CLKC_LEAF_CLK_EN4_SET, 15, 0, 0, &leaf4_gate_lock }, - { 118, "memc_mem", "mempll_clk1", CLK_IGNORE_UNUSED, SIRFSOC_CLKC_LEAF_CLK_EN5_SET, 0, 0, 0, &leaf5_gate_lock }, - { 119, "dapa_mem", "mempll_clk1", 0, SIRFSOC_CLKC_LEAF_CLK_EN5_SET, 1, 0, 0, &leaf5_gate_lock }, - { 120, "nocddrm_nocr", "ddrm_nocr", 0, SIRFSOC_CLKC_LEAF_CLK_EN5_SET, 2, 0, 0, &leaf5_gate_lock }, - { 121, "thddrm_nocr", "ddrm_nocr", 0, SIRFSOC_CLKC_LEAF_CLK_EN5_SET, 3, 0, 0, &leaf5_gate_lock }, - { 122, "spram1_cpudiv2", "cpum_cpu", 0, SIRFSOC_CLKC_LEAF_CLK_EN6_SET, 0, CLK_UNIT_NOC_SOCKET, 9, &leaf6_gate_lock }, - { 123, "spram2_cpudiv2", "cpum_cpu", 0, SIRFSOC_CLKC_LEAF_CLK_EN6_SET, 1, CLK_UNIT_NOC_SOCKET, 10, &leaf6_gate_lock }, - { 124, "coresight_cpudiv2", "cpum_cpu", 0, SIRFSOC_CLKC_LEAF_CLK_EN6_SET, 2, 0, 0, &leaf6_gate_lock }, - { 125, "coresight_tpiu", "cpum_tpiu", 0, SIRFSOC_CLKC_LEAF_CLK_EN6_SET, 3, 0, 0, &leaf6_gate_lock }, - { 126, "graphic_gpu", "gpum_gpu", 0, SIRFSOC_CLKC_LEAF_CLK_EN7_SET, 0, CLK_UNIT_NOC_CLOCK, 0, &leaf7_gate_lock }, - { 127, "vss_sdr", "gpum_sdr", 0, SIRFSOC_CLKC_LEAF_CLK_EN7_SET, 1, CLK_UNIT_NOC_CLOCK, 11, &leaf7_gate_lock }, - { 128, "thgpum_nocr", "gpum_nocr", 0, SIRFSOC_CLKC_LEAF_CLK_EN7_SET, 2, 0, 0, &leaf7_gate_lock }, - { 129, "a7ca_btss", "btm_btss", 0, SIRFSOC_CLKC_LEAF_CLK_EN8_SET, 1, 0, 0, &leaf8_gate_lock }, - { 130, "dmac4_io", "a7ca_io", 0, SIRFSOC_CLKC_LEAF_CLK_EN8_SET, 2, 0, 0, &leaf8_gate_lock }, - { 131, "uart6_io", "dmac4_io", 0, SIRFSOC_CLKC_LEAF_CLK_EN8_SET, 3, 0, 0, &leaf8_gate_lock }, - { 132, "usp3_io", "dmac4_io", 0, SIRFSOC_CLKC_LEAF_CLK_EN8_SET, 4, 0, 0, &leaf8_gate_lock }, - { 133, "a7ca_io", "noc_btm_io", 0, SIRFSOC_CLKC_LEAF_CLK_EN8_SET, 5, 0, 0, &leaf8_gate_lock }, - { 134, "noc_btm_io", "btm_io", 0, SIRFSOC_CLKC_LEAF_CLK_EN8_SET, 6, 0, 0, &leaf8_gate_lock }, - { 135, "thbtm_io", "btm_io", 0, SIRFSOC_CLKC_LEAF_CLK_EN8_SET, 7, 0, 0, &leaf8_gate_lock }, - { 136, "btslow", "xinw_fixdiv_btslow", 0, SIRFSOC_CLKC_ROOT_CLK_EN1_SET, 25, 0, 0, &root1_gate_lock }, - { 137, "a7ca_btslow", "btslow", 0, SIRFSOC_CLKC_LEAF_CLK_EN8_SET, 0, 0, 0, &leaf8_gate_lock }, - { 138, "pwm_io", "io_mux", 0, SIRFSOC_CLKC_LEAF_CLK_EN0_SET, 0, 0, 0, &leaf0_gate_lock }, - { 139, "pwm_xin", "xin", 0, SIRFSOC_CLKC_LEAF_CLK_EN0_SET, 1, 0, 0, &leaf0_gate_lock }, - { 140, "pwm_xinw", "xinw", 0, SIRFSOC_CLKC_LEAF_CLK_EN0_SET, 2, 0, 0, &leaf0_gate_lock }, - { 141, "thcgum_sys", "sys_mux", 0, SIRFSOC_CLKC_LEAF_CLK_EN0_SET, 3, 0, 0, &leaf0_gate_lock }, -}; - -static struct clk *atlas7_clks[ARRAY_SIZE(unit_list) + ARRAY_SIZE(mux_list)]; - -static int unit_clk_is_enabled(struct clk_hw *hw) -{ - struct clk_unit *clk = to_unitclk(hw); - u32 reg; - - reg = clk->regofs + SIRFSOC_CLKC_ROOT_CLK_EN0_STAT - SIRFSOC_CLKC_ROOT_CLK_EN0_SET; - - return !!(clkc_readl(reg) & BIT(clk->bit)); -} - -static int unit_clk_enable(struct clk_hw *hw) -{ - u32 reg; - struct clk_unit *clk = to_unitclk(hw); - unsigned long flags; - - reg = clk->regofs; - - spin_lock_irqsave(clk->lock, flags); - clkc_writel(BIT(clk->bit), reg); - if (clk->type == CLK_UNIT_NOC_CLOCK) - clkc_writel(BIT(clk->idle_bit), SIRFSOC_NOC_CLK_IDLEREQ_CLR); - else if (clk->type == CLK_UNIT_NOC_SOCKET) - clkc_writel(BIT(clk->idle_bit), SIRFSOC_NOC_CLK_SLVRDY_SET); - - spin_unlock_irqrestore(clk->lock, flags); - return 0; -} - -static void unit_clk_disable(struct clk_hw *hw) -{ - u32 reg; - u32 i = 0; - struct clk_unit *clk = to_unitclk(hw); - unsigned long flags; - - reg = clk->regofs + SIRFSOC_CLKC_ROOT_CLK_EN0_CLR - SIRFSOC_CLKC_ROOT_CLK_EN0_SET; - spin_lock_irqsave(clk->lock, flags); - if (clk->type == CLK_UNIT_NOC_CLOCK) { - clkc_writel(BIT(clk->idle_bit), SIRFSOC_NOC_CLK_IDLEREQ_SET); - while (!(clkc_readl(SIRFSOC_NOC_CLK_IDLE_STATUS) & - BIT(clk->idle_bit)) && (i++ < 100)) { - cpu_relax(); - udelay(10); - } - - if (i == 100) { - pr_err("unit NoC Clock disconnect Error:timeout\n"); - /*once timeout, undo idlereq by CLR*/ - clkc_writel(BIT(clk->idle_bit), SIRFSOC_NOC_CLK_IDLEREQ_CLR); - goto err; - } - - } else if (clk->type == CLK_UNIT_NOC_SOCKET) - clkc_writel(BIT(clk->idle_bit), SIRFSOC_NOC_CLK_SLVRDY_CLR); - - clkc_writel(BIT(clk->bit), reg); -err: - spin_unlock_irqrestore(clk->lock, flags); -} - -static const struct clk_ops unit_clk_ops = { - .is_enabled = unit_clk_is_enabled, - .enable = unit_clk_enable, - .disable = unit_clk_disable, -}; - -static struct clk * __init -atlas7_unit_clk_register(struct device *dev, const char *name, - const char * const parent_name, unsigned long flags, - u32 regofs, u8 bit, u32 type, u8 idle_bit, spinlock_t *lock) -{ - struct clk *clk; - struct clk_unit *unit; - struct clk_init_data init; - - unit = kzalloc(sizeof(*unit), GFP_KERNEL); - if (!unit) - return ERR_PTR(-ENOMEM); - - init.name = name; - init.parent_names = &parent_name; - init.num_parents = 1; - init.ops = &unit_clk_ops; - init.flags = flags; - - unit->hw.init = &init; - unit->regofs = regofs; - unit->bit = bit; - - unit->type = type; - unit->idle_bit = idle_bit; - unit->lock = lock; - - clk = clk_register(dev, &unit->hw); - if (IS_ERR(clk)) - kfree(unit); - - return clk; -} - -static struct atlas7_reset_desc atlas7_reset_unit[] = { - { "PWM", 0x0244, 0, 0x0320, 0, &leaf0_gate_lock }, /* 0-5 */ - { "THCGUM", 0x0244, 3, 0x0320, 1, &leaf0_gate_lock }, - { "CVD", 0x04A0, 0, 0x032C, 0, &leaf1_gate_lock }, - { "TIMER", 0x04A0, 1, 0x032C, 1, &leaf1_gate_lock }, - { "PULSEC", 0x04A0, 2, 0x032C, 2, &leaf1_gate_lock }, - { "TSC", 0x04A0, 3, 0x032C, 3, &leaf1_gate_lock }, - { "IOCTOP", 0x04A0, 4, 0x032C, 4, &leaf1_gate_lock }, /* 6-10 */ - { "RSC", 0x04A0, 5, 0x032C, 5, &leaf1_gate_lock }, - { "DVM", 0x04A0, 6, 0x032C, 6, &leaf1_gate_lock }, - { "LVDS", 0x04A0, 7, 0x032C, 7, &leaf1_gate_lock }, - { "KAS", 0x04A0, 8, 0x032C, 8, &leaf1_gate_lock }, - { "AC97", 0x04A0, 9, 0x032C, 9, &leaf1_gate_lock }, /* 11-15 */ - { "USP0", 0x04A0, 10, 0x032C, 10, &leaf1_gate_lock }, - { "USP1", 0x04A0, 11, 0x032C, 11, &leaf1_gate_lock }, - { "USP2", 0x04A0, 12, 0x032C, 12, &leaf1_gate_lock }, - { "DMAC2", 0x04A0, 13, 0x032C, 13, &leaf1_gate_lock }, - { "DMAC3", 0x04A0, 14, 0x032C, 14, &leaf1_gate_lock }, /* 16-20 */ - { "AUDIO", 0x04A0, 15, 0x032C, 15, &leaf1_gate_lock }, - { "I2S1", 0x04A0, 17, 0x032C, 16, &leaf1_gate_lock }, - { "PMU_AUDIO", 0x04A0, 22, 0x032C, 17, &leaf1_gate_lock }, - { "THAUDMSCM", 0x04A0, 23, 0x032C, 18, &leaf1_gate_lock }, - { "SYS2PCI", 0x04B8, 0, 0x0338, 0, &leaf2_gate_lock }, /* 21-25 */ - { "PCIARB", 0x04B8, 1, 0x0338, 1, &leaf2_gate_lock }, - { "PCICOPY", 0x04B8, 2, 0x0338, 2, &leaf2_gate_lock }, - { "ROM", 0x04B8, 3, 0x0338, 3, &leaf2_gate_lock }, - { "SDIO23", 0x04B8, 4, 0x0338, 4, &leaf2_gate_lock }, - { "SDIO45", 0x04B8, 5, 0x0338, 5, &leaf2_gate_lock }, /* 26-30 */ - { "SDIO67", 0x04B8, 6, 0x0338, 6, &leaf2_gate_lock }, - { "VIP1", 0x04B8, 7, 0x0338, 7, &leaf2_gate_lock }, - { "VPP0", 0x04B8, 11, 0x0338, 8, &leaf2_gate_lock }, - { "LCD0", 0x04B8, 12, 0x0338, 9, &leaf2_gate_lock }, - { "VPP1", 0x04B8, 13, 0x0338, 10, &leaf2_gate_lock }, /* 31-35 */ - { "LCD1", 0x04B8, 14, 0x0338, 11, &leaf2_gate_lock }, - { "DCU", 0x04B8, 15, 0x0338, 12, &leaf2_gate_lock }, - { "GPIO", 0x04B8, 18, 0x0338, 13, &leaf2_gate_lock }, - { "DAPA_VDIFM", 0x04B8, 17, 0x0338, 15, &leaf2_gate_lock }, - { "THVDIFM", 0x04B8, 19, 0x0338, 16, &leaf2_gate_lock }, /* 36-40 */ - { "RGMII", 0x04D0, 0, 0x0344, 0, &leaf3_gate_lock }, - { "GMAC", 0x04D0, 1, 0x0344, 1, &leaf3_gate_lock }, - { "UART1", 0x04D0, 2, 0x0344, 2, &leaf3_gate_lock }, - { "DMAC0", 0x04D0, 3, 0x0344, 3, &leaf3_gate_lock }, - { "UART0", 0x04D0, 4, 0x0344, 4, &leaf3_gate_lock }, /* 41-45 */ - { "UART2", 0x04D0, 5, 0x0344, 5, &leaf3_gate_lock }, - { "UART3", 0x04D0, 6, 0x0344, 6, &leaf3_gate_lock }, - { "UART4", 0x04D0, 7, 0x0344, 7, &leaf3_gate_lock }, - { "UART5", 0x04D0, 8, 0x0344, 8, &leaf3_gate_lock }, - { "SPI1", 0x04D0, 9, 0x0344, 9, &leaf3_gate_lock }, /* 46-50 */ - { "GNSS_SYS_M0", 0x04D0, 10, 0x0344, 10, &leaf3_gate_lock }, - { "CANBUS1", 0x04D0, 12, 0x0344, 11, &leaf3_gate_lock }, - { "CCSEC", 0x04D0, 15, 0x0344, 12, &leaf3_gate_lock }, - { "CCPUB", 0x04D0, 16, 0x0344, 13, &leaf3_gate_lock }, - { "DAPA_GNSSM", 0x04D0, 13, 0x0344, 14, &leaf3_gate_lock }, /* 51-55 */ - { "THGNSSM", 0x04D0, 14, 0x0344, 15, &leaf3_gate_lock }, - { "VDEC", 0x04E8, 0, 0x0350, 0, &leaf4_gate_lock }, - { "JPENC", 0x04E8, 1, 0x0350, 1, &leaf4_gate_lock }, - { "G2D", 0x04E8, 2, 0x0350, 2, &leaf4_gate_lock }, - { "I2C0", 0x04E8, 3, 0x0350, 3, &leaf4_gate_lock }, /* 56-60 */ - { "I2C1", 0x04E8, 4, 0x0350, 4, &leaf4_gate_lock }, - { "GPIO0", 0x04E8, 5, 0x0350, 5, &leaf4_gate_lock }, - { "NAND", 0x04E8, 6, 0x0350, 6, &leaf4_gate_lock }, - { "SDIO01", 0x04E8, 7, 0x0350, 7, &leaf4_gate_lock }, - { "SYS2PCI2", 0x04E8, 8, 0x0350, 8, &leaf4_gate_lock }, /* 61-65 */ - { "USB0", 0x04E8, 11, 0x0350, 9, &leaf4_gate_lock }, - { "USB1", 0x04E8, 12, 0x0350, 10, &leaf4_gate_lock }, - { "THMEDIAM", 0x04E8, 15, 0x0350, 11, &leaf4_gate_lock }, - { "MEMC_DDRPHY", 0x0500, 0, 0x035C, 0, &leaf5_gate_lock }, - { "MEMC_UPCTL", 0x0500, 0, 0x035C, 1, &leaf5_gate_lock }, /* 66-70 */ - { "DAPA_MEM", 0x0500, 1, 0x035C, 2, &leaf5_gate_lock }, - { "MEMC_MEMDIV", 0x0500, 0, 0x035C, 3, &leaf5_gate_lock }, - { "THDDRM", 0x0500, 3, 0x035C, 4, &leaf5_gate_lock }, - { "CORESIGHT", 0x0518, 3, 0x0368, 13, &leaf6_gate_lock }, - { "THCPUM", 0x0518, 4, 0x0368, 17, &leaf6_gate_lock }, /* 71-75 */ - { "GRAPHIC", 0x0530, 0, 0x0374, 0, &leaf7_gate_lock }, - { "VSS_SDR", 0x0530, 1, 0x0374, 1, &leaf7_gate_lock }, - { "THGPUM", 0x0530, 2, 0x0374, 2, &leaf7_gate_lock }, - { "DMAC4", 0x0548, 2, 0x0380, 1, &leaf8_gate_lock }, - { "UART6", 0x0548, 3, 0x0380, 2, &leaf8_gate_lock }, /* 76- */ - { "USP3", 0x0548, 4, 0x0380, 3, &leaf8_gate_lock }, - { "THBTM", 0x0548, 5, 0x0380, 5, &leaf8_gate_lock }, - { "A7CA", 0x0548, 1, 0x0380, 0, &leaf8_gate_lock }, - { "A7CA_APB", 0x0548, 5, 0x0380, 4, &leaf8_gate_lock }, -}; - -static int atlas7_reset_module(struct reset_controller_dev *rcdev, - unsigned long reset_idx) -{ - struct atlas7_reset_desc *reset = &atlas7_reset_unit[reset_idx]; - unsigned long flags; - - /* - * HW suggest unit reset sequence: - * assert sw reset (0) - * setting sw clk_en to if the clock was disabled before reset - * delay 16 clocks - * disable clock (sw clk_en = 0) - * de-assert reset (1) - * after this sequence, restore clock or not is decided by SW - */ - - spin_lock_irqsave(reset->lock, flags); - /* clock enable or not */ - if (clkc_readl(reset->clk_ofs + 8) & (1 << reset->clk_bit)) { - clkc_writel(1 << reset->rst_bit, reset->rst_ofs + 4); - udelay(2); - clkc_writel(1 << reset->clk_bit, reset->clk_ofs + 4); - clkc_writel(1 << reset->rst_bit, reset->rst_ofs); - /* restore clock enable */ - clkc_writel(1 << reset->clk_bit, reset->clk_ofs); - } else { - clkc_writel(1 << reset->rst_bit, reset->rst_ofs + 4); - clkc_writel(1 << reset->clk_bit, reset->clk_ofs); - udelay(2); - clkc_writel(1 << reset->clk_bit, reset->clk_ofs + 4); - clkc_writel(1 << reset->rst_bit, reset->rst_ofs); - } - spin_unlock_irqrestore(reset->lock, flags); - - return 0; -} - -static const struct reset_control_ops atlas7_rst_ops = { - .reset = atlas7_reset_module, -}; - -static struct reset_controller_dev atlas7_rst_ctlr = { - .ops = &atlas7_rst_ops, - .owner = THIS_MODULE, - .of_reset_n_cells = 1, -}; - -static void __init atlas7_clk_init(struct device_node *np) -{ - struct clk *clk; - struct atlas7_div_init_data *div; - struct atlas7_mux_init_data *mux; - struct atlas7_unit_init_data *unit; - int i; - int ret; - - sirfsoc_clk_vbase = of_iomap(np, 0); - if (!sirfsoc_clk_vbase) - panic("unable to map clkc registers\n"); - - of_node_put(np); - - clk = clk_register(NULL, &clk_cpupll.hw); - BUG_ON(!clk); - clk = clk_register(NULL, &clk_mempll.hw); - BUG_ON(!clk); - clk = clk_register(NULL, &clk_sys0pll.hw); - BUG_ON(!clk); - clk = clk_register(NULL, &clk_sys1pll.hw); - BUG_ON(!clk); - clk = clk_register(NULL, &clk_sys2pll.hw); - BUG_ON(!clk); - clk = clk_register(NULL, &clk_sys3pll.hw); - BUG_ON(!clk); - - clk = clk_register_divider_table(NULL, "cpupll_div1", "cpupll_vco", 0, - sirfsoc_clk_vbase + SIRFSOC_CLKC_CPUPLL_AB_CTRL1, 0, 3, 0, - pll_div_table, &cpupll_ctrl1_lock); - BUG_ON(!clk); - clk = clk_register_divider_table(NULL, "cpupll_div2", "cpupll_vco", 0, - sirfsoc_clk_vbase + SIRFSOC_CLKC_CPUPLL_AB_CTRL1, 4, 3, 0, - pll_div_table, &cpupll_ctrl1_lock); - BUG_ON(!clk); - clk = clk_register_divider_table(NULL, "cpupll_div3", "cpupll_vco", 0, - sirfsoc_clk_vbase + SIRFSOC_CLKC_CPUPLL_AB_CTRL1, 8, 3, 0, - pll_div_table, &cpupll_ctrl1_lock); - BUG_ON(!clk); - - clk = clk_register_divider_table(NULL, "mempll_div1", "mempll_vco", 0, - sirfsoc_clk_vbase + SIRFSOC_CLKC_MEMPLL_AB_CTRL1, 0, 3, 0, - pll_div_table, &mempll_ctrl1_lock); - BUG_ON(!clk); - clk = clk_register_divider_table(NULL, "mempll_div2", "mempll_vco", 0, - sirfsoc_clk_vbase + SIRFSOC_CLKC_MEMPLL_AB_CTRL1, 4, 3, 0, - pll_div_table, &mempll_ctrl1_lock); - BUG_ON(!clk); - clk = clk_register_divider_table(NULL, "mempll_div3", "mempll_vco", 0, - sirfsoc_clk_vbase + SIRFSOC_CLKC_MEMPLL_AB_CTRL1, 8, 3, 0, - pll_div_table, &mempll_ctrl1_lock); - BUG_ON(!clk); - - clk = clk_register_divider_table(NULL, "sys0pll_div1", "sys0pll_vco", 0, - sirfsoc_clk_vbase + SIRFSOC_CLKC_SYS0PLL_AB_CTRL1, 0, 3, 0, - pll_div_table, &sys0pll_ctrl1_lock); - BUG_ON(!clk); - clk = clk_register_divider_table(NULL, "sys0pll_div2", "sys0pll_vco", 0, - sirfsoc_clk_vbase + SIRFSOC_CLKC_SYS0PLL_AB_CTRL1, 4, 3, 0, - pll_div_table, &sys0pll_ctrl1_lock); - BUG_ON(!clk); - clk = clk_register_divider_table(NULL, "sys0pll_div3", "sys0pll_vco", 0, - sirfsoc_clk_vbase + SIRFSOC_CLKC_SYS0PLL_AB_CTRL1, 8, 3, 0, - pll_div_table, &sys0pll_ctrl1_lock); - BUG_ON(!clk); - clk = clk_register_fixed_factor(NULL, "sys0pll_fixdiv", "sys0pll_vco", - CLK_SET_RATE_PARENT, 1, 2); - - clk = clk_register_divider_table(NULL, "sys1pll_div1", "sys1pll_vco", 0, - sirfsoc_clk_vbase + SIRFSOC_CLKC_SYS1PLL_AB_CTRL1, 0, 3, 0, - pll_div_table, &sys1pll_ctrl1_lock); - BUG_ON(!clk); - clk = clk_register_divider_table(NULL, "sys1pll_div2", "sys1pll_vco", 0, - sirfsoc_clk_vbase + SIRFSOC_CLKC_SYS1PLL_AB_CTRL1, 4, 3, 0, - pll_div_table, &sys1pll_ctrl1_lock); - BUG_ON(!clk); - clk = clk_register_divider_table(NULL, "sys1pll_div3", "sys1pll_vco", 0, - sirfsoc_clk_vbase + SIRFSOC_CLKC_SYS1PLL_AB_CTRL1, 8, 3, 0, - pll_div_table, &sys1pll_ctrl1_lock); - BUG_ON(!clk); - clk = clk_register_fixed_factor(NULL, "sys1pll_fixdiv", "sys1pll_vco", - CLK_SET_RATE_PARENT, 1, 2); - - clk = clk_register_divider_table(NULL, "sys2pll_div1", "sys2pll_vco", 0, - sirfsoc_clk_vbase + SIRFSOC_CLKC_SYS2PLL_AB_CTRL1, 0, 3, 0, - pll_div_table, &sys2pll_ctrl1_lock); - BUG_ON(!clk); - clk = clk_register_divider_table(NULL, "sys2pll_div2", "sys2pll_vco", 0, - sirfsoc_clk_vbase + SIRFSOC_CLKC_SYS2PLL_AB_CTRL1, 4, 3, 0, - pll_div_table, &sys2pll_ctrl1_lock); - BUG_ON(!clk); - clk = clk_register_divider_table(NULL, "sys2pll_div3", "sys2pll_vco", 0, - sirfsoc_clk_vbase + SIRFSOC_CLKC_SYS2PLL_AB_CTRL1, 8, 3, 0, - pll_div_table, &sys2pll_ctrl1_lock); - BUG_ON(!clk); - clk = clk_register_fixed_factor(NULL, "sys2pll_fixdiv", "sys2pll_vco", - CLK_SET_RATE_PARENT, 1, 2); - - clk = clk_register_divider_table(NULL, "sys3pll_div1", "sys3pll_vco", 0, - sirfsoc_clk_vbase + SIRFSOC_CLKC_SYS3PLL_AB_CTRL1, 0, 3, 0, - pll_div_table, &sys3pll_ctrl1_lock); - BUG_ON(!clk); - clk = clk_register_divider_table(NULL, "sys3pll_div2", "sys3pll_vco", 0, - sirfsoc_clk_vbase + SIRFSOC_CLKC_SYS3PLL_AB_CTRL1, 4, 3, 0, - pll_div_table, &sys3pll_ctrl1_lock); - BUG_ON(!clk); - clk = clk_register_divider_table(NULL, "sys3pll_div3", "sys3pll_vco", 0, - sirfsoc_clk_vbase + SIRFSOC_CLKC_SYS3PLL_AB_CTRL1, 8, 3, 0, - pll_div_table, &sys3pll_ctrl1_lock); - BUG_ON(!clk); - clk = clk_register_fixed_factor(NULL, "sys3pll_fixdiv", "sys3pll_vco", - CLK_SET_RATE_PARENT, 1, 2); - - BUG_ON(!clk); - clk = clk_register_fixed_factor(NULL, "xinw_fixdiv_btslow", "xinw", - CLK_SET_RATE_PARENT, 1, 4); - - BUG_ON(!clk); - clk = clk_register_gate(NULL, "cpupll_clk1", "cpupll_div1", - CLK_SET_RATE_PARENT, sirfsoc_clk_vbase + SIRFSOC_CLKC_CPUPLL_AB_CTRL1, - 12, 0, &cpupll_ctrl1_lock); - BUG_ON(!clk); - clk = clk_register_gate(NULL, "cpupll_clk2", "cpupll_div2", - CLK_SET_RATE_PARENT, sirfsoc_clk_vbase + SIRFSOC_CLKC_CPUPLL_AB_CTRL1, - 13, 0, &cpupll_ctrl1_lock); - BUG_ON(!clk); - clk = clk_register_gate(NULL, "cpupll_clk3", "cpupll_div3", - CLK_SET_RATE_PARENT, sirfsoc_clk_vbase + SIRFSOC_CLKC_CPUPLL_AB_CTRL1, - 14, 0, &cpupll_ctrl1_lock); - BUG_ON(!clk); - - clk = clk_register_gate(NULL, "mempll_clk1", "mempll_div1", - CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED, - sirfsoc_clk_vbase + SIRFSOC_CLKC_MEMPLL_AB_CTRL1, - 12, 0, &mempll_ctrl1_lock); - BUG_ON(!clk); - clk = clk_register_gate(NULL, "mempll_clk2", "mempll_div2", - CLK_SET_RATE_PARENT, sirfsoc_clk_vbase + SIRFSOC_CLKC_MEMPLL_AB_CTRL1, - 13, 0, &mempll_ctrl1_lock); - BUG_ON(!clk); - clk = clk_register_gate(NULL, "mempll_clk3", "mempll_div3", - CLK_SET_RATE_PARENT, sirfsoc_clk_vbase + SIRFSOC_CLKC_MEMPLL_AB_CTRL1, - 14, 0, &mempll_ctrl1_lock); - BUG_ON(!clk); - - clk = clk_register_gate(NULL, "sys0pll_clk1", "sys0pll_div1", - CLK_SET_RATE_PARENT, sirfsoc_clk_vbase + SIRFSOC_CLKC_SYS0PLL_AB_CTRL1, - 12, 0, &sys0pll_ctrl1_lock); - BUG_ON(!clk); - clk = clk_register_gate(NULL, "sys0pll_clk2", "sys0pll_div2", - CLK_SET_RATE_PARENT, sirfsoc_clk_vbase + SIRFSOC_CLKC_SYS0PLL_AB_CTRL1, - 13, 0, &sys0pll_ctrl1_lock); - BUG_ON(!clk); - clk = clk_register_gate(NULL, "sys0pll_clk3", "sys0pll_div3", - CLK_SET_RATE_PARENT, sirfsoc_clk_vbase + SIRFSOC_CLKC_SYS0PLL_AB_CTRL1, - 14, 0, &sys0pll_ctrl1_lock); - BUG_ON(!clk); - - clk = clk_register_gate(NULL, "sys1pll_clk1", "sys1pll_div1", - CLK_SET_RATE_PARENT, sirfsoc_clk_vbase + SIRFSOC_CLKC_SYS1PLL_AB_CTRL1, - 12, 0, &sys1pll_ctrl1_lock); - BUG_ON(!clk); - clk = clk_register_gate(NULL, "sys1pll_clk2", "sys1pll_div2", - CLK_SET_RATE_PARENT, sirfsoc_clk_vbase + SIRFSOC_CLKC_SYS1PLL_AB_CTRL1, - 13, 0, &sys1pll_ctrl1_lock); - BUG_ON(!clk); - clk = clk_register_gate(NULL, "sys1pll_clk3", "sys1pll_div3", - CLK_SET_RATE_PARENT, sirfsoc_clk_vbase + SIRFSOC_CLKC_SYS1PLL_AB_CTRL1, - 14, 0, &sys1pll_ctrl1_lock); - BUG_ON(!clk); - - clk = clk_register_gate(NULL, "sys2pll_clk1", "sys2pll_div1", - CLK_SET_RATE_PARENT, sirfsoc_clk_vbase + SIRFSOC_CLKC_SYS2PLL_AB_CTRL1, - 12, 0, &sys2pll_ctrl1_lock); - BUG_ON(!clk); - clk = clk_register_gate(NULL, "sys2pll_clk2", "sys2pll_div2", - CLK_SET_RATE_PARENT, sirfsoc_clk_vbase + SIRFSOC_CLKC_SYS2PLL_AB_CTRL1, - 13, 0, &sys2pll_ctrl1_lock); - BUG_ON(!clk); - clk = clk_register_gate(NULL, "sys2pll_clk3", "sys2pll_div3", - CLK_SET_RATE_PARENT, sirfsoc_clk_vbase + SIRFSOC_CLKC_SYS2PLL_AB_CTRL1, - 14, 0, &sys2pll_ctrl1_lock); - BUG_ON(!clk); - - clk = clk_register_gate(NULL, "sys3pll_clk1", "sys3pll_div1", - CLK_SET_RATE_PARENT, sirfsoc_clk_vbase + SIRFSOC_CLKC_SYS3PLL_AB_CTRL1, - 12, 0, &sys3pll_ctrl1_lock); - BUG_ON(!clk); - clk = clk_register_gate(NULL, "sys3pll_clk2", "sys3pll_div2", - CLK_SET_RATE_PARENT, sirfsoc_clk_vbase + SIRFSOC_CLKC_SYS3PLL_AB_CTRL1, - 13, 0, &sys3pll_ctrl1_lock); - BUG_ON(!clk); - clk = clk_register_gate(NULL, "sys3pll_clk3", "sys3pll_div3", - CLK_SET_RATE_PARENT, sirfsoc_clk_vbase + SIRFSOC_CLKC_SYS3PLL_AB_CTRL1, - 14, 0, &sys3pll_ctrl1_lock); - BUG_ON(!clk); - - clk = clk_register(NULL, &clk_audio_dto.hw); - BUG_ON(!clk); - - clk = clk_register(NULL, &clk_disp0_dto.hw); - BUG_ON(!clk); - - clk = clk_register(NULL, &clk_disp1_dto.hw); - BUG_ON(!clk); - - for (i = 0; i < ARRAY_SIZE(divider_list); i++) { - div = ÷r_list[i]; - clk = clk_register_divider(NULL, div->div_name, - div->parent_name, div->divider_flags, sirfsoc_clk_vbase + div->div_offset, - div->shift, div->width, 0, div->lock); - BUG_ON(!clk); - clk = clk_register_gate(NULL, div->gate_name, div->div_name, - div->gate_flags, sirfsoc_clk_vbase + div->gate_offset, - div->gate_bit, 0, div->lock); - BUG_ON(!clk); - } - /* ignore selector status register check */ - for (i = 0; i < ARRAY_SIZE(mux_list); i++) { - mux = &mux_list[i]; - clk = clk_register_mux(NULL, mux->mux_name, mux->parent_names, - mux->parent_num, mux->flags, - sirfsoc_clk_vbase + mux->mux_offset, - mux->shift, mux->width, - mux->mux_flags, NULL); - atlas7_clks[ARRAY_SIZE(unit_list) + i] = clk; - BUG_ON(!clk); - } - - for (i = 0; i < ARRAY_SIZE(unit_list); i++) { - unit = &unit_list[i]; - atlas7_clks[i] = atlas7_unit_clk_register(NULL, unit->unit_name, unit->parent_name, - unit->flags, unit->regofs, unit->bit, unit->type, unit->idle_bit, unit->lock); - BUG_ON(!atlas7_clks[i]); - } - - clk_data.clks = atlas7_clks; - clk_data.clk_num = ARRAY_SIZE(unit_list) + ARRAY_SIZE(mux_list); - - ret = of_clk_add_provider(np, of_clk_src_onecell_get, &clk_data); - BUG_ON(ret); - - atlas7_rst_ctlr.of_node = np; - atlas7_rst_ctlr.nr_resets = ARRAY_SIZE(atlas7_reset_unit); - reset_controller_register(&atlas7_rst_ctlr); -} -CLK_OF_DECLARE(atlas7_clk, "sirf,atlas7-car", atlas7_clk_init); diff --git a/drivers/clk/sirf/clk-common.c b/drivers/clk/sirf/clk-common.c deleted file mode 100644 index dcf4e25a02168501d71811f82d255bfc303e187c..0000000000000000000000000000000000000000 --- a/drivers/clk/sirf/clk-common.c +++ /dev/null @@ -1,1037 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * common clks module for all SiRF SoCs - * - * Copyright (c) 2011 - 2014 Cambridge Silicon Radio Limited, a CSR plc group - * company. - */ - -#include - -#define KHZ 1000 -#define MHZ (KHZ * KHZ) - -static void __iomem *sirfsoc_clk_vbase; -static void __iomem *sirfsoc_rsc_vbase; -static struct clk_onecell_data clk_data; - -/* - * SiRFprimaII clock controller - * - 2 oscillators: osc-26MHz, rtc-32.768KHz - * - 3 standard configurable plls: pll1, pll2 & pll3 - * - 2 exclusive plls: usb phy pll and sata phy pll - * - 8 clock domains: cpu/cpudiv, mem/memdiv, sys/io, dsp, graphic, multimedia, - * display and sdphy. - * Each clock domain can select its own clock source from five clock sources, - * X_XIN, X_XINW, PLL1, PLL2 and PLL3. The domain clock is used as the source - * clock of the group clock. - * - dsp domain: gps, mf - * - io domain: dmac, nand, audio, uart, i2c, spi, usp, pwm, pulse - * - sys domain: security - */ - -struct clk_pll { - struct clk_hw hw; - unsigned short regofs; /* register offset */ -}; - -#define to_pllclk(_hw) container_of(_hw, struct clk_pll, hw) - -struct clk_dmn { - struct clk_hw hw; - signed char enable_bit; /* enable bit: 0 ~ 63 */ - unsigned short regofs; /* register offset */ -}; - -#define to_dmnclk(_hw) container_of(_hw, struct clk_dmn, hw) - -struct clk_std { - struct clk_hw hw; - signed char enable_bit; /* enable bit: 0 ~ 63 */ -}; - -#define to_stdclk(_hw) container_of(_hw, struct clk_std, hw) - -static int std_clk_is_enabled(struct clk_hw *hw); -static int std_clk_enable(struct clk_hw *hw); -static void std_clk_disable(struct clk_hw *hw); - -static inline unsigned long clkc_readl(unsigned reg) -{ - return readl(sirfsoc_clk_vbase + reg); -} - -static inline void clkc_writel(u32 val, unsigned reg) -{ - writel(val, sirfsoc_clk_vbase + reg); -} - -/* - * std pll - */ - -static unsigned long pll_clk_recalc_rate(struct clk_hw *hw, - unsigned long parent_rate) -{ - unsigned long fin = parent_rate; - struct clk_pll *clk = to_pllclk(hw); - u32 regcfg2 = clk->regofs + SIRFSOC_CLKC_PLL1_CFG2 - - SIRFSOC_CLKC_PLL1_CFG0; - - if (clkc_readl(regcfg2) & BIT(2)) { - /* pll bypass mode */ - return fin; - } else { - /* fout = fin * nf / nr / od */ - u32 cfg0 = clkc_readl(clk->regofs); - u32 nf = (cfg0 & (BIT(13) - 1)) + 1; - u32 nr = ((cfg0 >> 13) & (BIT(6) - 1)) + 1; - u32 od = ((cfg0 >> 19) & (BIT(4) - 1)) + 1; - WARN_ON(fin % MHZ); - return fin / MHZ * nf / nr / od * MHZ; - } -} - -static long pll_clk_round_rate(struct clk_hw *hw, unsigned long rate, - unsigned long *parent_rate) -{ - unsigned long fin, nf, nr, od; - u64 dividend; - - /* - * fout = fin * nf / (nr * od); - * set od = 1, nr = fin/MHz, so fout = nf * MHz - */ - rate = rate - rate % MHZ; - - nf = rate / MHZ; - if (nf > BIT(13)) - nf = BIT(13); - if (nf < 1) - nf = 1; - - fin = *parent_rate; - - nr = fin / MHZ; - if (nr > BIT(6)) - nr = BIT(6); - od = 1; - - dividend = (u64)fin * nf; - do_div(dividend, nr * od); - - return (long)dividend; -} - -static int pll_clk_set_rate(struct clk_hw *hw, unsigned long rate, - unsigned long parent_rate) -{ - struct clk_pll *clk = to_pllclk(hw); - unsigned long fin, nf, nr, od, reg; - - /* - * fout = fin * nf / (nr * od); - * set od = 1, nr = fin/MHz, so fout = nf * MHz - */ - - nf = rate / MHZ; - if (unlikely((rate % MHZ) || nf > BIT(13) || nf < 1)) - return -EINVAL; - - fin = parent_rate; - BUG_ON(fin < MHZ); - - nr = fin / MHZ; - BUG_ON((fin % MHZ) || nr > BIT(6)); - - od = 1; - - reg = (nf - 1) | ((nr - 1) << 13) | ((od - 1) << 19); - clkc_writel(reg, clk->regofs); - - reg = clk->regofs + SIRFSOC_CLKC_PLL1_CFG1 - SIRFSOC_CLKC_PLL1_CFG0; - clkc_writel((nf >> 1) - 1, reg); - - reg = clk->regofs + SIRFSOC_CLKC_PLL1_CFG2 - SIRFSOC_CLKC_PLL1_CFG0; - while (!(clkc_readl(reg) & BIT(6))) - cpu_relax(); - - return 0; -} - -static long cpu_clk_round_rate(struct clk_hw *hw, unsigned long rate, - unsigned long *parent_rate) -{ - /* - * SiRF SoC has not cpu clock control, - * So bypass to it's parent pll. - */ - struct clk_hw *parent_clk = clk_hw_get_parent(hw); - struct clk_hw *pll_parent_clk = clk_hw_get_parent(parent_clk); - unsigned long pll_parent_rate = clk_hw_get_rate(pll_parent_clk); - return pll_clk_round_rate(parent_clk, rate, &pll_parent_rate); -} - -static unsigned long cpu_clk_recalc_rate(struct clk_hw *hw, - unsigned long parent_rate) -{ - /* - * SiRF SoC has not cpu clock control, - * So return the parent pll rate. - */ - struct clk_hw *parent_clk = clk_hw_get_parent(hw); - return clk_hw_get_rate(parent_clk); -} - -static const struct clk_ops std_pll_ops = { - .recalc_rate = pll_clk_recalc_rate, - .round_rate = pll_clk_round_rate, - .set_rate = pll_clk_set_rate, -}; - -static const char * const pll_clk_parents[] = { - "osc", -}; - -static const struct clk_init_data clk_pll1_init = { - .name = "pll1", - .ops = &std_pll_ops, - .parent_names = pll_clk_parents, - .num_parents = ARRAY_SIZE(pll_clk_parents), -}; - -static const struct clk_init_data clk_pll2_init = { - .name = "pll2", - .ops = &std_pll_ops, - .parent_names = pll_clk_parents, - .num_parents = ARRAY_SIZE(pll_clk_parents), -}; - -static const struct clk_init_data clk_pll3_init = { - .name = "pll3", - .ops = &std_pll_ops, - .parent_names = pll_clk_parents, - .num_parents = ARRAY_SIZE(pll_clk_parents), -}; - -static struct clk_pll clk_pll1 = { - .regofs = SIRFSOC_CLKC_PLL1_CFG0, - .hw = { - .init = &clk_pll1_init, - }, -}; - -static struct clk_pll clk_pll2 = { - .regofs = SIRFSOC_CLKC_PLL2_CFG0, - .hw = { - .init = &clk_pll2_init, - }, -}; - -static struct clk_pll clk_pll3 = { - .regofs = SIRFSOC_CLKC_PLL3_CFG0, - .hw = { - .init = &clk_pll3_init, - }, -}; - -/* - * usb uses specified pll - */ - -static int usb_pll_clk_enable(struct clk_hw *hw) -{ - u32 reg = readl(sirfsoc_rsc_vbase + SIRFSOC_USBPHY_PLL_CTRL); - reg &= ~(SIRFSOC_USBPHY_PLL_POWERDOWN | SIRFSOC_USBPHY_PLL_BYPASS); - writel(reg, sirfsoc_rsc_vbase + SIRFSOC_USBPHY_PLL_CTRL); - while (!(readl(sirfsoc_rsc_vbase + SIRFSOC_USBPHY_PLL_CTRL) & - SIRFSOC_USBPHY_PLL_LOCK)) - cpu_relax(); - - return 0; -} - -static void usb_pll_clk_disable(struct clk_hw *clk) -{ - u32 reg = readl(sirfsoc_rsc_vbase + SIRFSOC_USBPHY_PLL_CTRL); - reg |= (SIRFSOC_USBPHY_PLL_POWERDOWN | SIRFSOC_USBPHY_PLL_BYPASS); - writel(reg, sirfsoc_rsc_vbase + SIRFSOC_USBPHY_PLL_CTRL); -} - -static unsigned long usb_pll_clk_recalc_rate(struct clk_hw *hw, unsigned long parent_rate) -{ - u32 reg = readl(sirfsoc_rsc_vbase + SIRFSOC_USBPHY_PLL_CTRL); - return (reg & SIRFSOC_USBPHY_PLL_BYPASS) ? parent_rate : 48*MHZ; -} - -static const struct clk_ops usb_pll_ops = { - .enable = usb_pll_clk_enable, - .disable = usb_pll_clk_disable, - .recalc_rate = usb_pll_clk_recalc_rate, -}; - -static const struct clk_init_data clk_usb_pll_init = { - .name = "usb_pll", - .ops = &usb_pll_ops, - .parent_names = pll_clk_parents, - .num_parents = ARRAY_SIZE(pll_clk_parents), -}; - -static struct clk_hw usb_pll_clk_hw = { - .init = &clk_usb_pll_init, -}; - -/* - * clock domains - cpu, mem, sys/io, dsp, gfx - */ - -static const char * const dmn_clk_parents[] = { - "rtc", - "osc", - "pll1", - "pll2", - "pll3", -}; - -static u8 dmn_clk_get_parent(struct clk_hw *hw) -{ - struct clk_dmn *clk = to_dmnclk(hw); - u32 cfg = clkc_readl(clk->regofs); - const char *name = clk_hw_get_name(hw); - - /* parent of io domain can only be pll3 */ - if (strcmp(name, "io") == 0) - return 4; - - WARN_ON((cfg & (BIT(3) - 1)) > 4); - - return cfg & (BIT(3) - 1); -} - -static int dmn_clk_set_parent(struct clk_hw *hw, u8 parent) -{ - struct clk_dmn *clk = to_dmnclk(hw); - u32 cfg = clkc_readl(clk->regofs); - const char *name = clk_hw_get_name(hw); - - /* parent of io domain can only be pll3 */ - if (strcmp(name, "io") == 0) - return -EINVAL; - - cfg &= ~(BIT(3) - 1); - clkc_writel(cfg | parent, clk->regofs); - /* BIT(3) - switching status: 1 - busy, 0 - done */ - while (clkc_readl(clk->regofs) & BIT(3)) - cpu_relax(); - - return 0; -} - -static unsigned long dmn_clk_recalc_rate(struct clk_hw *hw, - unsigned long parent_rate) - -{ - unsigned long fin = parent_rate; - struct clk_dmn *clk = to_dmnclk(hw); - - u32 cfg = clkc_readl(clk->regofs); - - if (cfg & BIT(24)) { - /* fcd bypass mode */ - return fin; - } else { - /* - * wait count: bit[19:16], hold count: bit[23:20] - */ - u32 wait = (cfg >> 16) & (BIT(4) - 1); - u32 hold = (cfg >> 20) & (BIT(4) - 1); - - return fin / (wait + hold + 2); - } -} - -static long dmn_clk_round_rate(struct clk_hw *hw, unsigned long rate, - unsigned long *parent_rate) -{ - unsigned long fin; - unsigned ratio, wait, hold; - const char *name = clk_hw_get_name(hw); - unsigned bits = (strcmp(name, "mem") == 0) ? 3 : 4; - - fin = *parent_rate; - ratio = fin / rate; - - if (ratio < 2) - ratio = 2; - if (ratio > BIT(bits + 1)) - ratio = BIT(bits + 1); - - wait = (ratio >> 1) - 1; - hold = ratio - wait - 2; - - return fin / (wait + hold + 2); -} - -static int dmn_clk_set_rate(struct clk_hw *hw, unsigned long rate, - unsigned long parent_rate) -{ - struct clk_dmn *clk = to_dmnclk(hw); - unsigned long fin; - unsigned ratio, wait, hold, reg; - const char *name = clk_hw_get_name(hw); - unsigned bits = (strcmp(name, "mem") == 0) ? 3 : 4; - - fin = parent_rate; - ratio = fin / rate; - - if (unlikely(ratio < 2 || ratio > BIT(bits + 1))) - return -EINVAL; - - WARN_ON(fin % rate); - - wait = (ratio >> 1) - 1; - hold = ratio - wait - 2; - - reg = clkc_readl(clk->regofs); - reg &= ~(((BIT(bits) - 1) << 16) | ((BIT(bits) - 1) << 20)); - reg |= (wait << 16) | (hold << 20) | BIT(25); - clkc_writel(reg, clk->regofs); - - /* waiting FCD been effective */ - while (clkc_readl(clk->regofs) & BIT(25)) - cpu_relax(); - - return 0; -} - -static int cpu_clk_set_rate(struct clk_hw *hw, unsigned long rate, - unsigned long parent_rate) -{ - int ret1, ret2; - struct clk *cur_parent; - - if (rate == clk_get_rate(clk_pll1.hw.clk)) { - ret1 = clk_set_parent(hw->clk, clk_pll1.hw.clk); - return ret1; - } - - if (rate == clk_get_rate(clk_pll2.hw.clk)) { - ret1 = clk_set_parent(hw->clk, clk_pll2.hw.clk); - return ret1; - } - - if (rate == clk_get_rate(clk_pll3.hw.clk)) { - ret1 = clk_set_parent(hw->clk, clk_pll3.hw.clk); - return ret1; - } - - cur_parent = clk_get_parent(hw->clk); - - /* switch to tmp pll before setting parent clock's rate */ - if (cur_parent == clk_pll1.hw.clk) { - ret1 = clk_set_parent(hw->clk, clk_pll2.hw.clk); - BUG_ON(ret1); - } - - ret2 = clk_set_rate(clk_pll1.hw.clk, rate); - - ret1 = clk_set_parent(hw->clk, clk_pll1.hw.clk); - - return ret2 ? ret2 : ret1; -} - -static const struct clk_ops msi_ops = { - .set_rate = dmn_clk_set_rate, - .round_rate = dmn_clk_round_rate, - .recalc_rate = dmn_clk_recalc_rate, - .set_parent = dmn_clk_set_parent, - .get_parent = dmn_clk_get_parent, -}; - -static const struct clk_init_data clk_mem_init = { - .name = "mem", - .ops = &msi_ops, - .parent_names = dmn_clk_parents, - .num_parents = ARRAY_SIZE(dmn_clk_parents), -}; - -static struct clk_dmn clk_mem = { - .regofs = SIRFSOC_CLKC_MEM_CFG, - .hw = { - .init = &clk_mem_init, - }, -}; - -static const struct clk_init_data clk_sys_init = { - .name = "sys", - .ops = &msi_ops, - .parent_names = dmn_clk_parents, - .num_parents = ARRAY_SIZE(dmn_clk_parents), - .flags = CLK_SET_RATE_GATE, -}; - -static struct clk_dmn clk_sys = { - .regofs = SIRFSOC_CLKC_SYS_CFG, - .hw = { - .init = &clk_sys_init, - }, -}; - -static const struct clk_init_data clk_io_init = { - .name = "io", - .ops = &msi_ops, - .parent_names = dmn_clk_parents, - .num_parents = ARRAY_SIZE(dmn_clk_parents), -}; - -static struct clk_dmn clk_io = { - .regofs = SIRFSOC_CLKC_IO_CFG, - .hw = { - .init = &clk_io_init, - }, -}; - -static const struct clk_ops cpu_ops = { - .set_parent = dmn_clk_set_parent, - .get_parent = dmn_clk_get_parent, - .set_rate = cpu_clk_set_rate, - .round_rate = cpu_clk_round_rate, - .recalc_rate = cpu_clk_recalc_rate, -}; - -static const struct clk_init_data clk_cpu_init = { - .name = "cpu", - .ops = &cpu_ops, - .parent_names = dmn_clk_parents, - .num_parents = ARRAY_SIZE(dmn_clk_parents), - .flags = CLK_SET_RATE_PARENT, -}; - -static struct clk_dmn clk_cpu = { - .regofs = SIRFSOC_CLKC_CPU_CFG, - .hw = { - .init = &clk_cpu_init, - }, -}; - -static const struct clk_ops dmn_ops = { - .is_enabled = std_clk_is_enabled, - .enable = std_clk_enable, - .disable = std_clk_disable, - .set_rate = dmn_clk_set_rate, - .round_rate = dmn_clk_round_rate, - .recalc_rate = dmn_clk_recalc_rate, - .set_parent = dmn_clk_set_parent, - .get_parent = dmn_clk_get_parent, -}; - -/* dsp, gfx, mm, lcd and vpp domain */ - -static const struct clk_init_data clk_dsp_init = { - .name = "dsp", - .ops = &dmn_ops, - .parent_names = dmn_clk_parents, - .num_parents = ARRAY_SIZE(dmn_clk_parents), -}; - -static struct clk_dmn clk_dsp = { - .regofs = SIRFSOC_CLKC_DSP_CFG, - .enable_bit = 0, - .hw = { - .init = &clk_dsp_init, - }, -}; - -static const struct clk_init_data clk_gfx_init = { - .name = "gfx", - .ops = &dmn_ops, - .parent_names = dmn_clk_parents, - .num_parents = ARRAY_SIZE(dmn_clk_parents), -}; - -static struct clk_dmn clk_gfx = { - .regofs = SIRFSOC_CLKC_GFX_CFG, - .enable_bit = 8, - .hw = { - .init = &clk_gfx_init, - }, -}; - -static const struct clk_init_data clk_mm_init = { - .name = "mm", - .ops = &dmn_ops, - .parent_names = dmn_clk_parents, - .num_parents = ARRAY_SIZE(dmn_clk_parents), -}; - -static struct clk_dmn clk_mm = { - .regofs = SIRFSOC_CLKC_MM_CFG, - .enable_bit = 9, - .hw = { - .init = &clk_mm_init, - }, -}; - -/* - * for atlas6, gfx2d holds the bit of prima2's clk_mm - */ -#define clk_gfx2d clk_mm - -static const struct clk_init_data clk_lcd_init = { - .name = "lcd", - .ops = &dmn_ops, - .parent_names = dmn_clk_parents, - .num_parents = ARRAY_SIZE(dmn_clk_parents), -}; - -static struct clk_dmn clk_lcd = { - .regofs = SIRFSOC_CLKC_LCD_CFG, - .enable_bit = 10, - .hw = { - .init = &clk_lcd_init, - }, -}; - -static const struct clk_init_data clk_vpp_init = { - .name = "vpp", - .ops = &dmn_ops, - .parent_names = dmn_clk_parents, - .num_parents = ARRAY_SIZE(dmn_clk_parents), -}; - -static struct clk_dmn clk_vpp = { - .regofs = SIRFSOC_CLKC_LCD_CFG, - .enable_bit = 11, - .hw = { - .init = &clk_vpp_init, - }, -}; - -static const struct clk_init_data clk_mmc01_init = { - .name = "mmc01", - .ops = &dmn_ops, - .parent_names = dmn_clk_parents, - .num_parents = ARRAY_SIZE(dmn_clk_parents), -}; - -static const struct clk_init_data clk_mmc23_init = { - .name = "mmc23", - .ops = &dmn_ops, - .parent_names = dmn_clk_parents, - .num_parents = ARRAY_SIZE(dmn_clk_parents), -}; - -static const struct clk_init_data clk_mmc45_init = { - .name = "mmc45", - .ops = &dmn_ops, - .parent_names = dmn_clk_parents, - .num_parents = ARRAY_SIZE(dmn_clk_parents), -}; - -/* - * peripheral controllers in io domain - */ - -static int std_clk_is_enabled(struct clk_hw *hw) -{ - u32 reg; - int bit; - struct clk_std *clk = to_stdclk(hw); - - bit = clk->enable_bit % 32; - reg = clk->enable_bit / 32; - reg = SIRFSOC_CLKC_CLK_EN0 + reg * sizeof(reg); - - return !!(clkc_readl(reg) & BIT(bit)); -} - -static int std_clk_enable(struct clk_hw *hw) -{ - u32 val, reg; - int bit; - struct clk_std *clk = to_stdclk(hw); - - BUG_ON(clk->enable_bit < 0 || clk->enable_bit > 63); - - bit = clk->enable_bit % 32; - reg = clk->enable_bit / 32; - reg = SIRFSOC_CLKC_CLK_EN0 + reg * sizeof(reg); - - val = clkc_readl(reg) | BIT(bit); - clkc_writel(val, reg); - return 0; -} - -static void std_clk_disable(struct clk_hw *hw) -{ - u32 val, reg; - int bit; - struct clk_std *clk = to_stdclk(hw); - - BUG_ON(clk->enable_bit < 0 || clk->enable_bit > 63); - - bit = clk->enable_bit % 32; - reg = clk->enable_bit / 32; - reg = SIRFSOC_CLKC_CLK_EN0 + reg * sizeof(reg); - - val = clkc_readl(reg) & ~BIT(bit); - clkc_writel(val, reg); -} - -static const char * const std_clk_io_parents[] = { - "io", -}; - -static const struct clk_ops ios_ops = { - .is_enabled = std_clk_is_enabled, - .enable = std_clk_enable, - .disable = std_clk_disable, -}; - -static const struct clk_init_data clk_cphif_init = { - .name = "cphif", - .ops = &ios_ops, - .parent_names = std_clk_io_parents, - .num_parents = ARRAY_SIZE(std_clk_io_parents), -}; - -static struct clk_std clk_cphif = { - .enable_bit = 20, - .hw = { - .init = &clk_cphif_init, - }, -}; - -static const struct clk_init_data clk_dmac0_init = { - .name = "dmac0", - .ops = &ios_ops, - .parent_names = std_clk_io_parents, - .num_parents = ARRAY_SIZE(std_clk_io_parents), -}; - -static struct clk_std clk_dmac0 = { - .enable_bit = 32, - .hw = { - .init = &clk_dmac0_init, - }, -}; - -static const struct clk_init_data clk_dmac1_init = { - .name = "dmac1", - .ops = &ios_ops, - .parent_names = std_clk_io_parents, - .num_parents = ARRAY_SIZE(std_clk_io_parents), -}; - -static struct clk_std clk_dmac1 = { - .enable_bit = 33, - .hw = { - .init = &clk_dmac1_init, - }, -}; - -static const struct clk_init_data clk_audio_init = { - .name = "audio", - .ops = &ios_ops, - .parent_names = std_clk_io_parents, - .num_parents = ARRAY_SIZE(std_clk_io_parents), -}; - -static struct clk_std clk_audio = { - .enable_bit = 35, - .hw = { - .init = &clk_audio_init, - }, -}; - -static const struct clk_init_data clk_uart0_init = { - .name = "uart0", - .ops = &ios_ops, - .parent_names = std_clk_io_parents, - .num_parents = ARRAY_SIZE(std_clk_io_parents), -}; - -static struct clk_std clk_uart0 = { - .enable_bit = 36, - .hw = { - .init = &clk_uart0_init, - }, -}; - -static const struct clk_init_data clk_uart1_init = { - .name = "uart1", - .ops = &ios_ops, - .parent_names = std_clk_io_parents, - .num_parents = ARRAY_SIZE(std_clk_io_parents), -}; - -static struct clk_std clk_uart1 = { - .enable_bit = 37, - .hw = { - .init = &clk_uart1_init, - }, -}; - -static const struct clk_init_data clk_uart2_init = { - .name = "uart2", - .ops = &ios_ops, - .parent_names = std_clk_io_parents, - .num_parents = ARRAY_SIZE(std_clk_io_parents), -}; - -static struct clk_std clk_uart2 = { - .enable_bit = 38, - .hw = { - .init = &clk_uart2_init, - }, -}; - -static const struct clk_init_data clk_usp0_init = { - .name = "usp0", - .ops = &ios_ops, - .parent_names = std_clk_io_parents, - .num_parents = ARRAY_SIZE(std_clk_io_parents), -}; - -static struct clk_std clk_usp0 = { - .enable_bit = 39, - .hw = { - .init = &clk_usp0_init, - }, -}; - -static const struct clk_init_data clk_usp1_init = { - .name = "usp1", - .ops = &ios_ops, - .parent_names = std_clk_io_parents, - .num_parents = ARRAY_SIZE(std_clk_io_parents), -}; - -static struct clk_std clk_usp1 = { - .enable_bit = 40, - .hw = { - .init = &clk_usp1_init, - }, -}; - -static const struct clk_init_data clk_usp2_init = { - .name = "usp2", - .ops = &ios_ops, - .parent_names = std_clk_io_parents, - .num_parents = ARRAY_SIZE(std_clk_io_parents), -}; - -static struct clk_std clk_usp2 = { - .enable_bit = 41, - .hw = { - .init = &clk_usp2_init, - }, -}; - -static const struct clk_init_data clk_vip_init = { - .name = "vip", - .ops = &ios_ops, - .parent_names = std_clk_io_parents, - .num_parents = ARRAY_SIZE(std_clk_io_parents), -}; - -static struct clk_std clk_vip = { - .enable_bit = 42, - .hw = { - .init = &clk_vip_init, - }, -}; - -static const struct clk_init_data clk_spi0_init = { - .name = "spi0", - .ops = &ios_ops, - .parent_names = std_clk_io_parents, - .num_parents = ARRAY_SIZE(std_clk_io_parents), -}; - -static struct clk_std clk_spi0 = { - .enable_bit = 43, - .hw = { - .init = &clk_spi0_init, - }, -}; - -static const struct clk_init_data clk_spi1_init = { - .name = "spi1", - .ops = &ios_ops, - .parent_names = std_clk_io_parents, - .num_parents = ARRAY_SIZE(std_clk_io_parents), -}; - -static struct clk_std clk_spi1 = { - .enable_bit = 44, - .hw = { - .init = &clk_spi1_init, - }, -}; - -static const struct clk_init_data clk_tsc_init = { - .name = "tsc", - .ops = &ios_ops, - .parent_names = std_clk_io_parents, - .num_parents = ARRAY_SIZE(std_clk_io_parents), -}; - -static struct clk_std clk_tsc = { - .enable_bit = 45, - .hw = { - .init = &clk_tsc_init, - }, -}; - -static const struct clk_init_data clk_i2c0_init = { - .name = "i2c0", - .ops = &ios_ops, - .parent_names = std_clk_io_parents, - .num_parents = ARRAY_SIZE(std_clk_io_parents), -}; - -static struct clk_std clk_i2c0 = { - .enable_bit = 46, - .hw = { - .init = &clk_i2c0_init, - }, -}; - -static const struct clk_init_data clk_i2c1_init = { - .name = "i2c1", - .ops = &ios_ops, - .parent_names = std_clk_io_parents, - .num_parents = ARRAY_SIZE(std_clk_io_parents), -}; - -static struct clk_std clk_i2c1 = { - .enable_bit = 47, - .hw = { - .init = &clk_i2c1_init, - }, -}; - -static const struct clk_init_data clk_pwmc_init = { - .name = "pwmc", - .ops = &ios_ops, - .parent_names = std_clk_io_parents, - .num_parents = ARRAY_SIZE(std_clk_io_parents), -}; - -static struct clk_std clk_pwmc = { - .enable_bit = 48, - .hw = { - .init = &clk_pwmc_init, - }, -}; - -static const struct clk_init_data clk_efuse_init = { - .name = "efuse", - .ops = &ios_ops, - .parent_names = std_clk_io_parents, - .num_parents = ARRAY_SIZE(std_clk_io_parents), -}; - -static struct clk_std clk_efuse = { - .enable_bit = 49, - .hw = { - .init = &clk_efuse_init, - }, -}; - -static const struct clk_init_data clk_pulse_init = { - .name = "pulse", - .ops = &ios_ops, - .parent_names = std_clk_io_parents, - .num_parents = ARRAY_SIZE(std_clk_io_parents), -}; - -static struct clk_std clk_pulse = { - .enable_bit = 50, - .hw = { - .init = &clk_pulse_init, - }, -}; - -static const char * const std_clk_dsp_parents[] = { - "dsp", -}; - -static const struct clk_init_data clk_gps_init = { - .name = "gps", - .ops = &ios_ops, - .parent_names = std_clk_dsp_parents, - .num_parents = ARRAY_SIZE(std_clk_dsp_parents), -}; - -static struct clk_std clk_gps = { - .enable_bit = 1, - .hw = { - .init = &clk_gps_init, - }, -}; - -static const struct clk_init_data clk_mf_init = { - .name = "mf", - .ops = &ios_ops, - .parent_names = std_clk_io_parents, - .num_parents = ARRAY_SIZE(std_clk_io_parents), -}; - -static struct clk_std clk_mf = { - .enable_bit = 2, - .hw = { - .init = &clk_mf_init, - }, -}; - -static const char * const std_clk_sys_parents[] = { - "sys", -}; - -static const struct clk_init_data clk_security_init = { - .name = "security", - .ops = &ios_ops, - .parent_names = std_clk_sys_parents, - .num_parents = ARRAY_SIZE(std_clk_sys_parents), -}; - -static struct clk_std clk_security = { - .enable_bit = 19, - .hw = { - .init = &clk_security_init, - }, -}; - -static const char * const std_clk_usb_parents[] = { - "usb_pll", -}; - -static const struct clk_init_data clk_usb0_init = { - .name = "usb0", - .ops = &ios_ops, - .parent_names = std_clk_usb_parents, - .num_parents = ARRAY_SIZE(std_clk_usb_parents), -}; - -static struct clk_std clk_usb0 = { - .enable_bit = 16, - .hw = { - .init = &clk_usb0_init, - }, -}; - -static const struct clk_init_data clk_usb1_init = { - .name = "usb1", - .ops = &ios_ops, - .parent_names = std_clk_usb_parents, - .num_parents = ARRAY_SIZE(std_clk_usb_parents), -}; - -static struct clk_std clk_usb1 = { - .enable_bit = 17, - .hw = { - .init = &clk_usb1_init, - }, -}; diff --git a/drivers/clk/sirf/clk-prima2.c b/drivers/clk/sirf/clk-prima2.c deleted file mode 100644 index d17b345f4d2dbe1ace006351dfcdeefe1bab2f4e..0000000000000000000000000000000000000000 --- a/drivers/clk/sirf/clk-prima2.c +++ /dev/null @@ -1,149 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Clock tree for CSR SiRFprimaII - * - * Copyright (c) 2011 - 2014 Cambridge Silicon Radio Limited, a CSR plc group - * company. - */ - -#include -#include -#include -#include -#include -#include -#include - -#include "prima2.h" -#include "clk-common.c" - -static struct clk_dmn clk_mmc01 = { - .regofs = SIRFSOC_CLKC_MMC_CFG, - .enable_bit = 59, - .hw = { - .init = &clk_mmc01_init, - }, -}; - -static struct clk_dmn clk_mmc23 = { - .regofs = SIRFSOC_CLKC_MMC_CFG, - .enable_bit = 60, - .hw = { - .init = &clk_mmc23_init, - }, -}; - -static struct clk_dmn clk_mmc45 = { - .regofs = SIRFSOC_CLKC_MMC_CFG, - .enable_bit = 61, - .hw = { - .init = &clk_mmc45_init, - }, -}; - -static const struct clk_init_data clk_nand_init = { - .name = "nand", - .ops = &ios_ops, - .parent_names = std_clk_io_parents, - .num_parents = ARRAY_SIZE(std_clk_io_parents), -}; - -static struct clk_std clk_nand = { - .enable_bit = 34, - .hw = { - .init = &clk_nand_init, - }, -}; - -enum prima2_clk_index { - /* 0 1 2 3 4 5 6 7 8 9 */ - rtc, osc, pll1, pll2, pll3, mem, sys, security, dsp, gps, - mf, io, cpu, uart0, uart1, uart2, tsc, i2c0, i2c1, spi0, - spi1, pwmc, efuse, pulse, dmac0, dmac1, nand, audio, usp0, usp1, - usp2, vip, gfx, mm, lcd, vpp, mmc01, mmc23, mmc45, usbpll, - usb0, usb1, cphif, maxclk, -}; - -static __initdata struct clk_hw *prima2_clk_hw_array[maxclk] = { - NULL, /* dummy */ - NULL, - &clk_pll1.hw, - &clk_pll2.hw, - &clk_pll3.hw, - &clk_mem.hw, - &clk_sys.hw, - &clk_security.hw, - &clk_dsp.hw, - &clk_gps.hw, - &clk_mf.hw, - &clk_io.hw, - &clk_cpu.hw, - &clk_uart0.hw, - &clk_uart1.hw, - &clk_uart2.hw, - &clk_tsc.hw, - &clk_i2c0.hw, - &clk_i2c1.hw, - &clk_spi0.hw, - &clk_spi1.hw, - &clk_pwmc.hw, - &clk_efuse.hw, - &clk_pulse.hw, - &clk_dmac0.hw, - &clk_dmac1.hw, - &clk_nand.hw, - &clk_audio.hw, - &clk_usp0.hw, - &clk_usp1.hw, - &clk_usp2.hw, - &clk_vip.hw, - &clk_gfx.hw, - &clk_mm.hw, - &clk_lcd.hw, - &clk_vpp.hw, - &clk_mmc01.hw, - &clk_mmc23.hw, - &clk_mmc45.hw, - &usb_pll_clk_hw, - &clk_usb0.hw, - &clk_usb1.hw, - &clk_cphif.hw, -}; - -static struct clk *prima2_clks[maxclk]; - -static void __init prima2_clk_init(struct device_node *np) -{ - struct device_node *rscnp; - int i; - - rscnp = of_find_compatible_node(NULL, NULL, "sirf,prima2-rsc"); - sirfsoc_rsc_vbase = of_iomap(rscnp, 0); - if (!sirfsoc_rsc_vbase) - panic("unable to map rsc registers\n"); - of_node_put(rscnp); - - sirfsoc_clk_vbase = of_iomap(np, 0); - if (!sirfsoc_clk_vbase) - panic("unable to map clkc registers\n"); - - /* These are always available (RTC and 26MHz OSC)*/ - prima2_clks[rtc] = clk_register_fixed_rate(NULL, "rtc", NULL, 0, 32768); - prima2_clks[osc] = clk_register_fixed_rate(NULL, "osc", NULL, 0, - 26000000); - - for (i = pll1; i < maxclk; i++) { - prima2_clks[i] = clk_register(NULL, prima2_clk_hw_array[i]); - BUG_ON(IS_ERR(prima2_clks[i])); - } - clk_register_clkdev(prima2_clks[cpu], NULL, "cpu"); - clk_register_clkdev(prima2_clks[io], NULL, "io"); - clk_register_clkdev(prima2_clks[mem], NULL, "mem"); - clk_register_clkdev(prima2_clks[mem], NULL, "osc"); - - clk_data.clks = prima2_clks; - clk_data.clk_num = maxclk; - - of_clk_add_provider(np, of_clk_src_onecell_get, &clk_data); -} -CLK_OF_DECLARE(prima2_clk, "sirf,prima2-clkc", prima2_clk_init); diff --git a/drivers/clk/sirf/prima2.h b/drivers/clk/sirf/prima2.h deleted file mode 100644 index 2fb56941795db42795aa6babdae83ef856e80572..0000000000000000000000000000000000000000 --- a/drivers/clk/sirf/prima2.h +++ /dev/null @@ -1,26 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#define SIRFSOC_CLKC_CLK_EN0 0x0000 -#define SIRFSOC_CLKC_CLK_EN1 0x0004 -#define SIRFSOC_CLKC_REF_CFG 0x0014 -#define SIRFSOC_CLKC_CPU_CFG 0x0018 -#define SIRFSOC_CLKC_MEM_CFG 0x001c -#define SIRFSOC_CLKC_SYS_CFG 0x0020 -#define SIRFSOC_CLKC_IO_CFG 0x0024 -#define SIRFSOC_CLKC_DSP_CFG 0x0028 -#define SIRFSOC_CLKC_GFX_CFG 0x002c -#define SIRFSOC_CLKC_MM_CFG 0x0030 -#define SIRFSOC_CLKC_LCD_CFG 0x0034 -#define SIRFSOC_CLKC_MMC_CFG 0x0038 -#define SIRFSOC_CLKC_PLL1_CFG0 0x0040 -#define SIRFSOC_CLKC_PLL2_CFG0 0x0044 -#define SIRFSOC_CLKC_PLL3_CFG0 0x0048 -#define SIRFSOC_CLKC_PLL1_CFG1 0x004c -#define SIRFSOC_CLKC_PLL2_CFG1 0x0050 -#define SIRFSOC_CLKC_PLL3_CFG1 0x0054 -#define SIRFSOC_CLKC_PLL1_CFG2 0x0058 -#define SIRFSOC_CLKC_PLL2_CFG2 0x005c -#define SIRFSOC_CLKC_PLL3_CFG2 0x0060 -#define SIRFSOC_USBPHY_PLL_CTRL 0x0008 -#define SIRFSOC_USBPHY_PLL_POWERDOWN BIT(1) -#define SIRFSOC_USBPHY_PLL_BYPASS BIT(2) -#define SIRFSOC_USBPHY_PLL_LOCK BIT(3) diff --git a/drivers/clk/socfpga/clk-agilex.c b/drivers/clk/socfpga/clk-agilex.c index bb3e80928ebe84460ccf44b8ab12be13bb96a4bf..7689bdd0a914bc11ab24a5783e76bc2f33d2bcd4 100644 --- a/drivers/clk/socfpga/clk-agilex.c +++ b/drivers/clk/socfpga/clk-agilex.c @@ -196,6 +196,17 @@ static const struct stratix10_pll_clock agilex_pll_clks[] = { 0, 0x9c}, }; +static const struct n5x_perip_c_clock n5x_main_perip_c_clks[] = { + { AGILEX_MAIN_PLL_C0_CLK, "main_pll_c0", "main_pll", NULL, 1, 0, 0x54, 0}, + { AGILEX_MAIN_PLL_C1_CLK, "main_pll_c1", "main_pll", NULL, 1, 0, 0x54, 8}, + { AGILEX_MAIN_PLL_C2_CLK, "main_pll_c2", "main_pll", NULL, 1, 0, 0x54, 16}, + { AGILEX_MAIN_PLL_C3_CLK, "main_pll_c3", "main_pll", NULL, 1, 0, 0x54, 24}, + { AGILEX_PERIPH_PLL_C0_CLK, "peri_pll_c0", "periph_pll", NULL, 1, 0, 0xA8, 0}, + { AGILEX_PERIPH_PLL_C1_CLK, "peri_pll_c1", "periph_pll", NULL, 1, 0, 0xA8, 8}, + { AGILEX_PERIPH_PLL_C2_CLK, "peri_pll_c2", "periph_pll", NULL, 1, 0, 0xA8, 16}, + { AGILEX_PERIPH_PLL_C3_CLK, "peri_pll_c3", "periph_pll", NULL, 1, 0, 0xA8, 24}, +}; + static const struct stratix10_perip_c_clock agilex_main_perip_c_clks[] = { { AGILEX_MAIN_PLL_C0_CLK, "main_pll_c0", "main_pll", NULL, 1, 0, 0x58}, { AGILEX_MAIN_PLL_C1_CLK, "main_pll_c1", "main_pll", NULL, 1, 0, 0x5C}, @@ -289,6 +300,25 @@ static const struct stratix10_gate_clock agilex_gate_clks[] = { 10, 0, 0, 0, 0, 0, 4}, }; +static int n5x_clk_register_c_perip(const struct n5x_perip_c_clock *clks, + int nums, struct stratix10_clock_data *data) +{ + struct clk *clk; + void __iomem *base = data->base; + int i; + + for (i = 0; i < nums; i++) { + clk = n5x_register_periph(&clks[i], base); + if (IS_ERR(clk)) { + pr_err("%s: failed to register clock %s\n", + __func__, clks[i].name); + continue; + } + data->clk_data.clks[clks[i].id] = clk; + } + return 0; +} + static int agilex_clk_register_c_perip(const struct stratix10_perip_c_clock *clks, int nums, struct stratix10_clock_data *data) { @@ -367,6 +397,26 @@ static int agilex_clk_register_pll(const struct stratix10_pll_clock *clks, return 0; } +static int n5x_clk_register_pll(const struct stratix10_pll_clock *clks, + int nums, struct stratix10_clock_data *data) +{ + struct clk *clk; + void __iomem *base = data->base; + int i; + + for (i = 0; i < nums; i++) { + clk = n5x_register_pll(&clks[i], base); + if (IS_ERR(clk)) { + pr_err("%s: failed to register clock %s\n", + __func__, clks[i].name); + continue; + } + data->clk_data.clks[clks[i].id] = clk; + } + + return 0; +} + static struct stratix10_clock_data *__socfpga_agilex_clk_init(struct platform_device *pdev, int nr_clks) { @@ -401,7 +451,7 @@ static struct stratix10_clock_data *__socfpga_agilex_clk_init(struct platform_de return clk_data; } -static int agilex_clkmgr_probe(struct platform_device *pdev) +static int agilex_clkmgr_init(struct platform_device *pdev) { struct stratix10_clock_data *clk_data; @@ -423,9 +473,43 @@ static int agilex_clkmgr_probe(struct platform_device *pdev) return 0; } +static int n5x_clkmgr_init(struct platform_device *pdev) +{ + struct stratix10_clock_data *clk_data; + + clk_data = __socfpga_agilex_clk_init(pdev, AGILEX_NUM_CLKS); + if (IS_ERR(clk_data)) + return PTR_ERR(clk_data); + + n5x_clk_register_pll(agilex_pll_clks, ARRAY_SIZE(agilex_pll_clks), clk_data); + + n5x_clk_register_c_perip(n5x_main_perip_c_clks, + ARRAY_SIZE(n5x_main_perip_c_clks), clk_data); + + agilex_clk_register_cnt_perip(agilex_main_perip_cnt_clks, + ARRAY_SIZE(agilex_main_perip_cnt_clks), + clk_data); + + agilex_clk_register_gate(agilex_gate_clks, ARRAY_SIZE(agilex_gate_clks), + clk_data); + return 0; +} + +static int agilex_clkmgr_probe(struct platform_device *pdev) +{ + int (*probe_func)(struct platform_device *init_func); + + probe_func = of_device_get_match_data(&pdev->dev); + if (!probe_func) + return -ENODEV; + return probe_func(pdev); +} + static const struct of_device_id agilex_clkmgr_match_table[] = { { .compatible = "intel,agilex-clkmgr", - .data = agilex_clkmgr_probe }, + .data = agilex_clkmgr_init }, + { .compatible = "intel,easic-n5x-clkmgr", + .data = n5x_clkmgr_init }, { } }; diff --git a/drivers/clk/socfpga/clk-periph-s10.c b/drivers/clk/socfpga/clk-periph-s10.c index 397b77b89b166eabd43b4a5ec41f098a06fc39a7..0ff2b9d240353a9b99199000cf73d0dd9eef97b5 100644 --- a/drivers/clk/socfpga/clk-periph-s10.c +++ b/drivers/clk/socfpga/clk-periph-s10.c @@ -15,6 +15,21 @@ #define to_periph_clk(p) container_of(p, struct socfpga_periph_clk, hw.hw) +static unsigned long n5x_clk_peri_c_clk_recalc_rate(struct clk_hw *hwclk, + unsigned long parent_rate) +{ + struct socfpga_periph_clk *socfpgaclk = to_periph_clk(hwclk); + unsigned long div; + unsigned long shift = socfpgaclk->shift; + u32 val; + + val = readl(socfpgaclk->hw.reg); + val &= (0x1f << shift); + div = (val >> shift) + 1; + + return parent_rate / div; +} + static unsigned long clk_peri_c_clk_recalc_rate(struct clk_hw *hwclk, unsigned long parent_rate) { @@ -63,6 +78,11 @@ static u8 clk_periclk_get_parent(struct clk_hw *hwclk) return parent; } +static const struct clk_ops n5x_peri_c_clk_ops = { + .recalc_rate = n5x_clk_peri_c_clk_recalc_rate, + .get_parent = clk_periclk_get_parent, +}; + static const struct clk_ops peri_c_clk_ops = { .recalc_rate = clk_peri_c_clk_recalc_rate, .get_parent = clk_periclk_get_parent, @@ -107,6 +127,39 @@ struct clk *s10_register_periph(const struct stratix10_perip_c_clock *clks, return clk; } +struct clk *n5x_register_periph(const struct n5x_perip_c_clock *clks, + void __iomem *regbase) +{ + struct clk *clk; + struct socfpga_periph_clk *periph_clk; + struct clk_init_data init; + const char *name = clks->name; + const char *parent_name = clks->parent_name; + + periph_clk = kzalloc(sizeof(*periph_clk), GFP_KERNEL); + if (WARN_ON(!periph_clk)) + return NULL; + + periph_clk->hw.reg = regbase + clks->offset; + periph_clk->shift = clks->shift; + + init.name = name; + init.ops = &n5x_peri_c_clk_ops; + init.flags = clks->flags; + + init.num_parents = clks->num_parents; + init.parent_names = parent_name ? &parent_name : NULL; + + periph_clk->hw.hw.init = &init; + + clk = clk_register(NULL, &periph_clk->hw.hw); + if (WARN_ON(IS_ERR(clk))) { + kfree(periph_clk); + return NULL; + } + return clk; +} + struct clk *s10_register_cnt_periph(const struct stratix10_perip_cnt_clock *clks, void __iomem *regbase) { diff --git a/drivers/clk/socfpga/clk-pll-a10.c b/drivers/clk/socfpga/clk-pll-a10.c index db54f7d806a0965868ea7ac4210dd4927e4c788d..3338f054fe98c37ebef34a963f416afa58692c04 100644 --- a/drivers/clk/socfpga/clk-pll-a10.c +++ b/drivers/clk/socfpga/clk-pll-a10.c @@ -73,7 +73,6 @@ static struct clk * __init __socfpga_pll_init(struct device_node *node, const char *parent_name[SOCFGPA_MAX_PARENTS]; struct clk_init_data init; struct device_node *clkmgr_np; - int rc; int i = 0; of_property_read_u32(node, "reg", ®); @@ -108,7 +107,7 @@ static struct clk * __init __socfpga_pll_init(struct device_node *node, kfree(pll_clk); return NULL; } - rc = of_clk_add_provider(node, of_clk_src_simple_get, clk); + of_clk_add_provider(node, of_clk_src_simple_get, clk); return clk; } diff --git a/drivers/clk/socfpga/clk-pll-s10.c b/drivers/clk/socfpga/clk-pll-s10.c index 4e268953b7da2c9dc7828364242cbb7f584da479..f6f66e08e1f455d394eae46ae41a1b62b8057cdf 100644 --- a/drivers/clk/socfpga/clk-pll-s10.c +++ b/drivers/clk/socfpga/clk-pll-s10.c @@ -27,10 +27,37 @@ #define SWCTRLBTCLKSEL_MASK 0x200 #define SWCTRLBTCLKSEL_SHIFT 9 +#define SOCFPGA_N5X_PLLDIV_FDIV_MASK GENMASK(16, 8) +#define SOCFPGA_N5X_PLLDIV_FDIV_SHIFT 8 +#define SOCFPGA_N5X_PLLDIV_RDIV_MASK GENMASK(5, 0) +#define SOCFPGA_N5X_PLLDIV_QDIV_MASK GENMASK(26, 24) +#define SOCFPGA_N5X_PLLDIV_QDIV_SHIFT 24 + #define SOCFPGA_BOOT_CLK "boot_clk" #define to_socfpga_clk(p) container_of(p, struct socfpga_pll, hw.hw) +static unsigned long n5x_clk_pll_recalc_rate(struct clk_hw *hwclk, + unsigned long parent_rate) +{ + struct socfpga_pll *socfpgaclk = to_socfpga_clk(hwclk); + unsigned long fdiv, reg, rdiv, qdiv; + u32 power = 1; + + /* read VCO1 reg for numerator and denominator */ + reg = readl(socfpgaclk->hw.reg + 0x8); + fdiv = (reg & SOCFPGA_N5X_PLLDIV_FDIV_MASK) >> SOCFPGA_N5X_PLLDIV_FDIV_SHIFT; + rdiv = (reg & SOCFPGA_N5X_PLLDIV_RDIV_MASK); + qdiv = (reg & SOCFPGA_N5X_PLLDIV_QDIV_MASK) >> SOCFPGA_N5X_PLLDIV_QDIV_SHIFT; + + while (qdiv) { + power *= 2; + qdiv--; + } + + return ((parent_rate * 2 * (fdiv + 1)) / ((rdiv + 1) * power)); +} + static unsigned long agilex_clk_pll_recalc_rate(struct clk_hw *hwclk, unsigned long parent_rate) { @@ -123,6 +150,25 @@ static int clk_pll_prepare(struct clk_hw *hwclk) return 0; } +static int n5x_clk_pll_prepare(struct clk_hw *hwclk) +{ + struct socfpga_pll *socfpgaclk = to_socfpga_clk(hwclk); + u32 reg; + + /* Bring PLL out of reset */ + reg = readl(socfpgaclk->hw.reg + 0x4); + reg |= SOCFPGA_PLL_RESET_MASK; + writel(reg, socfpgaclk->hw.reg + 0x4); + + return 0; +} + +static const struct clk_ops n5x_clk_pll_ops = { + .recalc_rate = n5x_clk_pll_recalc_rate, + .get_parent = clk_pll_get_parent, + .prepare = n5x_clk_pll_prepare, +}; + static const struct clk_ops agilex_clk_pll_ops = { .recalc_rate = agilex_clk_pll_recalc_rate, .get_parent = clk_pll_get_parent, @@ -214,3 +260,40 @@ struct clk *agilex_register_pll(const struct stratix10_pll_clock *clks, } return clk; } + +struct clk *n5x_register_pll(const struct stratix10_pll_clock *clks, + void __iomem *reg) +{ + struct clk *clk; + struct socfpga_pll *pll_clk; + struct clk_init_data init; + const char *name = clks->name; + + pll_clk = kzalloc(sizeof(*pll_clk), GFP_KERNEL); + if (WARN_ON(!pll_clk)) + return NULL; + + pll_clk->hw.reg = reg + clks->offset; + + if (streq(name, SOCFPGA_BOOT_CLK)) + init.ops = &clk_boot_ops; + else + init.ops = &n5x_clk_pll_ops; + + init.name = name; + init.flags = clks->flags; + + init.num_parents = clks->num_parents; + init.parent_names = NULL; + init.parent_data = clks->parent_data; + pll_clk->hw.hw.init = &init; + + pll_clk->hw.bit_idx = SOCFPGA_PLL_POWER; + + clk = clk_register(NULL, &pll_clk->hw.hw); + if (WARN_ON(IS_ERR(clk))) { + kfree(pll_clk); + return NULL; + } + return clk; +} diff --git a/drivers/clk/socfpga/clk-pll.c b/drivers/clk/socfpga/clk-pll.c index e5fb786843f39c3f44a3eae7da502ce5f1f41dde..3cf99df7d005634e4d4a3cdf74dae1a77307f6bd 100644 --- a/drivers/clk/socfpga/clk-pll.c +++ b/drivers/clk/socfpga/clk-pll.c @@ -80,7 +80,6 @@ static __init struct clk *__socfpga_pll_init(struct device_node *node, const char *parent_name[SOCFPGA_MAX_PARENTS]; struct clk_init_data init; struct device_node *clkmgr_np; - int rc; of_property_read_u32(node, "reg", ®); @@ -111,7 +110,7 @@ static __init struct clk *__socfpga_pll_init(struct device_node *node, kfree(pll_clk); return NULL; } - rc = of_clk_add_provider(node, of_clk_src_simple_get, clk); + of_clk_add_provider(node, of_clk_src_simple_get, clk); return clk; } diff --git a/drivers/clk/socfpga/stratix10-clk.h b/drivers/clk/socfpga/stratix10-clk.h index f9d5d724c694984cc61ad90754c4e5b8053fa32b..420deed677ce2ac383d5db4806034dbe402e0070 100644 --- a/drivers/clk/socfpga/stratix10-clk.h +++ b/drivers/clk/socfpga/stratix10-clk.h @@ -30,6 +30,17 @@ struct stratix10_perip_c_clock { unsigned long offset; }; +struct n5x_perip_c_clock { + unsigned int id; + const char *name; + const char *parent_name; + const char *const *parent_names; + u8 num_parents; + unsigned long flags; + unsigned long offset; + unsigned long shift; +}; + struct stratix10_perip_cnt_clock { unsigned int id; const char *name; @@ -64,8 +75,12 @@ struct clk *s10_register_pll(const struct stratix10_pll_clock *, void __iomem *); struct clk *agilex_register_pll(const struct stratix10_pll_clock *, void __iomem *); +struct clk *n5x_register_pll(const struct stratix10_pll_clock *clks, + void __iomem *reg); struct clk *s10_register_periph(const struct stratix10_perip_c_clock *, - void __iomem *); + void __iomem *reg); +struct clk *n5x_register_periph(const struct n5x_perip_c_clock *clks, + void __iomem *reg); struct clk *s10_register_cnt_periph(const struct stratix10_perip_cnt_clock *, void __iomem *); struct clk *s10_register_gate(const struct stratix10_gate_clock *, diff --git a/drivers/clk/spear/spear1310_clock.c b/drivers/clk/spear/spear1310_clock.c index 591248c9a88e797295a712abcb9b26b6d00a95e9..8c897486678990bc7d85b36ec22f4a519eae1c47 100644 --- a/drivers/clk/spear/spear1310_clock.c +++ b/drivers/clk/spear/spear1310_clock.c @@ -12,6 +12,7 @@ */ #include +#include #include #include #include diff --git a/drivers/clk/spear/spear1340_clock.c b/drivers/clk/spear/spear1340_clock.c index 9163bbb464112e46973eb958d48babaea4673250..c0dc94355c8735711d8a45381ee6b00e06f16bf6 100644 --- a/drivers/clk/spear/spear1340_clock.c +++ b/drivers/clk/spear/spear1340_clock.c @@ -12,6 +12,7 @@ */ #include +#include #include #include #include diff --git a/drivers/clk/st/clkgen-fsyn.c b/drivers/clk/st/clkgen-fsyn.c index f1adc858b5907d4ee480be6b73d9e0d51a10f1e0..dd6062e043e023152333517b973997412bd9b920 100644 --- a/drivers/clk/st/clkgen-fsyn.c +++ b/drivers/clk/st/clkgen-fsyn.c @@ -172,10 +172,10 @@ static const struct clkgen_quadfs_data st_fs660c32_D = { * ST quad channel frequency synthesizer block * * @hw: handle between common and hardware-specific interfaces. - * @ndiv: regmap field for the ndiv control. * @regs_base: base address of the configuration registers. * @lock: spinlock. - * + * @data: local driver data + * @ndiv: regmap field for the ndiv control. */ struct st_clk_quadfs_pll { struct clk_hw hw; @@ -426,7 +426,7 @@ static struct clk * __init st_clk_register_quadfs_pll( * parent - fixed parent. No clk_set_parent support */ -/** +/* * struct st_clk_quadfs_fsynth - One clock output from a four channel digital * frequency synthesizer (fsynth) block. * diff --git a/drivers/clk/st/clkgen-pll.c b/drivers/clk/st/clkgen-pll.c index c3952f2c42ba26da19a904cfd878b2a9f83834fa..119c5b33080cf849e9597cd2e73fd79049c17336 100644 --- a/drivers/clk/st/clkgen-pll.c +++ b/drivers/clk/st/clkgen-pll.c @@ -130,12 +130,11 @@ static struct clkgen_pll_data st_pll4600c28_418_a9 = { * parent - fixed parent. No clk_set_parent support */ -/** +/* * PLL clock that is integrated in the ClockGenA instances on the STiH415 * and STiH416. * * @hw: handle between common and hardware-specific interfaces. - * @type: PLL instance type. * @regs_base: base of the PLL configuration register(s). * */ diff --git a/drivers/clk/sunxi-ng/Kconfig b/drivers/clk/sunxi-ng/Kconfig index ce5f5847d5d3991876e6152f590050a300828129..cd46d8853876cee54d82e1b9e316e441542a5de4 100644 --- a/drivers/clk/sunxi-ng/Kconfig +++ b/drivers/clk/sunxi-ng/Kconfig @@ -32,8 +32,13 @@ config SUN50I_H6_CCU default ARM64 && ARCH_SUNXI depends on (ARM64 && ARCH_SUNXI) || COMPILE_TEST +config SUN50I_H616_CCU + bool "Support for the Allwinner H616 CCU" + default ARM64 && ARCH_SUNXI + depends on (ARM64 && ARCH_SUNXI) || COMPILE_TEST + config SUN50I_H6_R_CCU - bool "Support for the Allwinner H6 PRCM CCU" + bool "Support for the Allwinner H6 and H616 PRCM CCU" default ARM64 && ARCH_SUNXI depends on (ARM64 && ARCH_SUNXI) || COMPILE_TEST diff --git a/drivers/clk/sunxi-ng/Makefile b/drivers/clk/sunxi-ng/Makefile index 3eb5cff40eac430d04a4612659ba08e6e462f873..96c324306d97fa28170c37cd5f860ba39b0ebfe0 100644 --- a/drivers/clk/sunxi-ng/Makefile +++ b/drivers/clk/sunxi-ng/Makefile @@ -26,6 +26,7 @@ obj-$(CONFIG_SUN50I_A64_CCU) += ccu-sun50i-a64.o obj-$(CONFIG_SUN50I_A100_CCU) += ccu-sun50i-a100.o obj-$(CONFIG_SUN50I_A100_R_CCU) += ccu-sun50i-a100-r.o obj-$(CONFIG_SUN50I_H6_CCU) += ccu-sun50i-h6.o +obj-$(CONFIG_SUN50I_H616_CCU) += ccu-sun50i-h616.o obj-$(CONFIG_SUN50I_H6_R_CCU) += ccu-sun50i-h6-r.o obj-$(CONFIG_SUN4I_A10_CCU) += ccu-sun4i-a10.o obj-$(CONFIG_SUN5I_CCU) += ccu-sun5i.o diff --git a/drivers/clk/sunxi-ng/ccu-sun50i-h6-r.c b/drivers/clk/sunxi-ng/ccu-sun50i-h6-r.c index 50f8d1bc7046552804a1bad0576be558e9b89cb6..f8909a7ed5539a0f73eea5e7c14f06531e394900 100644 --- a/drivers/clk/sunxi-ng/ccu-sun50i-h6-r.c +++ b/drivers/clk/sunxi-ng/ccu-sun50i-h6-r.c @@ -91,6 +91,8 @@ static SUNXI_CCU_GATE(r_apb2_uart_clk, "r-apb2-uart", "r-apb2", 0x18c, BIT(0), 0); static SUNXI_CCU_GATE(r_apb2_i2c_clk, "r-apb2-i2c", "r-apb2", 0x19c, BIT(0), 0); +static SUNXI_CCU_GATE(r_apb2_rsb_clk, "r-apb2-rsb", "r-apb2", + 0x1bc, BIT(0), 0); static SUNXI_CCU_GATE(r_apb1_ir_clk, "r-apb1-ir", "r-apb1", 0x1cc, BIT(0), 0); static SUNXI_CCU_GATE(r_apb1_w1_clk, "r-apb1-w1", "r-apb1", @@ -130,12 +132,23 @@ static struct ccu_common *sun50i_h6_r_ccu_clks[] = { &r_apb1_pwm_clk.common, &r_apb2_uart_clk.common, &r_apb2_i2c_clk.common, + &r_apb2_rsb_clk.common, &r_apb1_ir_clk.common, &r_apb1_w1_clk.common, &ir_clk.common, &w1_clk.common, }; +static struct ccu_common *sun50i_h616_r_ccu_clks[] = { + &r_apb1_clk.common, + &r_apb2_clk.common, + &r_apb1_twd_clk.common, + &r_apb2_i2c_clk.common, + &r_apb2_rsb_clk.common, + &r_apb1_ir_clk.common, + &ir_clk.common, +}; + static struct clk_hw_onecell_data sun50i_h6_r_hw_clks = { .hws = { [CLK_AR100] = &ar100_clk.common.hw, @@ -147,6 +160,7 @@ static struct clk_hw_onecell_data sun50i_h6_r_hw_clks = { [CLK_R_APB1_PWM] = &r_apb1_pwm_clk.common.hw, [CLK_R_APB2_UART] = &r_apb2_uart_clk.common.hw, [CLK_R_APB2_I2C] = &r_apb2_i2c_clk.common.hw, + [CLK_R_APB2_RSB] = &r_apb2_rsb_clk.common.hw, [CLK_R_APB1_IR] = &r_apb1_ir_clk.common.hw, [CLK_R_APB1_W1] = &r_apb1_w1_clk.common.hw, [CLK_IR] = &ir_clk.common.hw, @@ -155,16 +169,38 @@ static struct clk_hw_onecell_data sun50i_h6_r_hw_clks = { .num = CLK_NUMBER, }; +static struct clk_hw_onecell_data sun50i_h616_r_hw_clks = { + .hws = { + [CLK_R_AHB] = &r_ahb_clk.hw, + [CLK_R_APB1] = &r_apb1_clk.common.hw, + [CLK_R_APB2] = &r_apb2_clk.common.hw, + [CLK_R_APB1_TWD] = &r_apb1_twd_clk.common.hw, + [CLK_R_APB2_I2C] = &r_apb2_i2c_clk.common.hw, + [CLK_R_APB2_RSB] = &r_apb2_rsb_clk.common.hw, + [CLK_R_APB1_IR] = &r_apb1_ir_clk.common.hw, + [CLK_IR] = &ir_clk.common.hw, + }, + .num = CLK_NUMBER, +}; + static struct ccu_reset_map sun50i_h6_r_ccu_resets[] = { [RST_R_APB1_TIMER] = { 0x11c, BIT(16) }, [RST_R_APB1_TWD] = { 0x12c, BIT(16) }, [RST_R_APB1_PWM] = { 0x13c, BIT(16) }, [RST_R_APB2_UART] = { 0x18c, BIT(16) }, [RST_R_APB2_I2C] = { 0x19c, BIT(16) }, + [RST_R_APB2_RSB] = { 0x1bc, BIT(16) }, [RST_R_APB1_IR] = { 0x1cc, BIT(16) }, [RST_R_APB1_W1] = { 0x1ec, BIT(16) }, }; +static struct ccu_reset_map sun50i_h616_r_ccu_resets[] = { + [RST_R_APB1_TWD] = { 0x12c, BIT(16) }, + [RST_R_APB2_I2C] = { 0x19c, BIT(16) }, + [RST_R_APB2_RSB] = { 0x1bc, BIT(16) }, + [RST_R_APB1_IR] = { 0x1cc, BIT(16) }, +}; + static const struct sunxi_ccu_desc sun50i_h6_r_ccu_desc = { .ccu_clks = sun50i_h6_r_ccu_clks, .num_ccu_clks = ARRAY_SIZE(sun50i_h6_r_ccu_clks), @@ -175,6 +211,16 @@ static const struct sunxi_ccu_desc sun50i_h6_r_ccu_desc = { .num_resets = ARRAY_SIZE(sun50i_h6_r_ccu_resets), }; +static const struct sunxi_ccu_desc sun50i_h616_r_ccu_desc = { + .ccu_clks = sun50i_h616_r_ccu_clks, + .num_ccu_clks = ARRAY_SIZE(sun50i_h616_r_ccu_clks), + + .hw_clks = &sun50i_h616_r_hw_clks, + + .resets = sun50i_h616_r_ccu_resets, + .num_resets = ARRAY_SIZE(sun50i_h616_r_ccu_resets), +}; + static void __init sunxi_r_ccu_init(struct device_node *node, const struct sunxi_ccu_desc *desc) { @@ -195,3 +241,10 @@ static void __init sun50i_h6_r_ccu_setup(struct device_node *node) } CLK_OF_DECLARE(sun50i_h6_r_ccu, "allwinner,sun50i-h6-r-ccu", sun50i_h6_r_ccu_setup); + +static void __init sun50i_h616_r_ccu_setup(struct device_node *node) +{ + sunxi_r_ccu_init(node, &sun50i_h616_r_ccu_desc); +} +CLK_OF_DECLARE(sun50i_h616_r_ccu, "allwinner,sun50i-h616-r-ccu", + sun50i_h616_r_ccu_setup); diff --git a/drivers/clk/sunxi-ng/ccu-sun50i-h6-r.h b/drivers/clk/sunxi-ng/ccu-sun50i-h6-r.h index 782117dc0b28c4c64d678139254a2eecd076b6b3..7e290b840803564b62e5743fb314fe38179fe4cd 100644 --- a/drivers/clk/sunxi-ng/ccu-sun50i-h6-r.h +++ b/drivers/clk/sunxi-ng/ccu-sun50i-h6-r.h @@ -14,6 +14,6 @@ #define CLK_R_APB2 3 -#define CLK_NUMBER (CLK_W1 + 1) +#define CLK_NUMBER (CLK_R_APB2_RSB + 1) #endif /* _CCU_SUN50I_H6_R_H */ diff --git a/drivers/clk/sunxi-ng/ccu-sun50i-h6.c b/drivers/clk/sunxi-ng/ccu-sun50i-h6.c index f2497d0a4683a36fd739c8e591ba720bba98f30d..bff446b7829071df891d74dd4a5eda8740cc1c42 100644 --- a/drivers/clk/sunxi-ng/ccu-sun50i-h6.c +++ b/drivers/clk/sunxi-ng/ccu-sun50i-h6.c @@ -237,7 +237,7 @@ static const char * const psi_ahb1_ahb2_parents[] = { "osc24M", "osc32k", static SUNXI_CCU_MP_WITH_MUX(psi_ahb1_ahb2_clk, "psi-ahb1-ahb2", psi_ahb1_ahb2_parents, 0x510, - 0, 5, /* M */ + 0, 2, /* M */ 8, 2, /* P */ 24, 2, /* mux */ 0); @@ -246,19 +246,19 @@ static const char * const ahb3_apb1_apb2_parents[] = { "osc24M", "osc32k", "psi-ahb1-ahb2", "pll-periph0" }; static SUNXI_CCU_MP_WITH_MUX(ahb3_clk, "ahb3", ahb3_apb1_apb2_parents, 0x51c, - 0, 5, /* M */ + 0, 2, /* M */ 8, 2, /* P */ 24, 2, /* mux */ 0); static SUNXI_CCU_MP_WITH_MUX(apb1_clk, "apb1", ahb3_apb1_apb2_parents, 0x520, - 0, 5, /* M */ + 0, 2, /* M */ 8, 2, /* P */ 24, 2, /* mux */ 0); static SUNXI_CCU_MP_WITH_MUX(apb2_clk, "apb2", ahb3_apb1_apb2_parents, 0x524, - 0, 5, /* M */ + 0, 2, /* M */ 8, 2, /* P */ 24, 2, /* mux */ 0); @@ -682,7 +682,7 @@ static struct ccu_mux hdmi_cec_clk = { .common = { .reg = 0xb10, - .features = CCU_FEATURE_VARIABLE_PREDIV, + .features = CCU_FEATURE_FIXED_PREDIV, .hw.init = CLK_HW_INIT_PARENTS("hdmi-cec", hdmi_cec_parents, &ccu_mux_ops, diff --git a/drivers/clk/sunxi-ng/ccu-sun50i-h616.c b/drivers/clk/sunxi-ng/ccu-sun50i-h616.c new file mode 100644 index 0000000000000000000000000000000000000000..225307305880e25054ad57ac9f1b75cb556dbde8 --- /dev/null +++ b/drivers/clk/sunxi-ng/ccu-sun50i-h616.c @@ -0,0 +1,1150 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2020 Arm Ltd. + * Based on the H6 CCU driver, which is: + * Copyright (c) 2017 Icenowy Zheng + */ + +#include +#include +#include +#include + +#include "ccu_common.h" +#include "ccu_reset.h" + +#include "ccu_div.h" +#include "ccu_gate.h" +#include "ccu_mp.h" +#include "ccu_mult.h" +#include "ccu_nk.h" +#include "ccu_nkm.h" +#include "ccu_nkmp.h" +#include "ccu_nm.h" + +#include "ccu-sun50i-h616.h" + +/* + * The CPU PLL is actually NP clock, with P being /1, /2 or /4. However + * P should only be used for output frequencies lower than 288 MHz. + * + * For now we can just model it as a multiplier clock, and force P to /1. + * + * The M factor is present in the register's description, but not in the + * frequency formula, and it's documented as "M is only used for backdoor + * testing", so it's not modelled and then force to 0. + */ +#define SUN50I_H616_PLL_CPUX_REG 0x000 +static struct ccu_mult pll_cpux_clk = { + .enable = BIT(31), + .lock = BIT(28), + .mult = _SUNXI_CCU_MULT_MIN(8, 8, 12), + .common = { + .reg = 0x000, + .hw.init = CLK_HW_INIT("pll-cpux", "osc24M", + &ccu_mult_ops, + CLK_SET_RATE_UNGATE), + }, +}; + +/* Some PLLs are input * N / div1 / P. Model them as NKMP with no K */ +#define SUN50I_H616_PLL_DDR0_REG 0x010 +static struct ccu_nkmp pll_ddr0_clk = { + .enable = BIT(31), + .lock = BIT(28), + .n = _SUNXI_CCU_MULT_MIN(8, 8, 12), + .m = _SUNXI_CCU_DIV(1, 1), /* input divider */ + .p = _SUNXI_CCU_DIV(0, 1), /* output divider */ + .common = { + .reg = 0x010, + .hw.init = CLK_HW_INIT("pll-ddr0", "osc24M", + &ccu_nkmp_ops, + CLK_SET_RATE_UNGATE), + }, +}; + +#define SUN50I_H616_PLL_DDR1_REG 0x018 +static struct ccu_nkmp pll_ddr1_clk = { + .enable = BIT(31), + .lock = BIT(28), + .n = _SUNXI_CCU_MULT_MIN(8, 8, 12), + .m = _SUNXI_CCU_DIV(1, 1), /* input divider */ + .p = _SUNXI_CCU_DIV(0, 1), /* output divider */ + .common = { + .reg = 0x018, + .hw.init = CLK_HW_INIT("pll-ddr1", "osc24M", + &ccu_nkmp_ops, + CLK_SET_RATE_UNGATE), + }, +}; + +#define SUN50I_H616_PLL_PERIPH0_REG 0x020 +static struct ccu_nkmp pll_periph0_clk = { + .enable = BIT(31), + .lock = BIT(28), + .n = _SUNXI_CCU_MULT_MIN(8, 8, 12), + .m = _SUNXI_CCU_DIV(1, 1), /* input divider */ + .p = _SUNXI_CCU_DIV(0, 1), /* output divider */ + .fixed_post_div = 2, + .common = { + .reg = 0x020, + .features = CCU_FEATURE_FIXED_POSTDIV, + .hw.init = CLK_HW_INIT("pll-periph0", "osc24M", + &ccu_nkmp_ops, + CLK_SET_RATE_UNGATE), + }, +}; + +#define SUN50I_H616_PLL_PERIPH1_REG 0x028 +static struct ccu_nkmp pll_periph1_clk = { + .enable = BIT(31), + .lock = BIT(28), + .n = _SUNXI_CCU_MULT_MIN(8, 8, 12), + .m = _SUNXI_CCU_DIV(1, 1), /* input divider */ + .p = _SUNXI_CCU_DIV(0, 1), /* output divider */ + .fixed_post_div = 2, + .common = { + .reg = 0x028, + .features = CCU_FEATURE_FIXED_POSTDIV, + .hw.init = CLK_HW_INIT("pll-periph1", "osc24M", + &ccu_nkmp_ops, + CLK_SET_RATE_UNGATE), + }, +}; + +#define SUN50I_H616_PLL_GPU_REG 0x030 +static struct ccu_nkmp pll_gpu_clk = { + .enable = BIT(31), + .lock = BIT(28), + .n = _SUNXI_CCU_MULT_MIN(8, 8, 12), + .m = _SUNXI_CCU_DIV(1, 1), /* input divider */ + .p = _SUNXI_CCU_DIV(0, 1), /* output divider */ + .common = { + .reg = 0x030, + .hw.init = CLK_HW_INIT("pll-gpu", "osc24M", + &ccu_nkmp_ops, + CLK_SET_RATE_UNGATE), + }, +}; + +/* + * For Video PLLs, the output divider is described as "used for testing" + * in the user manual. So it's not modelled and forced to 0. + */ +#define SUN50I_H616_PLL_VIDEO0_REG 0x040 +static struct ccu_nm pll_video0_clk = { + .enable = BIT(31), + .lock = BIT(28), + .n = _SUNXI_CCU_MULT_MIN(8, 8, 12), + .m = _SUNXI_CCU_DIV(1, 1), /* input divider */ + .fixed_post_div = 4, + .min_rate = 288000000, + .max_rate = 2400000000UL, + .common = { + .reg = 0x040, + .features = CCU_FEATURE_FIXED_POSTDIV, + .hw.init = CLK_HW_INIT("pll-video0", "osc24M", + &ccu_nm_ops, + CLK_SET_RATE_UNGATE), + }, +}; + +#define SUN50I_H616_PLL_VIDEO1_REG 0x048 +static struct ccu_nm pll_video1_clk = { + .enable = BIT(31), + .lock = BIT(28), + .n = _SUNXI_CCU_MULT_MIN(8, 8, 12), + .m = _SUNXI_CCU_DIV(1, 1), /* input divider */ + .fixed_post_div = 4, + .min_rate = 288000000, + .max_rate = 2400000000UL, + .common = { + .reg = 0x048, + .features = CCU_FEATURE_FIXED_POSTDIV, + .hw.init = CLK_HW_INIT("pll-video1", "osc24M", + &ccu_nm_ops, + CLK_SET_RATE_UNGATE), + }, +}; + +#define SUN50I_H616_PLL_VIDEO2_REG 0x050 +static struct ccu_nm pll_video2_clk = { + .enable = BIT(31), + .lock = BIT(28), + .n = _SUNXI_CCU_MULT_MIN(8, 8, 12), + .m = _SUNXI_CCU_DIV(1, 1), /* input divider */ + .fixed_post_div = 4, + .min_rate = 288000000, + .max_rate = 2400000000UL, + .common = { + .reg = 0x050, + .features = CCU_FEATURE_FIXED_POSTDIV, + .hw.init = CLK_HW_INIT("pll-video2", "osc24M", + &ccu_nm_ops, + CLK_SET_RATE_UNGATE), + }, +}; + +#define SUN50I_H616_PLL_VE_REG 0x058 +static struct ccu_nkmp pll_ve_clk = { + .enable = BIT(31), + .lock = BIT(28), + .n = _SUNXI_CCU_MULT_MIN(8, 8, 12), + .m = _SUNXI_CCU_DIV(1, 1), /* input divider */ + .p = _SUNXI_CCU_DIV(0, 1), /* output divider */ + .common = { + .reg = 0x058, + .hw.init = CLK_HW_INIT("pll-ve", "osc24M", + &ccu_nkmp_ops, + CLK_SET_RATE_UNGATE), + }, +}; + +#define SUN50I_H616_PLL_DE_REG 0x060 +static struct ccu_nkmp pll_de_clk = { + .enable = BIT(31), + .lock = BIT(28), + .n = _SUNXI_CCU_MULT_MIN(8, 8, 12), + .m = _SUNXI_CCU_DIV(1, 1), /* input divider */ + .p = _SUNXI_CCU_DIV(0, 1), /* output divider */ + .common = { + .reg = 0x060, + .hw.init = CLK_HW_INIT("pll-de", "osc24M", + &ccu_nkmp_ops, + CLK_SET_RATE_UNGATE), + }, +}; + +/* + * TODO: Determine SDM settings for the audio PLL. The manual suggests + * PLL_FACTOR_N=16, PLL_POST_DIV_P=2, OUTPUT_DIV=2, pattern=0xe000c49b + * for 24.576 MHz, and PLL_FACTOR_N=22, PLL_POST_DIV_P=3, OUTPUT_DIV=2, + * pattern=0xe001288c for 22.5792 MHz. + * This clashes with our fixed PLL_POST_DIV_P. + */ +#define SUN50I_H616_PLL_AUDIO_REG 0x078 +static struct ccu_nm pll_audio_hs_clk = { + .enable = BIT(31), + .lock = BIT(28), + .n = _SUNXI_CCU_MULT_MIN(8, 8, 12), + .m = _SUNXI_CCU_DIV(1, 1), /* input divider */ + .common = { + .reg = 0x078, + .hw.init = CLK_HW_INIT("pll-audio-hs", "osc24M", + &ccu_nm_ops, + CLK_SET_RATE_UNGATE), + }, +}; + +static const char * const cpux_parents[] = { "osc24M", "osc32k", + "iosc", "pll-cpux", "pll-periph0" }; +static SUNXI_CCU_MUX(cpux_clk, "cpux", cpux_parents, + 0x500, 24, 3, CLK_SET_RATE_PARENT | CLK_IS_CRITICAL); +static SUNXI_CCU_M(axi_clk, "axi", "cpux", 0x500, 0, 2, 0); +static SUNXI_CCU_M(cpux_apb_clk, "cpux-apb", "cpux", 0x500, 8, 2, 0); + +static const char * const psi_ahb1_ahb2_parents[] = { "osc24M", "osc32k", + "iosc", "pll-periph0" }; +static SUNXI_CCU_MP_WITH_MUX(psi_ahb1_ahb2_clk, "psi-ahb1-ahb2", + psi_ahb1_ahb2_parents, + 0x510, + 0, 2, /* M */ + 8, 2, /* P */ + 24, 2, /* mux */ + 0); + +static const char * const ahb3_apb1_apb2_parents[] = { "osc24M", "osc32k", + "psi-ahb1-ahb2", + "pll-periph0" }; +static SUNXI_CCU_MP_WITH_MUX(ahb3_clk, "ahb3", ahb3_apb1_apb2_parents, 0x51c, + 0, 2, /* M */ + 8, 2, /* P */ + 24, 2, /* mux */ + 0); + +static SUNXI_CCU_MP_WITH_MUX(apb1_clk, "apb1", ahb3_apb1_apb2_parents, 0x520, + 0, 2, /* M */ + 8, 2, /* P */ + 24, 2, /* mux */ + 0); + +static SUNXI_CCU_MP_WITH_MUX(apb2_clk, "apb2", ahb3_apb1_apb2_parents, 0x524, + 0, 2, /* M */ + 8, 2, /* P */ + 24, 2, /* mux */ + 0); + +static const char * const mbus_parents[] = { "osc24M", "pll-periph0-2x", + "pll-ddr0", "pll-ddr1" }; +static SUNXI_CCU_M_WITH_MUX_GATE(mbus_clk, "mbus", mbus_parents, 0x540, + 0, 3, /* M */ + 24, 2, /* mux */ + BIT(31), /* gate */ + CLK_IS_CRITICAL); + +static const char * const de_parents[] = { "pll-de", "pll-periph0-2x" }; +static SUNXI_CCU_M_WITH_MUX_GATE(de_clk, "de", de_parents, 0x600, + 0, 4, /* M */ + 24, 1, /* mux */ + BIT(31), /* gate */ + CLK_SET_RATE_PARENT); + +static SUNXI_CCU_GATE(bus_de_clk, "bus-de", "psi-ahb1-ahb2", + 0x60c, BIT(0), 0); + +static SUNXI_CCU_M_WITH_MUX_GATE(deinterlace_clk, "deinterlace", + de_parents, + 0x620, + 0, 4, /* M */ + 24, 1, /* mux */ + BIT(31), /* gate */ + 0); + +static SUNXI_CCU_GATE(bus_deinterlace_clk, "bus-deinterlace", "psi-ahb1-ahb2", + 0x62c, BIT(0), 0); + +static SUNXI_CCU_M_WITH_MUX_GATE(g2d_clk, "g2d", de_parents, 0x630, + 0, 4, /* M */ + 24, 1, /* mux */ + BIT(31), /* gate */ + 0); + +static SUNXI_CCU_GATE(bus_g2d_clk, "bus-g2d", "psi-ahb1-ahb2", + 0x63c, BIT(0), 0); + +static const char * const gpu0_parents[] = { "pll-gpu", "gpu1" }; +static SUNXI_CCU_M_WITH_MUX_GATE(gpu0_clk, "gpu0", gpu0_parents, 0x670, + 0, 2, /* M */ + 24, 1, /* mux */ + BIT(31), /* gate */ + CLK_SET_RATE_PARENT); +static SUNXI_CCU_M_WITH_GATE(gpu1_clk, "gpu1", "pll-periph0-2x", 0x674, + 0, 2, /* M */ + BIT(31),/* gate */ + 0); + +static SUNXI_CCU_GATE(bus_gpu_clk, "bus-gpu", "psi-ahb1-ahb2", + 0x67c, BIT(0), 0); + +static const char * const ce_parents[] = { "osc24M", "pll-periph0-2x" }; +static SUNXI_CCU_MP_WITH_MUX_GATE(ce_clk, "ce", ce_parents, 0x680, + 0, 4, /* M */ + 8, 2, /* N */ + 24, 1, /* mux */ + BIT(31),/* gate */ + 0); + +static SUNXI_CCU_GATE(bus_ce_clk, "bus-ce", "psi-ahb1-ahb2", + 0x68c, BIT(0), 0); + +static const char * const ve_parents[] = { "pll-ve" }; +static SUNXI_CCU_M_WITH_MUX_GATE(ve_clk, "ve", ve_parents, 0x690, + 0, 3, /* M */ + 24, 1, /* mux */ + BIT(31), /* gate */ + CLK_SET_RATE_PARENT); + +static SUNXI_CCU_GATE(bus_ve_clk, "bus-ve", "psi-ahb1-ahb2", + 0x69c, BIT(0), 0); + +static SUNXI_CCU_GATE(bus_dma_clk, "bus-dma", "psi-ahb1-ahb2", + 0x70c, BIT(0), 0); + +static SUNXI_CCU_GATE(bus_hstimer_clk, "bus-hstimer", "psi-ahb1-ahb2", + 0x73c, BIT(0), 0); + +static SUNXI_CCU_GATE(avs_clk, "avs", "osc24M", 0x740, BIT(31), 0); + +static SUNXI_CCU_GATE(bus_dbg_clk, "bus-dbg", "psi-ahb1-ahb2", + 0x78c, BIT(0), 0); + +static SUNXI_CCU_GATE(bus_psi_clk, "bus-psi", "psi-ahb1-ahb2", + 0x79c, BIT(0), 0); + +static SUNXI_CCU_GATE(bus_pwm_clk, "bus-pwm", "apb1", 0x7ac, BIT(0), 0); + +static SUNXI_CCU_GATE(bus_iommu_clk, "bus-iommu", "apb1", 0x7bc, BIT(0), 0); + +static const char * const dram_parents[] = { "pll-ddr0", "pll-ddr1" }; +static struct ccu_div dram_clk = { + .div = _SUNXI_CCU_DIV(0, 2), + .mux = _SUNXI_CCU_MUX(24, 2), + .common = { + .reg = 0x800, + .hw.init = CLK_HW_INIT_PARENTS("dram", + dram_parents, + &ccu_div_ops, + CLK_IS_CRITICAL), + }, +}; + +static SUNXI_CCU_GATE(mbus_dma_clk, "mbus-dma", "mbus", + 0x804, BIT(0), 0); +static SUNXI_CCU_GATE(mbus_ve_clk, "mbus-ve", "mbus", + 0x804, BIT(1), 0); +static SUNXI_CCU_GATE(mbus_ce_clk, "mbus-ce", "mbus", + 0x804, BIT(2), 0); +static SUNXI_CCU_GATE(mbus_ts_clk, "mbus-ts", "mbus", + 0x804, BIT(3), 0); +static SUNXI_CCU_GATE(mbus_nand_clk, "mbus-nand", "mbus", + 0x804, BIT(5), 0); +static SUNXI_CCU_GATE(mbus_g2d_clk, "mbus-g2d", "mbus", + 0x804, BIT(10), 0); + +static SUNXI_CCU_GATE(bus_dram_clk, "bus-dram", "psi-ahb1-ahb2", + 0x80c, BIT(0), CLK_IS_CRITICAL); + +static const char * const nand_spi_parents[] = { "osc24M", "pll-periph0", + "pll-periph1", "pll-periph0-2x", + "pll-periph1-2x" }; +static SUNXI_CCU_MP_WITH_MUX_GATE(nand0_clk, "nand0", nand_spi_parents, 0x810, + 0, 4, /* M */ + 8, 2, /* N */ + 24, 3, /* mux */ + BIT(31),/* gate */ + 0); + +static SUNXI_CCU_MP_WITH_MUX_GATE(nand1_clk, "nand1", nand_spi_parents, 0x814, + 0, 4, /* M */ + 8, 2, /* N */ + 24, 3, /* mux */ + BIT(31),/* gate */ + 0); + +static SUNXI_CCU_GATE(bus_nand_clk, "bus-nand", "ahb3", 0x82c, BIT(0), 0); + +static const char * const mmc_parents[] = { "osc24M", "pll-periph0-2x", + "pll-periph1-2x" }; +static SUNXI_CCU_MP_WITH_MUX_GATE_POSTDIV(mmc0_clk, "mmc0", mmc_parents, 0x830, + 0, 4, /* M */ + 8, 2, /* N */ + 24, 2, /* mux */ + BIT(31), /* gate */ + 2, /* post-div */ + 0); + +static SUNXI_CCU_MP_WITH_MUX_GATE_POSTDIV(mmc1_clk, "mmc1", mmc_parents, 0x834, + 0, 4, /* M */ + 8, 2, /* N */ + 24, 2, /* mux */ + BIT(31), /* gate */ + 2, /* post-div */ + 0); + +static SUNXI_CCU_MP_WITH_MUX_GATE_POSTDIV(mmc2_clk, "mmc2", mmc_parents, 0x838, + 0, 4, /* M */ + 8, 2, /* N */ + 24, 2, /* mux */ + BIT(31), /* gate */ + 2, /* post-div */ + 0); + +static SUNXI_CCU_GATE(bus_mmc0_clk, "bus-mmc0", "ahb3", 0x84c, BIT(0), 0); +static SUNXI_CCU_GATE(bus_mmc1_clk, "bus-mmc1", "ahb3", 0x84c, BIT(1), 0); +static SUNXI_CCU_GATE(bus_mmc2_clk, "bus-mmc2", "ahb3", 0x84c, BIT(2), 0); + +static SUNXI_CCU_GATE(bus_uart0_clk, "bus-uart0", "apb2", 0x90c, BIT(0), 0); +static SUNXI_CCU_GATE(bus_uart1_clk, "bus-uart1", "apb2", 0x90c, BIT(1), 0); +static SUNXI_CCU_GATE(bus_uart2_clk, "bus-uart2", "apb2", 0x90c, BIT(2), 0); +static SUNXI_CCU_GATE(bus_uart3_clk, "bus-uart3", "apb2", 0x90c, BIT(3), 0); +static SUNXI_CCU_GATE(bus_uart4_clk, "bus-uart4", "apb2", 0x90c, BIT(4), 0); +static SUNXI_CCU_GATE(bus_uart5_clk, "bus-uart5", "apb2", 0x90c, BIT(5), 0); + +static SUNXI_CCU_GATE(bus_i2c0_clk, "bus-i2c0", "apb2", 0x91c, BIT(0), 0); +static SUNXI_CCU_GATE(bus_i2c1_clk, "bus-i2c1", "apb2", 0x91c, BIT(1), 0); +static SUNXI_CCU_GATE(bus_i2c2_clk, "bus-i2c2", "apb2", 0x91c, BIT(2), 0); +static SUNXI_CCU_GATE(bus_i2c3_clk, "bus-i2c3", "apb2", 0x91c, BIT(3), 0); +static SUNXI_CCU_GATE(bus_i2c4_clk, "bus-i2c4", "apb2", 0x91c, BIT(4), 0); + +static SUNXI_CCU_MP_WITH_MUX_GATE(spi0_clk, "spi0", nand_spi_parents, 0x940, + 0, 4, /* M */ + 8, 2, /* N */ + 24, 3, /* mux */ + BIT(31),/* gate */ + 0); + +static SUNXI_CCU_MP_WITH_MUX_GATE(spi1_clk, "spi1", nand_spi_parents, 0x944, + 0, 4, /* M */ + 8, 2, /* N */ + 24, 3, /* mux */ + BIT(31),/* gate */ + 0); + +static SUNXI_CCU_GATE(bus_spi0_clk, "bus-spi0", "ahb3", 0x96c, BIT(0), 0); +static SUNXI_CCU_GATE(bus_spi1_clk, "bus-spi1", "ahb3", 0x96c, BIT(1), 0); + +static SUNXI_CCU_GATE(emac_25m_clk, "emac-25m", "ahb3", 0x970, + BIT(31) | BIT(30), 0); + +static SUNXI_CCU_GATE(bus_emac0_clk, "bus-emac0", "ahb3", 0x97c, BIT(0), 0); +static SUNXI_CCU_GATE(bus_emac1_clk, "bus-emac1", "ahb3", 0x97c, BIT(1), 0); + +static const char * const ts_parents[] = { "osc24M", "pll-periph0" }; +static SUNXI_CCU_MP_WITH_MUX_GATE(ts_clk, "ts", ts_parents, 0x9b0, + 0, 4, /* M */ + 8, 2, /* N */ + 24, 1, /* mux */ + BIT(31),/* gate */ + 0); + +static SUNXI_CCU_GATE(bus_ts_clk, "bus-ts", "ahb3", 0x9bc, BIT(0), 0); + +static SUNXI_CCU_GATE(bus_ths_clk, "bus-ths", "apb1", 0x9fc, BIT(0), 0); + +static const char * const audio_parents[] = { "pll-audio-1x", "pll-audio-2x", + "pll-audio-4x", "pll-audio-hs" }; +static struct ccu_div spdif_clk = { + .enable = BIT(31), + .div = _SUNXI_CCU_DIV_FLAGS(8, 2, CLK_DIVIDER_POWER_OF_TWO), + .mux = _SUNXI_CCU_MUX(24, 2), + .common = { + .reg = 0xa20, + .hw.init = CLK_HW_INIT_PARENTS("spdif", + audio_parents, + &ccu_div_ops, + 0), + }, +}; + +static SUNXI_CCU_GATE(bus_spdif_clk, "bus-spdif", "apb1", 0xa2c, BIT(0), 0); + +static struct ccu_div dmic_clk = { + .enable = BIT(31), + .div = _SUNXI_CCU_DIV_FLAGS(8, 2, CLK_DIVIDER_POWER_OF_TWO), + .mux = _SUNXI_CCU_MUX(24, 2), + .common = { + .reg = 0xa40, + .hw.init = CLK_HW_INIT_PARENTS("dmic", + audio_parents, + &ccu_div_ops, + 0), + }, +}; + +static SUNXI_CCU_GATE(bus_dmic_clk, "bus-dmic", "apb1", 0xa4c, BIT(0), 0); + +static SUNXI_CCU_M_WITH_MUX_GATE(audio_codec_1x_clk, "audio-codec-1x", + audio_parents, 0xa50, + 0, 4, /* M */ + 24, 2, /* mux */ + BIT(31), /* gate */ + CLK_SET_RATE_PARENT); +static SUNXI_CCU_M_WITH_MUX_GATE(audio_codec_4x_clk, "audio-codec-4x", + audio_parents, 0xa54, + 0, 4, /* M */ + 24, 2, /* mux */ + BIT(31), /* gate */ + CLK_SET_RATE_PARENT); + +static SUNXI_CCU_GATE(bus_audio_codec_clk, "bus-audio-codec", "apb1", 0xa5c, + BIT(0), 0); + +static struct ccu_div audio_hub_clk = { + .enable = BIT(31), + .div = _SUNXI_CCU_DIV_FLAGS(8, 2, CLK_DIVIDER_POWER_OF_TWO), + .mux = _SUNXI_CCU_MUX(24, 2), + .common = { + .reg = 0xa60, + .hw.init = CLK_HW_INIT_PARENTS("audio-hub", + audio_parents, + &ccu_div_ops, + 0), + }, +}; + +static SUNXI_CCU_GATE(bus_audio_hub_clk, "bus-audio-hub", "apb1", 0xa6c, BIT(0), 0); + +/* + * There are OHCI 12M clock source selection bits for the four USB 2.0 ports. + * We will force them to 0 (12M divided from 48M). + */ +#define SUN50I_H616_USB0_CLK_REG 0xa70 +#define SUN50I_H616_USB1_CLK_REG 0xa74 +#define SUN50I_H616_USB2_CLK_REG 0xa78 +#define SUN50I_H616_USB3_CLK_REG 0xa7c + +static SUNXI_CCU_GATE(usb_ohci0_clk, "usb-ohci0", "osc12M", 0xa70, BIT(31), 0); +static SUNXI_CCU_GATE(usb_phy0_clk, "usb-phy0", "osc24M", 0xa70, BIT(29), 0); + +static SUNXI_CCU_GATE(usb_ohci1_clk, "usb-ohci1", "osc12M", 0xa74, BIT(31), 0); +static SUNXI_CCU_GATE(usb_phy1_clk, "usb-phy1", "osc24M", 0xa74, BIT(29), 0); + +static SUNXI_CCU_GATE(usb_ohci2_clk, "usb-ohci2", "osc12M", 0xa78, BIT(31), 0); +static SUNXI_CCU_GATE(usb_phy2_clk, "usb-phy2", "osc24M", 0xa78, BIT(29), 0); + +static SUNXI_CCU_GATE(usb_ohci3_clk, "usb-ohci3", "osc12M", 0xa7c, BIT(31), 0); +static SUNXI_CCU_GATE(usb_phy3_clk, "usb-phy3", "osc24M", 0xa7c, BIT(29), 0); + +static SUNXI_CCU_GATE(bus_ohci0_clk, "bus-ohci0", "ahb3", 0xa8c, BIT(0), 0); +static SUNXI_CCU_GATE(bus_ohci1_clk, "bus-ohci1", "ahb3", 0xa8c, BIT(1), 0); +static SUNXI_CCU_GATE(bus_ohci2_clk, "bus-ohci2", "ahb3", 0xa8c, BIT(2), 0); +static SUNXI_CCU_GATE(bus_ohci3_clk, "bus-ohci3", "ahb3", 0xa8c, BIT(3), 0); +static SUNXI_CCU_GATE(bus_ehci0_clk, "bus-ehci0", "ahb3", 0xa8c, BIT(4), 0); +static SUNXI_CCU_GATE(bus_ehci1_clk, "bus-ehci1", "ahb3", 0xa8c, BIT(5), 0); +static SUNXI_CCU_GATE(bus_ehci2_clk, "bus-ehci2", "ahb3", 0xa8c, BIT(6), 0); +static SUNXI_CCU_GATE(bus_ehci3_clk, "bus-ehci3", "ahb3", 0xa8c, BIT(7), 0); +static SUNXI_CCU_GATE(bus_otg_clk, "bus-otg", "ahb3", 0xa8c, BIT(8), 0); + +static SUNXI_CCU_GATE(bus_keyadc_clk, "bus-keyadc", "apb1", 0xa9c, BIT(0), 0); + +static const char * const hdmi_parents[] = { "pll-video0", "pll-video0-4x", + "pll-video2", "pll-video2-4x" }; +static SUNXI_CCU_M_WITH_MUX_GATE(hdmi_clk, "hdmi", hdmi_parents, 0xb00, + 0, 4, /* M */ + 24, 2, /* mux */ + BIT(31), /* gate */ + 0); + +static SUNXI_CCU_GATE(hdmi_slow_clk, "hdmi-slow", "osc24M", 0xb04, BIT(31), 0); + +static const char * const hdmi_cec_parents[] = { "osc32k", "pll-periph0-2x" }; +static const struct ccu_mux_fixed_prediv hdmi_cec_predivs[] = { + { .index = 1, .div = 36621 }, +}; + +#define SUN50I_H616_HDMI_CEC_CLK_REG 0xb10 +static struct ccu_mux hdmi_cec_clk = { + .enable = BIT(31) | BIT(30), + + .mux = { + .shift = 24, + .width = 2, + + .fixed_predivs = hdmi_cec_predivs, + .n_predivs = ARRAY_SIZE(hdmi_cec_predivs), + }, + + .common = { + .reg = 0xb10, + .features = CCU_FEATURE_FIXED_PREDIV, + .hw.init = CLK_HW_INIT_PARENTS("hdmi-cec", + hdmi_cec_parents, + &ccu_mux_ops, + 0), + }, +}; + +static SUNXI_CCU_GATE(bus_hdmi_clk, "bus-hdmi", "ahb3", 0xb1c, BIT(0), 0); + +static SUNXI_CCU_GATE(bus_tcon_top_clk, "bus-tcon-top", "ahb3", + 0xb5c, BIT(0), 0); + +static const char * const tcon_tv_parents[] = { "pll-video0", + "pll-video0-4x", + "pll-video1", + "pll-video1-4x" }; +static SUNXI_CCU_MP_WITH_MUX_GATE(tcon_tv0_clk, "tcon-tv0", + tcon_tv_parents, 0xb80, + 0, 4, /* M */ + 8, 2, /* P */ + 24, 3, /* mux */ + BIT(31), /* gate */ + CLK_SET_RATE_PARENT); +static SUNXI_CCU_MP_WITH_MUX_GATE(tcon_tv1_clk, "tcon-tv1", + tcon_tv_parents, 0xb84, + 0, 4, /* M */ + 8, 2, /* P */ + 24, 3, /* mux */ + BIT(31), /* gate */ + CLK_SET_RATE_PARENT); + +static SUNXI_CCU_GATE(bus_tcon_tv0_clk, "bus-tcon-tv0", "ahb3", + 0xb9c, BIT(0), 0); +static SUNXI_CCU_GATE(bus_tcon_tv1_clk, "bus-tcon-tv1", "ahb3", + 0xb9c, BIT(1), 0); + +static SUNXI_CCU_MP_WITH_MUX_GATE(tve0_clk, "tve0", + tcon_tv_parents, 0xbb0, + 0, 4, /* M */ + 8, 2, /* P */ + 24, 3, /* mux */ + BIT(31), /* gate */ + CLK_SET_RATE_PARENT); + +static SUNXI_CCU_GATE(bus_tve_top_clk, "bus-tve-top", "ahb3", + 0xbbc, BIT(0), 0); +static SUNXI_CCU_GATE(bus_tve0_clk, "bus-tve0", "ahb3", + 0xbbc, BIT(1), 0); + +static const char * const hdcp_parents[] = { "pll-periph0", "pll-periph1" }; +static SUNXI_CCU_M_WITH_MUX_GATE(hdcp_clk, "hdcp", hdcp_parents, 0xc40, + 0, 4, /* M */ + 24, 2, /* mux */ + BIT(31), /* gate */ + 0); + +static SUNXI_CCU_GATE(bus_hdcp_clk, "bus-hdcp", "ahb3", 0xc4c, BIT(0), 0); + +/* Fixed factor clocks */ +static CLK_FIXED_FACTOR_FW_NAME(osc12M_clk, "osc12M", "hosc", 2, 1, 0); + +static const struct clk_hw *clk_parent_pll_audio[] = { + &pll_audio_hs_clk.common.hw +}; + +/* + * The divider of pll-audio is fixed to 24 for now, so 24576000 and 22579200 + * rates can be set exactly in conjunction with sigma-delta modulation. + */ +static CLK_FIXED_FACTOR_HWS(pll_audio_1x_clk, "pll-audio-1x", + clk_parent_pll_audio, + 96, 1, CLK_SET_RATE_PARENT); +static CLK_FIXED_FACTOR_HWS(pll_audio_2x_clk, "pll-audio-2x", + clk_parent_pll_audio, + 48, 1, CLK_SET_RATE_PARENT); +static CLK_FIXED_FACTOR_HWS(pll_audio_4x_clk, "pll-audio-4x", + clk_parent_pll_audio, + 24, 1, CLK_SET_RATE_PARENT); + +static const struct clk_hw *pll_periph0_parents[] = { + &pll_periph0_clk.common.hw +}; + +static CLK_FIXED_FACTOR_HWS(pll_periph0_2x_clk, "pll-periph0-2x", + pll_periph0_parents, + 1, 2, 0); + +static const struct clk_hw *pll_periph1_parents[] = { + &pll_periph1_clk.common.hw +}; + +static CLK_FIXED_FACTOR_HWS(pll_periph1_2x_clk, "pll-periph1-2x", + pll_periph1_parents, + 1, 2, 0); + +static CLK_FIXED_FACTOR_HW(pll_video0_4x_clk, "pll-video0-4x", + &pll_video0_clk.common.hw, + 1, 4, CLK_SET_RATE_PARENT); +static CLK_FIXED_FACTOR_HW(pll_video1_4x_clk, "pll-video1-4x", + &pll_video1_clk.common.hw, + 1, 4, CLK_SET_RATE_PARENT); +static CLK_FIXED_FACTOR_HW(pll_video2_4x_clk, "pll-video2-4x", + &pll_video2_clk.common.hw, + 1, 4, CLK_SET_RATE_PARENT); + +static struct ccu_common *sun50i_h616_ccu_clks[] = { + &pll_cpux_clk.common, + &pll_ddr0_clk.common, + &pll_ddr1_clk.common, + &pll_periph0_clk.common, + &pll_periph1_clk.common, + &pll_gpu_clk.common, + &pll_video0_clk.common, + &pll_video1_clk.common, + &pll_video2_clk.common, + &pll_ve_clk.common, + &pll_de_clk.common, + &pll_audio_hs_clk.common, + &cpux_clk.common, + &axi_clk.common, + &cpux_apb_clk.common, + &psi_ahb1_ahb2_clk.common, + &ahb3_clk.common, + &apb1_clk.common, + &apb2_clk.common, + &mbus_clk.common, + &de_clk.common, + &bus_de_clk.common, + &deinterlace_clk.common, + &bus_deinterlace_clk.common, + &g2d_clk.common, + &bus_g2d_clk.common, + &gpu0_clk.common, + &bus_gpu_clk.common, + &gpu1_clk.common, + &ce_clk.common, + &bus_ce_clk.common, + &ve_clk.common, + &bus_ve_clk.common, + &bus_dma_clk.common, + &bus_hstimer_clk.common, + &avs_clk.common, + &bus_dbg_clk.common, + &bus_psi_clk.common, + &bus_pwm_clk.common, + &bus_iommu_clk.common, + &dram_clk.common, + &mbus_dma_clk.common, + &mbus_ve_clk.common, + &mbus_ce_clk.common, + &mbus_ts_clk.common, + &mbus_nand_clk.common, + &mbus_g2d_clk.common, + &bus_dram_clk.common, + &nand0_clk.common, + &nand1_clk.common, + &bus_nand_clk.common, + &mmc0_clk.common, + &mmc1_clk.common, + &mmc2_clk.common, + &bus_mmc0_clk.common, + &bus_mmc1_clk.common, + &bus_mmc2_clk.common, + &bus_uart0_clk.common, + &bus_uart1_clk.common, + &bus_uart2_clk.common, + &bus_uart3_clk.common, + &bus_uart4_clk.common, + &bus_uart5_clk.common, + &bus_i2c0_clk.common, + &bus_i2c1_clk.common, + &bus_i2c2_clk.common, + &bus_i2c3_clk.common, + &bus_i2c4_clk.common, + &spi0_clk.common, + &spi1_clk.common, + &bus_spi0_clk.common, + &bus_spi1_clk.common, + &emac_25m_clk.common, + &bus_emac0_clk.common, + &bus_emac1_clk.common, + &ts_clk.common, + &bus_ts_clk.common, + &bus_ths_clk.common, + &spdif_clk.common, + &bus_spdif_clk.common, + &dmic_clk.common, + &bus_dmic_clk.common, + &audio_codec_1x_clk.common, + &audio_codec_4x_clk.common, + &bus_audio_codec_clk.common, + &audio_hub_clk.common, + &bus_audio_hub_clk.common, + &usb_ohci0_clk.common, + &usb_phy0_clk.common, + &usb_ohci1_clk.common, + &usb_phy1_clk.common, + &usb_ohci2_clk.common, + &usb_phy2_clk.common, + &usb_ohci3_clk.common, + &usb_phy3_clk.common, + &bus_ohci0_clk.common, + &bus_ohci1_clk.common, + &bus_ohci2_clk.common, + &bus_ohci3_clk.common, + &bus_ehci0_clk.common, + &bus_ehci1_clk.common, + &bus_ehci2_clk.common, + &bus_ehci3_clk.common, + &bus_otg_clk.common, + &bus_keyadc_clk.common, + &hdmi_clk.common, + &hdmi_slow_clk.common, + &hdmi_cec_clk.common, + &bus_hdmi_clk.common, + &bus_tcon_top_clk.common, + &tcon_tv0_clk.common, + &tcon_tv1_clk.common, + &bus_tcon_tv0_clk.common, + &bus_tcon_tv1_clk.common, + &tve0_clk.common, + &bus_tve_top_clk.common, + &bus_tve0_clk.common, + &hdcp_clk.common, + &bus_hdcp_clk.common, +}; + +static struct clk_hw_onecell_data sun50i_h616_hw_clks = { + .hws = { + [CLK_OSC12M] = &osc12M_clk.hw, + [CLK_PLL_CPUX] = &pll_cpux_clk.common.hw, + [CLK_PLL_DDR0] = &pll_ddr0_clk.common.hw, + [CLK_PLL_DDR1] = &pll_ddr1_clk.common.hw, + [CLK_PLL_PERIPH0] = &pll_periph0_clk.common.hw, + [CLK_PLL_PERIPH0_2X] = &pll_periph0_2x_clk.hw, + [CLK_PLL_PERIPH1] = &pll_periph1_clk.common.hw, + [CLK_PLL_PERIPH1_2X] = &pll_periph1_2x_clk.hw, + [CLK_PLL_GPU] = &pll_gpu_clk.common.hw, + [CLK_PLL_VIDEO0] = &pll_video0_clk.common.hw, + [CLK_PLL_VIDEO0_4X] = &pll_video0_4x_clk.hw, + [CLK_PLL_VIDEO1] = &pll_video1_clk.common.hw, + [CLK_PLL_VIDEO1_4X] = &pll_video1_4x_clk.hw, + [CLK_PLL_VIDEO2] = &pll_video2_clk.common.hw, + [CLK_PLL_VIDEO2_4X] = &pll_video2_4x_clk.hw, + [CLK_PLL_VE] = &pll_ve_clk.common.hw, + [CLK_PLL_DE] = &pll_de_clk.common.hw, + [CLK_PLL_AUDIO_HS] = &pll_audio_hs_clk.common.hw, + [CLK_PLL_AUDIO_1X] = &pll_audio_1x_clk.hw, + [CLK_PLL_AUDIO_2X] = &pll_audio_2x_clk.hw, + [CLK_PLL_AUDIO_4X] = &pll_audio_4x_clk.hw, + [CLK_CPUX] = &cpux_clk.common.hw, + [CLK_AXI] = &axi_clk.common.hw, + [CLK_CPUX_APB] = &cpux_apb_clk.common.hw, + [CLK_PSI_AHB1_AHB2] = &psi_ahb1_ahb2_clk.common.hw, + [CLK_AHB3] = &ahb3_clk.common.hw, + [CLK_APB1] = &apb1_clk.common.hw, + [CLK_APB2] = &apb2_clk.common.hw, + [CLK_MBUS] = &mbus_clk.common.hw, + [CLK_DE] = &de_clk.common.hw, + [CLK_BUS_DE] = &bus_de_clk.common.hw, + [CLK_DEINTERLACE] = &deinterlace_clk.common.hw, + [CLK_BUS_DEINTERLACE] = &bus_deinterlace_clk.common.hw, + [CLK_G2D] = &g2d_clk.common.hw, + [CLK_BUS_G2D] = &bus_g2d_clk.common.hw, + [CLK_GPU0] = &gpu0_clk.common.hw, + [CLK_BUS_GPU] = &bus_gpu_clk.common.hw, + [CLK_GPU1] = &gpu1_clk.common.hw, + [CLK_CE] = &ce_clk.common.hw, + [CLK_BUS_CE] = &bus_ce_clk.common.hw, + [CLK_VE] = &ve_clk.common.hw, + [CLK_BUS_VE] = &bus_ve_clk.common.hw, + [CLK_BUS_DMA] = &bus_dma_clk.common.hw, + [CLK_BUS_HSTIMER] = &bus_hstimer_clk.common.hw, + [CLK_AVS] = &avs_clk.common.hw, + [CLK_BUS_DBG] = &bus_dbg_clk.common.hw, + [CLK_BUS_PSI] = &bus_psi_clk.common.hw, + [CLK_BUS_PWM] = &bus_pwm_clk.common.hw, + [CLK_BUS_IOMMU] = &bus_iommu_clk.common.hw, + [CLK_DRAM] = &dram_clk.common.hw, + [CLK_MBUS_DMA] = &mbus_dma_clk.common.hw, + [CLK_MBUS_VE] = &mbus_ve_clk.common.hw, + [CLK_MBUS_CE] = &mbus_ce_clk.common.hw, + [CLK_MBUS_TS] = &mbus_ts_clk.common.hw, + [CLK_MBUS_NAND] = &mbus_nand_clk.common.hw, + [CLK_MBUS_G2D] = &mbus_g2d_clk.common.hw, + [CLK_BUS_DRAM] = &bus_dram_clk.common.hw, + [CLK_NAND0] = &nand0_clk.common.hw, + [CLK_NAND1] = &nand1_clk.common.hw, + [CLK_BUS_NAND] = &bus_nand_clk.common.hw, + [CLK_MMC0] = &mmc0_clk.common.hw, + [CLK_MMC1] = &mmc1_clk.common.hw, + [CLK_MMC2] = &mmc2_clk.common.hw, + [CLK_BUS_MMC0] = &bus_mmc0_clk.common.hw, + [CLK_BUS_MMC1] = &bus_mmc1_clk.common.hw, + [CLK_BUS_MMC2] = &bus_mmc2_clk.common.hw, + [CLK_BUS_UART0] = &bus_uart0_clk.common.hw, + [CLK_BUS_UART1] = &bus_uart1_clk.common.hw, + [CLK_BUS_UART2] = &bus_uart2_clk.common.hw, + [CLK_BUS_UART3] = &bus_uart3_clk.common.hw, + [CLK_BUS_UART4] = &bus_uart4_clk.common.hw, + [CLK_BUS_UART5] = &bus_uart5_clk.common.hw, + [CLK_BUS_I2C0] = &bus_i2c0_clk.common.hw, + [CLK_BUS_I2C1] = &bus_i2c1_clk.common.hw, + [CLK_BUS_I2C2] = &bus_i2c2_clk.common.hw, + [CLK_BUS_I2C3] = &bus_i2c3_clk.common.hw, + [CLK_BUS_I2C4] = &bus_i2c4_clk.common.hw, + [CLK_SPI0] = &spi0_clk.common.hw, + [CLK_SPI1] = &spi1_clk.common.hw, + [CLK_BUS_SPI0] = &bus_spi0_clk.common.hw, + [CLK_BUS_SPI1] = &bus_spi1_clk.common.hw, + [CLK_EMAC_25M] = &emac_25m_clk.common.hw, + [CLK_BUS_EMAC0] = &bus_emac0_clk.common.hw, + [CLK_BUS_EMAC1] = &bus_emac1_clk.common.hw, + [CLK_TS] = &ts_clk.common.hw, + [CLK_BUS_TS] = &bus_ts_clk.common.hw, + [CLK_BUS_THS] = &bus_ths_clk.common.hw, + [CLK_SPDIF] = &spdif_clk.common.hw, + [CLK_BUS_SPDIF] = &bus_spdif_clk.common.hw, + [CLK_DMIC] = &dmic_clk.common.hw, + [CLK_BUS_DMIC] = &bus_dmic_clk.common.hw, + [CLK_AUDIO_CODEC_1X] = &audio_codec_1x_clk.common.hw, + [CLK_AUDIO_CODEC_4X] = &audio_codec_4x_clk.common.hw, + [CLK_BUS_AUDIO_CODEC] = &bus_audio_codec_clk.common.hw, + [CLK_AUDIO_HUB] = &audio_hub_clk.common.hw, + [CLK_BUS_AUDIO_HUB] = &bus_audio_hub_clk.common.hw, + [CLK_USB_OHCI0] = &usb_ohci0_clk.common.hw, + [CLK_USB_PHY0] = &usb_phy0_clk.common.hw, + [CLK_USB_OHCI1] = &usb_ohci1_clk.common.hw, + [CLK_USB_PHY1] = &usb_phy1_clk.common.hw, + [CLK_USB_OHCI2] = &usb_ohci2_clk.common.hw, + [CLK_USB_PHY2] = &usb_phy2_clk.common.hw, + [CLK_USB_OHCI3] = &usb_ohci3_clk.common.hw, + [CLK_USB_PHY3] = &usb_phy3_clk.common.hw, + [CLK_BUS_OHCI0] = &bus_ohci0_clk.common.hw, + [CLK_BUS_OHCI1] = &bus_ohci1_clk.common.hw, + [CLK_BUS_OHCI2] = &bus_ohci2_clk.common.hw, + [CLK_BUS_OHCI3] = &bus_ohci3_clk.common.hw, + [CLK_BUS_EHCI0] = &bus_ehci0_clk.common.hw, + [CLK_BUS_EHCI1] = &bus_ehci1_clk.common.hw, + [CLK_BUS_EHCI2] = &bus_ehci2_clk.common.hw, + [CLK_BUS_EHCI3] = &bus_ehci3_clk.common.hw, + [CLK_BUS_OTG] = &bus_otg_clk.common.hw, + [CLK_BUS_KEYADC] = &bus_keyadc_clk.common.hw, + [CLK_HDMI] = &hdmi_clk.common.hw, + [CLK_HDMI_SLOW] = &hdmi_slow_clk.common.hw, + [CLK_HDMI_CEC] = &hdmi_cec_clk.common.hw, + [CLK_BUS_HDMI] = &bus_hdmi_clk.common.hw, + [CLK_BUS_TCON_TOP] = &bus_tcon_top_clk.common.hw, + [CLK_TCON_TV0] = &tcon_tv0_clk.common.hw, + [CLK_TCON_TV1] = &tcon_tv1_clk.common.hw, + [CLK_BUS_TCON_TV0] = &bus_tcon_tv0_clk.common.hw, + [CLK_BUS_TCON_TV1] = &bus_tcon_tv1_clk.common.hw, + [CLK_TVE0] = &tve0_clk.common.hw, + [CLK_BUS_TVE_TOP] = &bus_tve_top_clk.common.hw, + [CLK_BUS_TVE0] = &bus_tve0_clk.common.hw, + [CLK_HDCP] = &hdcp_clk.common.hw, + [CLK_BUS_HDCP] = &bus_hdcp_clk.common.hw, + }, + .num = CLK_NUMBER, +}; + +static struct ccu_reset_map sun50i_h616_ccu_resets[] = { + [RST_MBUS] = { 0x540, BIT(30) }, + + [RST_BUS_DE] = { 0x60c, BIT(16) }, + [RST_BUS_DEINTERLACE] = { 0x62c, BIT(16) }, + [RST_BUS_GPU] = { 0x67c, BIT(16) }, + [RST_BUS_CE] = { 0x68c, BIT(16) }, + [RST_BUS_VE] = { 0x69c, BIT(16) }, + [RST_BUS_DMA] = { 0x70c, BIT(16) }, + [RST_BUS_HSTIMER] = { 0x73c, BIT(16) }, + [RST_BUS_DBG] = { 0x78c, BIT(16) }, + [RST_BUS_PSI] = { 0x79c, BIT(16) }, + [RST_BUS_PWM] = { 0x7ac, BIT(16) }, + [RST_BUS_IOMMU] = { 0x7bc, BIT(16) }, + [RST_BUS_DRAM] = { 0x80c, BIT(16) }, + [RST_BUS_NAND] = { 0x82c, BIT(16) }, + [RST_BUS_MMC0] = { 0x84c, BIT(16) }, + [RST_BUS_MMC1] = { 0x84c, BIT(17) }, + [RST_BUS_MMC2] = { 0x84c, BIT(18) }, + [RST_BUS_UART0] = { 0x90c, BIT(16) }, + [RST_BUS_UART1] = { 0x90c, BIT(17) }, + [RST_BUS_UART2] = { 0x90c, BIT(18) }, + [RST_BUS_UART3] = { 0x90c, BIT(19) }, + [RST_BUS_UART4] = { 0x90c, BIT(20) }, + [RST_BUS_UART5] = { 0x90c, BIT(21) }, + [RST_BUS_I2C0] = { 0x91c, BIT(16) }, + [RST_BUS_I2C1] = { 0x91c, BIT(17) }, + [RST_BUS_I2C2] = { 0x91c, BIT(18) }, + [RST_BUS_I2C3] = { 0x91c, BIT(19) }, + [RST_BUS_I2C4] = { 0x91c, BIT(20) }, + [RST_BUS_SPI0] = { 0x96c, BIT(16) }, + [RST_BUS_SPI1] = { 0x96c, BIT(17) }, + [RST_BUS_EMAC0] = { 0x97c, BIT(16) }, + [RST_BUS_EMAC1] = { 0x97c, BIT(17) }, + [RST_BUS_TS] = { 0x9bc, BIT(16) }, + [RST_BUS_THS] = { 0x9fc, BIT(16) }, + [RST_BUS_SPDIF] = { 0xa2c, BIT(16) }, + [RST_BUS_DMIC] = { 0xa4c, BIT(16) }, + [RST_BUS_AUDIO_CODEC] = { 0xa5c, BIT(16) }, + [RST_BUS_AUDIO_HUB] = { 0xa6c, BIT(16) }, + + [RST_USB_PHY0] = { 0xa70, BIT(30) }, + [RST_USB_PHY1] = { 0xa74, BIT(30) }, + [RST_USB_PHY2] = { 0xa78, BIT(30) }, + [RST_USB_PHY3] = { 0xa7c, BIT(30) }, + [RST_BUS_OHCI0] = { 0xa8c, BIT(16) }, + [RST_BUS_OHCI1] = { 0xa8c, BIT(17) }, + [RST_BUS_OHCI2] = { 0xa8c, BIT(18) }, + [RST_BUS_OHCI3] = { 0xa8c, BIT(19) }, + [RST_BUS_EHCI0] = { 0xa8c, BIT(20) }, + [RST_BUS_EHCI1] = { 0xa8c, BIT(21) }, + [RST_BUS_EHCI2] = { 0xa8c, BIT(22) }, + [RST_BUS_EHCI3] = { 0xa8c, BIT(23) }, + [RST_BUS_OTG] = { 0xa8c, BIT(24) }, + [RST_BUS_KEYADC] = { 0xa9c, BIT(16) }, + + [RST_BUS_HDMI] = { 0xb1c, BIT(16) }, + [RST_BUS_HDMI_SUB] = { 0xb1c, BIT(17) }, + [RST_BUS_TCON_TOP] = { 0xb5c, BIT(16) }, + [RST_BUS_TCON_TV0] = { 0xb9c, BIT(16) }, + [RST_BUS_TCON_TV1] = { 0xb9c, BIT(17) }, + [RST_BUS_TVE_TOP] = { 0xbbc, BIT(16) }, + [RST_BUS_TVE0] = { 0xbbc, BIT(17) }, + [RST_BUS_HDCP] = { 0xc4c, BIT(16) }, +}; + +static const struct sunxi_ccu_desc sun50i_h616_ccu_desc = { + .ccu_clks = sun50i_h616_ccu_clks, + .num_ccu_clks = ARRAY_SIZE(sun50i_h616_ccu_clks), + + .hw_clks = &sun50i_h616_hw_clks, + + .resets = sun50i_h616_ccu_resets, + .num_resets = ARRAY_SIZE(sun50i_h616_ccu_resets), +}; + +static const u32 pll_regs[] = { + SUN50I_H616_PLL_CPUX_REG, + SUN50I_H616_PLL_DDR0_REG, + SUN50I_H616_PLL_DDR1_REG, + SUN50I_H616_PLL_PERIPH0_REG, + SUN50I_H616_PLL_PERIPH1_REG, + SUN50I_H616_PLL_GPU_REG, + SUN50I_H616_PLL_VIDEO0_REG, + SUN50I_H616_PLL_VIDEO1_REG, + SUN50I_H616_PLL_VIDEO2_REG, + SUN50I_H616_PLL_VE_REG, + SUN50I_H616_PLL_DE_REG, + SUN50I_H616_PLL_AUDIO_REG, +}; + +static const u32 pll_video_regs[] = { + SUN50I_H616_PLL_VIDEO0_REG, + SUN50I_H616_PLL_VIDEO1_REG, + SUN50I_H616_PLL_VIDEO2_REG, +}; + +static const u32 usb2_clk_regs[] = { + SUN50I_H616_USB0_CLK_REG, + SUN50I_H616_USB1_CLK_REG, + SUN50I_H616_USB2_CLK_REG, + SUN50I_H616_USB3_CLK_REG, +}; + +static void __init sun50i_h616_ccu_setup(struct device_node *node) +{ + void __iomem *reg; + u32 val; + int i; + + reg = of_io_request_and_map(node, 0, of_node_full_name(node)); + if (IS_ERR(reg)) { + pr_err("%pOF: Could not map clock registers\n", node); + return; + } + + /* Enable the lock bits and the output enable bits on all PLLs */ + for (i = 0; i < ARRAY_SIZE(pll_regs); i++) { + val = readl(reg + pll_regs[i]); + val |= BIT(29) | BIT(27); + writel(val, reg + pll_regs[i]); + } + + /* + * Force the output divider of video PLLs to 0. + * + * See the comment before pll-video0 definition for the reason. + */ + for (i = 0; i < ARRAY_SIZE(pll_video_regs); i++) { + val = readl(reg + pll_video_regs[i]); + val &= ~BIT(0); + writel(val, reg + pll_video_regs[i]); + } + + /* + * Force OHCI 12M clock sources to 00 (12MHz divided from 48MHz) + * + * This clock mux is still mysterious, and the code just enforces + * it to have a valid clock parent. + */ + for (i = 0; i < ARRAY_SIZE(usb2_clk_regs); i++) { + val = readl(reg + usb2_clk_regs[i]); + val &= ~GENMASK(25, 24); + writel(val, reg + usb2_clk_regs[i]); + } + + /* + * Force the post-divider of pll-audio to 12 and the output divider + * of it to 2, so 24576000 and 22579200 rates can be set exactly. + */ + val = readl(reg + SUN50I_H616_PLL_AUDIO_REG); + val &= ~(GENMASK(21, 16) | BIT(0)); + writel(val | (11 << 16) | BIT(0), reg + SUN50I_H616_PLL_AUDIO_REG); + + /* + * First clock parent (osc32K) is unusable for CEC. But since there + * is no good way to force parent switch (both run with same frequency), + * just set second clock parent here. + */ + val = readl(reg + SUN50I_H616_HDMI_CEC_CLK_REG); + val |= BIT(24); + writel(val, reg + SUN50I_H616_HDMI_CEC_CLK_REG); + + i = sunxi_ccu_probe(node, reg, &sun50i_h616_ccu_desc); + if (i) + pr_err("%pOF: probing clocks fails: %d\n", node, i); +} + +CLK_OF_DECLARE(sun50i_h616_ccu, "allwinner,sun50i-h616-ccu", + sun50i_h616_ccu_setup); diff --git a/drivers/clk/sunxi-ng/ccu-sun50i-h616.h b/drivers/clk/sunxi-ng/ccu-sun50i-h616.h new file mode 100644 index 0000000000000000000000000000000000000000..dd671b413f22446255c83ab6242bcf886549ddfc --- /dev/null +++ b/drivers/clk/sunxi-ng/ccu-sun50i-h616.h @@ -0,0 +1,56 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright 2020 Arm Ltd. + */ + +#ifndef _CCU_SUN50I_H616_H_ +#define _CCU_SUN50I_H616_H_ + +#include +#include + +#define CLK_OSC12M 0 +#define CLK_PLL_CPUX 1 +#define CLK_PLL_DDR0 2 +#define CLK_PLL_DDR1 3 + +/* PLL_PERIPH0 exported for PRCM */ + +#define CLK_PLL_PERIPH0_2X 5 +#define CLK_PLL_PERIPH1 6 +#define CLK_PLL_PERIPH1_2X 7 +#define CLK_PLL_GPU 8 +#define CLK_PLL_VIDEO0 9 +#define CLK_PLL_VIDEO0_4X 10 +#define CLK_PLL_VIDEO1 11 +#define CLK_PLL_VIDEO1_4X 12 +#define CLK_PLL_VIDEO2 13 +#define CLK_PLL_VIDEO2_4X 14 +#define CLK_PLL_VE 15 +#define CLK_PLL_DE 16 +#define CLK_PLL_AUDIO_HS 17 +#define CLK_PLL_AUDIO_1X 18 +#define CLK_PLL_AUDIO_2X 19 +#define CLK_PLL_AUDIO_4X 20 + +/* CPUX clock exported for DVFS */ + +#define CLK_AXI 22 +#define CLK_CPUX_APB 23 +#define CLK_PSI_AHB1_AHB2 24 +#define CLK_AHB3 25 + +/* APB1 clock exported for PIO */ + +#define CLK_APB2 27 +#define CLK_MBUS 28 + +/* All module clocks and bus gates are exported except DRAM */ + +#define CLK_DRAM 49 + +#define CLK_BUS_DRAM 56 + +#define CLK_NUMBER (CLK_BUS_HDCP + 1) + +#endif /* _CCU_SUN50I_H616_H_ */ diff --git a/drivers/clk/sunxi/clk-a10-ve.c b/drivers/clk/sunxi/clk-a10-ve.c index cb5daa4b37db712f7eb841c2f6b8f732d5fd2eac..65810937a13a154aaf3e3897a21c43fdb110d55b 100644 --- a/drivers/clk/sunxi/clk-a10-ve.c +++ b/drivers/clk/sunxi/clk-a10-ve.c @@ -20,7 +20,7 @@ static DEFINE_SPINLOCK(ve_lock); #define SUN4I_VE_DIVIDER_WIDTH 3 #define SUN4I_VE_RESET 0 -/** +/* * sunxi_ve_reset... - reset bit in ve clk registers handling */ diff --git a/drivers/clk/sunxi/clk-mod0.c b/drivers/clk/sunxi/clk-mod0.c index 0cca91e075a53b807de627b14363904b907cd7fb..f9d715ec990876f418be13022d28a6893d7ca8e1 100644 --- a/drivers/clk/sunxi/clk-mod0.c +++ b/drivers/clk/sunxi/clk-mod0.c @@ -14,7 +14,7 @@ #include "clk-factors.h" -/** +/* * sun4i_a10_get_mod0_factors() - calculates m, n factors for MOD0-style clocks * MOD0 rate is calculated as follows * rate = (parent_rate >> p) / (m + 1); diff --git a/drivers/clk/sunxi/clk-sunxi.c b/drivers/clk/sunxi/clk-sunxi.c index e1aa1fbac48a07b55b0a25c1fd9d22c3406007ee..5fe7049ea693a2257019074d9df430c5aa90e759 100644 --- a/drivers/clk/sunxi/clk-sunxi.c +++ b/drivers/clk/sunxi/clk-sunxi.c @@ -23,7 +23,7 @@ static DEFINE_SPINLOCK(clk_lock); /* Maximum number of parents our clocks have */ #define SUNXI_MAX_PARENTS 5 -/** +/* * sun4i_get_pll1_factors() - calculates n, k, m, p factors for PLL1 * PLL1 rate is calculated as follows * rate = (parent_rate * n * (k + 1) >> p) / (m + 1); @@ -71,7 +71,7 @@ static void sun4i_get_pll1_factors(struct factors_request *req) req->n = div / 4; } -/** +/* * sun6i_a31_get_pll1_factors() - calculates n, k and m factors for PLL1 * PLL1 rate is calculated as follows * rate = parent_rate * (n + 1) * (k + 1) / (m + 1); @@ -147,7 +147,7 @@ static void sun6i_a31_get_pll1_factors(struct factors_request *req) } } -/** +/* * sun8i_a23_get_pll1_factors() - calculates n, k, m, p factors for PLL1 * PLL1 rate is calculated as follows * rate = (parent_rate * (n + 1) * (k + 1) >> p) / (m + 1); @@ -191,7 +191,7 @@ static void sun8i_a23_get_pll1_factors(struct factors_request *req) req->n = div / 4 - 1; } -/** +/* * sun4i_get_pll5_factors() - calculates n, k factors for PLL5 * PLL5 rate is calculated as follows * rate = parent_rate * n * (k + 1) @@ -218,7 +218,7 @@ static void sun4i_get_pll5_factors(struct factors_request *req) req->n = DIV_ROUND_UP(div, (req->k + 1)); } -/** +/* * sun6i_a31_get_pll6_factors() - calculates n, k factors for A31 PLL6x2 * PLL6x2 rate is calculated as follows * rate = parent_rate * (n + 1) * (k + 1) @@ -240,7 +240,7 @@ static void sun6i_a31_get_pll6_factors(struct factors_request *req) req->n = DIV_ROUND_UP(div, (req->k + 1)) - 1; } -/** +/* * sun5i_a13_get_ahb_factors() - calculates m, p factors for AHB * AHB rate is calculated as follows * rate = parent_rate >> p @@ -276,7 +276,7 @@ static void sun5i_a13_get_ahb_factors(struct factors_request *req) #define SUN6I_AHB1_PARENT_PLL6 3 -/** +/* * sun6i_a31_get_ahb_factors() - calculates m, p factors for AHB * AHB rate is calculated as follows * rate = parent_rate >> p @@ -320,7 +320,7 @@ static void sun6i_get_ahb1_factors(struct factors_request *req) req->m = calcm - 1; } -/** +/* * sun6i_ahb1_recalc() - calculates AHB clock rate from m, p factors and * parent index */ @@ -336,7 +336,7 @@ static void sun6i_ahb1_recalc(struct factors_request *req) req->rate >>= req->p; } -/** +/* * sun4i_get_apb1_factors() - calculates m, p factors for APB1 * APB1 rate is calculated as follows * rate = (parent_rate >> p) / (m + 1); @@ -375,7 +375,7 @@ static void sun4i_get_apb1_factors(struct factors_request *req) -/** +/* * sun7i_a20_get_out_factors() - calculates m, p factors for CLK_OUT_A/B * CLK_OUT rate is calculated as follows * rate = (parent_rate >> p) / (m + 1); @@ -408,7 +408,7 @@ static void sun7i_a20_get_out_factors(struct factors_request *req) req->p = calcp; } -/** +/* * sunxi_factors_clk_setup() - Setup function for factor clocks */ @@ -625,7 +625,7 @@ CLK_OF_DECLARE(sun7i_out, "allwinner,sun7i-a20-out-clk", sun7i_out_clk_setup); -/** +/* * sunxi_mux_clk_setup() - Setup function for muxes */ @@ -717,7 +717,7 @@ CLK_OF_DECLARE(sun8i_ahb2, "allwinner,sun8i-h3-ahb2-clk", sun8i_ahb2_clk_setup); -/** +/* * sunxi_divider_clk_setup() - Setup function for simple divider clocks */ @@ -853,7 +853,7 @@ CLK_OF_DECLARE(sun8i_axi, "allwinner,sun8i-a23-axi-clk", -/** +/* * sunxi_gates_clk_setup() - Setup function for leaf gates on clocks */ @@ -863,7 +863,7 @@ struct gates_data { DECLARE_BITMAP(mask, SUNXI_GATES_MAX_SIZE); }; -/** +/* * sunxi_divs_clk_setup() helper data */ @@ -929,7 +929,7 @@ static const struct divs_data sun6i_a31_pll6_divs_data __initconst = { } }; -/** +/* * sunxi_divs_clk_setup() - Setup function for leaf divisors on clocks * * These clocks look something like this diff --git a/drivers/clk/tegra/clk-tegra30.c b/drivers/clk/tegra/clk-tegra30.c index 9cf249c344d9e415877c2f1a8b89ad1fa4d5f958..16dbf83d2f62a2cf6d355313133753b6f306fc61 100644 --- a/drivers/clk/tegra/clk-tegra30.c +++ b/drivers/clk/tegra/clk-tegra30.c @@ -1104,12 +1104,9 @@ static void tegra30_cpu_out_of_reset(u32 cpu) static void tegra30_enable_cpu_clock(u32 cpu) { - unsigned int reg; - writel(CPU_CLOCK(cpu), clk_base + TEGRA30_CLK_RST_CONTROLLER_CLK_CPU_CMPLX_CLR); - reg = readl(clk_base + - TEGRA30_CLK_RST_CONTROLLER_CLK_CPU_CMPLX_CLR); + readl(clk_base + TEGRA30_CLK_RST_CONTROLLER_CLK_CPU_CMPLX_CLR); } static void tegra30_disable_cpu_clock(u32 cpu) diff --git a/drivers/clk/tegra/cvb.c b/drivers/clk/tegra/cvb.c index 21115c4e5d3a161536f66062b2e790ad56cef425..a7fdc7622913c33870635b426a745498e3932eed 100644 --- a/drivers/clk/tegra/cvb.c +++ b/drivers/clk/tegra/cvb.c @@ -86,6 +86,7 @@ static int build_opp_table(struct device *dev, const struct cvb_table *table, * @dev: the struct device * for which the OPP table is built * @tables: array of CVB tables * @count: size of the previously mentioned array + * @align: parameters of the regulator step and offset * @process_id: process id of the HW module * @speedo_id: speedo id of the HW module * @speedo_value: speedo value of the HW module diff --git a/drivers/clk/ti/clkt_dpll.c b/drivers/clk/ti/clkt_dpll.c index 87ece6cd4226b02a3c5918aec499a0af10127869..dfaa4d1f0b64b8e3606d928af7cef701be3fc677 100644 --- a/drivers/clk/ti/clkt_dpll.c +++ b/drivers/clk/ti/clkt_dpll.c @@ -269,8 +269,9 @@ unsigned long omap2_get_dpll_rate(struct clk_hw_omap *clk) /** * omap2_dpll_round_rate - round a target rate for an OMAP DPLL - * @clk: struct clk * for a DPLL + * @hw: struct clk_hw containing the struct clk * for a DPLL * @target_rate: desired DPLL clock rate + * @parent_rate: parent's DPLL clock rate * * Given a DPLL and a desired target rate, round the target rate to a * possible, programmable rate for this DPLL. Attempts to select the diff --git a/drivers/clk/ti/clockdomain.c b/drivers/clk/ti/clockdomain.c index 700b7f44f67167236beddbbcd3d4a601a2538977..74831b2752b3b432220d658a13ff60179a84d408 100644 --- a/drivers/clk/ti/clockdomain.c +++ b/drivers/clk/ti/clockdomain.c @@ -97,7 +97,7 @@ void omap2_clkops_disable_clkdm(struct clk_hw *hw) /** * omap2_init_clk_clkdm - look up a clockdomain name, store pointer in clk - * @clk: OMAP clock struct ptr to use + * @hw: Pointer to clk_hw_omap used to obtain OMAP clock struct ptr to use * * Convert a clockdomain name stored in a struct clk 'clk' into a * clockdomain pointer, and save it into the struct clk. Intended to be diff --git a/drivers/clk/ti/dpll.c b/drivers/clk/ti/dpll.c index 247510e306e2a8a19f905c5700b83cf760325850..d6f1ac5b53e14f1cba0d2e27279ded3c69a8f87f 100644 --- a/drivers/clk/ti/dpll.c +++ b/drivers/clk/ti/dpll.c @@ -151,7 +151,7 @@ static const struct clk_ops dpll_x2_ck_ops = { /** * _register_dpll - low level registration of a DPLL clock - * @hw: hardware clock definition for the clock + * @user: pointer to the hardware clock definition for the clock * @node: device node for the clock * * Finalizes DPLL registration process. In case a failure (clk-ref or diff --git a/drivers/clk/ti/dpll3xxx.c b/drivers/clk/ti/dpll3xxx.c index 2490026948b4710c3ee1467d9449c6f695964cc9..6097b099a5dff6fd5d603f4a930a3f2a347033e5 100644 --- a/drivers/clk/ti/dpll3xxx.c +++ b/drivers/clk/ti/dpll3xxx.c @@ -125,7 +125,7 @@ static u16 _omap3_dpll_compute_freqsel(struct clk_hw_omap *clk, u8 n) return f; } -/* +/** * _omap3_noncore_dpll_lock - instruct a DPLL to lock and wait for readiness * @clk: pointer to a DPLL struct clk * @@ -168,7 +168,7 @@ static int _omap3_noncore_dpll_lock(struct clk_hw_omap *clk) return r; } -/* +/** * _omap3_noncore_dpll_bypass - instruct a DPLL to bypass and wait for readiness * @clk: pointer to a DPLL struct clk * @@ -204,7 +204,7 @@ static int _omap3_noncore_dpll_bypass(struct clk_hw_omap *clk) return r; } -/* +/** * _omap3_noncore_dpll_stop - instruct a DPLL to stop * @clk: pointer to a DPLL struct clk * @@ -291,7 +291,7 @@ static void _lookup_sddiv(struct clk_hw_omap *clk, u8 *sd_div, u16 m, u8 n) *sd_div = sd; } -/* +/** * _omap3_noncore_dpll_program - set non-core DPLL M,N values directly * @clk: struct clk * of DPLL to set * @freqsel: FREQSEL value to set @@ -406,7 +406,8 @@ static int omap3_noncore_dpll_program(struct clk_hw_omap *clk, u16 freqsel) /** * omap3_dpll_recalc - recalculate DPLL rate - * @clk: DPLL struct clk + * @hw: struct clk_hw containing the DPLL struct clk + * @parent_rate: clock rate of the DPLL parent * * Recalculate and propagate the DPLL rate. */ @@ -421,7 +422,7 @@ unsigned long omap3_dpll_recalc(struct clk_hw *hw, unsigned long parent_rate) /** * omap3_noncore_dpll_enable - instruct a DPLL to enter bypass or lock mode - * @clk: pointer to a DPLL struct clk + * @hw: struct clk_hw containing then pointer to a DPLL struct clk * * Instructs a non-CORE DPLL to enable, e.g., to enter bypass or lock. * The choice of modes depends on the DPLL's programmed rate: if it is @@ -470,7 +471,7 @@ int omap3_noncore_dpll_enable(struct clk_hw *hw) /** * omap3_noncore_dpll_disable - instruct a DPLL to enter low-power stop - * @clk: pointer to a DPLL struct clk + * @hw: struct clk_hw containing then pointer to a DPLL struct clk * * Instructs a non-CORE DPLL to enter low-power stop. This function is * intended for use in struct clkops. No return value. @@ -745,7 +746,8 @@ static struct clk_hw_omap *omap3_find_clkoutx2_dpll(struct clk_hw *hw) /** * omap3_clkoutx2_recalc - recalculate DPLL X2 output virtual clock rate - * @clk: DPLL output struct clk + * @hw: pointer struct clk_hw + * @parent_rate: clock rate of the DPLL parent * * Using parent clock DPLL data, look up DPLL state. If locked, set our * rate to the dpll_clk * 2; otherwise, just use dpll_clk. @@ -913,7 +915,7 @@ const struct clk_hw_omap_ops clkhwops_omap3_dpll = { * omap3_dpll4_set_rate - set rate for omap3 per-dpll * @hw: clock to change * @rate: target rate for clock - * @parent_rate: rate of the parent clock + * @parent_rate: clock rate of the DPLL parent * * Check if the current SoC supports the per-dpll reprogram operation * or not, and then do the rate change if supported. Returns -EINVAL diff --git a/drivers/clk/ti/dpll44xx.c b/drivers/clk/ti/dpll44xx.c index 89c3ed1a24b829b9f246abc4cbac9817f92df150..3fc2cab69a3fe6cfadb3d6bbb3ff87d434c32e6a 100644 --- a/drivers/clk/ti/dpll44xx.c +++ b/drivers/clk/ti/dpll44xx.c @@ -102,7 +102,8 @@ static void omap4_dpll_lpmode_recalc(struct dpll_data *dd) /** * omap4_dpll_regm4xen_recalc - compute DPLL rate, considering REGM4XEN bit - * @clk: struct clk * of the DPLL to compute the rate for + * @hw: pointer to the clock to compute the rate for + * @parent_rate: clock rate of the DPLL parent * * Compute the output rate for the OMAP4 DPLL represented by @clk. * Takes the REGM4XEN bit into consideration, which is needed for the @@ -134,8 +135,9 @@ unsigned long omap4_dpll_regm4xen_recalc(struct clk_hw *hw, /** * omap4_dpll_regm4xen_round_rate - round DPLL rate, considering REGM4XEN bit - * @clk: struct clk * of the DPLL to round a rate for + * @hw: struct hw_clk containing the struct clk * of the DPLL to round a rate for * @target_rate: the desired rate of the DPLL + * @parent_rate: clock rate of the DPLL parent * * Compute the rate that would be programmed into the DPLL hardware * for @clk if set_rate() were to be provided with the rate diff --git a/drivers/clk/ti/gate.c b/drivers/clk/ti/gate.c index 42389558418c58339ee51c292fc5b29f509f6d77..b1d0fdb40a75a0382db4ce1d0cf073953f720ad9 100644 --- a/drivers/clk/ti/gate.c +++ b/drivers/clk/ti/gate.c @@ -55,7 +55,7 @@ static const struct clk_ops omap_gate_clk_hsdiv_restore_ops = { /** * omap36xx_gate_clk_enable_with_hsdiv_restore - enable clocks suffering * from HSDivider PWRDN problem Implements Errata ID: i556. - * @clk: DPLL output struct clk + * @hw: DPLL output struct clk_hw * * 3630 only: dpll3_m3_ck, dpll4_m2_ck, dpll4_m3_ck, dpll4_m4_ck, * dpll4_m5_ck & dpll4_m6_ck dividers gets loaded with reset diff --git a/drivers/clk/versatile/clk-icst.c b/drivers/clk/versatile/clk-icst.c index 692be2fd9261a8ca9341cfca583161c52591b8b1..fdd6aa3cb1fc85544cb6345beb5e8e18eabf2c99 100644 --- a/drivers/clk/versatile/clk-icst.c +++ b/drivers/clk/versatile/clk-icst.c @@ -36,8 +36,9 @@ /** * struct clk_icst - ICST VCO clock wrapper * @hw: corresponding clock hardware entry - * @vcoreg: VCO register address - * @lockreg: VCO lock register address + * @map: register map + * @vcoreg_off: VCO register address + * @lockreg_off: VCO lock register address * @params: parameters for this ICST instance * @rate: current rate * @ctype: the type of control register for the ICST @@ -428,7 +429,7 @@ static const struct icst_params icst307_params = { .idx2s = icst307_idx2s, }; -/** +/* * The core modules on the Integrator/AP and Integrator/CP have * especially crippled ICST525 control. */ diff --git a/drivers/clk/xilinx/Kconfig b/drivers/clk/xilinx/Kconfig new file mode 100644 index 0000000000000000000000000000000000000000..5224114176ed60db3b4168467894c1453b481473 --- /dev/null +++ b/drivers/clk/xilinx/Kconfig @@ -0,0 +1,19 @@ +# SPDX-License-Identifier: GPL-2.0 + +config XILINX_VCU + tristate "Xilinx VCU logicoreIP Init" + depends on HAS_IOMEM && COMMON_CLK + select REGMAP_MMIO + help + Provides the driver to enable and disable the isolation between the + processing system and programmable logic part by using the logicoreIP + register set. This driver also configures the frequency based on the + clock information from the logicoreIP register set. + + If you say yes here you get support for the logicoreIP. + + If unsure, say N. + + To compile this driver as a module, choose M here: the + module will be called xlnx_vcu. + diff --git a/drivers/clk/xilinx/Makefile b/drivers/clk/xilinx/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..dee8fd51e303281968e2c2f198d403a5ac573473 --- /dev/null +++ b/drivers/clk/xilinx/Makefile @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_XILINX_VCU) += xlnx_vcu.o diff --git a/drivers/clk/xilinx/xlnx_vcu.c b/drivers/clk/xilinx/xlnx_vcu.c new file mode 100644 index 0000000000000000000000000000000000000000..d66b1315114e652370c16768dc3e3b0228800843 --- /dev/null +++ b/drivers/clk/xilinx/xlnx_vcu.c @@ -0,0 +1,743 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Xilinx VCU Init + * + * Copyright (C) 2016 - 2017 Xilinx, Inc. + * + * Contacts Dhaval Shah + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#define VCU_PLL_CTRL 0x24 +#define VCU_PLL_CTRL_RESET BIT(0) +#define VCU_PLL_CTRL_POR_IN BIT(1) +#define VCU_PLL_CTRL_PWR_POR BIT(2) +#define VCU_PLL_CTRL_BYPASS BIT(3) +#define VCU_PLL_CTRL_FBDIV GENMASK(14, 8) +#define VCU_PLL_CTRL_CLKOUTDIV GENMASK(18, 16) + +#define VCU_PLL_CFG 0x28 +#define VCU_PLL_CFG_RES GENMASK(3, 0) +#define VCU_PLL_CFG_CP GENMASK(8, 5) +#define VCU_PLL_CFG_LFHF GENMASK(12, 10) +#define VCU_PLL_CFG_LOCK_CNT GENMASK(22, 13) +#define VCU_PLL_CFG_LOCK_DLY GENMASK(31, 25) +#define VCU_ENC_CORE_CTRL 0x30 +#define VCU_ENC_MCU_CTRL 0x34 +#define VCU_DEC_CORE_CTRL 0x38 +#define VCU_DEC_MCU_CTRL 0x3c +#define VCU_PLL_STATUS 0x60 +#define VCU_PLL_STATUS_LOCK_STATUS BIT(0) + +#define MHZ 1000000 +#define FVCO_MIN (1500U * MHZ) +#define FVCO_MAX (3000U * MHZ) + +/** + * struct xvcu_device - Xilinx VCU init device structure + * @dev: Platform device + * @pll_ref: pll ref clock source + * @aclk: axi clock source + * @logicore_reg_ba: logicore reg base address + * @vcu_slcr_ba: vcu_slcr Register base address + * @pll: handle for the VCU PLL + * @pll_post: handle for the VCU PLL post divider + * @clk_data: clocks provided by the vcu clock provider + */ +struct xvcu_device { + struct device *dev; + struct clk *pll_ref; + struct clk *aclk; + struct regmap *logicore_reg_ba; + void __iomem *vcu_slcr_ba; + struct clk_hw *pll; + struct clk_hw *pll_post; + struct clk_hw_onecell_data *clk_data; +}; + +static struct regmap_config vcu_settings_regmap_config = { + .name = "regmap", + .reg_bits = 32, + .val_bits = 32, + .reg_stride = 4, + .max_register = 0xfff, + .cache_type = REGCACHE_NONE, +}; + +/** + * struct xvcu_pll_cfg - Helper data + * @fbdiv: The integer portion of the feedback divider to the PLL + * @cp: PLL charge pump control + * @res: PLL loop filter resistor control + * @lfhf: PLL loop filter high frequency capacitor control + * @lock_dly: Lock circuit configuration settings for lock windowsize + * @lock_cnt: Lock circuit counter setting + */ +struct xvcu_pll_cfg { + u32 fbdiv; + u32 cp; + u32 res; + u32 lfhf; + u32 lock_dly; + u32 lock_cnt; +}; + +static const struct xvcu_pll_cfg xvcu_pll_cfg[] = { + { 25, 3, 10, 3, 63, 1000 }, + { 26, 3, 10, 3, 63, 1000 }, + { 27, 4, 6, 3, 63, 1000 }, + { 28, 4, 6, 3, 63, 1000 }, + { 29, 4, 6, 3, 63, 1000 }, + { 30, 4, 6, 3, 63, 1000 }, + { 31, 6, 1, 3, 63, 1000 }, + { 32, 6, 1, 3, 63, 1000 }, + { 33, 4, 10, 3, 63, 1000 }, + { 34, 5, 6, 3, 63, 1000 }, + { 35, 5, 6, 3, 63, 1000 }, + { 36, 5, 6, 3, 63, 1000 }, + { 37, 5, 6, 3, 63, 1000 }, + { 38, 5, 6, 3, 63, 975 }, + { 39, 3, 12, 3, 63, 950 }, + { 40, 3, 12, 3, 63, 925 }, + { 41, 3, 12, 3, 63, 900 }, + { 42, 3, 12, 3, 63, 875 }, + { 43, 3, 12, 3, 63, 850 }, + { 44, 3, 12, 3, 63, 850 }, + { 45, 3, 12, 3, 63, 825 }, + { 46, 3, 12, 3, 63, 800 }, + { 47, 3, 12, 3, 63, 775 }, + { 48, 3, 12, 3, 63, 775 }, + { 49, 3, 12, 3, 63, 750 }, + { 50, 3, 12, 3, 63, 750 }, + { 51, 3, 2, 3, 63, 725 }, + { 52, 3, 2, 3, 63, 700 }, + { 53, 3, 2, 3, 63, 700 }, + { 54, 3, 2, 3, 63, 675 }, + { 55, 3, 2, 3, 63, 675 }, + { 56, 3, 2, 3, 63, 650 }, + { 57, 3, 2, 3, 63, 650 }, + { 58, 3, 2, 3, 63, 625 }, + { 59, 3, 2, 3, 63, 625 }, + { 60, 3, 2, 3, 63, 625 }, + { 61, 3, 2, 3, 63, 600 }, + { 62, 3, 2, 3, 63, 600 }, + { 63, 3, 2, 3, 63, 600 }, + { 64, 3, 2, 3, 63, 600 }, + { 65, 3, 2, 3, 63, 600 }, + { 66, 3, 2, 3, 63, 600 }, + { 67, 3, 2, 3, 63, 600 }, + { 68, 3, 2, 3, 63, 600 }, + { 69, 3, 2, 3, 63, 600 }, + { 70, 3, 2, 3, 63, 600 }, + { 71, 3, 2, 3, 63, 600 }, + { 72, 3, 2, 3, 63, 600 }, + { 73, 3, 2, 3, 63, 600 }, + { 74, 3, 2, 3, 63, 600 }, + { 75, 3, 2, 3, 63, 600 }, + { 76, 3, 2, 3, 63, 600 }, + { 77, 3, 2, 3, 63, 600 }, + { 78, 3, 2, 3, 63, 600 }, + { 79, 3, 2, 3, 63, 600 }, + { 80, 3, 2, 3, 63, 600 }, + { 81, 3, 2, 3, 63, 600 }, + { 82, 3, 2, 3, 63, 600 }, + { 83, 4, 2, 3, 63, 600 }, + { 84, 4, 2, 3, 63, 600 }, + { 85, 4, 2, 3, 63, 600 }, + { 86, 4, 2, 3, 63, 600 }, + { 87, 4, 2, 3, 63, 600 }, + { 88, 4, 2, 3, 63, 600 }, + { 89, 4, 2, 3, 63, 600 }, + { 90, 4, 2, 3, 63, 600 }, + { 91, 4, 2, 3, 63, 600 }, + { 92, 4, 2, 3, 63, 600 }, + { 93, 4, 2, 3, 63, 600 }, + { 94, 4, 2, 3, 63, 600 }, + { 95, 4, 2, 3, 63, 600 }, + { 96, 4, 2, 3, 63, 600 }, + { 97, 4, 2, 3, 63, 600 }, + { 98, 4, 2, 3, 63, 600 }, + { 99, 4, 2, 3, 63, 600 }, + { 100, 4, 2, 3, 63, 600 }, + { 101, 4, 2, 3, 63, 600 }, + { 102, 4, 2, 3, 63, 600 }, + { 103, 5, 2, 3, 63, 600 }, + { 104, 5, 2, 3, 63, 600 }, + { 105, 5, 2, 3, 63, 600 }, + { 106, 5, 2, 3, 63, 600 }, + { 107, 3, 4, 3, 63, 600 }, + { 108, 3, 4, 3, 63, 600 }, + { 109, 3, 4, 3, 63, 600 }, + { 110, 3, 4, 3, 63, 600 }, + { 111, 3, 4, 3, 63, 600 }, + { 112, 3, 4, 3, 63, 600 }, + { 113, 3, 4, 3, 63, 600 }, + { 114, 3, 4, 3, 63, 600 }, + { 115, 3, 4, 3, 63, 600 }, + { 116, 3, 4, 3, 63, 600 }, + { 117, 3, 4, 3, 63, 600 }, + { 118, 3, 4, 3, 63, 600 }, + { 119, 3, 4, 3, 63, 600 }, + { 120, 3, 4, 3, 63, 600 }, + { 121, 3, 4, 3, 63, 600 }, + { 122, 3, 4, 3, 63, 600 }, + { 123, 3, 4, 3, 63, 600 }, + { 124, 3, 4, 3, 63, 600 }, + { 125, 3, 4, 3, 63, 600 }, +}; + +/** + * xvcu_read - Read from the VCU register space + * @iomem: vcu reg space base address + * @offset: vcu reg offset from base + * + * Return: Returns 32bit value from VCU register specified + * + */ +static inline u32 xvcu_read(void __iomem *iomem, u32 offset) +{ + return ioread32(iomem + offset); +} + +/** + * xvcu_write - Write to the VCU register space + * @iomem: vcu reg space base address + * @offset: vcu reg offset from base + * @value: Value to write + */ +static inline void xvcu_write(void __iomem *iomem, u32 offset, u32 value) +{ + iowrite32(value, iomem + offset); +} + +#define to_vcu_pll(_hw) container_of(_hw, struct vcu_pll, hw) + +struct vcu_pll { + struct clk_hw hw; + void __iomem *reg_base; + unsigned long fvco_min; + unsigned long fvco_max; +}; + +static int xvcu_pll_wait_for_lock(struct vcu_pll *pll) +{ + void __iomem *base = pll->reg_base; + unsigned long timeout; + u32 lock_status; + + timeout = jiffies + msecs_to_jiffies(2000); + do { + lock_status = xvcu_read(base, VCU_PLL_STATUS); + if (lock_status & VCU_PLL_STATUS_LOCK_STATUS) + return 0; + } while (!time_after(jiffies, timeout)); + + return -ETIMEDOUT; +} + +static struct clk_hw *xvcu_register_pll_post(struct device *dev, + const char *name, + const struct clk_hw *parent_hw, + void __iomem *reg_base) +{ + u32 div; + u32 vcu_pll_ctrl; + + /* + * The output divider of the PLL must be set to 1/2 to meet the + * timing in the design. + */ + vcu_pll_ctrl = xvcu_read(reg_base, VCU_PLL_CTRL); + div = FIELD_GET(VCU_PLL_CTRL_CLKOUTDIV, vcu_pll_ctrl); + if (div != 1) + return ERR_PTR(-EINVAL); + + return clk_hw_register_fixed_factor(dev, "vcu_pll_post", + clk_hw_get_name(parent_hw), + CLK_SET_RATE_PARENT, 1, 2); +} + +static const struct xvcu_pll_cfg *xvcu_find_cfg(int div) +{ + const struct xvcu_pll_cfg *cfg = NULL; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(xvcu_pll_cfg) - 1; i++) + if (xvcu_pll_cfg[i].fbdiv == div) + cfg = &xvcu_pll_cfg[i]; + + return cfg; +} + +static int xvcu_pll_set_div(struct vcu_pll *pll, int div) +{ + void __iomem *base = pll->reg_base; + const struct xvcu_pll_cfg *cfg = NULL; + u32 vcu_pll_ctrl; + u32 cfg_val; + + cfg = xvcu_find_cfg(div); + if (!cfg) + return -EINVAL; + + vcu_pll_ctrl = xvcu_read(base, VCU_PLL_CTRL); + vcu_pll_ctrl &= ~VCU_PLL_CTRL_FBDIV; + vcu_pll_ctrl |= FIELD_PREP(VCU_PLL_CTRL_FBDIV, cfg->fbdiv); + xvcu_write(base, VCU_PLL_CTRL, vcu_pll_ctrl); + + cfg_val = FIELD_PREP(VCU_PLL_CFG_RES, cfg->res) | + FIELD_PREP(VCU_PLL_CFG_CP, cfg->cp) | + FIELD_PREP(VCU_PLL_CFG_LFHF, cfg->lfhf) | + FIELD_PREP(VCU_PLL_CFG_LOCK_CNT, cfg->lock_cnt) | + FIELD_PREP(VCU_PLL_CFG_LOCK_DLY, cfg->lock_dly); + xvcu_write(base, VCU_PLL_CFG, cfg_val); + + return 0; +} + +static long xvcu_pll_round_rate(struct clk_hw *hw, + unsigned long rate, unsigned long *parent_rate) +{ + struct vcu_pll *pll = to_vcu_pll(hw); + unsigned int feedback_div; + + rate = clamp_t(unsigned long, rate, pll->fvco_min, pll->fvco_max); + + feedback_div = DIV_ROUND_CLOSEST_ULL(rate, *parent_rate); + feedback_div = clamp_t(unsigned int, feedback_div, 25, 125); + + return *parent_rate * feedback_div; +} + +static unsigned long xvcu_pll_recalc_rate(struct clk_hw *hw, + unsigned long parent_rate) +{ + struct vcu_pll *pll = to_vcu_pll(hw); + void __iomem *base = pll->reg_base; + unsigned int div; + u32 vcu_pll_ctrl; + + vcu_pll_ctrl = xvcu_read(base, VCU_PLL_CTRL); + div = FIELD_GET(VCU_PLL_CTRL_FBDIV, vcu_pll_ctrl); + + return div * parent_rate; +} + +static int xvcu_pll_set_rate(struct clk_hw *hw, + unsigned long rate, unsigned long parent_rate) +{ + struct vcu_pll *pll = to_vcu_pll(hw); + + return xvcu_pll_set_div(pll, rate / parent_rate); +} + +static int xvcu_pll_enable(struct clk_hw *hw) +{ + struct vcu_pll *pll = to_vcu_pll(hw); + void __iomem *base = pll->reg_base; + u32 vcu_pll_ctrl; + int ret; + + vcu_pll_ctrl = xvcu_read(base, VCU_PLL_CTRL); + vcu_pll_ctrl |= VCU_PLL_CTRL_BYPASS; + xvcu_write(base, VCU_PLL_CTRL, vcu_pll_ctrl); + + vcu_pll_ctrl = xvcu_read(base, VCU_PLL_CTRL); + vcu_pll_ctrl &= ~VCU_PLL_CTRL_POR_IN; + vcu_pll_ctrl &= ~VCU_PLL_CTRL_PWR_POR; + vcu_pll_ctrl &= ~VCU_PLL_CTRL_RESET; + xvcu_write(base, VCU_PLL_CTRL, vcu_pll_ctrl); + + ret = xvcu_pll_wait_for_lock(pll); + if (ret) { + pr_err("VCU PLL is not locked\n"); + goto err; + } + + vcu_pll_ctrl = xvcu_read(base, VCU_PLL_CTRL); + vcu_pll_ctrl &= ~VCU_PLL_CTRL_BYPASS; + xvcu_write(base, VCU_PLL_CTRL, vcu_pll_ctrl); + +err: + return ret; +} + +static void xvcu_pll_disable(struct clk_hw *hw) +{ + struct vcu_pll *pll = to_vcu_pll(hw); + void __iomem *base = pll->reg_base; + u32 vcu_pll_ctrl; + + vcu_pll_ctrl = xvcu_read(base, VCU_PLL_CTRL); + vcu_pll_ctrl |= VCU_PLL_CTRL_POR_IN; + vcu_pll_ctrl |= VCU_PLL_CTRL_PWR_POR; + vcu_pll_ctrl |= VCU_PLL_CTRL_RESET; + xvcu_write(base, VCU_PLL_CTRL, vcu_pll_ctrl); +} + +static const struct clk_ops vcu_pll_ops = { + .enable = xvcu_pll_enable, + .disable = xvcu_pll_disable, + .round_rate = xvcu_pll_round_rate, + .recalc_rate = xvcu_pll_recalc_rate, + .set_rate = xvcu_pll_set_rate, +}; + +static struct clk_hw *xvcu_register_pll(struct device *dev, + void __iomem *reg_base, + const char *name, const char *parent, + unsigned long flags) +{ + struct vcu_pll *pll; + struct clk_hw *hw; + struct clk_init_data init; + int ret; + + init.name = name; + init.parent_names = &parent; + init.ops = &vcu_pll_ops; + init.num_parents = 1; + init.flags = flags; + + pll = devm_kmalloc(dev, sizeof(*pll), GFP_KERNEL); + if (!pll) + return ERR_PTR(-ENOMEM); + + pll->hw.init = &init; + pll->reg_base = reg_base; + pll->fvco_min = FVCO_MIN; + pll->fvco_max = FVCO_MAX; + + hw = &pll->hw; + ret = devm_clk_hw_register(dev, hw); + if (ret) + return ERR_PTR(ret); + + clk_hw_set_rate_range(hw, pll->fvco_min, pll->fvco_max); + + return hw; +} + +static struct clk_hw *xvcu_clk_hw_register_leaf(struct device *dev, + const char *name, + const struct clk_parent_data *parent_data, + u8 num_parents, + void __iomem *reg) +{ + u8 mux_flags = CLK_MUX_ROUND_CLOSEST; + u8 divider_flags = CLK_DIVIDER_ONE_BASED | CLK_DIVIDER_ALLOW_ZERO | + CLK_DIVIDER_ROUND_CLOSEST; + struct clk_hw *mux = NULL; + struct clk_hw *divider = NULL; + struct clk_hw *gate = NULL; + char *name_mux; + char *name_div; + int err; + /* Protect register shared by clocks */ + spinlock_t *lock; + + lock = devm_kzalloc(dev, sizeof(*lock), GFP_KERNEL); + if (!lock) + return ERR_PTR(-ENOMEM); + spin_lock_init(lock); + + name_mux = devm_kasprintf(dev, GFP_KERNEL, "%s%s", name, "_mux"); + if (!name_mux) + return ERR_PTR(-ENOMEM); + mux = clk_hw_register_mux_parent_data(dev, name_mux, + parent_data, num_parents, + CLK_SET_RATE_PARENT, + reg, 0, 1, mux_flags, lock); + if (IS_ERR(mux)) + return mux; + + name_div = devm_kasprintf(dev, GFP_KERNEL, "%s%s", name, "_div"); + if (!name_div) { + err = -ENOMEM; + goto unregister_mux; + } + divider = clk_hw_register_divider_parent_hw(dev, name_div, mux, + CLK_SET_RATE_PARENT, + reg, 4, 6, divider_flags, + lock); + if (IS_ERR(divider)) { + err = PTR_ERR(divider); + goto unregister_mux; + } + + gate = clk_hw_register_gate_parent_hw(dev, name, divider, + CLK_SET_RATE_PARENT, reg, 12, 0, + lock); + if (IS_ERR(gate)) { + err = PTR_ERR(gate); + goto unregister_divider; + } + + return gate; + +unregister_divider: + clk_hw_unregister_divider(divider); +unregister_mux: + clk_hw_unregister_mux(mux); + + return ERR_PTR(err); +} + +static void xvcu_clk_hw_unregister_leaf(struct clk_hw *hw) +{ + struct clk_hw *gate = hw; + struct clk_hw *divider; + struct clk_hw *mux; + + if (!gate) + return; + + divider = clk_hw_get_parent(gate); + clk_hw_unregister_gate(gate); + if (!divider) + return; + + mux = clk_hw_get_parent(divider); + clk_hw_unregister_mux(mux); + if (!divider) + return; + + clk_hw_unregister_divider(divider); +} + +static int xvcu_register_clock_provider(struct xvcu_device *xvcu) +{ + struct device *dev = xvcu->dev; + struct clk_parent_data parent_data[2] = { 0 }; + struct clk_hw_onecell_data *data; + struct clk_hw **hws; + struct clk_hw *hw; + void __iomem *reg_base = xvcu->vcu_slcr_ba; + + data = devm_kzalloc(dev, struct_size(data, hws, CLK_XVCU_NUM_CLOCKS), GFP_KERNEL); + if (!data) + return -ENOMEM; + data->num = CLK_XVCU_NUM_CLOCKS; + hws = data->hws; + + xvcu->clk_data = data; + + hw = xvcu_register_pll(dev, reg_base, + "vcu_pll", __clk_get_name(xvcu->pll_ref), + CLK_SET_RATE_NO_REPARENT | CLK_OPS_PARENT_ENABLE); + if (IS_ERR(hw)) + return PTR_ERR(hw); + xvcu->pll = hw; + + hw = xvcu_register_pll_post(dev, "vcu_pll_post", xvcu->pll, reg_base); + if (IS_ERR(hw)) + return PTR_ERR(hw); + xvcu->pll_post = hw; + + parent_data[0].fw_name = "pll_ref"; + parent_data[1].hw = xvcu->pll_post; + + hws[CLK_XVCU_ENC_CORE] = + xvcu_clk_hw_register_leaf(dev, "venc_core_clk", + parent_data, + ARRAY_SIZE(parent_data), + reg_base + VCU_ENC_CORE_CTRL); + hws[CLK_XVCU_ENC_MCU] = + xvcu_clk_hw_register_leaf(dev, "venc_mcu_clk", + parent_data, + ARRAY_SIZE(parent_data), + reg_base + VCU_ENC_MCU_CTRL); + hws[CLK_XVCU_DEC_CORE] = + xvcu_clk_hw_register_leaf(dev, "vdec_core_clk", + parent_data, + ARRAY_SIZE(parent_data), + reg_base + VCU_DEC_CORE_CTRL); + hws[CLK_XVCU_DEC_MCU] = + xvcu_clk_hw_register_leaf(dev, "vdec_mcu_clk", + parent_data, + ARRAY_SIZE(parent_data), + reg_base + VCU_DEC_MCU_CTRL); + + return devm_of_clk_add_hw_provider(dev, of_clk_hw_onecell_get, data); +} + +static void xvcu_unregister_clock_provider(struct xvcu_device *xvcu) +{ + struct clk_hw_onecell_data *data = xvcu->clk_data; + struct clk_hw **hws = data->hws; + + if (!IS_ERR_OR_NULL(hws[CLK_XVCU_DEC_MCU])) + xvcu_clk_hw_unregister_leaf(hws[CLK_XVCU_DEC_MCU]); + if (!IS_ERR_OR_NULL(hws[CLK_XVCU_DEC_CORE])) + xvcu_clk_hw_unregister_leaf(hws[CLK_XVCU_DEC_CORE]); + if (!IS_ERR_OR_NULL(hws[CLK_XVCU_ENC_MCU])) + xvcu_clk_hw_unregister_leaf(hws[CLK_XVCU_ENC_MCU]); + if (!IS_ERR_OR_NULL(hws[CLK_XVCU_ENC_CORE])) + xvcu_clk_hw_unregister_leaf(hws[CLK_XVCU_ENC_CORE]); + + clk_hw_unregister_fixed_factor(xvcu->pll_post); +} + +/** + * xvcu_probe - Probe existence of the logicoreIP + * and initialize PLL + * + * @pdev: Pointer to the platform_device structure + * + * Return: Returns 0 on success + * Negative error code otherwise + */ +static int xvcu_probe(struct platform_device *pdev) +{ + struct resource *res; + struct xvcu_device *xvcu; + void __iomem *regs; + int ret; + + xvcu = devm_kzalloc(&pdev->dev, sizeof(*xvcu), GFP_KERNEL); + if (!xvcu) + return -ENOMEM; + + xvcu->dev = &pdev->dev; + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "vcu_slcr"); + if (!res) { + dev_err(&pdev->dev, "get vcu_slcr memory resource failed.\n"); + return -ENODEV; + } + + xvcu->vcu_slcr_ba = devm_ioremap(&pdev->dev, res->start, + resource_size(res)); + if (!xvcu->vcu_slcr_ba) { + dev_err(&pdev->dev, "vcu_slcr register mapping failed.\n"); + return -ENOMEM; + } + + xvcu->logicore_reg_ba = + syscon_regmap_lookup_by_compatible("xlnx,vcu-settings"); + if (IS_ERR(xvcu->logicore_reg_ba)) { + dev_info(&pdev->dev, + "could not find xlnx,vcu-settings: trying direct register access\n"); + + res = platform_get_resource_byname(pdev, + IORESOURCE_MEM, "logicore"); + if (!res) { + dev_err(&pdev->dev, "get logicore memory resource failed.\n"); + return -ENODEV; + } + + regs = devm_ioremap(&pdev->dev, res->start, resource_size(res)); + if (!regs) { + dev_err(&pdev->dev, "logicore register mapping failed.\n"); + return -ENOMEM; + } + + xvcu->logicore_reg_ba = + devm_regmap_init_mmio(&pdev->dev, regs, + &vcu_settings_regmap_config); + if (IS_ERR(xvcu->logicore_reg_ba)) { + dev_err(&pdev->dev, "failed to init regmap\n"); + return PTR_ERR(xvcu->logicore_reg_ba); + } + } + + xvcu->aclk = devm_clk_get(&pdev->dev, "aclk"); + if (IS_ERR(xvcu->aclk)) { + dev_err(&pdev->dev, "Could not get aclk clock\n"); + return PTR_ERR(xvcu->aclk); + } + + xvcu->pll_ref = devm_clk_get(&pdev->dev, "pll_ref"); + if (IS_ERR(xvcu->pll_ref)) { + dev_err(&pdev->dev, "Could not get pll_ref clock\n"); + return PTR_ERR(xvcu->pll_ref); + } + + ret = clk_prepare_enable(xvcu->aclk); + if (ret) { + dev_err(&pdev->dev, "aclk clock enable failed\n"); + return ret; + } + + /* + * Do the Gasket isolation and put the VCU out of reset + * Bit 0 : Gasket isolation + * Bit 1 : put VCU out of reset + */ + regmap_write(xvcu->logicore_reg_ba, VCU_GASKET_INIT, VCU_GASKET_VALUE); + + ret = xvcu_register_clock_provider(xvcu); + if (ret) { + dev_err(&pdev->dev, "failed to register clock provider\n"); + goto error_clk_provider; + } + + dev_set_drvdata(&pdev->dev, xvcu); + + return 0; + +error_clk_provider: + xvcu_unregister_clock_provider(xvcu); + clk_disable_unprepare(xvcu->aclk); + return ret; +} + +/** + * xvcu_remove - Insert gasket isolation + * and disable the clock + * @pdev: Pointer to the platform_device structure + * + * Return: Returns 0 on success + * Negative error code otherwise + */ +static int xvcu_remove(struct platform_device *pdev) +{ + struct xvcu_device *xvcu; + + xvcu = platform_get_drvdata(pdev); + if (!xvcu) + return -ENODEV; + + xvcu_unregister_clock_provider(xvcu); + + /* Add the Gasket isolation and put the VCU in reset. */ + regmap_write(xvcu->logicore_reg_ba, VCU_GASKET_INIT, 0); + + clk_disable_unprepare(xvcu->aclk); + + return 0; +} + +static const struct of_device_id xvcu_of_id_table[] = { + { .compatible = "xlnx,vcu" }, + { .compatible = "xlnx,vcu-logicoreip-1.0" }, + { } +}; +MODULE_DEVICE_TABLE(of, xvcu_of_id_table); + +static struct platform_driver xvcu_driver = { + .driver = { + .name = "xilinx-vcu", + .of_match_table = xvcu_of_id_table, + }, + .probe = xvcu_probe, + .remove = xvcu_remove, +}; + +module_platform_driver(xvcu_driver); + +MODULE_AUTHOR("Dhaval Shah "); +MODULE_DESCRIPTION("Xilinx VCU init Driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/clk/zte/Makefile b/drivers/clk/zte/Makefile deleted file mode 100644 index f130643b695dd2d1c76c08b986599d865c3091ce..0000000000000000000000000000000000000000 --- a/drivers/clk/zte/Makefile +++ /dev/null @@ -1,4 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -obj-y := clk.o -obj-$(CONFIG_SOC_ZX296702) += clk-zx296702.o -obj-$(CONFIG_ARCH_ZX) += clk-zx296718.o diff --git a/drivers/clk/zte/clk-zx296702.c b/drivers/clk/zte/clk-zx296702.c deleted file mode 100644 index e846f2a34feb3fa20df845bc9a91537f2a659891..0000000000000000000000000000000000000000 --- a/drivers/clk/zte/clk-zx296702.c +++ /dev/null @@ -1,741 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Copyright 2014 Linaro Ltd. - * Copyright (C) 2014 ZTE Corporation. - */ - -#include -#include -#include -#include "clk.h" - -static DEFINE_SPINLOCK(reg_lock); - -static void __iomem *topcrm_base; -static void __iomem *lsp0crpm_base; -static void __iomem *lsp1crpm_base; - -static struct clk *topclk[ZX296702_TOPCLK_END]; -static struct clk *lsp0clk[ZX296702_LSP0CLK_END]; -static struct clk *lsp1clk[ZX296702_LSP1CLK_END]; - -static struct clk_onecell_data topclk_data; -static struct clk_onecell_data lsp0clk_data; -static struct clk_onecell_data lsp1clk_data; - -#define CLK_MUX (topcrm_base + 0x04) -#define CLK_DIV (topcrm_base + 0x08) -#define CLK_EN0 (topcrm_base + 0x0c) -#define CLK_EN1 (topcrm_base + 0x10) -#define VOU_LOCAL_CLKEN (topcrm_base + 0x68) -#define VOU_LOCAL_CLKSEL (topcrm_base + 0x70) -#define VOU_LOCAL_DIV2_SET (topcrm_base + 0x74) -#define CLK_MUX1 (topcrm_base + 0x8c) - -#define CLK_SDMMC1 (lsp0crpm_base + 0x0c) -#define CLK_GPIO (lsp0crpm_base + 0x2c) -#define CLK_SPDIF0 (lsp0crpm_base + 0x10) -#define SPDIF0_DIV (lsp0crpm_base + 0x14) -#define CLK_I2S0 (lsp0crpm_base + 0x18) -#define I2S0_DIV (lsp0crpm_base + 0x1c) -#define CLK_I2S1 (lsp0crpm_base + 0x20) -#define I2S1_DIV (lsp0crpm_base + 0x24) -#define CLK_I2S2 (lsp0crpm_base + 0x34) -#define I2S2_DIV (lsp0crpm_base + 0x38) - -#define CLK_UART0 (lsp1crpm_base + 0x20) -#define CLK_UART1 (lsp1crpm_base + 0x24) -#define CLK_SDMMC0 (lsp1crpm_base + 0x2c) -#define CLK_SPDIF1 (lsp1crpm_base + 0x30) -#define SPDIF1_DIV (lsp1crpm_base + 0x34) - -static const struct zx_pll_config pll_a9_config[] = { - { .rate = 700000000, .cfg0 = 0x800405d1, .cfg1 = 0x04555555 }, - { .rate = 800000000, .cfg0 = 0x80040691, .cfg1 = 0x04aaaaaa }, - { .rate = 900000000, .cfg0 = 0x80040791, .cfg1 = 0x04000000 }, - { .rate = 1000000000, .cfg0 = 0x80040851, .cfg1 = 0x04555555 }, - { .rate = 1100000000, .cfg0 = 0x80040911, .cfg1 = 0x04aaaaaa }, - { .rate = 1200000000, .cfg0 = 0x80040a11, .cfg1 = 0x04000000 }, -}; - -static const struct clk_div_table main_hlk_div[] = { - { .val = 1, .div = 2, }, - { .val = 3, .div = 4, }, - { /* sentinel */ } -}; - -static const struct clk_div_table a9_as1_aclk_divider[] = { - { .val = 0, .div = 1, }, - { .val = 1, .div = 2, }, - { .val = 3, .div = 4, }, - { /* sentinel */ } -}; - -static const struct clk_div_table sec_wclk_divider[] = { - { .val = 0, .div = 1, }, - { .val = 1, .div = 2, }, - { .val = 3, .div = 4, }, - { .val = 5, .div = 6, }, - { .val = 7, .div = 8, }, - { /* sentinel */ } -}; - -static const char * const matrix_aclk_sel[] = { - "pll_mm0_198M", - "osc", - "clk_148M5", - "pll_lsp_104M", -}; - -static const char * const a9_wclk_sel[] = { - "pll_a9", - "osc", - "clk_500", - "clk_250", -}; - -static const char * const a9_as1_aclk_sel[] = { - "clk_250", - "osc", - "pll_mm0_396M", - "pll_mac_333M", -}; - -static const char * const a9_trace_clkin_sel[] = { - "clk_74M25", - "pll_mm1_108M", - "clk_125", - "clk_148M5", -}; - -static const char * const decppu_aclk_sel[] = { - "clk_250", - "pll_mm0_198M", - "pll_lsp_104M", - "pll_audio_294M912", -}; - -static const char * const vou_main_wclk_sel[] = { - "clk_148M5", - "clk_74M25", - "clk_27", - "pll_mm1_54M", -}; - -static const char * const vou_scaler_wclk_sel[] = { - "clk_250", - "pll_mac_333M", - "pll_audio_294M912", - "pll_mm0_198M", -}; - -static const char * const r2d_wclk_sel[] = { - "pll_audio_294M912", - "pll_mac_333M", - "pll_a9_350M", - "pll_mm0_396M", -}; - -static const char * const ddr_wclk_sel[] = { - "pll_mac_333M", - "pll_ddr_266M", - "pll_audio_294M912", - "pll_mm0_198M", -}; - -static const char * const nand_wclk_sel[] = { - "pll_lsp_104M", - "osc", -}; - -static const char * const lsp_26_wclk_sel[] = { - "pll_lsp_26M", - "osc", -}; - -static const char * const vl0_sel[] = { - "vou_main_channel_div", - "vou_aux_channel_div", -}; - -static const char * const hdmi_sel[] = { - "vou_main_channel_wclk", - "vou_aux_channel_wclk", -}; - -static const char * const sdmmc0_wclk_sel[] = { - "lsp1_104M_wclk", - "lsp1_26M_wclk", -}; - -static const char * const sdmmc1_wclk_sel[] = { - "lsp0_104M_wclk", - "lsp0_26M_wclk", -}; - -static const char * const uart_wclk_sel[] = { - "lsp1_104M_wclk", - "lsp1_26M_wclk", -}; - -static const char * const spdif0_wclk_sel[] = { - "lsp0_104M_wclk", - "lsp0_26M_wclk", -}; - -static const char * const spdif1_wclk_sel[] = { - "lsp1_104M_wclk", - "lsp1_26M_wclk", -}; - -static const char * const i2s_wclk_sel[] = { - "lsp0_104M_wclk", - "lsp0_26M_wclk", -}; - -static inline struct clk *zx_divtbl(const char *name, const char *parent, - void __iomem *reg, u8 shift, u8 width, - const struct clk_div_table *table) -{ - return clk_register_divider_table(NULL, name, parent, 0, reg, shift, - width, 0, table, ®_lock); -} - -static inline struct clk *zx_div(const char *name, const char *parent, - void __iomem *reg, u8 shift, u8 width) -{ - return clk_register_divider(NULL, name, parent, 0, - reg, shift, width, 0, ®_lock); -} - -static inline struct clk *zx_mux(const char *name, const char * const *parents, - int num_parents, void __iomem *reg, u8 shift, u8 width) -{ - return clk_register_mux(NULL, name, parents, num_parents, - 0, reg, shift, width, 0, ®_lock); -} - -static inline struct clk *zx_gate(const char *name, const char *parent, - void __iomem *reg, u8 shift) -{ - return clk_register_gate(NULL, name, parent, CLK_IGNORE_UNUSED, - reg, shift, CLK_SET_RATE_PARENT, ®_lock); -} - -static void __init zx296702_top_clocks_init(struct device_node *np) -{ - struct clk **clk = topclk; - int i; - - topcrm_base = of_iomap(np, 0); - WARN_ON(!topcrm_base); - - clk[ZX296702_OSC] = - clk_register_fixed_rate(NULL, "osc", NULL, 0, 30000000); - clk[ZX296702_PLL_A9] = - clk_register_zx_pll("pll_a9", "osc", 0, topcrm_base - + 0x01c, pll_a9_config, - ARRAY_SIZE(pll_a9_config), ®_lock); - - /* TODO: pll_a9_350M look like changeble follow a9 pll */ - clk[ZX296702_PLL_A9_350M] = - clk_register_fixed_rate(NULL, "pll_a9_350M", "osc", 0, - 350000000); - clk[ZX296702_PLL_MAC_1000M] = - clk_register_fixed_rate(NULL, "pll_mac_1000M", "osc", 0, - 1000000000); - clk[ZX296702_PLL_MAC_333M] = - clk_register_fixed_rate(NULL, "pll_mac_333M", "osc", 0, - 333000000); - clk[ZX296702_PLL_MM0_1188M] = - clk_register_fixed_rate(NULL, "pll_mm0_1188M", "osc", 0, - 1188000000); - clk[ZX296702_PLL_MM0_396M] = - clk_register_fixed_rate(NULL, "pll_mm0_396M", "osc", 0, - 396000000); - clk[ZX296702_PLL_MM0_198M] = - clk_register_fixed_rate(NULL, "pll_mm0_198M", "osc", 0, - 198000000); - clk[ZX296702_PLL_MM1_108M] = - clk_register_fixed_rate(NULL, "pll_mm1_108M", "osc", 0, - 108000000); - clk[ZX296702_PLL_MM1_72M] = - clk_register_fixed_rate(NULL, "pll_mm1_72M", "osc", 0, - 72000000); - clk[ZX296702_PLL_MM1_54M] = - clk_register_fixed_rate(NULL, "pll_mm1_54M", "osc", 0, - 54000000); - clk[ZX296702_PLL_LSP_104M] = - clk_register_fixed_rate(NULL, "pll_lsp_104M", "osc", 0, - 104000000); - clk[ZX296702_PLL_LSP_26M] = - clk_register_fixed_rate(NULL, "pll_lsp_26M", "osc", 0, - 26000000); - clk[ZX296702_PLL_DDR_266M] = - clk_register_fixed_rate(NULL, "pll_ddr_266M", "osc", 0, - 266000000); - clk[ZX296702_PLL_AUDIO_294M912] = - clk_register_fixed_rate(NULL, "pll_audio_294M912", "osc", 0, - 294912000); - - /* bus clock */ - clk[ZX296702_MATRIX_ACLK] = - zx_mux("matrix_aclk", matrix_aclk_sel, - ARRAY_SIZE(matrix_aclk_sel), CLK_MUX, 2, 2); - clk[ZX296702_MAIN_HCLK] = - zx_divtbl("main_hclk", "matrix_aclk", CLK_DIV, 0, 2, - main_hlk_div); - clk[ZX296702_MAIN_PCLK] = - zx_divtbl("main_pclk", "matrix_aclk", CLK_DIV, 2, 2, - main_hlk_div); - - /* cpu clock */ - clk[ZX296702_CLK_500] = - clk_register_fixed_factor(NULL, "clk_500", "pll_mac_1000M", 0, - 1, 2); - clk[ZX296702_CLK_250] = - clk_register_fixed_factor(NULL, "clk_250", "pll_mac_1000M", 0, - 1, 4); - clk[ZX296702_CLK_125] = - clk_register_fixed_factor(NULL, "clk_125", "clk_250", 0, 1, 2); - clk[ZX296702_CLK_148M5] = - clk_register_fixed_factor(NULL, "clk_148M5", "pll_mm0_1188M", 0, - 1, 8); - clk[ZX296702_CLK_74M25] = - clk_register_fixed_factor(NULL, "clk_74M25", "pll_mm0_1188M", 0, - 1, 16); - clk[ZX296702_A9_WCLK] = - zx_mux("a9_wclk", a9_wclk_sel, ARRAY_SIZE(a9_wclk_sel), CLK_MUX, - 0, 2); - clk[ZX296702_A9_AS1_ACLK_MUX] = - zx_mux("a9_as1_aclk_mux", a9_as1_aclk_sel, - ARRAY_SIZE(a9_as1_aclk_sel), CLK_MUX, 4, 2); - clk[ZX296702_A9_TRACE_CLKIN_MUX] = - zx_mux("a9_trace_clkin_mux", a9_trace_clkin_sel, - ARRAY_SIZE(a9_trace_clkin_sel), CLK_MUX1, 0, 2); - clk[ZX296702_A9_AS1_ACLK_DIV] = - zx_divtbl("a9_as1_aclk_div", "a9_as1_aclk_mux", CLK_DIV, 4, 2, - a9_as1_aclk_divider); - - /* multi-media clock */ - clk[ZX296702_CLK_2] = - clk_register_fixed_factor(NULL, "clk_2", "pll_mm1_72M", 0, - 1, 36); - clk[ZX296702_CLK_27] = - clk_register_fixed_factor(NULL, "clk_27", "pll_mm1_54M", 0, - 1, 2); - clk[ZX296702_DECPPU_ACLK_MUX] = - zx_mux("decppu_aclk_mux", decppu_aclk_sel, - ARRAY_SIZE(decppu_aclk_sel), CLK_MUX, 6, 2); - clk[ZX296702_PPU_ACLK_MUX] = - zx_mux("ppu_aclk_mux", decppu_aclk_sel, - ARRAY_SIZE(decppu_aclk_sel), CLK_MUX, 8, 2); - clk[ZX296702_MALI400_ACLK_MUX] = - zx_mux("mali400_aclk_mux", decppu_aclk_sel, - ARRAY_SIZE(decppu_aclk_sel), CLK_MUX, 12, 2); - clk[ZX296702_VOU_ACLK_MUX] = - zx_mux("vou_aclk_mux", decppu_aclk_sel, - ARRAY_SIZE(decppu_aclk_sel), CLK_MUX, 10, 2); - clk[ZX296702_VOU_MAIN_WCLK_MUX] = - zx_mux("vou_main_wclk_mux", vou_main_wclk_sel, - ARRAY_SIZE(vou_main_wclk_sel), CLK_MUX, 14, 2); - clk[ZX296702_VOU_AUX_WCLK_MUX] = - zx_mux("vou_aux_wclk_mux", vou_main_wclk_sel, - ARRAY_SIZE(vou_main_wclk_sel), CLK_MUX, 16, 2); - clk[ZX296702_VOU_SCALER_WCLK_MUX] = - zx_mux("vou_scaler_wclk_mux", vou_scaler_wclk_sel, - ARRAY_SIZE(vou_scaler_wclk_sel), CLK_MUX, - 18, 2); - clk[ZX296702_R2D_ACLK_MUX] = - zx_mux("r2d_aclk_mux", decppu_aclk_sel, - ARRAY_SIZE(decppu_aclk_sel), CLK_MUX, 20, 2); - clk[ZX296702_R2D_WCLK_MUX] = - zx_mux("r2d_wclk_mux", r2d_wclk_sel, - ARRAY_SIZE(r2d_wclk_sel), CLK_MUX, 22, 2); - - /* other clock */ - clk[ZX296702_CLK_50] = - clk_register_fixed_factor(NULL, "clk_50", "pll_mac_1000M", - 0, 1, 20); - clk[ZX296702_CLK_25] = - clk_register_fixed_factor(NULL, "clk_25", "pll_mac_1000M", - 0, 1, 40); - clk[ZX296702_CLK_12] = - clk_register_fixed_factor(NULL, "clk_12", "pll_mm1_72M", - 0, 1, 6); - clk[ZX296702_CLK_16M384] = - clk_register_fixed_factor(NULL, "clk_16M384", - "pll_audio_294M912", 0, 1, 18); - clk[ZX296702_CLK_32K768] = - clk_register_fixed_factor(NULL, "clk_32K768", "clk_16M384", - 0, 1, 500); - clk[ZX296702_SEC_WCLK_DIV] = - zx_divtbl("sec_wclk_div", "pll_lsp_104M", CLK_DIV, 6, 3, - sec_wclk_divider); - clk[ZX296702_DDR_WCLK_MUX] = - zx_mux("ddr_wclk_mux", ddr_wclk_sel, - ARRAY_SIZE(ddr_wclk_sel), CLK_MUX, 24, 2); - clk[ZX296702_NAND_WCLK_MUX] = - zx_mux("nand_wclk_mux", nand_wclk_sel, - ARRAY_SIZE(nand_wclk_sel), CLK_MUX, 24, 2); - clk[ZX296702_LSP_26_WCLK_MUX] = - zx_mux("lsp_26_wclk_mux", lsp_26_wclk_sel, - ARRAY_SIZE(lsp_26_wclk_sel), CLK_MUX, 27, 1); - - /* gates */ - clk[ZX296702_A9_AS0_ACLK] = - zx_gate("a9_as0_aclk", "matrix_aclk", CLK_EN0, 0); - clk[ZX296702_A9_AS1_ACLK] = - zx_gate("a9_as1_aclk", "a9_as1_aclk_div", CLK_EN0, 1); - clk[ZX296702_A9_TRACE_CLKIN] = - zx_gate("a9_trace_clkin", "a9_trace_clkin_mux", CLK_EN0, 2); - clk[ZX296702_DECPPU_AXI_M_ACLK] = - zx_gate("decppu_axi_m_aclk", "decppu_aclk_mux", CLK_EN0, 3); - clk[ZX296702_DECPPU_AHB_S_HCLK] = - zx_gate("decppu_ahb_s_hclk", "main_hclk", CLK_EN0, 4); - clk[ZX296702_PPU_AXI_M_ACLK] = - zx_gate("ppu_axi_m_aclk", "ppu_aclk_mux", CLK_EN0, 5); - clk[ZX296702_PPU_AHB_S_HCLK] = - zx_gate("ppu_ahb_s_hclk", "main_hclk", CLK_EN0, 6); - clk[ZX296702_VOU_AXI_M_ACLK] = - zx_gate("vou_axi_m_aclk", "vou_aclk_mux", CLK_EN0, 7); - clk[ZX296702_VOU_APB_PCLK] = - zx_gate("vou_apb_pclk", "main_pclk", CLK_EN0, 8); - clk[ZX296702_VOU_MAIN_CHANNEL_WCLK] = - zx_gate("vou_main_channel_wclk", "vou_main_wclk_mux", - CLK_EN0, 9); - clk[ZX296702_VOU_AUX_CHANNEL_WCLK] = - zx_gate("vou_aux_channel_wclk", "vou_aux_wclk_mux", - CLK_EN0, 10); - clk[ZX296702_VOU_HDMI_OSCLK_CEC] = - zx_gate("vou_hdmi_osclk_cec", "clk_2", CLK_EN0, 11); - clk[ZX296702_VOU_SCALER_WCLK] = - zx_gate("vou_scaler_wclk", "vou_scaler_wclk_mux", CLK_EN0, 12); - clk[ZX296702_MALI400_AXI_M_ACLK] = - zx_gate("mali400_axi_m_aclk", "mali400_aclk_mux", CLK_EN0, 13); - clk[ZX296702_MALI400_APB_PCLK] = - zx_gate("mali400_apb_pclk", "main_pclk", CLK_EN0, 14); - clk[ZX296702_R2D_WCLK] = - zx_gate("r2d_wclk", "r2d_wclk_mux", CLK_EN0, 15); - clk[ZX296702_R2D_AXI_M_ACLK] = - zx_gate("r2d_axi_m_aclk", "r2d_aclk_mux", CLK_EN0, 16); - clk[ZX296702_R2D_AHB_HCLK] = - zx_gate("r2d_ahb_hclk", "main_hclk", CLK_EN0, 17); - clk[ZX296702_DDR3_AXI_S0_ACLK] = - zx_gate("ddr3_axi_s0_aclk", "matrix_aclk", CLK_EN0, 18); - clk[ZX296702_DDR3_APB_PCLK] = - zx_gate("ddr3_apb_pclk", "main_pclk", CLK_EN0, 19); - clk[ZX296702_DDR3_WCLK] = - zx_gate("ddr3_wclk", "ddr_wclk_mux", CLK_EN0, 20); - clk[ZX296702_USB20_0_AHB_HCLK] = - zx_gate("usb20_0_ahb_hclk", "main_hclk", CLK_EN0, 21); - clk[ZX296702_USB20_0_EXTREFCLK] = - zx_gate("usb20_0_extrefclk", "clk_12", CLK_EN0, 22); - clk[ZX296702_USB20_1_AHB_HCLK] = - zx_gate("usb20_1_ahb_hclk", "main_hclk", CLK_EN0, 23); - clk[ZX296702_USB20_1_EXTREFCLK] = - zx_gate("usb20_1_extrefclk", "clk_12", CLK_EN0, 24); - clk[ZX296702_USB20_2_AHB_HCLK] = - zx_gate("usb20_2_ahb_hclk", "main_hclk", CLK_EN0, 25); - clk[ZX296702_USB20_2_EXTREFCLK] = - zx_gate("usb20_2_extrefclk", "clk_12", CLK_EN0, 26); - clk[ZX296702_GMAC_AXI_M_ACLK] = - zx_gate("gmac_axi_m_aclk", "matrix_aclk", CLK_EN0, 27); - clk[ZX296702_GMAC_APB_PCLK] = - zx_gate("gmac_apb_pclk", "main_pclk", CLK_EN0, 28); - clk[ZX296702_GMAC_125_CLKIN] = - zx_gate("gmac_125_clkin", "clk_125", CLK_EN0, 29); - clk[ZX296702_GMAC_RMII_CLKIN] = - zx_gate("gmac_rmii_clkin", "clk_50", CLK_EN0, 30); - clk[ZX296702_GMAC_25M_CLK] = - zx_gate("gmac_25M_clk", "clk_25", CLK_EN0, 31); - clk[ZX296702_NANDFLASH_AHB_HCLK] = - zx_gate("nandflash_ahb_hclk", "main_hclk", CLK_EN1, 0); - clk[ZX296702_NANDFLASH_WCLK] = - zx_gate("nandflash_wclk", "nand_wclk_mux", CLK_EN1, 1); - clk[ZX296702_LSP0_APB_PCLK] = - zx_gate("lsp0_apb_pclk", "main_pclk", CLK_EN1, 2); - clk[ZX296702_LSP0_AHB_HCLK] = - zx_gate("lsp0_ahb_hclk", "main_hclk", CLK_EN1, 3); - clk[ZX296702_LSP0_26M_WCLK] = - zx_gate("lsp0_26M_wclk", "lsp_26_wclk_mux", CLK_EN1, 4); - clk[ZX296702_LSP0_104M_WCLK] = - zx_gate("lsp0_104M_wclk", "pll_lsp_104M", CLK_EN1, 5); - clk[ZX296702_LSP0_16M384_WCLK] = - zx_gate("lsp0_16M384_wclk", "clk_16M384", CLK_EN1, 6); - clk[ZX296702_LSP1_APB_PCLK] = - zx_gate("lsp1_apb_pclk", "main_pclk", CLK_EN1, 7); - /* FIXME: wclk enable bit is bit8. We hack it as reserved 31 for - * UART does not work after parent clk is disabled/enabled */ - clk[ZX296702_LSP1_26M_WCLK] = - zx_gate("lsp1_26M_wclk", "lsp_26_wclk_mux", CLK_EN1, 31); - clk[ZX296702_LSP1_104M_WCLK] = - zx_gate("lsp1_104M_wclk", "pll_lsp_104M", CLK_EN1, 9); - clk[ZX296702_LSP1_32K_CLK] = - zx_gate("lsp1_32K_clk", "clk_32K768", CLK_EN1, 10); - clk[ZX296702_AON_HCLK] = - zx_gate("aon_hclk", "main_hclk", CLK_EN1, 11); - clk[ZX296702_SYS_CTRL_PCLK] = - zx_gate("sys_ctrl_pclk", "main_pclk", CLK_EN1, 12); - clk[ZX296702_DMA_PCLK] = - zx_gate("dma_pclk", "main_pclk", CLK_EN1, 13); - clk[ZX296702_DMA_ACLK] = - zx_gate("dma_aclk", "matrix_aclk", CLK_EN1, 14); - clk[ZX296702_SEC_HCLK] = - zx_gate("sec_hclk", "main_hclk", CLK_EN1, 15); - clk[ZX296702_AES_WCLK] = - zx_gate("aes_wclk", "sec_wclk_div", CLK_EN1, 16); - clk[ZX296702_DES_WCLK] = - zx_gate("des_wclk", "sec_wclk_div", CLK_EN1, 17); - clk[ZX296702_IRAM_ACLK] = - zx_gate("iram_aclk", "matrix_aclk", CLK_EN1, 18); - clk[ZX296702_IROM_ACLK] = - zx_gate("irom_aclk", "matrix_aclk", CLK_EN1, 19); - clk[ZX296702_BOOT_CTRL_HCLK] = - zx_gate("boot_ctrl_hclk", "main_hclk", CLK_EN1, 20); - clk[ZX296702_EFUSE_CLK_30] = - zx_gate("efuse_clk_30", "osc", CLK_EN1, 21); - - /* TODO: add VOU Local clocks */ - clk[ZX296702_VOU_MAIN_CHANNEL_DIV] = - zx_div("vou_main_channel_div", "vou_main_channel_wclk", - VOU_LOCAL_DIV2_SET, 1, 1); - clk[ZX296702_VOU_AUX_CHANNEL_DIV] = - zx_div("vou_aux_channel_div", "vou_aux_channel_wclk", - VOU_LOCAL_DIV2_SET, 0, 1); - clk[ZX296702_VOU_TV_ENC_HD_DIV] = - zx_div("vou_tv_enc_hd_div", "vou_tv_enc_hd_mux", - VOU_LOCAL_DIV2_SET, 3, 1); - clk[ZX296702_VOU_TV_ENC_SD_DIV] = - zx_div("vou_tv_enc_sd_div", "vou_tv_enc_sd_mux", - VOU_LOCAL_DIV2_SET, 2, 1); - clk[ZX296702_VL0_MUX] = - zx_mux("vl0_mux", vl0_sel, ARRAY_SIZE(vl0_sel), - VOU_LOCAL_CLKSEL, 8, 1); - clk[ZX296702_VL1_MUX] = - zx_mux("vl1_mux", vl0_sel, ARRAY_SIZE(vl0_sel), - VOU_LOCAL_CLKSEL, 9, 1); - clk[ZX296702_VL2_MUX] = - zx_mux("vl2_mux", vl0_sel, ARRAY_SIZE(vl0_sel), - VOU_LOCAL_CLKSEL, 10, 1); - clk[ZX296702_GL0_MUX] = - zx_mux("gl0_mux", vl0_sel, ARRAY_SIZE(vl0_sel), - VOU_LOCAL_CLKSEL, 5, 1); - clk[ZX296702_GL1_MUX] = - zx_mux("gl1_mux", vl0_sel, ARRAY_SIZE(vl0_sel), - VOU_LOCAL_CLKSEL, 6, 1); - clk[ZX296702_GL2_MUX] = - zx_mux("gl2_mux", vl0_sel, ARRAY_SIZE(vl0_sel), - VOU_LOCAL_CLKSEL, 7, 1); - clk[ZX296702_WB_MUX] = - zx_mux("wb_mux", vl0_sel, ARRAY_SIZE(vl0_sel), - VOU_LOCAL_CLKSEL, 11, 1); - clk[ZX296702_HDMI_MUX] = - zx_mux("hdmi_mux", hdmi_sel, ARRAY_SIZE(hdmi_sel), - VOU_LOCAL_CLKSEL, 4, 1); - clk[ZX296702_VOU_TV_ENC_HD_MUX] = - zx_mux("vou_tv_enc_hd_mux", hdmi_sel, ARRAY_SIZE(hdmi_sel), - VOU_LOCAL_CLKSEL, 3, 1); - clk[ZX296702_VOU_TV_ENC_SD_MUX] = - zx_mux("vou_tv_enc_sd_mux", hdmi_sel, ARRAY_SIZE(hdmi_sel), - VOU_LOCAL_CLKSEL, 2, 1); - clk[ZX296702_VL0_CLK] = - zx_gate("vl0_clk", "vl0_mux", VOU_LOCAL_CLKEN, 8); - clk[ZX296702_VL1_CLK] = - zx_gate("vl1_clk", "vl1_mux", VOU_LOCAL_CLKEN, 9); - clk[ZX296702_VL2_CLK] = - zx_gate("vl2_clk", "vl2_mux", VOU_LOCAL_CLKEN, 10); - clk[ZX296702_GL0_CLK] = - zx_gate("gl0_clk", "gl0_mux", VOU_LOCAL_CLKEN, 5); - clk[ZX296702_GL1_CLK] = - zx_gate("gl1_clk", "gl1_mux", VOU_LOCAL_CLKEN, 6); - clk[ZX296702_GL2_CLK] = - zx_gate("gl2_clk", "gl2_mux", VOU_LOCAL_CLKEN, 7); - clk[ZX296702_WB_CLK] = - zx_gate("wb_clk", "wb_mux", VOU_LOCAL_CLKEN, 11); - clk[ZX296702_CL_CLK] = - zx_gate("cl_clk", "vou_main_channel_div", VOU_LOCAL_CLKEN, 12); - clk[ZX296702_MAIN_MIX_CLK] = - zx_gate("main_mix_clk", "vou_main_channel_div", - VOU_LOCAL_CLKEN, 4); - clk[ZX296702_AUX_MIX_CLK] = - zx_gate("aux_mix_clk", "vou_aux_channel_div", - VOU_LOCAL_CLKEN, 3); - clk[ZX296702_HDMI_CLK] = - zx_gate("hdmi_clk", "hdmi_mux", VOU_LOCAL_CLKEN, 2); - clk[ZX296702_VOU_TV_ENC_HD_DAC_CLK] = - zx_gate("vou_tv_enc_hd_dac_clk", "vou_tv_enc_hd_div", - VOU_LOCAL_CLKEN, 1); - clk[ZX296702_VOU_TV_ENC_SD_DAC_CLK] = - zx_gate("vou_tv_enc_sd_dac_clk", "vou_tv_enc_sd_div", - VOU_LOCAL_CLKEN, 0); - - /* CA9 PERIPHCLK = a9_wclk / 2 */ - clk[ZX296702_A9_PERIPHCLK] = - clk_register_fixed_factor(NULL, "a9_periphclk", "a9_wclk", - 0, 1, 2); - - for (i = 0; i < ARRAY_SIZE(topclk); i++) { - if (IS_ERR(clk[i])) { - pr_err("zx296702 clk %d: register failed with %ld\n", - i, PTR_ERR(clk[i])); - return; - } - } - - topclk_data.clks = topclk; - topclk_data.clk_num = ARRAY_SIZE(topclk); - of_clk_add_provider(np, of_clk_src_onecell_get, &topclk_data); -} -CLK_OF_DECLARE(zx296702_top_clk, "zte,zx296702-topcrm-clk", - zx296702_top_clocks_init); - -static void __init zx296702_lsp0_clocks_init(struct device_node *np) -{ - struct clk **clk = lsp0clk; - int i; - - lsp0crpm_base = of_iomap(np, 0); - WARN_ON(!lsp0crpm_base); - - /* SDMMC1 */ - clk[ZX296702_SDMMC1_WCLK_MUX] = - zx_mux("sdmmc1_wclk_mux", sdmmc1_wclk_sel, - ARRAY_SIZE(sdmmc1_wclk_sel), CLK_SDMMC1, 4, 1); - clk[ZX296702_SDMMC1_WCLK_DIV] = - zx_div("sdmmc1_wclk_div", "sdmmc1_wclk_mux", CLK_SDMMC1, 12, 4); - clk[ZX296702_SDMMC1_WCLK] = - zx_gate("sdmmc1_wclk", "sdmmc1_wclk_div", CLK_SDMMC1, 1); - clk[ZX296702_SDMMC1_PCLK] = - zx_gate("sdmmc1_pclk", "lsp0_apb_pclk", CLK_SDMMC1, 0); - - clk[ZX296702_GPIO_CLK] = - zx_gate("gpio_clk", "lsp0_apb_pclk", CLK_GPIO, 0); - - /* SPDIF */ - clk[ZX296702_SPDIF0_WCLK_MUX] = - zx_mux("spdif0_wclk_mux", spdif0_wclk_sel, - ARRAY_SIZE(spdif0_wclk_sel), CLK_SPDIF0, 4, 1); - clk[ZX296702_SPDIF0_WCLK] = - zx_gate("spdif0_wclk", "spdif0_wclk_mux", CLK_SPDIF0, 1); - clk[ZX296702_SPDIF0_PCLK] = - zx_gate("spdif0_pclk", "lsp0_apb_pclk", CLK_SPDIF0, 0); - - clk[ZX296702_SPDIF0_DIV] = - clk_register_zx_audio("spdif0_div", "spdif0_wclk", 0, - SPDIF0_DIV); - - /* I2S */ - clk[ZX296702_I2S0_WCLK_MUX] = - zx_mux("i2s0_wclk_mux", i2s_wclk_sel, - ARRAY_SIZE(i2s_wclk_sel), CLK_I2S0, 4, 1); - clk[ZX296702_I2S0_WCLK] = - zx_gate("i2s0_wclk", "i2s0_wclk_mux", CLK_I2S0, 1); - clk[ZX296702_I2S0_PCLK] = - zx_gate("i2s0_pclk", "lsp0_apb_pclk", CLK_I2S0, 0); - - clk[ZX296702_I2S0_DIV] = - clk_register_zx_audio("i2s0_div", "i2s0_wclk", 0, I2S0_DIV); - - clk[ZX296702_I2S1_WCLK_MUX] = - zx_mux("i2s1_wclk_mux", i2s_wclk_sel, - ARRAY_SIZE(i2s_wclk_sel), CLK_I2S1, 4, 1); - clk[ZX296702_I2S1_WCLK] = - zx_gate("i2s1_wclk", "i2s1_wclk_mux", CLK_I2S1, 1); - clk[ZX296702_I2S1_PCLK] = - zx_gate("i2s1_pclk", "lsp0_apb_pclk", CLK_I2S1, 0); - - clk[ZX296702_I2S1_DIV] = - clk_register_zx_audio("i2s1_div", "i2s1_wclk", 0, I2S1_DIV); - - clk[ZX296702_I2S2_WCLK_MUX] = - zx_mux("i2s2_wclk_mux", i2s_wclk_sel, - ARRAY_SIZE(i2s_wclk_sel), CLK_I2S2, 4, 1); - clk[ZX296702_I2S2_WCLK] = - zx_gate("i2s2_wclk", "i2s2_wclk_mux", CLK_I2S2, 1); - clk[ZX296702_I2S2_PCLK] = - zx_gate("i2s2_pclk", "lsp0_apb_pclk", CLK_I2S2, 0); - - clk[ZX296702_I2S2_DIV] = - clk_register_zx_audio("i2s2_div", "i2s2_wclk", 0, I2S2_DIV); - - for (i = 0; i < ARRAY_SIZE(lsp0clk); i++) { - if (IS_ERR(clk[i])) { - pr_err("zx296702 clk %d: register failed with %ld\n", - i, PTR_ERR(clk[i])); - return; - } - } - - lsp0clk_data.clks = lsp0clk; - lsp0clk_data.clk_num = ARRAY_SIZE(lsp0clk); - of_clk_add_provider(np, of_clk_src_onecell_get, &lsp0clk_data); -} -CLK_OF_DECLARE(zx296702_lsp0_clk, "zte,zx296702-lsp0crpm-clk", - zx296702_lsp0_clocks_init); - -static void __init zx296702_lsp1_clocks_init(struct device_node *np) -{ - struct clk **clk = lsp1clk; - int i; - - lsp1crpm_base = of_iomap(np, 0); - WARN_ON(!lsp1crpm_base); - - /* UART0 */ - clk[ZX296702_UART0_WCLK_MUX] = - zx_mux("uart0_wclk_mux", uart_wclk_sel, - ARRAY_SIZE(uart_wclk_sel), CLK_UART0, 4, 1); - /* FIXME: uart wclk enable bit is bit1 in. We hack it as reserved 31 for - * UART does not work after parent clk is disabled/enabled */ - clk[ZX296702_UART0_WCLK] = - zx_gate("uart0_wclk", "uart0_wclk_mux", CLK_UART0, 31); - clk[ZX296702_UART0_PCLK] = - zx_gate("uart0_pclk", "lsp1_apb_pclk", CLK_UART0, 0); - - /* UART1 */ - clk[ZX296702_UART1_WCLK_MUX] = - zx_mux("uart1_wclk_mux", uart_wclk_sel, - ARRAY_SIZE(uart_wclk_sel), CLK_UART1, 4, 1); - clk[ZX296702_UART1_WCLK] = - zx_gate("uart1_wclk", "uart1_wclk_mux", CLK_UART1, 1); - clk[ZX296702_UART1_PCLK] = - zx_gate("uart1_pclk", "lsp1_apb_pclk", CLK_UART1, 0); - - /* SDMMC0 */ - clk[ZX296702_SDMMC0_WCLK_MUX] = - zx_mux("sdmmc0_wclk_mux", sdmmc0_wclk_sel, - ARRAY_SIZE(sdmmc0_wclk_sel), CLK_SDMMC0, 4, 1); - clk[ZX296702_SDMMC0_WCLK_DIV] = - zx_div("sdmmc0_wclk_div", "sdmmc0_wclk_mux", CLK_SDMMC0, 12, 4); - clk[ZX296702_SDMMC0_WCLK] = - zx_gate("sdmmc0_wclk", "sdmmc0_wclk_div", CLK_SDMMC0, 1); - clk[ZX296702_SDMMC0_PCLK] = - zx_gate("sdmmc0_pclk", "lsp1_apb_pclk", CLK_SDMMC0, 0); - - clk[ZX296702_SPDIF1_WCLK_MUX] = - zx_mux("spdif1_wclk_mux", spdif1_wclk_sel, - ARRAY_SIZE(spdif1_wclk_sel), CLK_SPDIF1, 4, 1); - clk[ZX296702_SPDIF1_WCLK] = - zx_gate("spdif1_wclk", "spdif1_wclk_mux", CLK_SPDIF1, 1); - clk[ZX296702_SPDIF1_PCLK] = - zx_gate("spdif1_pclk", "lsp1_apb_pclk", CLK_SPDIF1, 0); - - clk[ZX296702_SPDIF1_DIV] = - clk_register_zx_audio("spdif1_div", "spdif1_wclk", 0, - SPDIF1_DIV); - - for (i = 0; i < ARRAY_SIZE(lsp1clk); i++) { - if (IS_ERR(clk[i])) { - pr_err("zx296702 clk %d: register failed with %ld\n", - i, PTR_ERR(clk[i])); - return; - } - } - - lsp1clk_data.clks = lsp1clk; - lsp1clk_data.clk_num = ARRAY_SIZE(lsp1clk); - of_clk_add_provider(np, of_clk_src_onecell_get, &lsp1clk_data); -} -CLK_OF_DECLARE(zx296702_lsp1_clk, "zte,zx296702-lsp1crpm-clk", - zx296702_lsp1_clocks_init); diff --git a/drivers/clk/zte/clk-zx296718.c b/drivers/clk/zte/clk-zx296718.c deleted file mode 100644 index dd7045bc48c152f124635b2c885e054497702ff5..0000000000000000000000000000000000000000 --- a/drivers/clk/zte/clk-zx296718.c +++ /dev/null @@ -1,1074 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Copyright (C) 2015 - 2016 ZTE Corporation. - * Copyright (C) 2016 Linaro Ltd. - */ -#include -#include -#include -#include -#include -#include - -#include -#include "clk.h" - -/* TOP CRM */ -#define TOP_CLK_MUX0 0x04 -#define TOP_CLK_MUX1 0x08 -#define TOP_CLK_MUX2 0x0c -#define TOP_CLK_MUX3 0x10 -#define TOP_CLK_MUX4 0x14 -#define TOP_CLK_MUX5 0x18 -#define TOP_CLK_MUX6 0x1c -#define TOP_CLK_MUX7 0x20 -#define TOP_CLK_MUX9 0x28 - - -#define TOP_CLK_GATE0 0x34 -#define TOP_CLK_GATE1 0x38 -#define TOP_CLK_GATE2 0x3c -#define TOP_CLK_GATE3 0x40 -#define TOP_CLK_GATE4 0x44 -#define TOP_CLK_GATE5 0x48 -#define TOP_CLK_GATE6 0x4c - -#define TOP_CLK_DIV0 0x58 - -#define PLL_CPU_REG 0x80 -#define PLL_VGA_REG 0xb0 -#define PLL_DDR_REG 0xa0 - -/* LSP0 CRM */ -#define LSP0_TIMER3_CLK 0x4 -#define LSP0_TIMER4_CLK 0x8 -#define LSP0_TIMER5_CLK 0xc -#define LSP0_UART3_CLK 0x10 -#define LSP0_UART1_CLK 0x14 -#define LSP0_UART2_CLK 0x18 -#define LSP0_SPIFC0_CLK 0x1c -#define LSP0_I2C4_CLK 0x20 -#define LSP0_I2C5_CLK 0x24 -#define LSP0_SSP0_CLK 0x28 -#define LSP0_SSP1_CLK 0x2c -#define LSP0_USIM0_CLK 0x30 -#define LSP0_GPIO_CLK 0x34 -#define LSP0_I2C3_CLK 0x38 - -/* LSP1 CRM */ -#define LSP1_UART4_CLK 0x08 -#define LSP1_UART5_CLK 0x0c -#define LSP1_PWM_CLK 0x10 -#define LSP1_I2C2_CLK 0x14 -#define LSP1_SSP2_CLK 0x1c -#define LSP1_SSP3_CLK 0x20 -#define LSP1_SSP4_CLK 0x24 -#define LSP1_USIM1_CLK 0x28 - -/* audio lsp */ -#define AUDIO_I2S0_DIV_CFG1 0x10 -#define AUDIO_I2S0_DIV_CFG2 0x14 -#define AUDIO_I2S0_CLK 0x18 -#define AUDIO_I2S1_DIV_CFG1 0x20 -#define AUDIO_I2S1_DIV_CFG2 0x24 -#define AUDIO_I2S1_CLK 0x28 -#define AUDIO_I2S2_DIV_CFG1 0x30 -#define AUDIO_I2S2_DIV_CFG2 0x34 -#define AUDIO_I2S2_CLK 0x38 -#define AUDIO_I2S3_DIV_CFG1 0x40 -#define AUDIO_I2S3_DIV_CFG2 0x44 -#define AUDIO_I2S3_CLK 0x48 -#define AUDIO_I2C0_CLK 0x50 -#define AUDIO_SPDIF0_DIV_CFG1 0x60 -#define AUDIO_SPDIF0_DIV_CFG2 0x64 -#define AUDIO_SPDIF0_CLK 0x68 -#define AUDIO_SPDIF1_DIV_CFG1 0x70 -#define AUDIO_SPDIF1_DIV_CFG2 0x74 -#define AUDIO_SPDIF1_CLK 0x78 -#define AUDIO_TIMER_CLK 0x80 -#define AUDIO_TDM_CLK 0x90 -#define AUDIO_TS_CLK 0xa0 - -static DEFINE_SPINLOCK(clk_lock); - -static const struct zx_pll_config pll_cpu_table[] = { - PLL_RATE(1312000000, 0x00103621, 0x04aaaaaa), - PLL_RATE(1407000000, 0x00103a21, 0x04aaaaaa), - PLL_RATE(1503000000, 0x00103e21, 0x04aaaaaa), - PLL_RATE(1600000000, 0x00104221, 0x04aaaaaa), -}; - -static const struct zx_pll_config pll_vga_table[] = { - PLL_RATE(36000000, 0x00102464, 0x04000000), /* 800x600@56 */ - PLL_RATE(40000000, 0x00102864, 0x04000000), /* 800x600@60 */ - PLL_RATE(49500000, 0x00103164, 0x04800000), /* 800x600@75 */ - PLL_RATE(50000000, 0x00103264, 0x04000000), /* 800x600@72 */ - PLL_RATE(56250000, 0x00103864, 0x04400000), /* 800x600@85 */ - PLL_RATE(65000000, 0x00104164, 0x04000000), /* 1024x768@60 */ - PLL_RATE(74375000, 0x00104a64, 0x04600000), /* 1280x720@60 */ - PLL_RATE(75000000, 0x00104b64, 0x04800000), /* 1024x768@70 */ - PLL_RATE(78750000, 0x00104e64, 0x04c00000), /* 1024x768@75 */ - PLL_RATE(85500000, 0x00105564, 0x04800000), /* 1360x768@60 */ - PLL_RATE(106500000, 0x00106a64, 0x04800000), /* 1440x900@60 */ - PLL_RATE(108000000, 0x00106c64, 0x04000000), /* 1280x1024@60 */ - PLL_RATE(110000000, 0x00106e64, 0x04000000), /* 1024x768@85 */ - PLL_RATE(135000000, 0x00105a44, 0x04000000), /* 1280x1024@75 */ - PLL_RATE(136750000, 0x00104462, 0x04600000), /* 1440x900@75 */ - PLL_RATE(148500000, 0x00104a62, 0x04400000), /* 1920x1080@60 */ - PLL_RATE(157000000, 0x00104e62, 0x04800000), /* 1440x900@85 */ - PLL_RATE(157500000, 0x00104e62, 0x04c00000), /* 1280x1024@85 */ - PLL_RATE(162000000, 0x00105162, 0x04000000), /* 1600x1200@60 */ - PLL_RATE(193250000, 0x00106062, 0x04a00000), /* 1920x1200@60 */ -}; - -PNAME(osc) = { - "osc24m", - "osc32k", -}; - -PNAME(dbg_wclk_p) = { - "clk334m", - "clk466m", - "clk396m", - "clk250m", -}; - -PNAME(a72_coreclk_p) = { - "osc24m", - "pll_mm0_1188m", - "pll_mm1_1296m", - "clk1000m", - "clk648m", - "clk1600m", - "pll_audio_1800m", - "pll_vga_1800m", -}; - -PNAME(cpu_periclk_p) = { - "osc24m", - "clk500m", - "clk594m", - "clk466m", - "clk294m", - "clk334m", - "clk250m", - "clk125m", -}; - -PNAME(a53_coreclk_p) = { - "osc24m", - "clk1000m", - "pll_mm0_1188m", - "clk648m", - "clk500m", - "clk800m", - "clk1600m", - "pll_audio_1800m", -}; - -PNAME(sec_wclk_p) = { - "osc24m", - "clk396m", - "clk334m", - "clk297m", - "clk250m", - "clk198m", - "clk148m5", - "clk99m", -}; - -PNAME(sd_nand_wclk_p) = { - "osc24m", - "clk49m5", - "clk99m", - "clk198m", - "clk167m", - "clk148m5", - "clk125m", - "clk216m", -}; - -PNAME(emmc_wclk_p) = { - "osc24m", - "clk198m", - "clk99m", - "clk396m", - "clk334m", - "clk297m", - "clk250m", - "clk148m5", -}; - -PNAME(clk32_p) = { - "osc32k", - "clk32k768", -}; - -PNAME(usb_ref24m_p) = { - "osc32k", - "clk32k768", -}; - -PNAME(sys_noc_alck_p) = { - "osc24m", - "clk250m", - "clk198m", - "clk148m5", - "clk108m", - "clk54m", - "clk216m", - "clk240m", -}; - -PNAME(vde_aclk_p) = { - "clk334m", - "clk594m", - "clk500m", - "clk432m", - "clk480m", - "clk297m", - "clk_vga", /*600MHz*/ - "clk294m", -}; - -PNAME(vce_aclk_p) = { - "clk334m", - "clk594m", - "clk500m", - "clk432m", - "clk396m", - "clk297m", - "clk_vga", /*600MHz*/ - "clk294m", -}; - -PNAME(hde_aclk_p) = { - "clk334m", - "clk594m", - "clk500m", - "clk432m", - "clk396m", - "clk297m", - "clk_vga", /*600MHz*/ - "clk294m", -}; - -PNAME(gpu_aclk_p) = { - "clk334m", - "clk648m", - "clk594m", - "clk500m", - "clk396m", - "clk297m", - "clk_vga", /*600MHz*/ - "clk294m", -}; - -PNAME(sappu_aclk_p) = { - "clk396m", - "clk500m", - "clk250m", - "clk148m5", -}; - -PNAME(sappu_wclk_p) = { - "clk198m", - "clk396m", - "clk334m", - "clk297m", - "clk250m", - "clk148m5", - "clk125m", - "clk99m", -}; - -PNAME(vou_aclk_p) = { - "clk334m", - "clk594m", - "clk500m", - "clk432m", - "clk396m", - "clk297m", - "clk_vga", /*600MHz*/ - "clk294m", -}; - -PNAME(vou_main_wclk_p) = { - "clk108m", - "clk594m", - "clk297m", - "clk148m5", - "clk74m25", - "clk54m", - "clk27m", - "clk_vga", -}; - -PNAME(vou_aux_wclk_p) = { - "clk108m", - "clk148m5", - "clk74m25", - "clk54m", - "clk27m", - "clk_vga", - "clk54m_mm0", - "clk" -}; - -PNAME(vou_ppu_wclk_p) = { - "clk334m", - "clk432m", - "clk396m", - "clk297m", - "clk250m", - "clk125m", - "clk198m", - "clk99m", -}; - -PNAME(vga_i2c_wclk_p) = { - "osc24m", - "clk99m", -}; - -PNAME(viu_m0_aclk_p) = { - "clk334m", - "clk432m", - "clk396m", - "clk297m", - "clk250m", - "clk125m", - "clk198m", - "osc24m", -}; - -PNAME(viu_m1_aclk_p) = { - "clk198m", - "clk250m", - "clk297m", - "clk125m", - "clk396m", - "clk334m", - "clk148m5", - "osc24m", -}; - -PNAME(viu_clk_p) = { - "clk198m", - "clk334m", - "clk297m", - "clk250m", - "clk396m", - "clk125m", - "clk99m", - "clk148m5", -}; - -PNAME(viu_jpeg_clk_p) = { - "clk334m", - "clk480m", - "clk432m", - "clk396m", - "clk297m", - "clk250m", - "clk125m", - "clk198m", -}; - -PNAME(ts_sys_clk_p) = { - "clk192m", - "clk167m", - "clk125m", - "clk99m", -}; - -PNAME(wdt_ares_p) = { - "osc24m", - "clk32k" -}; - -static struct clk_zx_pll zx296718_pll_clk[] = { - ZX296718_PLL("pll_cpu", "osc24m", PLL_CPU_REG, pll_cpu_table), - ZX296718_PLL("pll_vga", "osc24m", PLL_VGA_REG, pll_vga_table), -}; - -static struct zx_clk_fixed_factor top_ffactor_clk[] = { - FFACTOR(0, "clk4m", "osc24m", 1, 6, 0), - FFACTOR(0, "clk2m", "osc24m", 1, 12, 0), - /* pll cpu */ - FFACTOR(0, "clk1600m", "pll_cpu", 1, 1, CLK_SET_RATE_PARENT), - FFACTOR(0, "clk800m", "pll_cpu", 1, 2, CLK_SET_RATE_PARENT), - /* pll mac */ - FFACTOR(0, "clk25m", "pll_mac", 1, 40, 0), - FFACTOR(0, "clk125m", "pll_mac", 1, 8, 0), - FFACTOR(0, "clk250m", "pll_mac", 1, 4, 0), - FFACTOR(0, "clk50m", "pll_mac", 1, 20, 0), - FFACTOR(0, "clk500m", "pll_mac", 1, 2, 0), - FFACTOR(0, "clk1000m", "pll_mac", 1, 1, 0), - FFACTOR(0, "clk334m", "pll_mac", 1, 3, 0), - FFACTOR(0, "clk167m", "pll_mac", 1, 6, 0), - /* pll mm */ - FFACTOR(0, "clk54m_mm0", "pll_mm0", 1, 22, 0), - FFACTOR(0, "clk74m25", "pll_mm0", 1, 16, 0), - FFACTOR(0, "clk148m5", "pll_mm0", 1, 8, 0), - FFACTOR(0, "clk297m", "pll_mm0", 1, 4, 0), - FFACTOR(0, "clk594m", "pll_mm0", 1, 2, 0), - FFACTOR(0, "pll_mm0_1188m", "pll_mm0", 1, 1, 0), - FFACTOR(0, "clk396m", "pll_mm0", 1, 3, 0), - FFACTOR(0, "clk198m", "pll_mm0", 1, 6, 0), - FFACTOR(0, "clk99m", "pll_mm0", 1, 12, 0), - FFACTOR(0, "clk49m5", "pll_mm0", 1, 24, 0), - /* pll mm */ - FFACTOR(0, "clk324m", "pll_mm1", 1, 4, 0), - FFACTOR(0, "clk648m", "pll_mm1", 1, 2, 0), - FFACTOR(0, "pll_mm1_1296m", "pll_mm1", 1, 1, 0), - FFACTOR(0, "clk216m", "pll_mm1", 1, 6, 0), - FFACTOR(0, "clk432m", "pll_mm1", 1, 3, 0), - FFACTOR(0, "clk108m", "pll_mm1", 1, 12, 0), - FFACTOR(0, "clk72m", "pll_mm1", 1, 18, 0), - FFACTOR(0, "clk27m", "pll_mm1", 1, 48, 0), - FFACTOR(0, "clk54m", "pll_mm1", 1, 24, 0), - /* vga */ - FFACTOR(0, "pll_vga_1800m", "pll_vga", 1, 1, 0), - FFACTOR(0, "clk_vga", "pll_vga", 1, 1, CLK_SET_RATE_PARENT), - /* pll ddr */ - FFACTOR(0, "clk466m", "pll_ddr", 1, 2, 0), - - /* pll audio */ - FFACTOR(0, "pll_audio_1800m", "pll_audio", 1, 1, 0), - FFACTOR(0, "clk32k768", "pll_audio", 1, 27000, 0), - FFACTOR(0, "clk16m384", "pll_audio", 1, 54, 0), - FFACTOR(0, "clk294m", "pll_audio", 1, 3, 0), - - /* pll hsic*/ - FFACTOR(0, "clk240m", "pll_hsic", 1, 4, 0), - FFACTOR(0, "clk480m", "pll_hsic", 1, 2, 0), - FFACTOR(0, "clk192m", "pll_hsic", 1, 5, 0), - FFACTOR(0, "clk_pll_24m", "pll_hsic", 1, 40, 0), - FFACTOR(0, "emmc_mux_div2", "emmc_mux", 1, 2, CLK_SET_RATE_PARENT), -}; - -static const struct clk_div_table noc_div_table[] = { - { .val = 1, .div = 2, }, - { .val = 3, .div = 4, }, -}; -static struct zx_clk_div top_div_clk[] = { - DIV_T(0, "sys_noc_hclk", "sys_noc_aclk", TOP_CLK_DIV0, 0, 2, 0, noc_div_table), - DIV_T(0, "sys_noc_pclk", "sys_noc_aclk", TOP_CLK_DIV0, 4, 2, 0, noc_div_table), -}; - -static struct zx_clk_mux top_mux_clk[] = { - MUX(0, "dbg_mux", dbg_wclk_p, TOP_CLK_MUX0, 12, 2), - MUX(0, "a72_mux", a72_coreclk_p, TOP_CLK_MUX0, 8, 3), - MUX(0, "cpu_peri_mux", cpu_periclk_p, TOP_CLK_MUX0, 4, 3), - MUX_F(0, "a53_mux", a53_coreclk_p, TOP_CLK_MUX0, 0, 3, CLK_SET_RATE_PARENT, 0), - MUX(0, "sys_noc_aclk", sys_noc_alck_p, TOP_CLK_MUX1, 0, 3), - MUX(0, "sec_mux", sec_wclk_p, TOP_CLK_MUX2, 16, 3), - MUX(0, "sd1_mux", sd_nand_wclk_p, TOP_CLK_MUX2, 12, 3), - MUX(0, "sd0_mux", sd_nand_wclk_p, TOP_CLK_MUX2, 8, 3), - MUX(0, "emmc_mux", emmc_wclk_p, TOP_CLK_MUX2, 4, 3), - MUX(0, "nand_mux", sd_nand_wclk_p, TOP_CLK_MUX2, 0, 3), - MUX(0, "usb_ref24m_mux", usb_ref24m_p, TOP_CLK_MUX9, 16, 1), - MUX(0, "clk32k", clk32_p, TOP_CLK_MUX9, 12, 1), - MUX_F(0, "wdt_mux", wdt_ares_p, TOP_CLK_MUX9, 8, 1, CLK_SET_RATE_PARENT, 0), - MUX(0, "timer_mux", osc, TOP_CLK_MUX9, 4, 1), - MUX(0, "vde_mux", vde_aclk_p, TOP_CLK_MUX4, 0, 3), - MUX(0, "vce_mux", vce_aclk_p, TOP_CLK_MUX4, 4, 3), - MUX(0, "hde_mux", hde_aclk_p, TOP_CLK_MUX4, 8, 3), - MUX(0, "gpu_mux", gpu_aclk_p, TOP_CLK_MUX5, 0, 3), - MUX(0, "sappu_a_mux", sappu_aclk_p, TOP_CLK_MUX5, 4, 2), - MUX(0, "sappu_w_mux", sappu_wclk_p, TOP_CLK_MUX5, 8, 3), - MUX(0, "vou_a_mux", vou_aclk_p, TOP_CLK_MUX7, 0, 3), - MUX_F(0, "vou_main_w_mux", vou_main_wclk_p, TOP_CLK_MUX7, 4, 3, CLK_SET_RATE_PARENT, 0), - MUX_F(0, "vou_aux_w_mux", vou_aux_wclk_p, TOP_CLK_MUX7, 8, 3, CLK_SET_RATE_PARENT, 0), - MUX(0, "vou_ppu_w_mux", vou_ppu_wclk_p, TOP_CLK_MUX7, 12, 3), - MUX(0, "vga_i2c_mux", vga_i2c_wclk_p, TOP_CLK_MUX7, 16, 1), - MUX(0, "viu_m0_a_mux", viu_m0_aclk_p, TOP_CLK_MUX6, 0, 3), - MUX(0, "viu_m1_a_mux", viu_m1_aclk_p, TOP_CLK_MUX6, 4, 3), - MUX(0, "viu_w_mux", viu_clk_p, TOP_CLK_MUX6, 8, 3), - MUX(0, "viu_jpeg_w_mux", viu_jpeg_clk_p, TOP_CLK_MUX6, 12, 3), - MUX(0, "ts_sys_mux", ts_sys_clk_p, TOP_CLK_MUX6, 16, 2), -}; - -static struct zx_clk_gate top_gate_clk[] = { - GATE(CPU_DBG_GATE, "dbg_wclk", "dbg_mux", TOP_CLK_GATE0, 4, CLK_SET_RATE_PARENT, 0), - GATE(A72_GATE, "a72_coreclk", "a72_mux", TOP_CLK_GATE0, 3, CLK_SET_RATE_PARENT, 0), - GATE(CPU_PERI_GATE, "cpu_peri", "cpu_peri_mux", TOP_CLK_GATE0, 1, CLK_SET_RATE_PARENT, 0), - GATE(A53_GATE, "a53_coreclk", "a53_mux", TOP_CLK_GATE0, 0, CLK_SET_RATE_PARENT, 0), - GATE(SD1_WCLK, "sd1_wclk", "sd1_mux", TOP_CLK_GATE1, 13, CLK_SET_RATE_PARENT, 0), - GATE(SD0_WCLK, "sd0_wclk", "sd0_mux", TOP_CLK_GATE1, 9, CLK_SET_RATE_PARENT, 0), - GATE(EMMC_WCLK, "emmc_wclk", "emmc_mux_div2", TOP_CLK_GATE0, 5, CLK_SET_RATE_PARENT, 0), - GATE(EMMC_NAND_AXI, "emmc_nand_aclk", "sys_noc_aclk", TOP_CLK_GATE1, 4, CLK_SET_RATE_PARENT, 0), - GATE(NAND_WCLK, "nand_wclk", "nand_mux", TOP_CLK_GATE0, 1, CLK_SET_RATE_PARENT, 0), - GATE(EMMC_NAND_AHB, "emmc_nand_hclk", "sys_noc_hclk", TOP_CLK_GATE1, 0, CLK_SET_RATE_PARENT, 0), - GATE(0, "lsp1_pclk", "sys_noc_pclk", TOP_CLK_GATE2, 31, 0, 0), - GATE(LSP1_148M5, "lsp1_148m5", "clk148m5", TOP_CLK_GATE2, 30, 0, 0), - GATE(LSP1_99M, "lsp1_99m", "clk99m", TOP_CLK_GATE2, 29, 0, 0), - GATE(LSP1_24M, "lsp1_24m", "osc24m", TOP_CLK_GATE2, 28, 0, 0), - GATE(LSP0_74M25, "lsp0_74m25", "clk74m25", TOP_CLK_GATE2, 25, 0, 0), - GATE(0, "lsp0_pclk", "sys_noc_pclk", TOP_CLK_GATE2, 24, 0, 0), - GATE(LSP0_32K, "lsp0_32k", "osc32k", TOP_CLK_GATE2, 23, 0, 0), - GATE(LSP0_148M5, "lsp0_148m5", "clk148m5", TOP_CLK_GATE2, 22, 0, 0), - GATE(LSP0_99M, "lsp0_99m", "clk99m", TOP_CLK_GATE2, 21, 0, 0), - GATE(LSP0_24M, "lsp0_24m", "osc24m", TOP_CLK_GATE2, 20, 0, 0), - GATE(AUDIO_99M, "audio_99m", "clk99m", TOP_CLK_GATE5, 27, 0, 0), - GATE(AUDIO_24M, "audio_24m", "osc24m", TOP_CLK_GATE5, 28, 0, 0), - GATE(AUDIO_16M384, "audio_16m384", "clk16m384", TOP_CLK_GATE5, 29, 0, 0), - GATE(AUDIO_32K, "audio_32k", "clk32k", TOP_CLK_GATE5, 30, 0, 0), - GATE(WDT_WCLK, "wdt_wclk", "wdt_mux", TOP_CLK_GATE6, 9, CLK_SET_RATE_PARENT, 0), - GATE(TIMER_WCLK, "timer_wclk", "timer_mux", TOP_CLK_GATE6, 5, CLK_SET_RATE_PARENT, 0), - GATE(VDE_ACLK, "vde_aclk", "vde_mux", TOP_CLK_GATE3, 0, CLK_SET_RATE_PARENT, 0), - GATE(VCE_ACLK, "vce_aclk", "vce_mux", TOP_CLK_GATE3, 4, CLK_SET_RATE_PARENT, 0), - GATE(HDE_ACLK, "hde_aclk", "hde_mux", TOP_CLK_GATE3, 8, CLK_SET_RATE_PARENT, 0), - GATE(GPU_ACLK, "gpu_aclk", "gpu_mux", TOP_CLK_GATE3, 16, CLK_SET_RATE_PARENT, 0), - GATE(SAPPU_ACLK, "sappu_aclk", "sappu_a_mux", TOP_CLK_GATE3, 20, CLK_SET_RATE_PARENT, 0), - GATE(SAPPU_WCLK, "sappu_wclk", "sappu_w_mux", TOP_CLK_GATE3, 22, CLK_SET_RATE_PARENT, 0), - GATE(VOU_ACLK, "vou_aclk", "vou_a_mux", TOP_CLK_GATE4, 16, CLK_SET_RATE_PARENT, 0), - GATE(VOU_MAIN_WCLK, "vou_main_wclk", "vou_main_w_mux", TOP_CLK_GATE4, 18, CLK_SET_RATE_PARENT, 0), - GATE(VOU_AUX_WCLK, "vou_aux_wclk", "vou_aux_w_mux", TOP_CLK_GATE4, 19, CLK_SET_RATE_PARENT, 0), - GATE(VOU_PPU_WCLK, "vou_ppu_wclk", "vou_ppu_w_mux", TOP_CLK_GATE4, 20, CLK_SET_RATE_PARENT, 0), - GATE(MIPI_CFG_CLK, "mipi_cfg_clk", "osc24m", TOP_CLK_GATE4, 21, 0, 0), - GATE(VGA_I2C_WCLK, "vga_i2c_wclk", "vga_i2c_mux", TOP_CLK_GATE4, 23, CLK_SET_RATE_PARENT, 0), - GATE(MIPI_REF_CLK, "mipi_ref_clk", "clk27m", TOP_CLK_GATE4, 24, 0, 0), - GATE(HDMI_OSC_CEC, "hdmi_osc_cec", "clk2m", TOP_CLK_GATE4, 22, 0, 0), - GATE(HDMI_OSC_CLK, "hdmi_osc_clk", "clk240m", TOP_CLK_GATE4, 25, 0, 0), - GATE(HDMI_XCLK, "hdmi_xclk", "osc24m", TOP_CLK_GATE4, 26, 0, 0), - GATE(VIU_M0_ACLK, "viu_m0_aclk", "viu_m0_a_mux", TOP_CLK_GATE4, 0, CLK_SET_RATE_PARENT, 0), - GATE(VIU_M1_ACLK, "viu_m1_aclk", "viu_m1_a_mux", TOP_CLK_GATE4, 1, CLK_SET_RATE_PARENT, 0), - GATE(VIU_WCLK, "viu_wclk", "viu_w_mux", TOP_CLK_GATE4, 2, CLK_SET_RATE_PARENT, 0), - GATE(VIU_JPEG_WCLK, "viu_jpeg_wclk", "viu_jpeg_w_mux", TOP_CLK_GATE4, 3, CLK_SET_RATE_PARENT, 0), - GATE(VIU_CFG_CLK, "viu_cfg_clk", "osc24m", TOP_CLK_GATE4, 6, 0, 0), - GATE(TS_SYS_WCLK, "ts_sys_wclk", "ts_sys_mux", TOP_CLK_GATE5, 2, CLK_SET_RATE_PARENT, 0), - GATE(TS_SYS_108M, "ts_sys_108m", "clk108m", TOP_CLK_GATE5, 3, 0, 0), - GATE(USB20_HCLK, "usb20_hclk", "sys_noc_hclk", TOP_CLK_GATE2, 12, 0, 0), - GATE(USB20_PHY_CLK, "usb20_phy_clk", "usb_ref24m_mux", TOP_CLK_GATE2, 13, 0, 0), - GATE(USB21_HCLK, "usb21_hclk", "sys_noc_hclk", TOP_CLK_GATE2, 14, 0, 0), - GATE(USB21_PHY_CLK, "usb21_phy_clk", "usb_ref24m_mux", TOP_CLK_GATE2, 15, 0, 0), - GATE(GMAC_RMIICLK, "gmac_rmii_clk", "clk50m", TOP_CLK_GATE2, 3, 0, 0), - GATE(GMAC_PCLK, "gmac_pclk", "clk198m", TOP_CLK_GATE2, 1, 0, 0), - GATE(GMAC_ACLK, "gmac_aclk", "clk49m5", TOP_CLK_GATE2, 0, 0, 0), - GATE(GMAC_RFCLK, "gmac_refclk", "clk25m", TOP_CLK_GATE2, 4, 0, 0), - GATE(SD1_AHB, "sd1_hclk", "sys_noc_hclk", TOP_CLK_GATE1, 12, 0, 0), - GATE(SD0_AHB, "sd0_hclk", "sys_noc_hclk", TOP_CLK_GATE1, 8, 0, 0), - GATE(TEMPSENSOR_GATE, "tempsensor_gate", "clk4m", TOP_CLK_GATE5, 31, 0, 0), -}; - -static struct clk_hw_onecell_data top_hw_onecell_data = { - .num = TOP_NR_CLKS, - .hws = { - [TOP_NR_CLKS - 1] = NULL, - }, -}; - -static int __init top_clocks_init(struct device_node *np) -{ - void __iomem *reg_base; - int i, ret; - const char *name; - - reg_base = of_iomap(np, 0); - if (!reg_base) { - pr_err("%s: Unable to map clk base\n", __func__); - return -ENXIO; - } - - for (i = 0; i < ARRAY_SIZE(zx296718_pll_clk); i++) { - zx296718_pll_clk[i].reg_base += (uintptr_t)reg_base; - name = zx296718_pll_clk[i].hw.init->name; - ret = clk_hw_register(NULL, &zx296718_pll_clk[i].hw); - if (ret) - pr_warn("top clk %s init error!\n", name); - } - - for (i = 0; i < ARRAY_SIZE(top_ffactor_clk); i++) { - if (top_ffactor_clk[i].id) - top_hw_onecell_data.hws[top_ffactor_clk[i].id] = - &top_ffactor_clk[i].factor.hw; - - name = top_ffactor_clk[i].factor.hw.init->name; - ret = clk_hw_register(NULL, &top_ffactor_clk[i].factor.hw); - if (ret) - pr_warn("top clk %s init error!\n", name); - } - - for (i = 0; i < ARRAY_SIZE(top_mux_clk); i++) { - if (top_mux_clk[i].id) - top_hw_onecell_data.hws[top_mux_clk[i].id] = - &top_mux_clk[i].mux.hw; - - top_mux_clk[i].mux.reg += (uintptr_t)reg_base; - name = top_mux_clk[i].mux.hw.init->name; - ret = clk_hw_register(NULL, &top_mux_clk[i].mux.hw); - if (ret) - pr_warn("top clk %s init error!\n", name); - } - - for (i = 0; i < ARRAY_SIZE(top_gate_clk); i++) { - if (top_gate_clk[i].id) - top_hw_onecell_data.hws[top_gate_clk[i].id] = - &top_gate_clk[i].gate.hw; - - top_gate_clk[i].gate.reg += (uintptr_t)reg_base; - name = top_gate_clk[i].gate.hw.init->name; - ret = clk_hw_register(NULL, &top_gate_clk[i].gate.hw); - if (ret) - pr_warn("top clk %s init error!\n", name); - } - - for (i = 0; i < ARRAY_SIZE(top_div_clk); i++) { - if (top_div_clk[i].id) - top_hw_onecell_data.hws[top_div_clk[i].id] = - &top_div_clk[i].div.hw; - - top_div_clk[i].div.reg += (uintptr_t)reg_base; - name = top_div_clk[i].div.hw.init->name; - ret = clk_hw_register(NULL, &top_div_clk[i].div.hw); - if (ret) - pr_warn("top clk %s init error!\n", name); - } - - ret = of_clk_add_hw_provider(np, of_clk_hw_onecell_get, - &top_hw_onecell_data); - if (ret) { - pr_err("failed to register top clk provider: %d\n", ret); - return ret; - } - - return 0; -} - -static const struct clk_div_table common_even_div_table[] = { - { .val = 0, .div = 1, }, - { .val = 1, .div = 2, }, - { .val = 3, .div = 4, }, - { .val = 5, .div = 6, }, - { .val = 7, .div = 8, }, - { .val = 9, .div = 10, }, - { .val = 11, .div = 12, }, - { .val = 13, .div = 14, }, - { .val = 15, .div = 16, }, -}; - -static const struct clk_div_table common_div_table[] = { - { .val = 0, .div = 1, }, - { .val = 1, .div = 2, }, - { .val = 2, .div = 3, }, - { .val = 3, .div = 4, }, - { .val = 4, .div = 5, }, - { .val = 5, .div = 6, }, - { .val = 6, .div = 7, }, - { .val = 7, .div = 8, }, - { .val = 8, .div = 9, }, - { .val = 9, .div = 10, }, - { .val = 10, .div = 11, }, - { .val = 11, .div = 12, }, - { .val = 12, .div = 13, }, - { .val = 13, .div = 14, }, - { .val = 14, .div = 15, }, - { .val = 15, .div = 16, }, -}; - -PNAME(lsp0_wclk_common_p) = { - "lsp0_24m", - "lsp0_99m", -}; - -PNAME(lsp0_wclk_timer3_p) = { - "timer3_div", - "lsp0_32k" -}; - -PNAME(lsp0_wclk_timer4_p) = { - "timer4_div", - "lsp0_32k" -}; - -PNAME(lsp0_wclk_timer5_p) = { - "timer5_div", - "lsp0_32k" -}; - -PNAME(lsp0_wclk_spifc0_p) = { - "lsp0_148m5", - "lsp0_24m", - "lsp0_99m", - "lsp0_74m25" -}; - -PNAME(lsp0_wclk_ssp_p) = { - "lsp0_148m5", - "lsp0_99m", - "lsp0_24m", -}; - -static struct zx_clk_mux lsp0_mux_clk[] = { - MUX(0, "timer3_wclk_mux", lsp0_wclk_timer3_p, LSP0_TIMER3_CLK, 4, 1), - MUX(0, "timer4_wclk_mux", lsp0_wclk_timer4_p, LSP0_TIMER4_CLK, 4, 1), - MUX(0, "timer5_wclk_mux", lsp0_wclk_timer5_p, LSP0_TIMER5_CLK, 4, 1), - MUX(0, "uart3_wclk_mux", lsp0_wclk_common_p, LSP0_UART3_CLK, 4, 1), - MUX(0, "uart1_wclk_mux", lsp0_wclk_common_p, LSP0_UART1_CLK, 4, 1), - MUX(0, "uart2_wclk_mux", lsp0_wclk_common_p, LSP0_UART2_CLK, 4, 1), - MUX(0, "spifc0_wclk_mux", lsp0_wclk_spifc0_p, LSP0_SPIFC0_CLK, 4, 2), - MUX(0, "i2c4_wclk_mux", lsp0_wclk_common_p, LSP0_I2C4_CLK, 4, 1), - MUX(0, "i2c5_wclk_mux", lsp0_wclk_common_p, LSP0_I2C5_CLK, 4, 1), - MUX(0, "ssp0_wclk_mux", lsp0_wclk_ssp_p, LSP0_SSP0_CLK, 4, 1), - MUX(0, "ssp1_wclk_mux", lsp0_wclk_ssp_p, LSP0_SSP1_CLK, 4, 1), - MUX(0, "i2c3_wclk_mux", lsp0_wclk_common_p, LSP0_I2C3_CLK, 4, 1), -}; - -static struct zx_clk_gate lsp0_gate_clk[] = { - GATE(LSP0_TIMER3_WCLK, "timer3_wclk", "timer3_wclk_mux", LSP0_TIMER3_CLK, 1, CLK_SET_RATE_PARENT, 0), - GATE(LSP0_TIMER4_WCLK, "timer4_wclk", "timer4_wclk_mux", LSP0_TIMER4_CLK, 1, CLK_SET_RATE_PARENT, 0), - GATE(LSP0_TIMER5_WCLK, "timer5_wclk", "timer5_wclk_mux", LSP0_TIMER5_CLK, 1, CLK_SET_RATE_PARENT, 0), - GATE(LSP0_UART3_WCLK, "uart3_wclk", "uart3_wclk_mux", LSP0_UART3_CLK, 1, CLK_SET_RATE_PARENT, 0), - GATE(LSP0_UART1_WCLK, "uart1_wclk", "uart1_wclk_mux", LSP0_UART1_CLK, 1, CLK_SET_RATE_PARENT, 0), - GATE(LSP0_UART2_WCLK, "uart2_wclk", "uart2_wclk_mux", LSP0_UART2_CLK, 1, CLK_SET_RATE_PARENT, 0), - GATE(LSP0_SPIFC0_WCLK, "spifc0_wclk", "spifc0_wclk_mux", LSP0_SPIFC0_CLK, 1, CLK_SET_RATE_PARENT, 0), - GATE(LSP0_I2C4_WCLK, "i2c4_wclk", "i2c4_wclk_mux", LSP0_I2C4_CLK, 1, CLK_SET_RATE_PARENT, 0), - GATE(LSP0_I2C5_WCLK, "i2c5_wclk", "i2c5_wclk_mux", LSP0_I2C5_CLK, 1, CLK_SET_RATE_PARENT, 0), - GATE(LSP0_SSP0_WCLK, "ssp0_wclk", "ssp0_div", LSP0_SSP0_CLK, 1, CLK_SET_RATE_PARENT, 0), - GATE(LSP0_SSP1_WCLK, "ssp1_wclk", "ssp1_div", LSP0_SSP1_CLK, 1, CLK_SET_RATE_PARENT, 0), - GATE(LSP0_I2C3_WCLK, "i2c3_wclk", "i2c3_wclk_mux", LSP0_I2C3_CLK, 1, CLK_SET_RATE_PARENT, 0), -}; - -static struct zx_clk_div lsp0_div_clk[] = { - DIV_T(0, "timer3_div", "lsp0_24m", LSP0_TIMER3_CLK, 12, 4, 0, common_even_div_table), - DIV_T(0, "timer4_div", "lsp0_24m", LSP0_TIMER4_CLK, 12, 4, 0, common_even_div_table), - DIV_T(0, "timer5_div", "lsp0_24m", LSP0_TIMER5_CLK, 12, 4, 0, common_even_div_table), - DIV_T(0, "ssp0_div", "ssp0_wclk_mux", LSP0_SSP0_CLK, 12, 4, 0, common_even_div_table), - DIV_T(0, "ssp1_div", "ssp1_wclk_mux", LSP0_SSP1_CLK, 12, 4, 0, common_even_div_table), -}; - -static struct clk_hw_onecell_data lsp0_hw_onecell_data = { - .num = LSP0_NR_CLKS, - .hws = { - [LSP0_NR_CLKS - 1] = NULL, - }, -}; - -static int __init lsp0_clocks_init(struct device_node *np) -{ - void __iomem *reg_base; - int i, ret; - const char *name; - - reg_base = of_iomap(np, 0); - if (!reg_base) { - pr_err("%s: Unable to map clk base\n", __func__); - return -ENXIO; - } - - for (i = 0; i < ARRAY_SIZE(lsp0_mux_clk); i++) { - if (lsp0_mux_clk[i].id) - lsp0_hw_onecell_data.hws[lsp0_mux_clk[i].id] = - &lsp0_mux_clk[i].mux.hw; - - lsp0_mux_clk[i].mux.reg += (uintptr_t)reg_base; - name = lsp0_mux_clk[i].mux.hw.init->name; - ret = clk_hw_register(NULL, &lsp0_mux_clk[i].mux.hw); - if (ret) - pr_warn("lsp0 clk %s init error!\n", name); - } - - for (i = 0; i < ARRAY_SIZE(lsp0_gate_clk); i++) { - if (lsp0_gate_clk[i].id) - lsp0_hw_onecell_data.hws[lsp0_gate_clk[i].id] = - &lsp0_gate_clk[i].gate.hw; - - lsp0_gate_clk[i].gate.reg += (uintptr_t)reg_base; - name = lsp0_gate_clk[i].gate.hw.init->name; - ret = clk_hw_register(NULL, &lsp0_gate_clk[i].gate.hw); - if (ret) - pr_warn("lsp0 clk %s init error!\n", name); - } - - for (i = 0; i < ARRAY_SIZE(lsp0_div_clk); i++) { - if (lsp0_div_clk[i].id) - lsp0_hw_onecell_data.hws[lsp0_div_clk[i].id] = - &lsp0_div_clk[i].div.hw; - - lsp0_div_clk[i].div.reg += (uintptr_t)reg_base; - name = lsp0_div_clk[i].div.hw.init->name; - ret = clk_hw_register(NULL, &lsp0_div_clk[i].div.hw); - if (ret) - pr_warn("lsp0 clk %s init error!\n", name); - } - - ret = of_clk_add_hw_provider(np, of_clk_hw_onecell_get, - &lsp0_hw_onecell_data); - if (ret) { - pr_err("failed to register lsp0 clk provider: %d\n", ret); - return ret; - } - - return 0; -} - -PNAME(lsp1_wclk_common_p) = { - "lsp1_24m", - "lsp1_99m", -}; - -PNAME(lsp1_wclk_ssp_p) = { - "lsp1_148m5", - "lsp1_99m", - "lsp1_24m", -}; - -static struct zx_clk_mux lsp1_mux_clk[] = { - MUX(0, "uart4_wclk_mux", lsp1_wclk_common_p, LSP1_UART4_CLK, 4, 1), - MUX(0, "uart5_wclk_mux", lsp1_wclk_common_p, LSP1_UART5_CLK, 4, 1), - MUX(0, "pwm_wclk_mux", lsp1_wclk_common_p, LSP1_PWM_CLK, 4, 1), - MUX(0, "i2c2_wclk_mux", lsp1_wclk_common_p, LSP1_I2C2_CLK, 4, 1), - MUX(0, "ssp2_wclk_mux", lsp1_wclk_ssp_p, LSP1_SSP2_CLK, 4, 2), - MUX(0, "ssp3_wclk_mux", lsp1_wclk_ssp_p, LSP1_SSP3_CLK, 4, 2), - MUX(0, "ssp4_wclk_mux", lsp1_wclk_ssp_p, LSP1_SSP4_CLK, 4, 2), - MUX(0, "usim1_wclk_mux", lsp1_wclk_common_p, LSP1_USIM1_CLK, 4, 1), -}; - -static struct zx_clk_div lsp1_div_clk[] = { - DIV_T(0, "pwm_div", "pwm_wclk_mux", LSP1_PWM_CLK, 12, 4, CLK_SET_RATE_PARENT, common_div_table), - DIV_T(0, "ssp2_div", "ssp2_wclk_mux", LSP1_SSP2_CLK, 12, 4, CLK_SET_RATE_PARENT, common_even_div_table), - DIV_T(0, "ssp3_div", "ssp3_wclk_mux", LSP1_SSP3_CLK, 12, 4, CLK_SET_RATE_PARENT, common_even_div_table), - DIV_T(0, "ssp4_div", "ssp4_wclk_mux", LSP1_SSP4_CLK, 12, 4, CLK_SET_RATE_PARENT, common_even_div_table), -}; - -static struct zx_clk_gate lsp1_gate_clk[] = { - GATE(LSP1_UART4_WCLK, "lsp1_uart4_wclk", "uart4_wclk_mux", LSP1_UART4_CLK, 1, CLK_SET_RATE_PARENT, 0), - GATE(LSP1_UART5_WCLK, "lsp1_uart5_wclk", "uart5_wclk_mux", LSP1_UART5_CLK, 1, CLK_SET_RATE_PARENT, 0), - GATE(LSP1_PWM_WCLK, "lsp1_pwm_wclk", "pwm_div", LSP1_PWM_CLK, 1, CLK_SET_RATE_PARENT, 0), - GATE(LSP1_PWM_PCLK, "lsp1_pwm_pclk", "lsp1_pclk", LSP1_PWM_CLK, 0, 0, 0), - GATE(LSP1_I2C2_WCLK, "lsp1_i2c2_wclk", "i2c2_wclk_mux", LSP1_I2C2_CLK, 1, CLK_SET_RATE_PARENT, 0), - GATE(LSP1_SSP2_WCLK, "lsp1_ssp2_wclk", "ssp2_div", LSP1_SSP2_CLK, 1, CLK_SET_RATE_PARENT, 0), - GATE(LSP1_SSP3_WCLK, "lsp1_ssp3_wclk", "ssp3_div", LSP1_SSP3_CLK, 1, CLK_SET_RATE_PARENT, 0), - GATE(LSP1_SSP4_WCLK, "lsp1_ssp4_wclk", "ssp4_div", LSP1_SSP4_CLK, 1, CLK_SET_RATE_PARENT, 0), - GATE(LSP1_USIM1_WCLK, "lsp1_usim1_wclk", "usim1_wclk_mux", LSP1_USIM1_CLK, 1, CLK_SET_RATE_PARENT, 0), -}; - -static struct clk_hw_onecell_data lsp1_hw_onecell_data = { - .num = LSP1_NR_CLKS, - .hws = { - [LSP1_NR_CLKS - 1] = NULL, - }, -}; - -static int __init lsp1_clocks_init(struct device_node *np) -{ - void __iomem *reg_base; - int i, ret; - const char *name; - - reg_base = of_iomap(np, 0); - if (!reg_base) { - pr_err("%s: Unable to map clk base\n", __func__); - return -ENXIO; - } - - for (i = 0; i < ARRAY_SIZE(lsp1_mux_clk); i++) { - if (lsp1_mux_clk[i].id) - lsp1_hw_onecell_data.hws[lsp1_mux_clk[i].id] = - &lsp0_mux_clk[i].mux.hw; - - lsp1_mux_clk[i].mux.reg += (uintptr_t)reg_base; - name = lsp1_mux_clk[i].mux.hw.init->name; - ret = clk_hw_register(NULL, &lsp1_mux_clk[i].mux.hw); - if (ret) - pr_warn("lsp1 clk %s init error!\n", name); - } - - for (i = 0; i < ARRAY_SIZE(lsp1_gate_clk); i++) { - if (lsp1_gate_clk[i].id) - lsp1_hw_onecell_data.hws[lsp1_gate_clk[i].id] = - &lsp1_gate_clk[i].gate.hw; - - lsp1_gate_clk[i].gate.reg += (uintptr_t)reg_base; - name = lsp1_gate_clk[i].gate.hw.init->name; - ret = clk_hw_register(NULL, &lsp1_gate_clk[i].gate.hw); - if (ret) - pr_warn("lsp1 clk %s init error!\n", name); - } - - for (i = 0; i < ARRAY_SIZE(lsp1_div_clk); i++) { - if (lsp1_div_clk[i].id) - lsp1_hw_onecell_data.hws[lsp1_div_clk[i].id] = - &lsp1_div_clk[i].div.hw; - - lsp1_div_clk[i].div.reg += (uintptr_t)reg_base; - name = lsp1_div_clk[i].div.hw.init->name; - ret = clk_hw_register(NULL, &lsp1_div_clk[i].div.hw); - if (ret) - pr_warn("lsp1 clk %s init error!\n", name); - } - - ret = of_clk_add_hw_provider(np, of_clk_hw_onecell_get, - &lsp1_hw_onecell_data); - if (ret) { - pr_err("failed to register lsp1 clk provider: %d\n", ret); - return ret; - } - - return 0; -} - -PNAME(audio_wclk_common_p) = { - "audio_99m", - "audio_24m", -}; - -PNAME(audio_timer_p) = { - "audio_24m", - "audio_32k", -}; - -static struct zx_clk_mux audio_mux_clk[] = { - MUX(I2S0_WCLK_MUX, "i2s0_wclk_mux", audio_wclk_common_p, AUDIO_I2S0_CLK, 0, 1), - MUX(I2S1_WCLK_MUX, "i2s1_wclk_mux", audio_wclk_common_p, AUDIO_I2S1_CLK, 0, 1), - MUX(I2S2_WCLK_MUX, "i2s2_wclk_mux", audio_wclk_common_p, AUDIO_I2S2_CLK, 0, 1), - MUX(I2S3_WCLK_MUX, "i2s3_wclk_mux", audio_wclk_common_p, AUDIO_I2S3_CLK, 0, 1), - MUX(0, "i2c0_wclk_mux", audio_wclk_common_p, AUDIO_I2C0_CLK, 0, 1), - MUX(0, "spdif0_wclk_mux", audio_wclk_common_p, AUDIO_SPDIF0_CLK, 0, 1), - MUX(0, "spdif1_wclk_mux", audio_wclk_common_p, AUDIO_SPDIF1_CLK, 0, 1), - MUX(0, "timer_wclk_mux", audio_timer_p, AUDIO_TIMER_CLK, 0, 1), -}; - -static struct clk_zx_audio_divider audio_adiv_clk[] = { - AUDIO_DIV(0, "i2s0_wclk_div", "i2s0_wclk_mux", AUDIO_I2S0_DIV_CFG1), - AUDIO_DIV(0, "i2s1_wclk_div", "i2s1_wclk_mux", AUDIO_I2S1_DIV_CFG1), - AUDIO_DIV(0, "i2s2_wclk_div", "i2s2_wclk_mux", AUDIO_I2S2_DIV_CFG1), - AUDIO_DIV(0, "i2s3_wclk_div", "i2s3_wclk_mux", AUDIO_I2S3_DIV_CFG1), - AUDIO_DIV(0, "spdif0_wclk_div", "spdif0_wclk_mux", AUDIO_SPDIF0_DIV_CFG1), - AUDIO_DIV(0, "spdif1_wclk_div", "spdif1_wclk_mux", AUDIO_SPDIF1_DIV_CFG1), -}; - -static struct zx_clk_div audio_div_clk[] = { - DIV_T(0, "tdm_wclk_div", "audio_16m384", AUDIO_TDM_CLK, 8, 4, 0, common_div_table), -}; - -static struct zx_clk_gate audio_gate_clk[] = { - GATE(AUDIO_I2S0_WCLK, "i2s0_wclk", "i2s0_wclk_div", AUDIO_I2S0_CLK, 9, CLK_SET_RATE_PARENT, 0), - GATE(AUDIO_I2S1_WCLK, "i2s1_wclk", "i2s1_wclk_div", AUDIO_I2S1_CLK, 9, CLK_SET_RATE_PARENT, 0), - GATE(AUDIO_I2S2_WCLK, "i2s2_wclk", "i2s2_wclk_div", AUDIO_I2S2_CLK, 9, CLK_SET_RATE_PARENT, 0), - GATE(AUDIO_I2S3_WCLK, "i2s3_wclk", "i2s3_wclk_div", AUDIO_I2S3_CLK, 9, CLK_SET_RATE_PARENT, 0), - GATE(AUDIO_I2S0_PCLK, "i2s0_pclk", "clk49m5", AUDIO_I2S0_CLK, 8, 0, 0), - GATE(AUDIO_I2S1_PCLK, "i2s1_pclk", "clk49m5", AUDIO_I2S1_CLK, 8, 0, 0), - GATE(AUDIO_I2S2_PCLK, "i2s2_pclk", "clk49m5", AUDIO_I2S2_CLK, 8, 0, 0), - GATE(AUDIO_I2S3_PCLK, "i2s3_pclk", "clk49m5", AUDIO_I2S3_CLK, 8, 0, 0), - GATE(AUDIO_I2C0_WCLK, "i2c0_wclk", "i2c0_wclk_mux", AUDIO_I2C0_CLK, 9, CLK_SET_RATE_PARENT, 0), - GATE(AUDIO_SPDIF0_WCLK, "spdif0_wclk", "spdif0_wclk_div", AUDIO_SPDIF0_CLK, 9, CLK_SET_RATE_PARENT, 0), - GATE(AUDIO_SPDIF1_WCLK, "spdif1_wclk", "spdif1_wclk_div", AUDIO_SPDIF1_CLK, 9, CLK_SET_RATE_PARENT, 0), - GATE(AUDIO_TDM_WCLK, "tdm_wclk", "tdm_wclk_div", AUDIO_TDM_CLK, 17, CLK_SET_RATE_PARENT, 0), - GATE(AUDIO_TS_PCLK, "tempsensor_pclk", "clk49m5", AUDIO_TS_CLK, 1, 0, 0), -}; - -static struct clk_hw_onecell_data audio_hw_onecell_data = { - .num = AUDIO_NR_CLKS, - .hws = { - [AUDIO_NR_CLKS - 1] = NULL, - }, -}; - -static int __init audio_clocks_init(struct device_node *np) -{ - void __iomem *reg_base; - int i, ret; - const char *name; - - reg_base = of_iomap(np, 0); - if (!reg_base) { - pr_err("%s: Unable to map audio clk base\n", __func__); - return -ENXIO; - } - - for (i = 0; i < ARRAY_SIZE(audio_mux_clk); i++) { - if (audio_mux_clk[i].id) - audio_hw_onecell_data.hws[audio_mux_clk[i].id] = - &audio_mux_clk[i].mux.hw; - - audio_mux_clk[i].mux.reg += (uintptr_t)reg_base; - name = audio_mux_clk[i].mux.hw.init->name; - ret = clk_hw_register(NULL, &audio_mux_clk[i].mux.hw); - if (ret) - pr_warn("audio clk %s init error!\n", name); - } - - for (i = 0; i < ARRAY_SIZE(audio_adiv_clk); i++) { - if (audio_adiv_clk[i].id) - audio_hw_onecell_data.hws[audio_adiv_clk[i].id] = - &audio_adiv_clk[i].hw; - - audio_adiv_clk[i].reg_base += (uintptr_t)reg_base; - name = audio_adiv_clk[i].hw.init->name; - ret = clk_hw_register(NULL, &audio_adiv_clk[i].hw); - if (ret) - pr_warn("audio clk %s init error!\n", name); - } - - for (i = 0; i < ARRAY_SIZE(audio_div_clk); i++) { - if (audio_div_clk[i].id) - audio_hw_onecell_data.hws[audio_div_clk[i].id] = - &audio_div_clk[i].div.hw; - - audio_div_clk[i].div.reg += (uintptr_t)reg_base; - name = audio_div_clk[i].div.hw.init->name; - ret = clk_hw_register(NULL, &audio_div_clk[i].div.hw); - if (ret) - pr_warn("audio clk %s init error!\n", name); - } - - for (i = 0; i < ARRAY_SIZE(audio_gate_clk); i++) { - if (audio_gate_clk[i].id) - audio_hw_onecell_data.hws[audio_gate_clk[i].id] = - &audio_gate_clk[i].gate.hw; - - audio_gate_clk[i].gate.reg += (uintptr_t)reg_base; - name = audio_gate_clk[i].gate.hw.init->name; - ret = clk_hw_register(NULL, &audio_gate_clk[i].gate.hw); - if (ret) - pr_warn("audio clk %s init error!\n", name); - } - - ret = of_clk_add_hw_provider(np, of_clk_hw_onecell_get, - &audio_hw_onecell_data); - if (ret) { - pr_err("failed to register audio clk provider: %d\n", ret); - return ret; - } - - return 0; -} - -static const struct of_device_id zx_clkc_match_table[] = { - { .compatible = "zte,zx296718-topcrm", .data = &top_clocks_init }, - { .compatible = "zte,zx296718-lsp0crm", .data = &lsp0_clocks_init }, - { .compatible = "zte,zx296718-lsp1crm", .data = &lsp1_clocks_init }, - { .compatible = "zte,zx296718-audiocrm", .data = &audio_clocks_init }, - { } -}; - -static int zx_clkc_probe(struct platform_device *pdev) -{ - int (*init_fn)(struct device_node *np); - struct device_node *np = pdev->dev.of_node; - - init_fn = of_device_get_match_data(&pdev->dev); - if (!init_fn) { - dev_err(&pdev->dev, "Error: No device match found\n"); - return -ENODEV; - } - - return init_fn(np); -} - -static struct platform_driver zx_clk_driver = { - .probe = zx_clkc_probe, - .driver = { - .name = "zx296718-clkc", - .of_match_table = zx_clkc_match_table, - }, -}; - -static int __init zx_clk_init(void) -{ - return platform_driver_register(&zx_clk_driver); -} -core_initcall(zx_clk_init); diff --git a/drivers/clk/zte/clk.c b/drivers/clk/zte/clk.c deleted file mode 100644 index 8bda6d41ad3a2ac76e222497a4e04ca11d6bc383..0000000000000000000000000000000000000000 --- a/drivers/clk/zte/clk.c +++ /dev/null @@ -1,446 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Copyright 2014 Linaro Ltd. - * Copyright (C) 2014 ZTE Corporation. - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "clk.h" - -#define to_clk_zx_pll(_hw) container_of(_hw, struct clk_zx_pll, hw) -#define to_clk_zx_audio(_hw) container_of(_hw, struct clk_zx_audio, hw) - -#define CFG0_CFG1_OFFSET 4 -#define LOCK_FLAG 30 -#define POWER_DOWN 31 - -static int rate_to_idx(struct clk_zx_pll *zx_pll, unsigned long rate) -{ - const struct zx_pll_config *config = zx_pll->lookup_table; - int i; - - for (i = 0; i < zx_pll->count; i++) { - if (config[i].rate > rate) - return i > 0 ? i - 1 : 0; - - if (config[i].rate == rate) - return i; - } - - return i - 1; -} - -static int hw_to_idx(struct clk_zx_pll *zx_pll) -{ - const struct zx_pll_config *config = zx_pll->lookup_table; - u32 hw_cfg0, hw_cfg1; - int i; - - hw_cfg0 = readl_relaxed(zx_pll->reg_base); - hw_cfg1 = readl_relaxed(zx_pll->reg_base + CFG0_CFG1_OFFSET); - - /* For matching the value in lookup table */ - hw_cfg0 &= ~BIT(zx_pll->lock_bit); - - /* Check availability of pd_bit */ - if (zx_pll->pd_bit < 32) - hw_cfg0 |= BIT(zx_pll->pd_bit); - - for (i = 0; i < zx_pll->count; i++) { - if (hw_cfg0 == config[i].cfg0 && hw_cfg1 == config[i].cfg1) - return i; - } - - return -EINVAL; -} - -static unsigned long zx_pll_recalc_rate(struct clk_hw *hw, - unsigned long parent_rate) -{ - struct clk_zx_pll *zx_pll = to_clk_zx_pll(hw); - int idx; - - idx = hw_to_idx(zx_pll); - if (unlikely(idx == -EINVAL)) - return 0; - - return zx_pll->lookup_table[idx].rate; -} - -static long zx_pll_round_rate(struct clk_hw *hw, unsigned long rate, - unsigned long *prate) -{ - struct clk_zx_pll *zx_pll = to_clk_zx_pll(hw); - int idx; - - idx = rate_to_idx(zx_pll, rate); - - return zx_pll->lookup_table[idx].rate; -} - -static int zx_pll_set_rate(struct clk_hw *hw, unsigned long rate, - unsigned long parent_rate) -{ - /* Assume current cpu is not running on current PLL */ - struct clk_zx_pll *zx_pll = to_clk_zx_pll(hw); - const struct zx_pll_config *config; - int idx; - - idx = rate_to_idx(zx_pll, rate); - config = &zx_pll->lookup_table[idx]; - - writel_relaxed(config->cfg0, zx_pll->reg_base); - writel_relaxed(config->cfg1, zx_pll->reg_base + CFG0_CFG1_OFFSET); - - return 0; -} - -static int zx_pll_enable(struct clk_hw *hw) -{ - struct clk_zx_pll *zx_pll = to_clk_zx_pll(hw); - u32 reg; - - /* If pd_bit is not available, simply return success. */ - if (zx_pll->pd_bit > 31) - return 0; - - reg = readl_relaxed(zx_pll->reg_base); - writel_relaxed(reg & ~BIT(zx_pll->pd_bit), zx_pll->reg_base); - - return readl_relaxed_poll_timeout(zx_pll->reg_base, reg, - reg & BIT(zx_pll->lock_bit), 0, 100); -} - -static void zx_pll_disable(struct clk_hw *hw) -{ - struct clk_zx_pll *zx_pll = to_clk_zx_pll(hw); - u32 reg; - - if (zx_pll->pd_bit > 31) - return; - - reg = readl_relaxed(zx_pll->reg_base); - writel_relaxed(reg | BIT(zx_pll->pd_bit), zx_pll->reg_base); -} - -static int zx_pll_is_enabled(struct clk_hw *hw) -{ - struct clk_zx_pll *zx_pll = to_clk_zx_pll(hw); - u32 reg; - - reg = readl_relaxed(zx_pll->reg_base); - - return !(reg & BIT(zx_pll->pd_bit)); -} - -const struct clk_ops zx_pll_ops = { - .recalc_rate = zx_pll_recalc_rate, - .round_rate = zx_pll_round_rate, - .set_rate = zx_pll_set_rate, - .enable = zx_pll_enable, - .disable = zx_pll_disable, - .is_enabled = zx_pll_is_enabled, -}; -EXPORT_SYMBOL(zx_pll_ops); - -struct clk *clk_register_zx_pll(const char *name, const char *parent_name, - unsigned long flags, void __iomem *reg_base, - const struct zx_pll_config *lookup_table, - int count, spinlock_t *lock) -{ - struct clk_zx_pll *zx_pll; - struct clk *clk; - struct clk_init_data init; - - zx_pll = kzalloc(sizeof(*zx_pll), GFP_KERNEL); - if (!zx_pll) - return ERR_PTR(-ENOMEM); - - init.name = name; - init.ops = &zx_pll_ops; - init.flags = flags; - init.parent_names = parent_name ? &parent_name : NULL; - init.num_parents = parent_name ? 1 : 0; - - zx_pll->reg_base = reg_base; - zx_pll->lookup_table = lookup_table; - zx_pll->count = count; - zx_pll->lock_bit = LOCK_FLAG; - zx_pll->pd_bit = POWER_DOWN; - zx_pll->lock = lock; - zx_pll->hw.init = &init; - - clk = clk_register(NULL, &zx_pll->hw); - if (IS_ERR(clk)) - kfree(zx_pll); - - return clk; -} - -#define BPAR 1000000 -static u32 calc_reg(u32 parent_rate, u32 rate) -{ - u32 sel, integ, fra_div, tmp; - u64 tmp64 = (u64)parent_rate * BPAR; - - do_div(tmp64, rate); - integ = (u32)tmp64 / BPAR; - integ = integ >> 1; - - tmp = (u32)tmp64 % BPAR; - sel = tmp / BPAR; - - tmp = tmp % BPAR; - fra_div = tmp * 0xff / BPAR; - tmp = (sel << 24) | (integ << 16) | (0xff << 8) | fra_div; - - /* Set I2S integer divider as 1. This bit is reserved for SPDIF - * and do no harm. - */ - tmp |= BIT(28); - return tmp; -} - -static u32 calc_rate(u32 reg, u32 parent_rate) -{ - u32 sel, integ, fra_div, tmp; - u64 tmp64 = (u64)parent_rate * BPAR; - - tmp = reg; - sel = (tmp >> 24) & BIT(0); - integ = (tmp >> 16) & 0xff; - fra_div = tmp & 0xff; - - tmp = fra_div * BPAR; - tmp = tmp / 0xff; - tmp += sel * BPAR; - tmp += 2 * integ * BPAR; - do_div(tmp64, tmp); - - return (u32)tmp64; -} - -static unsigned long zx_audio_recalc_rate(struct clk_hw *hw, - unsigned long parent_rate) -{ - struct clk_zx_audio *zx_audio = to_clk_zx_audio(hw); - u32 reg; - - reg = readl_relaxed(zx_audio->reg_base); - return calc_rate(reg, parent_rate); -} - -static long zx_audio_round_rate(struct clk_hw *hw, unsigned long rate, - unsigned long *prate) -{ - u32 reg; - - if (rate * 2 > *prate) - return -EINVAL; - - reg = calc_reg(*prate, rate); - return calc_rate(reg, *prate); -} - -static int zx_audio_set_rate(struct clk_hw *hw, unsigned long rate, - unsigned long parent_rate) -{ - struct clk_zx_audio *zx_audio = to_clk_zx_audio(hw); - u32 reg; - - reg = calc_reg(parent_rate, rate); - writel_relaxed(reg, zx_audio->reg_base); - - return 0; -} - -#define ZX_AUDIO_EN BIT(25) -static int zx_audio_enable(struct clk_hw *hw) -{ - struct clk_zx_audio *zx_audio = to_clk_zx_audio(hw); - u32 reg; - - reg = readl_relaxed(zx_audio->reg_base); - writel_relaxed(reg & ~ZX_AUDIO_EN, zx_audio->reg_base); - return 0; -} - -static void zx_audio_disable(struct clk_hw *hw) -{ - struct clk_zx_audio *zx_audio = to_clk_zx_audio(hw); - u32 reg; - - reg = readl_relaxed(zx_audio->reg_base); - writel_relaxed(reg | ZX_AUDIO_EN, zx_audio->reg_base); -} - -static const struct clk_ops zx_audio_ops = { - .recalc_rate = zx_audio_recalc_rate, - .round_rate = zx_audio_round_rate, - .set_rate = zx_audio_set_rate, - .enable = zx_audio_enable, - .disable = zx_audio_disable, -}; - -struct clk *clk_register_zx_audio(const char *name, - const char * const parent_name, - unsigned long flags, - void __iomem *reg_base) -{ - struct clk_zx_audio *zx_audio; - struct clk *clk; - struct clk_init_data init; - - zx_audio = kzalloc(sizeof(*zx_audio), GFP_KERNEL); - if (!zx_audio) - return ERR_PTR(-ENOMEM); - - init.name = name; - init.ops = &zx_audio_ops; - init.flags = flags; - init.parent_names = parent_name ? &parent_name : NULL; - init.num_parents = parent_name ? 1 : 0; - - zx_audio->reg_base = reg_base; - zx_audio->hw.init = &init; - - clk = clk_register(NULL, &zx_audio->hw); - if (IS_ERR(clk)) - kfree(zx_audio); - - return clk; -} - -#define CLK_AUDIO_DIV_FRAC BIT(0) -#define CLK_AUDIO_DIV_INT BIT(1) -#define CLK_AUDIO_DIV_UNCOMMON BIT(1) - -#define CLK_AUDIO_DIV_FRAC_NSHIFT 16 -#define CLK_AUDIO_DIV_INT_FRAC_RE BIT(16) -#define CLK_AUDIO_DIV_INT_FRAC_MAX (0xffff) -#define CLK_AUDIO_DIV_INT_FRAC_MIN (0x2) -#define CLK_AUDIO_DIV_INT_INT_SHIFT 24 -#define CLK_AUDIO_DIV_INT_INT_WIDTH 4 - -struct zx_clk_audio_div_table { - unsigned long rate; - unsigned int int_reg; - unsigned int frac_reg; -}; - -#define to_clk_zx_audio_div(_hw) container_of(_hw, struct clk_zx_audio_divider, hw) - -static unsigned long audio_calc_rate(struct clk_zx_audio_divider *audio_div, - u32 reg_frac, u32 reg_int, - unsigned long parent_rate) -{ - unsigned long rate, m, n; - - m = reg_frac & 0xffff; - n = (reg_frac >> 16) & 0xffff; - - m = (reg_int & 0xffff) * n + m; - rate = (parent_rate * n) / m; - - return rate; -} - -static void audio_calc_reg(struct clk_zx_audio_divider *audio_div, - struct zx_clk_audio_div_table *div_table, - unsigned long rate, unsigned long parent_rate) -{ - unsigned int reg_int, reg_frac; - unsigned long m, n, div; - - reg_int = parent_rate / rate; - - if (reg_int > CLK_AUDIO_DIV_INT_FRAC_MAX) - reg_int = CLK_AUDIO_DIV_INT_FRAC_MAX; - else if (reg_int < CLK_AUDIO_DIV_INT_FRAC_MIN) - reg_int = 0; - m = parent_rate - rate * reg_int; - n = rate; - - div = gcd(m, n); - m = m / div; - n = n / div; - - if ((m >> 16) || (n >> 16)) { - if (m > n) { - n = n * 0xffff / m; - m = 0xffff; - } else { - m = m * 0xffff / n; - n = 0xffff; - } - } - reg_frac = m | (n << 16); - - div_table->rate = parent_rate * n / (reg_int * n + m); - div_table->int_reg = reg_int; - div_table->frac_reg = reg_frac; -} - -static unsigned long zx_audio_div_recalc_rate(struct clk_hw *hw, - unsigned long parent_rate) -{ - struct clk_zx_audio_divider *zx_audio_div = to_clk_zx_audio_div(hw); - u32 reg_frac, reg_int; - - reg_frac = readl_relaxed(zx_audio_div->reg_base); - reg_int = readl_relaxed(zx_audio_div->reg_base + 0x4); - - return audio_calc_rate(zx_audio_div, reg_frac, reg_int, parent_rate); -} - -static long zx_audio_div_round_rate(struct clk_hw *hw, unsigned long rate, - unsigned long *prate) -{ - struct clk_zx_audio_divider *zx_audio_div = to_clk_zx_audio_div(hw); - struct zx_clk_audio_div_table divt; - - audio_calc_reg(zx_audio_div, &divt, rate, *prate); - - return audio_calc_rate(zx_audio_div, divt.frac_reg, divt.int_reg, *prate); -} - -static int zx_audio_div_set_rate(struct clk_hw *hw, unsigned long rate, - unsigned long parent_rate) -{ - struct clk_zx_audio_divider *zx_audio_div = to_clk_zx_audio_div(hw); - struct zx_clk_audio_div_table divt; - unsigned int val; - - audio_calc_reg(zx_audio_div, &divt, rate, parent_rate); - if (divt.rate != rate) - pr_debug("the real rate is:%ld", divt.rate); - - writel_relaxed(divt.frac_reg, zx_audio_div->reg_base); - - val = readl_relaxed(zx_audio_div->reg_base + 0x4); - val &= ~0xffff; - val |= divt.int_reg | CLK_AUDIO_DIV_INT_FRAC_RE; - writel_relaxed(val, zx_audio_div->reg_base + 0x4); - - mdelay(1); - - val = readl_relaxed(zx_audio_div->reg_base + 0x4); - val &= ~CLK_AUDIO_DIV_INT_FRAC_RE; - writel_relaxed(val, zx_audio_div->reg_base + 0x4); - - return 0; -} - -const struct clk_ops zx_audio_div_ops = { - .recalc_rate = zx_audio_div_recalc_rate, - .round_rate = zx_audio_div_round_rate, - .set_rate = zx_audio_div_set_rate, -}; diff --git a/drivers/clk/zte/clk.h b/drivers/clk/zte/clk.h deleted file mode 100644 index aeaf2a380ba6b86b07b8ef1ae29f084ab3653eca..0000000000000000000000000000000000000000 --- a/drivers/clk/zte/clk.h +++ /dev/null @@ -1,174 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright 2015 Linaro Ltd. - * Copyright (C) 2014 ZTE Corporation. - */ - -#ifndef __ZTE_CLK_H -#define __ZTE_CLK_H -#include -#include - -#define PNAME(x) static const char *x[] - -struct zx_pll_config { - unsigned long rate; - u32 cfg0; - u32 cfg1; -}; - -struct clk_zx_pll { - struct clk_hw hw; - void __iomem *reg_base; - const struct zx_pll_config *lookup_table; /* order by rate asc */ - int count; - spinlock_t *lock; - u8 pd_bit; /* power down bit */ - u8 lock_bit; /* pll lock flag bit */ -}; - -#define PLL_RATE(_rate, _cfg0, _cfg1) \ -{ \ - .rate = _rate, \ - .cfg0 = _cfg0, \ - .cfg1 = _cfg1, \ -} - -#define ZX_PLL(_name, _parent, _reg, _table, _pd, _lock) \ -{ \ - .reg_base = (void __iomem *) _reg, \ - .lookup_table = _table, \ - .count = ARRAY_SIZE(_table), \ - .pd_bit = _pd, \ - .lock_bit = _lock, \ - .hw.init = CLK_HW_INIT(_name, _parent, &zx_pll_ops, \ - CLK_GET_RATE_NOCACHE), \ -} - -/* - * The pd_bit is not available on ZX296718, so let's pass something - * bigger than 31, e.g. 0xff, to indicate that. - */ -#define ZX296718_PLL(_name, _parent, _reg, _table) \ -ZX_PLL(_name, _parent, _reg, _table, 0xff, 30) - -struct zx_clk_gate { - struct clk_gate gate; - u16 id; -}; - -#define GATE(_id, _name, _parent, _reg, _bit, _flag, _gflags) \ -{ \ - .gate = { \ - .reg = (void __iomem *) _reg, \ - .bit_idx = (_bit), \ - .flags = _gflags, \ - .lock = &clk_lock, \ - .hw.init = CLK_HW_INIT(_name, \ - _parent, \ - &clk_gate_ops, \ - _flag | CLK_IGNORE_UNUSED), \ - }, \ - .id = _id, \ -} - -struct zx_clk_fixed_factor { - struct clk_fixed_factor factor; - u16 id; -}; - -#define FFACTOR(_id, _name, _parent, _mult, _div, _flag) \ -{ \ - .factor = { \ - .div = _div, \ - .mult = _mult, \ - .hw.init = CLK_HW_INIT(_name, \ - _parent, \ - &clk_fixed_factor_ops, \ - _flag), \ - }, \ - .id = _id, \ -} - -struct zx_clk_mux { - struct clk_mux mux; - u16 id; -}; - -#define MUX_F(_id, _name, _parent, _reg, _shift, _width, _flag, _mflag) \ -{ \ - .mux = { \ - .reg = (void __iomem *) _reg, \ - .mask = BIT(_width) - 1, \ - .shift = _shift, \ - .flags = _mflag, \ - .lock = &clk_lock, \ - .hw.init = CLK_HW_INIT_PARENTS(_name, \ - _parent, \ - &clk_mux_ops, \ - _flag), \ - }, \ - .id = _id, \ -} - -#define MUX(_id, _name, _parent, _reg, _shift, _width) \ -MUX_F(_id, _name, _parent, _reg, _shift, _width, 0, 0) - -struct zx_clk_div { - struct clk_divider div; - u16 id; -}; - -#define DIV_T(_id, _name, _parent, _reg, _shift, _width, _flag, _table) \ -{ \ - .div = { \ - .reg = (void __iomem *) _reg, \ - .shift = _shift, \ - .width = _width, \ - .flags = 0, \ - .table = _table, \ - .lock = &clk_lock, \ - .hw.init = CLK_HW_INIT(_name, \ - _parent, \ - &clk_divider_ops, \ - _flag), \ - }, \ - .id = _id, \ -} - -struct clk_zx_audio_divider { - struct clk_hw hw; - void __iomem *reg_base; - unsigned int rate_count; - spinlock_t *lock; - u16 id; -}; - -#define AUDIO_DIV(_id, _name, _parent, _reg) \ -{ \ - .reg_base = (void __iomem *) _reg, \ - .lock = &clk_lock, \ - .hw.init = CLK_HW_INIT(_name, \ - _parent, \ - &zx_audio_div_ops, \ - 0), \ - .id = _id, \ -} - -struct clk *clk_register_zx_pll(const char *name, const char *parent_name, - unsigned long flags, void __iomem *reg_base, - const struct zx_pll_config *lookup_table, int count, spinlock_t *lock); - -struct clk_zx_audio { - struct clk_hw hw; - void __iomem *reg_base; -}; - -struct clk *clk_register_zx_audio(const char *name, - const char * const parent_name, - unsigned long flags, void __iomem *reg_base); - -extern const struct clk_ops zx_pll_ops; -extern const struct clk_ops zx_audio_div_ops; - -#endif diff --git a/drivers/clk/zynq/clkc.c b/drivers/clk/zynq/clkc.c index ffbb9008c1c997aefe2d1a8d44dc53e423f4e4d1..204b83d911b902865fbd130d31a48512ea5a1ca8 100644 --- a/drivers/clk/zynq/clkc.c +++ b/drivers/clk/zynq/clkc.c @@ -103,7 +103,6 @@ static void __init zynq_clk_register_fclk(enum zynq_clk fclk, const char *clk_name, void __iomem *fclk_ctrl_reg, const char **parents, int enable) { - struct clk *clk; u32 enable_reg; char *mux_name; char *div0_name; @@ -131,15 +130,15 @@ static void __init zynq_clk_register_fclk(enum zynq_clk fclk, if (!div1_name) goto err_div1_name; - clk = clk_register_mux(NULL, mux_name, parents, 4, + clk_register_mux(NULL, mux_name, parents, 4, CLK_SET_RATE_NO_REPARENT, fclk_ctrl_reg, 4, 2, 0, fclk_lock); - clk = clk_register_divider(NULL, div0_name, mux_name, + clk_register_divider(NULL, div0_name, mux_name, 0, fclk_ctrl_reg, 8, 6, CLK_DIVIDER_ONE_BASED | CLK_DIVIDER_ALLOW_ZERO, fclk_lock); - clk = clk_register_divider(NULL, div1_name, div0_name, + clk_register_divider(NULL, div1_name, div0_name, CLK_SET_RATE_PARENT, fclk_ctrl_reg, 20, 6, CLK_DIVIDER_ONE_BASED | CLK_DIVIDER_ALLOW_ZERO, fclk_lock); @@ -176,7 +175,6 @@ static void __init zynq_clk_register_periph_clk(enum zynq_clk clk0, const char *clk_name1, void __iomem *clk_ctrl, const char **parents, unsigned int two_gates) { - struct clk *clk; char *mux_name; char *div_name; spinlock_t *lock; @@ -189,10 +187,10 @@ static void __init zynq_clk_register_periph_clk(enum zynq_clk clk0, mux_name = kasprintf(GFP_KERNEL, "%s_mux", clk_name0); div_name = kasprintf(GFP_KERNEL, "%s_div", clk_name0); - clk = clk_register_mux(NULL, mux_name, parents, 4, + clk_register_mux(NULL, mux_name, parents, 4, CLK_SET_RATE_NO_REPARENT, clk_ctrl, 4, 2, 0, lock); - clk = clk_register_divider(NULL, div_name, mux_name, 0, clk_ctrl, 8, 6, + clk_register_divider(NULL, div_name, mux_name, 0, clk_ctrl, 8, 6, CLK_DIVIDER_ONE_BASED | CLK_DIVIDER_ALLOW_ZERO, lock); clks[clk0] = clk_register_gate(NULL, clk_name0, div_name, @@ -217,7 +215,6 @@ static void __init zynq_clk_setup(struct device_node *np) int i; u32 tmp; int ret; - struct clk *clk; char *clk_name; unsigned int fclk_enable = 0; const char *clk_output_name[clk_max]; @@ -257,19 +254,19 @@ static void __init zynq_clk_setup(struct device_node *np) ps_clk = clk_register_fixed_rate(NULL, "ps_clk", NULL, 0, tmp); /* PLLs */ - clk = clk_register_zynq_pll("armpll_int", "ps_clk", SLCR_ARMPLL_CTRL, + clk_register_zynq_pll("armpll_int", "ps_clk", SLCR_ARMPLL_CTRL, SLCR_PLL_STATUS, 0, &armpll_lock); clks[armpll] = clk_register_mux(NULL, clk_output_name[armpll], armpll_parents, 2, CLK_SET_RATE_NO_REPARENT, SLCR_ARMPLL_CTRL, 4, 1, 0, &armpll_lock); - clk = clk_register_zynq_pll("ddrpll_int", "ps_clk", SLCR_DDRPLL_CTRL, + clk_register_zynq_pll("ddrpll_int", "ps_clk", SLCR_DDRPLL_CTRL, SLCR_PLL_STATUS, 1, &ddrpll_lock); clks[ddrpll] = clk_register_mux(NULL, clk_output_name[ddrpll], ddrpll_parents, 2, CLK_SET_RATE_NO_REPARENT, SLCR_DDRPLL_CTRL, 4, 1, 0, &ddrpll_lock); - clk = clk_register_zynq_pll("iopll_int", "ps_clk", SLCR_IOPLL_CTRL, + clk_register_zynq_pll("iopll_int", "ps_clk", SLCR_IOPLL_CTRL, SLCR_PLL_STATUS, 2, &iopll_lock); clks[iopll] = clk_register_mux(NULL, clk_output_name[iopll], iopll_parents, 2, CLK_SET_RATE_NO_REPARENT, @@ -277,10 +274,10 @@ static void __init zynq_clk_setup(struct device_node *np) /* CPU clocks */ tmp = readl(SLCR_621_TRUE) & 1; - clk = clk_register_mux(NULL, "cpu_mux", cpu_parents, 4, + clk_register_mux(NULL, "cpu_mux", cpu_parents, 4, CLK_SET_RATE_NO_REPARENT, SLCR_ARM_CLK_CTRL, 4, 2, 0, &armclk_lock); - clk = clk_register_divider(NULL, "cpu_div", "cpu_mux", 0, + clk_register_divider(NULL, "cpu_div", "cpu_mux", 0, SLCR_ARM_CLK_CTRL, 8, 6, CLK_DIVIDER_ONE_BASED | CLK_DIVIDER_ALLOW_ZERO, &armclk_lock); @@ -288,20 +285,20 @@ static void __init zynq_clk_setup(struct device_node *np) "cpu_div", CLK_SET_RATE_PARENT | CLK_IGNORE_UNUSED, SLCR_ARM_CLK_CTRL, 24, 0, &armclk_lock); - clk = clk_register_fixed_factor(NULL, "cpu_3or2x_div", "cpu_div", 0, + clk_register_fixed_factor(NULL, "cpu_3or2x_div", "cpu_div", 0, 1, 2); clks[cpu_3or2x] = clk_register_gate(NULL, clk_output_name[cpu_3or2x], "cpu_3or2x_div", CLK_IGNORE_UNUSED, SLCR_ARM_CLK_CTRL, 25, 0, &armclk_lock); - clk = clk_register_fixed_factor(NULL, "cpu_2x_div", "cpu_div", 0, 1, + clk_register_fixed_factor(NULL, "cpu_2x_div", "cpu_div", 0, 1, 2 + tmp); clks[cpu_2x] = clk_register_gate(NULL, clk_output_name[cpu_2x], "cpu_2x_div", CLK_IGNORE_UNUSED, SLCR_ARM_CLK_CTRL, 26, 0, &armclk_lock); clk_prepare_enable(clks[cpu_2x]); - clk = clk_register_fixed_factor(NULL, "cpu_1x_div", "cpu_div", 0, 1, + clk_register_fixed_factor(NULL, "cpu_1x_div", "cpu_div", 0, 1, 4 + 2 * tmp); clks[cpu_1x] = clk_register_gate(NULL, clk_output_name[cpu_1x], "cpu_1x_div", CLK_IGNORE_UNUSED, SLCR_ARM_CLK_CTRL, 27, @@ -324,23 +321,23 @@ static void __init zynq_clk_setup(struct device_node *np) &swdtclk_lock); /* DDR clocks */ - clk = clk_register_divider(NULL, "ddr2x_div", "ddrpll", 0, + clk_register_divider(NULL, "ddr2x_div", "ddrpll", 0, SLCR_DDR_CLK_CTRL, 26, 6, CLK_DIVIDER_ONE_BASED | CLK_DIVIDER_ALLOW_ZERO, &ddrclk_lock); clks[ddr2x] = clk_register_gate(NULL, clk_output_name[ddr2x], "ddr2x_div", 0, SLCR_DDR_CLK_CTRL, 1, 0, &ddrclk_lock); clk_prepare_enable(clks[ddr2x]); - clk = clk_register_divider(NULL, "ddr3x_div", "ddrpll", 0, + clk_register_divider(NULL, "ddr3x_div", "ddrpll", 0, SLCR_DDR_CLK_CTRL, 20, 6, CLK_DIVIDER_ONE_BASED | CLK_DIVIDER_ALLOW_ZERO, &ddrclk_lock); clks[ddr3x] = clk_register_gate(NULL, clk_output_name[ddr3x], "ddr3x_div", 0, SLCR_DDR_CLK_CTRL, 0, 0, &ddrclk_lock); clk_prepare_enable(clks[ddr3x]); - clk = clk_register_divider(NULL, "dci_div0", "ddrpll", 0, + clk_register_divider(NULL, "dci_div0", "ddrpll", 0, SLCR_DCI_CLK_CTRL, 8, 6, CLK_DIVIDER_ONE_BASED | CLK_DIVIDER_ALLOW_ZERO, &dciclk_lock); - clk = clk_register_divider(NULL, "dci_div1", "dci_div0", + clk_register_divider(NULL, "dci_div1", "dci_div0", CLK_SET_RATE_PARENT, SLCR_DCI_CLK_CTRL, 20, 6, CLK_DIVIDER_ONE_BASED | CLK_DIVIDER_ALLOW_ZERO, &dciclk_lock); @@ -385,17 +382,17 @@ static void __init zynq_clk_setup(struct device_node *np) gem0_mux_parents[i + 1] = of_clk_get_parent_name(np, idx); } - clk = clk_register_mux(NULL, "gem0_mux", periph_parents, 4, + clk_register_mux(NULL, "gem0_mux", periph_parents, 4, CLK_SET_RATE_NO_REPARENT, SLCR_GEM0_CLK_CTRL, 4, 2, 0, &gem0clk_lock); - clk = clk_register_divider(NULL, "gem0_div0", "gem0_mux", 0, + clk_register_divider(NULL, "gem0_div0", "gem0_mux", 0, SLCR_GEM0_CLK_CTRL, 8, 6, CLK_DIVIDER_ONE_BASED | CLK_DIVIDER_ALLOW_ZERO, &gem0clk_lock); - clk = clk_register_divider(NULL, "gem0_div1", "gem0_div0", + clk_register_divider(NULL, "gem0_div1", "gem0_div0", CLK_SET_RATE_PARENT, SLCR_GEM0_CLK_CTRL, 20, 6, CLK_DIVIDER_ONE_BASED | CLK_DIVIDER_ALLOW_ZERO, &gem0clk_lock); - clk = clk_register_mux(NULL, "gem0_emio_mux", gem0_mux_parents, 2, + clk_register_mux(NULL, "gem0_emio_mux", gem0_mux_parents, 2, CLK_SET_RATE_PARENT | CLK_SET_RATE_NO_REPARENT, SLCR_GEM0_CLK_CTRL, 6, 1, 0, &gem0clk_lock); @@ -410,17 +407,17 @@ static void __init zynq_clk_setup(struct device_node *np) gem1_mux_parents[i + 1] = of_clk_get_parent_name(np, idx); } - clk = clk_register_mux(NULL, "gem1_mux", periph_parents, 4, + clk_register_mux(NULL, "gem1_mux", periph_parents, 4, CLK_SET_RATE_NO_REPARENT, SLCR_GEM1_CLK_CTRL, 4, 2, 0, &gem1clk_lock); - clk = clk_register_divider(NULL, "gem1_div0", "gem1_mux", 0, + clk_register_divider(NULL, "gem1_div0", "gem1_mux", 0, SLCR_GEM1_CLK_CTRL, 8, 6, CLK_DIVIDER_ONE_BASED | CLK_DIVIDER_ALLOW_ZERO, &gem1clk_lock); - clk = clk_register_divider(NULL, "gem1_div1", "gem1_div0", + clk_register_divider(NULL, "gem1_div1", "gem1_div0", CLK_SET_RATE_PARENT, SLCR_GEM1_CLK_CTRL, 20, 6, CLK_DIVIDER_ONE_BASED | CLK_DIVIDER_ALLOW_ZERO, &gem1clk_lock); - clk = clk_register_mux(NULL, "gem1_emio_mux", gem1_mux_parents, 2, + clk_register_mux(NULL, "gem1_emio_mux", gem1_mux_parents, 2, CLK_SET_RATE_PARENT | CLK_SET_RATE_NO_REPARENT, SLCR_GEM1_CLK_CTRL, 6, 1, 0, &gem1clk_lock); @@ -442,27 +439,27 @@ static void __init zynq_clk_setup(struct device_node *np) can_mio_mux_parents[i] = dummy_nm; } kfree(clk_name); - clk = clk_register_mux(NULL, "can_mux", periph_parents, 4, + clk_register_mux(NULL, "can_mux", periph_parents, 4, CLK_SET_RATE_NO_REPARENT, SLCR_CAN_CLK_CTRL, 4, 2, 0, &canclk_lock); - clk = clk_register_divider(NULL, "can_div0", "can_mux", 0, + clk_register_divider(NULL, "can_div0", "can_mux", 0, SLCR_CAN_CLK_CTRL, 8, 6, CLK_DIVIDER_ONE_BASED | CLK_DIVIDER_ALLOW_ZERO, &canclk_lock); - clk = clk_register_divider(NULL, "can_div1", "can_div0", + clk_register_divider(NULL, "can_div1", "can_div0", CLK_SET_RATE_PARENT, SLCR_CAN_CLK_CTRL, 20, 6, CLK_DIVIDER_ONE_BASED | CLK_DIVIDER_ALLOW_ZERO, &canclk_lock); - clk = clk_register_gate(NULL, "can0_gate", "can_div1", + clk_register_gate(NULL, "can0_gate", "can_div1", CLK_SET_RATE_PARENT, SLCR_CAN_CLK_CTRL, 0, 0, &canclk_lock); - clk = clk_register_gate(NULL, "can1_gate", "can_div1", + clk_register_gate(NULL, "can1_gate", "can_div1", CLK_SET_RATE_PARENT, SLCR_CAN_CLK_CTRL, 1, 0, &canclk_lock); - clk = clk_register_mux(NULL, "can0_mio_mux", + clk_register_mux(NULL, "can0_mio_mux", can_mio_mux_parents, 54, CLK_SET_RATE_PARENT | CLK_SET_RATE_NO_REPARENT, SLCR_CAN_MIOCLK_CTRL, 0, 6, 0, &canmioclk_lock); - clk = clk_register_mux(NULL, "can1_mio_mux", + clk_register_mux(NULL, "can1_mio_mux", can_mio_mux_parents, 54, CLK_SET_RATE_PARENT | CLK_SET_RATE_NO_REPARENT, SLCR_CAN_MIOCLK_CTRL, 16, 6, 0, &canmioclk_lock); @@ -482,13 +479,13 @@ static void __init zynq_clk_setup(struct device_node *np) dbg_emio_mux_parents[i + 1] = of_clk_get_parent_name(np, idx); } - clk = clk_register_mux(NULL, "dbg_mux", periph_parents, 4, + clk_register_mux(NULL, "dbg_mux", periph_parents, 4, CLK_SET_RATE_NO_REPARENT, SLCR_DBG_CLK_CTRL, 4, 2, 0, &dbgclk_lock); - clk = clk_register_divider(NULL, "dbg_div", "dbg_mux", 0, + clk_register_divider(NULL, "dbg_div", "dbg_mux", 0, SLCR_DBG_CLK_CTRL, 8, 6, CLK_DIVIDER_ONE_BASED | CLK_DIVIDER_ALLOW_ZERO, &dbgclk_lock); - clk = clk_register_mux(NULL, "dbg_emio_mux", dbg_emio_mux_parents, 2, + clk_register_mux(NULL, "dbg_emio_mux", dbg_emio_mux_parents, 2, CLK_SET_RATE_NO_REPARENT, SLCR_DBG_CLK_CTRL, 6, 1, 0, &dbgclk_lock); clks[dbg_trc] = clk_register_gate(NULL, clk_output_name[dbg_trc], diff --git a/drivers/clk/zynq/pll.c b/drivers/clk/zynq/pll.c index dcb2037a95964b7537cff435717974dca656dff2..54f4184de89af291107aa696f20b746b2484a7bd 100644 --- a/drivers/clk/zynq/pll.c +++ b/drivers/clk/zynq/pll.c @@ -173,12 +173,12 @@ static const struct clk_ops zynq_pll_ops = { /** * clk_register_zynq_pll() - Register PLL with the clock framework - * @name PLL name - * @parent Parent clock name - * @pll_ctrl Pointer to PLL control register - * @pll_status Pointer to PLL status register - * @lock_index Bit index to this PLL's lock status bit in @pll_status - * @lock Register lock + * @name: PLL name + * @parent: Parent clock name + * @pll_ctrl: Pointer to PLL control register + * @pll_status: Pointer to PLL status register + * @lock_index: Bit index to this PLL's lock status bit in @pll_status + * @lock: Register lock * Returns handle to the registered clock. */ struct clk *clk_register_zynq_pll(const char *name, const char *parent, diff --git a/drivers/clk/zynqmp/divider.c b/drivers/clk/zynqmp/divider.c index 66da02b83d3936246130fd924169be23e6816363..e9bf7958b8218e0ff58108dfd6724ed3a9bfd24c 100644 --- a/drivers/clk/zynqmp/divider.c +++ b/drivers/clk/zynqmp/divider.c @@ -35,6 +35,7 @@ * @is_frac: The divider is a fractional divider * @clk_id: Id of clock * @div_type: divisor type (TYPE_DIV1 or TYPE_DIV2) + * @max_div: maximum supported divisor (fetched from firmware) */ struct zynqmp_clk_divider { struct clk_hw hw; diff --git a/drivers/clocksource/Kconfig b/drivers/clocksource/Kconfig index d184000eb09e1e36e696c410c9e5147cfa9dafb1..39aa21d01e054a124b6839c5deceadd0c2bfb8d0 100644 --- a/drivers/clocksource/Kconfig +++ b/drivers/clocksource/Kconfig @@ -79,6 +79,7 @@ config IXP4XX_TIMER bool "Intel XScale IXP4xx timer driver" if COMPILE_TEST depends on HAS_IOMEM select CLKSRC_MMIO + select TIMER_OF if OF help Enables support for the Intel XScale IXP4xx SoC timer. diff --git a/drivers/clocksource/mxs_timer.c b/drivers/clocksource/mxs_timer.c index bc96a4cbf26c64c82bb7c1bd1c8b1a1f4fe212fb..e52e12d27d2aa0321020992ae617daecad584641 100644 --- a/drivers/clocksource/mxs_timer.c +++ b/drivers/clocksource/mxs_timer.c @@ -131,10 +131,7 @@ static void mxs_irq_clear(char *state) /* Clear pending interrupt */ timrot_irq_acknowledge(); - -#ifdef DEBUG - pr_info("%s: changing mode to %s\n", __func__, state) -#endif /* DEBUG */ + pr_debug("%s: changing mode to %s\n", __func__, state); } static int mxs_shutdown(struct clock_event_device *evt) diff --git a/drivers/clocksource/sh_cmt.c b/drivers/clocksource/sh_cmt.c index e258230d432c00029acf930bc4dbd803b948e04a..c98f8851fd6804540bfb8e40d96703df7c736407 100644 --- a/drivers/clocksource/sh_cmt.c +++ b/drivers/clocksource/sh_cmt.c @@ -235,6 +235,8 @@ static const struct sh_cmt_info sh_cmt_info[] = { #define CMCNT 1 /* channel register */ #define CMCOR 2 /* channel register */ +#define CMCLKE 0x1000 /* CLK Enable Register (R-Car Gen2) */ + static inline u32 sh_cmt_read_cmstr(struct sh_cmt_channel *ch) { if (ch->iostart) @@ -853,6 +855,7 @@ static int sh_cmt_setup_channel(struct sh_cmt_channel *ch, unsigned int index, unsigned int hwidx, bool clockevent, bool clocksource, struct sh_cmt_device *cmt) { + u32 value; int ret; /* Skip unused channels. */ @@ -882,6 +885,11 @@ static int sh_cmt_setup_channel(struct sh_cmt_channel *ch, unsigned int index, ch->iostart = cmt->mapbase + ch->hwidx * 0x100; ch->ioctrl = ch->iostart + 0x10; ch->timer_bit = 0; + + /* Enable the clock supply to the channel */ + value = ioread32(cmt->mapbase + CMCLKE); + value |= BIT(hwidx); + iowrite32(value, cmt->mapbase + CMCLKE); break; } @@ -1014,12 +1022,10 @@ static int sh_cmt_setup(struct sh_cmt_device *cmt, struct platform_device *pdev) else cmt->rate = clk_get_rate(cmt->clk) / 8; - clk_disable(cmt->clk); - /* Map the memory resource(s). */ ret = sh_cmt_map_memory(cmt); if (ret < 0) - goto err_clk_unprepare; + goto err_clk_disable; /* Allocate and setup the channels. */ cmt->num_channels = hweight8(cmt->hw_channels); @@ -1047,6 +1053,8 @@ static int sh_cmt_setup(struct sh_cmt_device *cmt, struct platform_device *pdev) mask &= ~(1 << hwidx); } + clk_disable(cmt->clk); + platform_set_drvdata(pdev, cmt); return 0; @@ -1054,6 +1062,8 @@ static int sh_cmt_setup(struct sh_cmt_device *cmt, struct platform_device *pdev) err_unmap: kfree(cmt->channels); iounmap(cmt->mapbase); +err_clk_disable: + clk_disable(cmt->clk); err_clk_unprepare: clk_unprepare(cmt->clk); err_clk_put: diff --git a/drivers/cpufreq/Kconfig.x86 b/drivers/cpufreq/Kconfig.x86 index 39952628932041831fd556de62e37548799e9d24..92701a18bdd91f6a85c2e821c3e13ad5fc59cffe 100644 --- a/drivers/cpufreq/Kconfig.x86 +++ b/drivers/cpufreq/Kconfig.x86 @@ -62,16 +62,6 @@ config X86_ACPI_CPUFREQ_CPB By enabling this option the acpi_cpufreq driver provides the old entry in addition to the new boost ones, for compatibility reasons. -config X86_SFI_CPUFREQ - tristate "SFI Performance-States driver" - depends on X86_INTEL_MID && SFI - help - This adds a CPUFreq driver for some Silvermont based Intel Atom - architectures like Z34xx and Z35xx which enumerate processor - performance states through SFI. - - If in doubt, say N. - config ELAN_CPUFREQ tristate "AMD Elan SC400 and SC410" depends on MELAN diff --git a/drivers/cpufreq/Makefile b/drivers/cpufreq/Makefile index 1ab9b1536304c9269d7776b65f21fc3bb9b5de8c..27d3bd7ea9d4e79e0fd0334d70ae843ec1aa17b0 100644 --- a/drivers/cpufreq/Makefile +++ b/drivers/cpufreq/Makefile @@ -43,7 +43,6 @@ obj-$(CONFIG_X86_P4_CLOCKMOD) += p4-clockmod.o obj-$(CONFIG_X86_CPUFREQ_NFORCE2) += cpufreq-nforce2.o obj-$(CONFIG_X86_INTEL_PSTATE) += intel_pstate.o obj-$(CONFIG_X86_AMD_FREQ_SENSITIVITY) += amd_freq_sensitivity.o -obj-$(CONFIG_X86_SFI_CPUFREQ) += sfi-cpufreq.o ################################################################################## # ARM SoC drivers diff --git a/drivers/cpufreq/acpi-cpufreq.c b/drivers/cpufreq/acpi-cpufreq.c index d3e5a6fceb61b0ab3913750b37e5111f694f269d..d1bbc16fba4b4c9881ba88c3e039cc63ddd91b1a 100644 --- a/drivers/cpufreq/acpi-cpufreq.c +++ b/drivers/cpufreq/acpi-cpufreq.c @@ -54,7 +54,6 @@ struct acpi_cpufreq_data { unsigned int resume; unsigned int cpu_feature; unsigned int acpi_perf_cpu; - unsigned int first_perf_state; cpumask_var_t freqdomain_cpus; void (*cpu_freq_write)(struct acpi_pct_register *reg, u32 val); u32 (*cpu_freq_read)(struct acpi_pct_register *reg); @@ -223,10 +222,10 @@ static unsigned extract_msr(struct cpufreq_policy *policy, u32 msr) perf = to_perf_data(data); - cpufreq_for_each_entry(pos, policy->freq_table + data->first_perf_state) + cpufreq_for_each_entry(pos, policy->freq_table) if (msr == perf->states[pos->driver_data].status) return pos->frequency; - return policy->freq_table[data->first_perf_state].frequency; + return policy->freq_table[0].frequency; } static unsigned extract_freq(struct cpufreq_policy *policy, u32 val) @@ -365,7 +364,6 @@ static unsigned int get_cur_freq_on_cpu(unsigned int cpu) struct cpufreq_policy *policy; unsigned int freq; unsigned int cached_freq; - unsigned int state; pr_debug("%s (%d)\n", __func__, cpu); @@ -377,11 +375,7 @@ static unsigned int get_cur_freq_on_cpu(unsigned int cpu) if (unlikely(!data || !policy->freq_table)) return 0; - state = to_perf_data(data)->state; - if (state < data->first_perf_state) - state = data->first_perf_state; - - cached_freq = policy->freq_table[state].frequency; + cached_freq = policy->freq_table[to_perf_data(data)->state].frequency; freq = extract_freq(policy, get_cur_val(cpumask_of(cpu), data)); if (freq != cached_freq) { /* @@ -680,7 +674,6 @@ static int acpi_cpufreq_cpu_init(struct cpufreq_policy *policy) struct cpuinfo_x86 *c = &cpu_data(cpu); unsigned int valid_states = 0; unsigned int result = 0; - unsigned int state_count; u64 max_boost_ratio; unsigned int i; #ifdef CONFIG_SMP @@ -795,28 +788,8 @@ static int acpi_cpufreq_cpu_init(struct cpufreq_policy *policy) goto err_unreg; } - state_count = perf->state_count + 1; - - max_boost_ratio = get_max_boost_ratio(cpu); - if (max_boost_ratio) { - /* - * Make a room for one more entry to represent the highest - * available "boost" frequency. - */ - state_count++; - valid_states++; - data->first_perf_state = valid_states; - } else { - /* - * If the maximum "boost" frequency is unknown, ask the arch - * scale-invariance code to use the "nominal" performance for - * CPU utilization scaling so as to prevent the schedutil - * governor from selecting inadequate CPU frequencies. - */ - arch_set_max_freq_ratio(true); - } - - freq_table = kcalloc(state_count, sizeof(*freq_table), GFP_KERNEL); + freq_table = kcalloc(perf->state_count + 1, sizeof(*freq_table), + GFP_KERNEL); if (!freq_table) { result = -ENOMEM; goto err_unreg; @@ -851,27 +824,25 @@ static int acpi_cpufreq_cpu_init(struct cpufreq_policy *policy) } freq_table[valid_states].frequency = CPUFREQ_TABLE_END; + max_boost_ratio = get_max_boost_ratio(cpu); if (max_boost_ratio) { - unsigned int state = data->first_perf_state; - unsigned int freq = freq_table[state].frequency; + unsigned int freq = freq_table[0].frequency; /* * Because the loop above sorts the freq_table entries in the * descending order, freq is the maximum frequency in the table. * Assume that it corresponds to the CPPC nominal frequency and - * use it to populate the frequency field of the extra "boost" - * frequency entry. + * use it to set cpuinfo.max_freq. */ - freq_table[0].frequency = freq * max_boost_ratio >> SCHED_CAPACITY_SHIFT; + policy->cpuinfo.max_freq = freq * max_boost_ratio >> SCHED_CAPACITY_SHIFT; + } else { /* - * The purpose of the extra "boost" frequency entry is to make - * the rest of cpufreq aware of the real maximum frequency, but - * the way to request it is the same as for the first_perf_state - * entry that is expected to cover the entire range of "boost" - * frequencies of the CPU, so copy the driver_data value from - * that entry. + * If the maximum "boost" frequency is unknown, ask the arch + * scale-invariance code to use the "nominal" performance for + * CPU utilization scaling so as to prevent the schedutil + * governor from selecting inadequate CPU frequencies. */ - freq_table[0].driver_data = freq_table[state].driver_data; + arch_set_max_freq_ratio(true); } policy->freq_table = freq_table; @@ -947,8 +918,7 @@ static void acpi_cpufreq_cpu_ready(struct cpufreq_policy *policy) { struct acpi_processor_performance *perf = per_cpu_ptr(acpi_perf_data, policy->cpu); - struct acpi_cpufreq_data *data = policy->driver_data; - unsigned int freq = policy->freq_table[data->first_perf_state].frequency; + unsigned int freq = policy->freq_table[0].frequency; if (perf->states[0].core_frequency * 1000 != freq) pr_warn(FW_WARN "P-state 0 is not max freq\n"); diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index 7d0ae968def74c03f92f2379f1eb0694c12c25bb..1d1b563cea4b8dd52ddb34c75ce148b0b7cb5c72 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -2101,7 +2101,7 @@ EXPORT_SYMBOL_GPL(cpufreq_driver_fast_switch); * cpufreq_driver_adjust_perf - Adjust CPU performance level in one go. * @cpu: Target CPU. * @min_perf: Minimum (required) performance level (units of @capacity). - * @target_perf: Terget (desired) performance level (units of @capacity). + * @target_perf: Target (desired) performance level (units of @capacity). * @capacity: Capacity of the target CPU. * * Carry out a fast performance level switch of @cpu without sleeping. diff --git a/drivers/cpufreq/freq_table.c b/drivers/cpufreq/freq_table.c index f839dc9852c08d05b826d06c6b4b71813172815a..d3f756f7b5a0594c9543d76c902f9f422fd571e1 100644 --- a/drivers/cpufreq/freq_table.c +++ b/drivers/cpufreq/freq_table.c @@ -52,7 +52,13 @@ int cpufreq_frequency_table_cpuinfo(struct cpufreq_policy *policy, } policy->min = policy->cpuinfo.min_freq = min_freq; - policy->max = policy->cpuinfo.max_freq = max_freq; + policy->max = max_freq; + /* + * If the driver has set its own cpuinfo.max_freq above max_freq, leave + * it as is. + */ + if (policy->cpuinfo.max_freq < max_freq) + policy->max = policy->cpuinfo.max_freq = max_freq; if (policy->min == ~0) return -EINVAL; diff --git a/drivers/cpufreq/qcom-cpufreq-hw.c b/drivers/cpufreq/qcom-cpufreq-hw.c index 356244510b18a307bbe5a050df723930f04250d8..d3c23447b892d3e87c0fadb25351c5404aa3485b 100644 --- a/drivers/cpufreq/qcom-cpufreq-hw.c +++ b/drivers/cpufreq/qcom-cpufreq-hw.c @@ -32,6 +32,7 @@ struct qcom_cpufreq_soc_data { struct qcom_cpufreq_data { void __iomem *base; + struct resource *res; const struct qcom_cpufreq_soc_data *soc_data; }; @@ -280,6 +281,7 @@ static int qcom_cpufreq_hw_cpu_init(struct cpufreq_policy *policy) struct of_phandle_args args; struct device_node *cpu_np; struct device *cpu_dev; + struct resource *res; void __iomem *base; struct qcom_cpufreq_data *data; int ret, index; @@ -303,18 +305,33 @@ static int qcom_cpufreq_hw_cpu_init(struct cpufreq_policy *policy) index = args.args[0]; - base = devm_platform_ioremap_resource(pdev, index); - if (IS_ERR(base)) - return PTR_ERR(base); + res = platform_get_resource(pdev, IORESOURCE_MEM, index); + if (!res) { + dev_err(dev, "failed to get mem resource %d\n", index); + return -ENODEV; + } + + if (!request_mem_region(res->start, resource_size(res), res->name)) { + dev_err(dev, "failed to request resource %pR\n", res); + return -EBUSY; + } - data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL); + base = ioremap(res->start, resource_size(res)); + if (IS_ERR(base)) { + dev_err(dev, "failed to map resource %pR\n", res); + ret = PTR_ERR(base); + goto release_region; + } + + data = kzalloc(sizeof(*data), GFP_KERNEL); if (!data) { ret = -ENOMEM; - goto error; + goto unmap_base; } data->soc_data = of_device_get_match_data(&pdev->dev); data->base = base; + data->res = res; /* HW should be in enabled state to proceed */ if (!(readl_relaxed(base + data->soc_data->reg_enable) & 0x1)) { @@ -355,7 +372,11 @@ static int qcom_cpufreq_hw_cpu_init(struct cpufreq_policy *policy) return 0; error: - devm_iounmap(dev, base); + kfree(data); +unmap_base: + iounmap(data->base); +release_region: + release_mem_region(res->start, resource_size(res)); return ret; } @@ -363,12 +384,15 @@ static int qcom_cpufreq_hw_cpu_exit(struct cpufreq_policy *policy) { struct device *cpu_dev = get_cpu_device(policy->cpu); struct qcom_cpufreq_data *data = policy->driver_data; - struct platform_device *pdev = cpufreq_get_driver_data(); + struct resource *res = data->res; + void __iomem *base = data->base; dev_pm_opp_remove_all_dynamic(cpu_dev); dev_pm_opp_of_cpumask_remove_table(policy->related_cpus); kfree(policy->freq_table); - devm_iounmap(&pdev->dev, data->base); + kfree(data); + iounmap(base); + release_mem_region(res->start, resource_size(res)); return 0; } diff --git a/drivers/cpufreq/sfi-cpufreq.c b/drivers/cpufreq/sfi-cpufreq.c deleted file mode 100644 index 45cfdf67cf03d7c2932bf63e24cf08aee643f2d7..0000000000000000000000000000000000000000 --- a/drivers/cpufreq/sfi-cpufreq.c +++ /dev/null @@ -1,127 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * SFI Performance States Driver - * - * Author: Vishwesh M Rudramuni - * Author: Srinidhi Kasagar - */ - -#include -#include -#include -#include -#include -#include -#include - -#include - -static struct cpufreq_frequency_table *freq_table; -static struct sfi_freq_table_entry *sfi_cpufreq_array; -static int num_freq_table_entries; - -static int sfi_parse_freq(struct sfi_table_header *table) -{ - struct sfi_table_simple *sb; - struct sfi_freq_table_entry *pentry; - int totallen; - - sb = (struct sfi_table_simple *)table; - num_freq_table_entries = SFI_GET_NUM_ENTRIES(sb, - struct sfi_freq_table_entry); - if (num_freq_table_entries <= 1) { - pr_err("No p-states discovered\n"); - return -ENODEV; - } - - pentry = (struct sfi_freq_table_entry *)sb->pentry; - totallen = num_freq_table_entries * sizeof(*pentry); - - sfi_cpufreq_array = kmemdup(pentry, totallen, GFP_KERNEL); - if (!sfi_cpufreq_array) - return -ENOMEM; - - return 0; -} - -static int sfi_cpufreq_target(struct cpufreq_policy *policy, unsigned int index) -{ - unsigned int next_perf_state = 0; /* Index into perf table */ - u32 lo, hi; - - next_perf_state = policy->freq_table[index].driver_data; - - rdmsr_on_cpu(policy->cpu, MSR_IA32_PERF_CTL, &lo, &hi); - lo = (lo & ~INTEL_PERF_CTL_MASK) | - ((u32) sfi_cpufreq_array[next_perf_state].ctrl_val & - INTEL_PERF_CTL_MASK); - wrmsr_on_cpu(policy->cpu, MSR_IA32_PERF_CTL, lo, hi); - - return 0; -} - -static int sfi_cpufreq_cpu_init(struct cpufreq_policy *policy) -{ - policy->shared_type = CPUFREQ_SHARED_TYPE_HW; - policy->cpuinfo.transition_latency = 100000; /* 100us */ - policy->freq_table = freq_table; - - return 0; -} - -static struct cpufreq_driver sfi_cpufreq_driver = { - .flags = CPUFREQ_CONST_LOOPS, - .verify = cpufreq_generic_frequency_table_verify, - .target_index = sfi_cpufreq_target, - .init = sfi_cpufreq_cpu_init, - .name = "sfi-cpufreq", - .attr = cpufreq_generic_attr, -}; - -static int __init sfi_cpufreq_init(void) -{ - int ret, i; - - /* parse the freq table from SFI */ - ret = sfi_table_parse(SFI_SIG_FREQ, NULL, NULL, sfi_parse_freq); - if (ret) - return ret; - - freq_table = kcalloc(num_freq_table_entries + 1, sizeof(*freq_table), - GFP_KERNEL); - if (!freq_table) { - ret = -ENOMEM; - goto err_free_array; - } - - for (i = 0; i < num_freq_table_entries; i++) { - freq_table[i].driver_data = i; - freq_table[i].frequency = sfi_cpufreq_array[i].freq_mhz * 1000; - } - freq_table[i].frequency = CPUFREQ_TABLE_END; - - ret = cpufreq_register_driver(&sfi_cpufreq_driver); - if (ret) - goto err_free_tbl; - - return ret; - -err_free_tbl: - kfree(freq_table); -err_free_array: - kfree(sfi_cpufreq_array); - return ret; -} -late_initcall(sfi_cpufreq_init); - -static void __exit sfi_cpufreq_exit(void) -{ - cpufreq_unregister_driver(&sfi_cpufreq_driver); - kfree(freq_table); - kfree(sfi_cpufreq_array); -} -module_exit(sfi_cpufreq_exit); - -MODULE_AUTHOR("Vishwesh M Rudramuni "); -MODULE_DESCRIPTION("SFI Performance-States Driver"); -MODULE_LICENSE("GPL"); diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig index e535f28a80283e36db6ca9d1a9d484078ca40c0d..9a4c275a13350df22336e66d6f1095dc2bc9f4ba 100644 --- a/drivers/crypto/Kconfig +++ b/drivers/crypto/Kconfig @@ -348,7 +348,7 @@ config CRYPTO_DEV_PPC4XX config HW_RANDOM_PPC4XX bool "PowerPC 4xx generic true random number generator support" - depends on CRYPTO_DEV_PPC4XX && HW_RANDOM + depends on CRYPTO_DEV_PPC4XX && HW_RANDOM=y default y help This option provides the kernel-side support for the TRNG hardware @@ -404,24 +404,6 @@ config CRYPTO_DEV_OMAP_DES endif # CRYPTO_DEV_OMAP -config CRYPTO_DEV_PICOXCELL - tristate "Support for picoXcell IPSEC and Layer2 crypto engines" - depends on (ARCH_PICOXCELL || COMPILE_TEST) && HAVE_CLK - select CRYPTO_AEAD - select CRYPTO_AES - select CRYPTO_AUTHENC - select CRYPTO_SKCIPHER - select CRYPTO_LIB_DES - select CRYPTO_CBC - select CRYPTO_ECB - select CRYPTO_SEQIV - help - This option enables support for the hardware offload engines in the - Picochip picoXcell SoC devices. Select this for IPSEC ESP offload - and for 3gpp Layer 2 ciphering support. - - Saying m here will build a module named picoxcell_crypto. - config CRYPTO_DEV_SAHARA tristate "Support for SAHARA crypto accelerator" depends on ARCH_MXC && OF @@ -773,21 +755,6 @@ config CRYPTO_DEV_ZYNQMP_AES accelerator. Select this if you want to use the ZynqMP module for AES algorithms. -config CRYPTO_DEV_MEDIATEK - tristate "MediaTek's EIP97 Cryptographic Engine driver" - depends on (ARM && ARCH_MEDIATEK) || COMPILE_TEST - select CRYPTO_LIB_AES - select CRYPTO_AEAD - select CRYPTO_SKCIPHER - select CRYPTO_SHA1 - select CRYPTO_SHA256 - select CRYPTO_SHA512 - select CRYPTO_HMAC - help - This driver allows you to utilize the hardware crypto accelerator - EIP97 which can be found on the MT7623 MT2701, MT8521p, etc .... - Select this if you want to use it for AES/SHA1/SHA2 algorithms. - source "drivers/crypto/chelsio/Kconfig" source "drivers/crypto/virtio/Kconfig" diff --git a/drivers/crypto/Makefile b/drivers/crypto/Makefile index fff9a70348e123fabe1febc51e883a5c4f882b78..fa22cb19e242a254736025aa25a03c56b8490857 100644 --- a/drivers/crypto/Makefile +++ b/drivers/crypto/Makefile @@ -19,7 +19,6 @@ obj-$(CONFIG_CRYPTO_DEV_HIFN_795X) += hifn_795x.o obj-$(CONFIG_CRYPTO_DEV_IMGTEC_HASH) += img-hash.o obj-$(CONFIG_CRYPTO_DEV_IXP4XX) += ixp4xx_crypto.o obj-$(CONFIG_CRYPTO_DEV_MARVELL) += marvell/ -obj-$(CONFIG_CRYPTO_DEV_MEDIATEK) += mediatek/ obj-$(CONFIG_CRYPTO_DEV_MXS_DCP) += mxs-dcp.o obj-$(CONFIG_CRYPTO_DEV_NIAGARA2) += n2_crypto.o n2_crypto-y := n2_core.o n2_asm.o @@ -31,7 +30,6 @@ obj-$(CONFIG_CRYPTO_DEV_OMAP_DES) += omap-des.o obj-$(CONFIG_CRYPTO_DEV_OMAP_SHAM) += omap-sham.o obj-$(CONFIG_CRYPTO_DEV_PADLOCK_AES) += padlock-aes.o obj-$(CONFIG_CRYPTO_DEV_PADLOCK_SHA) += padlock-sha.o -obj-$(CONFIG_CRYPTO_DEV_PICOXCELL) += picoxcell_crypto.o obj-$(CONFIG_CRYPTO_DEV_PPC4XX) += amcc/ obj-$(CONFIG_CRYPTO_DEV_QAT) += qat/ obj-$(CONFIG_CRYPTO_DEV_QCE) += qce/ diff --git a/drivers/crypto/allwinner/Kconfig b/drivers/crypto/allwinner/Kconfig index 180c8a9db819d5e6e465e11ea16ff8c8e212a3cc..856fb20456566ab8dc1fbe78b0727c6c6693fd17 100644 --- a/drivers/crypto/allwinner/Kconfig +++ b/drivers/crypto/allwinner/Kconfig @@ -32,6 +32,15 @@ config CRYPTO_DEV_SUN4I_SS_PRNG Select this option if you want to provide kernel-side support for the Pseudo-Random Number Generator found in the Security System. +config CRYPTO_DEV_SUN4I_SS_DEBUG + bool "Enable sun4i-ss stats" + depends on CRYPTO_DEV_SUN4I_SS + depends on DEBUG_FS + help + Say y to enable sun4i-ss debug stats. + This will create /sys/kernel/debug/sun4i-ss/stats for displaying + the number of requests per algorithm. + config CRYPTO_DEV_SUN8I_CE tristate "Support for Allwinner Crypto Engine cryptographic offloader" select CRYPTO_SKCIPHER diff --git a/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-cipher.c b/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-cipher.c index b72de8939497b98f99703cb8ea766af8ead15758..c2e6f5ed1d79744a7bc0b0e4179fdf81c9cd8ec7 100644 --- a/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-cipher.c +++ b/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-cipher.c @@ -20,6 +20,7 @@ static int noinline_for_stack sun4i_ss_opti_poll(struct skcipher_request *areq) unsigned int ivsize = crypto_skcipher_ivsize(tfm); struct sun4i_cipher_req_ctx *ctx = skcipher_request_ctx(areq); u32 mode = ctx->mode; + void *backup_iv = NULL; /* when activating SS, the default FIFO space is SS_RX_DEFAULT(32) */ u32 rx_cnt = SS_RX_DEFAULT; u32 tx_cnt = 0; @@ -30,9 +31,13 @@ static int noinline_for_stack sun4i_ss_opti_poll(struct skcipher_request *areq) unsigned int ileft = areq->cryptlen; unsigned int oleft = areq->cryptlen; unsigned int todo; + unsigned long pi = 0, po = 0; /* progress for in and out */ + bool miter_err; struct sg_mapping_iter mi, mo; unsigned int oi, oo; /* offset for in and out */ unsigned long flags; + struct skcipher_alg *alg = crypto_skcipher_alg(tfm); + struct sun4i_ss_alg_template *algt; if (!areq->cryptlen) return 0; @@ -42,52 +47,77 @@ static int noinline_for_stack sun4i_ss_opti_poll(struct skcipher_request *areq) return -EINVAL; } + if (areq->iv && ivsize > 0 && mode & SS_DECRYPTION) { + backup_iv = kzalloc(ivsize, GFP_KERNEL); + if (!backup_iv) + return -ENOMEM; + scatterwalk_map_and_copy(backup_iv, areq->src, areq->cryptlen - ivsize, ivsize, 0); + } + + if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN4I_SS_DEBUG)) { + algt = container_of(alg, struct sun4i_ss_alg_template, alg.crypto); + algt->stat_opti++; + algt->stat_bytes += areq->cryptlen; + } + spin_lock_irqsave(&ss->slock, flags); - for (i = 0; i < op->keylen; i += 4) - writel(*(op->key + i / 4), ss->base + SS_KEY0 + i); + for (i = 0; i < op->keylen / 4; i++) + writesl(ss->base + SS_KEY0 + i * 4, &op->key[i], 1); if (areq->iv) { for (i = 0; i < 4 && i < ivsize / 4; i++) { v = *(u32 *)(areq->iv + i * 4); - writel(v, ss->base + SS_IV0 + i * 4); + writesl(ss->base + SS_IV0 + i * 4, &v, 1); } } writel(mode, ss->base + SS_CTL); - sg_miter_start(&mi, areq->src, sg_nents(areq->src), - SG_MITER_FROM_SG | SG_MITER_ATOMIC); - sg_miter_start(&mo, areq->dst, sg_nents(areq->dst), - SG_MITER_TO_SG | SG_MITER_ATOMIC); - sg_miter_next(&mi); - sg_miter_next(&mo); - if (!mi.addr || !mo.addr) { - dev_err_ratelimited(ss->dev, "ERROR: sg_miter return null\n"); - err = -EINVAL; - goto release_ss; - } ileft = areq->cryptlen / 4; oleft = areq->cryptlen / 4; oi = 0; oo = 0; do { - todo = min(rx_cnt, ileft); - todo = min_t(size_t, todo, (mi.length - oi) / 4); - if (todo) { - ileft -= todo; - writesl(ss->base + SS_RXFIFO, mi.addr + oi, todo); - oi += todo * 4; - } - if (oi == mi.length) { - sg_miter_next(&mi); - oi = 0; + if (ileft) { + sg_miter_start(&mi, areq->src, sg_nents(areq->src), + SG_MITER_FROM_SG | SG_MITER_ATOMIC); + if (pi) + sg_miter_skip(&mi, pi); + miter_err = sg_miter_next(&mi); + if (!miter_err || !mi.addr) { + dev_err_ratelimited(ss->dev, "ERROR: sg_miter return null\n"); + err = -EINVAL; + goto release_ss; + } + todo = min(rx_cnt, ileft); + todo = min_t(size_t, todo, (mi.length - oi) / 4); + if (todo) { + ileft -= todo; + writesl(ss->base + SS_RXFIFO, mi.addr + oi, todo); + oi += todo * 4; + } + if (oi == mi.length) { + pi += mi.length; + oi = 0; + } + sg_miter_stop(&mi); } spaces = readl(ss->base + SS_FCSR); rx_cnt = SS_RXFIFO_SPACES(spaces); tx_cnt = SS_TXFIFO_SPACES(spaces); + sg_miter_start(&mo, areq->dst, sg_nents(areq->dst), + SG_MITER_TO_SG | SG_MITER_ATOMIC); + if (po) + sg_miter_skip(&mo, po); + miter_err = sg_miter_next(&mo); + if (!miter_err || !mo.addr) { + dev_err_ratelimited(ss->dev, "ERROR: sg_miter return null\n"); + err = -EINVAL; + goto release_ss; + } todo = min(tx_cnt, oleft); todo = min_t(size_t, todo, (mo.length - oo) / 4); if (todo) { @@ -96,33 +126,41 @@ static int noinline_for_stack sun4i_ss_opti_poll(struct skcipher_request *areq) oo += todo * 4; } if (oo == mo.length) { - sg_miter_next(&mo); oo = 0; + po += mo.length; } + sg_miter_stop(&mo); } while (oleft); if (areq->iv) { - for (i = 0; i < 4 && i < ivsize / 4; i++) { - v = readl(ss->base + SS_IV0 + i * 4); - *(u32 *)(areq->iv + i * 4) = v; + if (mode & SS_DECRYPTION) { + memcpy(areq->iv, backup_iv, ivsize); + kfree_sensitive(backup_iv); + } else { + scatterwalk_map_and_copy(areq->iv, areq->dst, areq->cryptlen - ivsize, + ivsize, 0); } } release_ss: - sg_miter_stop(&mi); - sg_miter_stop(&mo); writel(0, ss->base + SS_CTL); spin_unlock_irqrestore(&ss->slock, flags); return err; } - static int noinline_for_stack sun4i_ss_cipher_poll_fallback(struct skcipher_request *areq) { struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm); struct sun4i_cipher_req_ctx *ctx = skcipher_request_ctx(areq); int err; + struct skcipher_alg *alg = crypto_skcipher_alg(tfm); + struct sun4i_ss_alg_template *algt; + + if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN4I_SS_DEBUG)) { + algt = container_of(alg, struct sun4i_ss_alg_template, alg.crypto); + algt->stat_fb++; + } skcipher_request_set_tfm(&ctx->fallback_req, op->fallback_tfm); skcipher_request_set_callback(&ctx->fallback_req, areq->base.flags, @@ -161,13 +199,16 @@ static int sun4i_ss_cipher_poll(struct skcipher_request *areq) unsigned int ileft = areq->cryptlen; unsigned int oleft = areq->cryptlen; unsigned int todo; + void *backup_iv = NULL; struct sg_mapping_iter mi, mo; + unsigned long pi = 0, po = 0; /* progress for in and out */ + bool miter_err; unsigned int oi, oo; /* offset for in and out */ unsigned int ob = 0; /* offset in buf */ unsigned int obo = 0; /* offset in bufo*/ unsigned int obl = 0; /* length of data in bufo */ unsigned long flags; - bool need_fallback; + bool need_fallback = false; if (!areq->cryptlen) return 0; @@ -186,12 +227,12 @@ static int sun4i_ss_cipher_poll(struct skcipher_request *areq) * we can use the SS optimized function */ while (in_sg && no_chunk == 1) { - if (in_sg->length % 4) + if ((in_sg->length | in_sg->offset) & 3u) no_chunk = 0; in_sg = sg_next(in_sg); } while (out_sg && no_chunk == 1) { - if (out_sg->length % 4) + if ((out_sg->length | out_sg->offset) & 3u) no_chunk = 0; out_sg = sg_next(out_sg); } @@ -202,30 +243,31 @@ static int sun4i_ss_cipher_poll(struct skcipher_request *areq) if (need_fallback) return sun4i_ss_cipher_poll_fallback(areq); + if (areq->iv && ivsize > 0 && mode & SS_DECRYPTION) { + backup_iv = kzalloc(ivsize, GFP_KERNEL); + if (!backup_iv) + return -ENOMEM; + scatterwalk_map_and_copy(backup_iv, areq->src, areq->cryptlen - ivsize, ivsize, 0); + } + + if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN4I_SS_DEBUG)) { + algt->stat_req++; + algt->stat_bytes += areq->cryptlen; + } + spin_lock_irqsave(&ss->slock, flags); - for (i = 0; i < op->keylen; i += 4) - writel(*(op->key + i / 4), ss->base + SS_KEY0 + i); + for (i = 0; i < op->keylen / 4; i++) + writesl(ss->base + SS_KEY0 + i * 4, &op->key[i], 1); if (areq->iv) { for (i = 0; i < 4 && i < ivsize / 4; i++) { v = *(u32 *)(areq->iv + i * 4); - writel(v, ss->base + SS_IV0 + i * 4); + writesl(ss->base + SS_IV0 + i * 4, &v, 1); } } writel(mode, ss->base + SS_CTL); - sg_miter_start(&mi, areq->src, sg_nents(areq->src), - SG_MITER_FROM_SG | SG_MITER_ATOMIC); - sg_miter_start(&mo, areq->dst, sg_nents(areq->dst), - SG_MITER_TO_SG | SG_MITER_ATOMIC); - sg_miter_next(&mi); - sg_miter_next(&mo); - if (!mi.addr || !mo.addr) { - dev_err_ratelimited(ss->dev, "ERROR: sg_miter return null\n"); - err = -EINVAL; - goto release_ss; - } ileft = areq->cryptlen; oleft = areq->cryptlen; oi = 0; @@ -233,8 +275,16 @@ static int sun4i_ss_cipher_poll(struct skcipher_request *areq) while (oleft) { if (ileft) { - char buf[4 * SS_RX_MAX];/* buffer for linearize SG src */ - + sg_miter_start(&mi, areq->src, sg_nents(areq->src), + SG_MITER_FROM_SG | SG_MITER_ATOMIC); + if (pi) + sg_miter_skip(&mi, pi); + miter_err = sg_miter_next(&mi); + if (!miter_err || !mi.addr) { + dev_err_ratelimited(ss->dev, "ERROR: sg_miter return null\n"); + err = -EINVAL; + goto release_ss; + } /* * todo is the number of consecutive 4byte word that we * can read from current SG @@ -256,52 +306,57 @@ static int sun4i_ss_cipher_poll(struct skcipher_request *areq) */ todo = min(rx_cnt * 4 - ob, ileft); todo = min_t(size_t, todo, mi.length - oi); - memcpy(buf + ob, mi.addr + oi, todo); + memcpy(ss->buf + ob, mi.addr + oi, todo); ileft -= todo; oi += todo; ob += todo; if (!(ob % 4)) { - writesl(ss->base + SS_RXFIFO, buf, + writesl(ss->base + SS_RXFIFO, ss->buf, ob / 4); ob = 0; } } if (oi == mi.length) { - sg_miter_next(&mi); + pi += mi.length; oi = 0; } + sg_miter_stop(&mi); } spaces = readl(ss->base + SS_FCSR); rx_cnt = SS_RXFIFO_SPACES(spaces); tx_cnt = SS_TXFIFO_SPACES(spaces); - dev_dbg(ss->dev, - "%x %u/%zu %u/%u cnt=%u %u/%zu %u/%u cnt=%u %u\n", - mode, - oi, mi.length, ileft, areq->cryptlen, rx_cnt, - oo, mo.length, oleft, areq->cryptlen, tx_cnt, ob); if (!tx_cnt) continue; + sg_miter_start(&mo, areq->dst, sg_nents(areq->dst), + SG_MITER_TO_SG | SG_MITER_ATOMIC); + if (po) + sg_miter_skip(&mo, po); + miter_err = sg_miter_next(&mo); + if (!miter_err || !mo.addr) { + dev_err_ratelimited(ss->dev, "ERROR: sg_miter return null\n"); + err = -EINVAL; + goto release_ss; + } /* todo in 4bytes word */ todo = min(tx_cnt, oleft / 4); todo = min_t(size_t, todo, (mo.length - oo) / 4); + if (todo) { readsl(ss->base + SS_TXFIFO, mo.addr + oo, todo); oleft -= todo * 4; oo += todo * 4; if (oo == mo.length) { - sg_miter_next(&mo); + po += mo.length; oo = 0; } } else { - char bufo[4 * SS_TX_MAX]; /* buffer for linearize SG dst */ - /* * read obl bytes in bufo, we read at maximum for * emptying the device */ - readsl(ss->base + SS_TXFIFO, bufo, tx_cnt); + readsl(ss->base + SS_TXFIFO, ss->bufo, tx_cnt); obl = tx_cnt * 4; obo = 0; do { @@ -313,28 +368,31 @@ static int sun4i_ss_cipher_poll(struct skcipher_request *areq) */ todo = min_t(size_t, mo.length - oo, obl - obo); - memcpy(mo.addr + oo, bufo + obo, todo); + memcpy(mo.addr + oo, ss->bufo + obo, todo); oleft -= todo; obo += todo; oo += todo; if (oo == mo.length) { + po += mo.length; sg_miter_next(&mo); oo = 0; } } while (obo < obl); /* bufo must be fully used here */ } + sg_miter_stop(&mo); } if (areq->iv) { - for (i = 0; i < 4 && i < ivsize / 4; i++) { - v = readl(ss->base + SS_IV0 + i * 4); - *(u32 *)(areq->iv + i * 4) = v; + if (mode & SS_DECRYPTION) { + memcpy(areq->iv, backup_iv, ivsize); + kfree_sensitive(backup_iv); + } else { + scatterwalk_map_and_copy(areq->iv, areq->dst, areq->cryptlen - ivsize, + ivsize, 0); } } release_ss: - sg_miter_stop(&mi); - sg_miter_stop(&mo); writel(0, ss->base + SS_CTL); spin_unlock_irqrestore(&ss->slock, flags); @@ -503,7 +561,6 @@ int sun4i_ss_cipher_init(struct crypto_tfm *tfm) sizeof(struct sun4i_cipher_req_ctx) + crypto_skcipher_reqsize(op->fallback_tfm)); - err = pm_runtime_get_sync(op->ss->dev); if (err < 0) goto error_pm; @@ -590,5 +647,4 @@ int sun4i_ss_des3_setkey(struct crypto_skcipher *tfm, const u8 *key, crypto_skcipher_set_flags(op->fallback_tfm, tfm->base.crt_flags & CRYPTO_TFM_REQ_MASK); return crypto_skcipher_setkey(op->fallback_tfm, key, keylen); - } diff --git a/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-core.c b/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-core.c index a2b67f7f8a8141777013f84e499d4807f2806659..709905ec46806d3303bcd79faa7878bc471c3dd4 100644 --- a/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-core.c +++ b/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-core.c @@ -10,6 +10,7 @@ */ #include #include +#include #include #include #include @@ -234,6 +235,51 @@ static struct sun4i_ss_alg_template ss_algs[] = { #endif }; +static int sun4i_ss_dbgfs_read(struct seq_file *seq, void *v) +{ + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(ss_algs); i++) { + if (!ss_algs[i].ss) + continue; + switch (ss_algs[i].type) { + case CRYPTO_ALG_TYPE_SKCIPHER: + seq_printf(seq, "%s %s reqs=%lu opti=%lu fallback=%lu tsize=%lu\n", + ss_algs[i].alg.crypto.base.cra_driver_name, + ss_algs[i].alg.crypto.base.cra_name, + ss_algs[i].stat_req, ss_algs[i].stat_opti, ss_algs[i].stat_fb, + ss_algs[i].stat_bytes); + break; + case CRYPTO_ALG_TYPE_RNG: + seq_printf(seq, "%s %s reqs=%lu tsize=%lu\n", + ss_algs[i].alg.rng.base.cra_driver_name, + ss_algs[i].alg.rng.base.cra_name, + ss_algs[i].stat_req, ss_algs[i].stat_bytes); + break; + case CRYPTO_ALG_TYPE_AHASH: + seq_printf(seq, "%s %s reqs=%lu\n", + ss_algs[i].alg.hash.halg.base.cra_driver_name, + ss_algs[i].alg.hash.halg.base.cra_name, + ss_algs[i].stat_req); + break; + } + } + return 0; +} + +static int sun4i_ss_dbgfs_open(struct inode *inode, struct file *file) +{ + return single_open(file, sun4i_ss_dbgfs_read, inode->i_private); +} + +static const struct file_operations sun4i_ss_debugfs_fops = { + .owner = THIS_MODULE, + .open = sun4i_ss_dbgfs_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + /* * Power management strategy: The device is suspended unless a TFM exists for * one of the algorithms proposed by this driver. @@ -454,6 +500,12 @@ static int sun4i_ss_probe(struct platform_device *pdev) break; } } + + /* Ignore error of debugfs */ + ss->dbgfs_dir = debugfs_create_dir("sun4i-ss", NULL); + ss->dbgfs_stats = debugfs_create_file("stats", 0444, ss->dbgfs_dir, ss, + &sun4i_ss_debugfs_fops); + return 0; error_alg: i--; diff --git a/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-hash.c b/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-hash.c index 1dff48558f5389f0ad548552782221ae90775666..c1b4585e9bbc7d459bfb6c47d66adbf145735f9a 100644 --- a/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-hash.c +++ b/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-hash.c @@ -191,8 +191,10 @@ static int sun4i_hash(struct ahash_request *areq) u32 spaces, rx_cnt = SS_RX_DEFAULT, bf[32] = {0}, v, ivmode = 0; struct sun4i_req_ctx *op = ahash_request_ctx(areq); struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct ahash_alg *alg = __crypto_ahash_alg(tfm->base.__crt_alg); struct sun4i_tfm_ctx *tfmctx = crypto_ahash_ctx(tfm); struct sun4i_ss_ctx *ss = tfmctx->ss; + struct sun4i_ss_alg_template *algt; struct scatterlist *in_sg = areq->src; struct sg_mapping_iter mi; int in_r, err = 0; @@ -398,6 +400,10 @@ static int sun4i_hash(struct ahash_request *areq) */ hash_final: + if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN4I_SS_DEBUG)) { + algt = container_of(alg, struct sun4i_ss_alg_template, alg.hash); + algt->stat_req++; + } /* write the remaining words of the wait buffer */ if (op->len) { diff --git a/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-prng.c b/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-prng.c index 729aafdbea84eada50c3964995c02352a4718ad4..443160a114bb031a0d53e78106d82eb01c0e51a5 100644 --- a/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-prng.c +++ b/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-prng.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0-or-later #include "sun4i-ss.h" int sun4i_ss_prng_seed(struct crypto_rng *tfm, const u8 *seed, @@ -32,6 +33,11 @@ int sun4i_ss_prng_generate(struct crypto_rng *tfm, const u8 *src, if (err < 0) return err; + if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN4I_SS_DEBUG)) { + algt->stat_req++; + algt->stat_bytes += todo; + } + spin_lock_bh(&ss->slock); writel(mode, ss->base + SS_CTL); diff --git a/drivers/crypto/allwinner/sun4i-ss/sun4i-ss.h b/drivers/crypto/allwinner/sun4i-ss/sun4i-ss.h index 5c291e4a6857b6c4edc9357dd968a6a548c127a7..0fee6f4e2d90dc700ce393ae7df24bd67d759ae2 100644 --- a/drivers/crypto/allwinner/sun4i-ss/sun4i-ss.h +++ b/drivers/crypto/allwinner/sun4i-ss/sun4i-ss.h @@ -148,10 +148,14 @@ struct sun4i_ss_ctx { struct reset_control *reset; struct device *dev; struct resource *res; + char buf[4 * SS_RX_MAX];/* buffer for linearize SG src */ + char bufo[4 * SS_TX_MAX]; /* buffer for linearize SG dst */ spinlock_t slock; /* control the use of the device */ #ifdef CONFIG_CRYPTO_DEV_SUN4I_SS_PRNG u32 seed[SS_SEED_LEN / BITS_PER_LONG]; #endif + struct dentry *dbgfs_dir; + struct dentry *dbgfs_stats; }; struct sun4i_ss_alg_template { @@ -163,6 +167,10 @@ struct sun4i_ss_alg_template { struct rng_alg rng; } alg; struct sun4i_ss_ctx *ss; + unsigned long stat_req; + unsigned long stat_fb; + unsigned long stat_bytes; + unsigned long stat_opti; }; struct sun4i_tfm_ctx { diff --git a/drivers/crypto/bcm/cipher.c b/drivers/crypto/bcm/cipher.c index 30390a7324b2957aabc7b4b0d936ca7f34885124..851b149f71701d48658138cc23ccbc533b1f5417 100644 --- a/drivers/crypto/bcm/cipher.c +++ b/drivers/crypto/bcm/cipher.c @@ -42,7 +42,7 @@ /* ================= Device Structure ================== */ -struct device_private iproc_priv; +struct bcm_device_private iproc_priv; /* ==================== Parameters ===================== */ @@ -471,10 +471,8 @@ static int handle_skcipher_req(struct iproc_reqctx_s *rctx) static void handle_skcipher_resp(struct iproc_reqctx_s *rctx) { struct spu_hw *spu = &iproc_priv.spu; -#ifdef DEBUG struct crypto_async_request *areq = rctx->parent; struct skcipher_request *req = skcipher_request_cast(areq); -#endif struct iproc_ctx_s *ctx = rctx->ctx; u32 payload_len; @@ -996,13 +994,11 @@ static int ahash_req_done(struct iproc_reqctx_s *rctx) static void handle_ahash_resp(struct iproc_reqctx_s *rctx) { struct iproc_ctx_s *ctx = rctx->ctx; -#ifdef DEBUG struct crypto_async_request *areq = rctx->parent; struct ahash_request *req = ahash_request_cast(areq); struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); unsigned int blocksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(ahash)); -#endif /* * Save hash to use as input to next op if incremental. Might be copying * too much, but that's easier than figuring out actual digest size here diff --git a/drivers/crypto/bcm/cipher.h b/drivers/crypto/bcm/cipher.h index 0ad5892b445d383f8c29b782995280248290e27d..71281a3bdbdc07dca64ff201344b73a9e504426e 100644 --- a/drivers/crypto/bcm/cipher.h +++ b/drivers/crypto/bcm/cipher.h @@ -420,7 +420,7 @@ struct spu_hw { u32 num_chan; }; -struct device_private { +struct bcm_device_private { struct platform_device *pdev; struct spu_hw spu; @@ -467,6 +467,6 @@ struct device_private { struct mbox_chan **mbox; }; -extern struct device_private iproc_priv; +extern struct bcm_device_private iproc_priv; #endif diff --git a/drivers/crypto/bcm/spu.c b/drivers/crypto/bcm/spu.c index fe126f95c70261e32bd64de1ed58c5bc73a93780..007abf92cc05be6541ac11c26676abbb72f7a6ac 100644 --- a/drivers/crypto/bcm/spu.c +++ b/drivers/crypto/bcm/spu.c @@ -41,7 +41,7 @@ void spum_dump_msg_hdr(u8 *buf, unsigned int buf_len) packet_log("SPU Message header %p len: %u\n", buf, buf_len); /* ========== Decode MH ========== */ - packet_log(" MH 0x%08x\n", be32_to_cpu(*((u32 *)ptr))); + packet_log(" MH 0x%08x\n", be32_to_cpup((__be32 *)ptr)); if (spuh->mh.flags & MH_SCTX_PRES) packet_log(" SCTX present\n"); if (spuh->mh.flags & MH_BDESC_PRES) @@ -273,22 +273,21 @@ void spum_dump_msg_hdr(u8 *buf, unsigned int buf_len) /* ========== Decode BDESC ========== */ if (spuh->mh.flags & MH_BDESC_PRES) { -#ifdef DEBUG struct BDESC_HEADER *bdesc = (struct BDESC_HEADER *)ptr; -#endif - packet_log(" BDESC[0] 0x%08x\n", be32_to_cpu(*((u32 *)ptr))); + + packet_log(" BDESC[0] 0x%08x\n", be32_to_cpup((__be32 *)ptr)); packet_log(" OffsetMAC:%u LengthMAC:%u\n", be16_to_cpu(bdesc->offset_mac), be16_to_cpu(bdesc->length_mac)); ptr += sizeof(u32); - packet_log(" BDESC[1] 0x%08x\n", be32_to_cpu(*((u32 *)ptr))); + packet_log(" BDESC[1] 0x%08x\n", be32_to_cpup((__be32 *)ptr)); packet_log(" OffsetCrypto:%u LengthCrypto:%u\n", be16_to_cpu(bdesc->offset_crypto), be16_to_cpu(bdesc->length_crypto)); ptr += sizeof(u32); - packet_log(" BDESC[2] 0x%08x\n", be32_to_cpu(*((u32 *)ptr))); + packet_log(" BDESC[2] 0x%08x\n", be32_to_cpup((__be32 *)ptr)); packet_log(" OffsetICV:%u OffsetIV:%u\n", be16_to_cpu(bdesc->offset_icv), be16_to_cpu(bdesc->offset_iv)); @@ -297,10 +296,9 @@ void spum_dump_msg_hdr(u8 *buf, unsigned int buf_len) /* ========== Decode BD ========== */ if (spuh->mh.flags & MH_BD_PRES) { -#ifdef DEBUG struct BD_HEADER *bd = (struct BD_HEADER *)ptr; -#endif - packet_log(" BD[0] 0x%08x\n", be32_to_cpu(*((u32 *)ptr))); + + packet_log(" BD[0] 0x%08x\n", be32_to_cpup((__be32 *)ptr)); packet_log(" Size:%ubytes PrevLength:%u\n", be16_to_cpu(bd->size), be16_to_cpu(bd->prev_length)); ptr += 4; @@ -1056,9 +1054,9 @@ void spum_request_pad(u8 *pad_start, /* add the size at the end as required per alg */ if (auth_alg == HASH_ALG_MD5) - *(u64 *)ptr = cpu_to_le64((u64)total_sent * 8); + *(__le64 *)ptr = cpu_to_le64(total_sent * 8ull); else /* SHA1, SHA2-224, SHA2-256 */ - *(u64 *)ptr = cpu_to_be64((u64)total_sent * 8); + *(__be64 *)ptr = cpu_to_be64(total_sent * 8ull); ptr += sizeof(u64); } } diff --git a/drivers/crypto/bcm/spu2.c b/drivers/crypto/bcm/spu2.c index c860ffb0b4c38d7123c2884f0a6e45e4212498fb..2db35b5ccaa245f59fac09a39daccca394a65b8e 100644 --- a/drivers/crypto/bcm/spu2.c +++ b/drivers/crypto/bcm/spu2.c @@ -964,7 +964,6 @@ u32 spu2_create_request(u8 *spu_hdr, unsigned int cipher_offset = aead_parms->assoc_size + aead_parms->aad_pad_len + aead_parms->iv_len; -#ifdef DEBUG /* total size of the data following OMD (without STAT word padding) */ unsigned int real_db_size = spu_real_db_size(aead_parms->assoc_size, aead_parms->iv_len, @@ -973,7 +972,6 @@ u32 spu2_create_request(u8 *spu_hdr, aead_parms->aad_pad_len, aead_parms->data_pad_len, hash_parms->pad_len); -#endif unsigned int assoc_size = aead_parms->assoc_size; if (req_opts->is_aead && @@ -1263,9 +1261,9 @@ void spu2_request_pad(u8 *pad_start, u32 gcm_padding, u32 hash_pad_len, /* add the size at the end as required per alg */ if (auth_alg == HASH_ALG_MD5) - *(u64 *)ptr = cpu_to_le64((u64)total_sent * 8); + *(__le64 *)ptr = cpu_to_le64(total_sent * 8ull); else /* SHA1, SHA2-224, SHA2-256 */ - *(u64 *)ptr = cpu_to_be64((u64)total_sent * 8); + *(__be64 *)ptr = cpu_to_be64(total_sent * 8ull); ptr += sizeof(u64); } diff --git a/drivers/crypto/bcm/spu2.h b/drivers/crypto/bcm/spu2.h index 6e666bfb3cfcb69ffdc8e9a3521780a82987a5e0..a76d4e054466a8f010776d423198b6a8a5aad880 100644 --- a/drivers/crypto/bcm/spu2.h +++ b/drivers/crypto/bcm/spu2.h @@ -73,10 +73,10 @@ enum spu2_ret_md_opts { /* Fixed Metadata format */ struct SPU2_FMD { - u64 ctrl0; - u64 ctrl1; - u64 ctrl2; - u64 ctrl3; + __le64 ctrl0; + __le64 ctrl1; + __le64 ctrl2; + __le64 ctrl3; }; #define FMD_SIZE sizeof(struct SPU2_FMD) diff --git a/drivers/crypto/bcm/spum.h b/drivers/crypto/bcm/spum.h index 6116ad1dd26e3b52f190b621a48980f22577eb44..f062f75808de243a2baf788c23d7cd976f187b03 100644 --- a/drivers/crypto/bcm/spum.h +++ b/drivers/crypto/bcm/spum.h @@ -69,18 +69,18 @@ /* Buffer Descriptor Header [BDESC]. SPU in big-endian mode. */ struct BDESC_HEADER { - u16 offset_mac; /* word 0 [31-16] */ - u16 length_mac; /* word 0 [15-0] */ - u16 offset_crypto; /* word 1 [31-16] */ - u16 length_crypto; /* word 1 [15-0] */ - u16 offset_icv; /* word 2 [31-16] */ - u16 offset_iv; /* word 2 [15-0] */ + __be16 offset_mac; /* word 0 [31-16] */ + __be16 length_mac; /* word 0 [15-0] */ + __be16 offset_crypto; /* word 1 [31-16] */ + __be16 length_crypto; /* word 1 [15-0] */ + __be16 offset_icv; /* word 2 [31-16] */ + __be16 offset_iv; /* word 2 [15-0] */ }; /* Buffer Data Header [BD]. SPU in big-endian mode. */ struct BD_HEADER { - u16 size; - u16 prev_length; + __be16 size; + __be16 prev_length; }; /* Command Context Header. SPU-M in big endian mode. */ @@ -144,13 +144,13 @@ struct MHEADER { /* Generic Mode Security Context Structure [SCTX] */ struct SCTX { /* word 0: protocol flags */ - u32 proto_flags; + __be32 proto_flags; /* word 1: cipher flags */ - u32 cipher_flags; + __be32 cipher_flags; /* word 2: Extended cipher flags */ - u32 ecf; + __be32 ecf; }; diff --git a/drivers/crypto/bcm/util.c b/drivers/crypto/bcm/util.c index 2b304fc780595c10ec589ea5bc0a14353017326c..c4669a96eaecf1258acdf3b57e9211c9d71c221f 100644 --- a/drivers/crypto/bcm/util.c +++ b/drivers/crypto/bcm/util.c @@ -268,6 +268,7 @@ int do_shash(unsigned char *name, unsigned char *result, return rc; } +#ifdef DEBUG /* Dump len bytes of a scatterlist starting at skip bytes into the sg */ void __dump_sg(struct scatterlist *sg, unsigned int skip, unsigned int len) { @@ -289,6 +290,7 @@ void __dump_sg(struct scatterlist *sg, unsigned int skip, unsigned int len) if (debug_logging_sleep) msleep(debug_logging_sleep); } +#endif /* Returns the name for a given cipher alg/mode */ char *spu_alg_name(enum spu_cipher_alg alg, enum spu_cipher_mode mode) @@ -348,7 +350,7 @@ char *spu_alg_name(enum spu_cipher_alg alg, enum spu_cipher_mode mode) static ssize_t spu_debugfs_read(struct file *filp, char __user *ubuf, size_t count, loff_t *offp) { - struct device_private *ipriv; + struct bcm_device_private *ipriv; char *buf; ssize_t ret, out_offset, out_count; int i; diff --git a/drivers/crypto/bcm/util.h b/drivers/crypto/bcm/util.h index a89b2b9c1f524b9dc3c2e9540b89d17585f48010..61c256384816f56c47bf26d762ac458bd272413a 100644 --- a/drivers/crypto/bcm/util.h +++ b/drivers/crypto/bcm/util.h @@ -58,12 +58,26 @@ void __dump_sg(struct scatterlist *sg, unsigned int skip, unsigned int len); #else /* !DEBUG_ON */ -#define flow_log(...) do {} while (0) -#define flow_dump(msg, var, var_len) do {} while (0) -#define packet_log(...) do {} while (0) -#define packet_dump(msg, var, var_len) do {} while (0) - -#define dump_sg(sg, skip, len) do {} while (0) +static inline void flow_log(const char *format, ...) +{ +} + +static inline void flow_dump(const char *msg, const void *var, size_t var_len) +{ +} + +static inline void packet_log(const char *format, ...) +{ +} + +static inline void packet_dump(const char *msg, const void *var, size_t var_len) +{ +} + +static inline void dump_sg(struct scatterlist *sg, unsigned int skip, + unsigned int len) +{ +} #endif /* DEBUG_ON */ diff --git a/drivers/crypto/caam/debugfs.c b/drivers/crypto/caam/debugfs.c index 8ebf18398166b6adf1ae4bf9c74365336776622e..806bb20d2aa196a2cef2ff526191b786e3973b99 100644 --- a/drivers/crypto/caam/debugfs.c +++ b/drivers/crypto/caam/debugfs.c @@ -19,8 +19,8 @@ static int caam_debugfs_u32_get(void *data, u64 *val) return 0; } -DEFINE_SIMPLE_ATTRIBUTE(caam_fops_u32_ro, caam_debugfs_u32_get, NULL, "%llu\n"); -DEFINE_SIMPLE_ATTRIBUTE(caam_fops_u64_ro, caam_debugfs_u64_get, NULL, "%llu\n"); +DEFINE_DEBUGFS_ATTRIBUTE(caam_fops_u32_ro, caam_debugfs_u32_get, NULL, "%llu\n"); +DEFINE_DEBUGFS_ATTRIBUTE(caam_fops_u64_ro, caam_debugfs_u64_get, NULL, "%llu\n"); #ifdef CONFIG_CAAM_QI /* diff --git a/drivers/crypto/cavium/cpt/cptvf_main.c b/drivers/crypto/cavium/cpt/cptvf_main.c index f016448e43bba56fca39755661b4d2ed77fee28c..112b12a32542b50655a0cd76273cbf332e57c2a3 100644 --- a/drivers/crypto/cavium/cpt/cptvf_main.c +++ b/drivers/crypto/cavium/cpt/cptvf_main.c @@ -233,10 +233,10 @@ static int alloc_command_queues(struct cpt_vf *cptvf, c_size = (rem_q_size > qcsize_bytes) ? qcsize_bytes : rem_q_size; - curr->head = (u8 *)dma_alloc_coherent(&pdev->dev, - c_size + CPT_NEXT_CHUNK_PTR_SIZE, - &curr->dma_addr, - GFP_KERNEL); + curr->head = dma_alloc_coherent(&pdev->dev, + c_size + CPT_NEXT_CHUNK_PTR_SIZE, + &curr->dma_addr, + GFP_KERNEL); if (!curr->head) { dev_err(&pdev->dev, "Command Q (%d) chunk (%d) allocation failed\n", i, queue->nchunks); diff --git a/drivers/crypto/ccree/cc_cipher.c b/drivers/crypto/ccree/cc_cipher.c index cdfee501fbd9fb5db50e92f730ca6c65b71e1c47..78833491f534d347c4f2ce8cd840363ce420d600 100644 --- a/drivers/crypto/ccree/cc_cipher.c +++ b/drivers/crypto/ccree/cc_cipher.c @@ -921,7 +921,7 @@ static int cc_cipher_process(struct skcipher_request *req, return crypto_skcipher_decrypt(subreq); } - /* The IV we are handed may be allocted from the stack so + /* The IV we are handed may be allocated from the stack so * we must copy it to a DMAable buffer before use. */ req_ctx->iv = kmemdup(iv, ivsize, flags); diff --git a/drivers/crypto/ccree/cc_driver.h b/drivers/crypto/ccree/cc_driver.h index 5f1d4602eb8f2daa6f106c67b15614c07a54fb24..f49579aa145297482d30a46a5615d9ee81da0f1e 100644 --- a/drivers/crypto/ccree/cc_driver.h +++ b/drivers/crypto/ccree/cc_driver.h @@ -23,7 +23,6 @@ #include #include #include -#include #include #include diff --git a/drivers/crypto/geode-aes.c b/drivers/crypto/geode-aes.c index f4f18bfc2247bfc550e082a2c8e7188122f1015a..4ee010f399124ae5df0afd14b510f1db4cc7569a 100644 --- a/drivers/crypto/geode-aes.c +++ b/drivers/crypto/geode-aes.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -434,3 +435,4 @@ module_pci_driver(geode_aes_driver); MODULE_AUTHOR("Advanced Micro Devices, Inc."); MODULE_DESCRIPTION("Geode LX Hardware AES driver"); MODULE_LICENSE("GPL"); +MODULE_IMPORT_NS(CRYPTO_INTERNAL); diff --git a/drivers/crypto/hisilicon/hpre/hpre.h b/drivers/crypto/hisilicon/hpre/hpre.h index f69252b24671b8c559cc6a56909152eeba6ec38f..181c109b19f7c4219be4db42739c04b4bd984382 100644 --- a/drivers/crypto/hisilicon/hpre/hpre.h +++ b/drivers/crypto/hisilicon/hpre/hpre.h @@ -14,8 +14,7 @@ enum { HPRE_CLUSTER0, HPRE_CLUSTER1, HPRE_CLUSTER2, - HPRE_CLUSTER3, - HPRE_CLUSTERS_NUM, + HPRE_CLUSTER3 }; enum hpre_ctrl_dbgfs_file { @@ -36,7 +35,10 @@ enum hpre_dfx_dbgfs_file { HPRE_DFX_FILE_NUM }; -#define HPRE_DEBUGFS_FILE_NUM (HPRE_DEBUG_FILE_NUM + HPRE_CLUSTERS_NUM - 1) +#define HPRE_CLUSTERS_NUM_V2 (HPRE_CLUSTER3 + 1) +#define HPRE_CLUSTERS_NUM_V3 1 +#define HPRE_CLUSTERS_NUM_MAX HPRE_CLUSTERS_NUM_V2 +#define HPRE_DEBUGFS_FILE_NUM (HPRE_DEBUG_FILE_NUM + HPRE_CLUSTERS_NUM_MAX - 1) struct hpre_debugfs_file { int index; diff --git a/drivers/crypto/hisilicon/hpre/hpre_main.c b/drivers/crypto/hisilicon/hpre/hpre_main.c index e5c991913f092f6dc075591031122cb66939bd24..e7a2c70eb9cf52f3cc70d6389418a781a62037e4 100644 --- a/drivers/crypto/hisilicon/hpre/hpre_main.c +++ b/drivers/crypto/hisilicon/hpre/hpre_main.c @@ -10,6 +10,7 @@ #include #include #include +#include #include "hpre.h" #define HPRE_QUEUE_NUM_V2 1024 @@ -29,13 +30,14 @@ #define HPRE_BD_ARUSR_CFG 0x301030 #define HPRE_BD_AWUSR_CFG 0x301034 #define HPRE_TYPES_ENB 0x301038 +#define HPRE_RSA_ENB BIT(0) +#define HPRE_ECC_ENB BIT(1) #define HPRE_DATA_RUSER_CFG 0x30103c #define HPRE_DATA_WUSER_CFG 0x301040 #define HPRE_INT_MASK 0x301400 #define HPRE_INT_STATUS 0x301800 #define HPRE_CORE_INT_ENABLE 0 #define HPRE_CORE_INT_DISABLE 0x003fffff -#define HPRE_RAS_ECC_1BIT_TH 0x30140c #define HPRE_RDCHN_INI_ST 0x301a00 #define HPRE_CLSTR_BASE 0x302000 #define HPRE_CORE_EN_OFFSET 0x04 @@ -45,7 +47,7 @@ #define HPRE_CORE_IS_SCHD_OFFSET 0x90 #define HPRE_RAS_CE_ENB 0x301410 -#define HPRE_HAC_RAS_CE_ENABLE 0x1 +#define HPRE_HAC_RAS_CE_ENABLE (BIT(0) | BIT(22) | BIT(23)) #define HPRE_RAS_NFE_ENB 0x301414 #define HPRE_HAC_RAS_NFE_ENABLE 0x3ffffe #define HPRE_RAS_FE_ENB 0x301418 @@ -73,7 +75,8 @@ #define HPRE_QM_AXI_CFG_MASK 0xffff #define HPRE_QM_VFG_AX_MASK 0xff #define HPRE_BD_USR_MASK 0x3 -#define HPRE_CLUSTER_CORE_MASK 0xf +#define HPRE_CLUSTER_CORE_MASK_V2 0xf +#define HPRE_CLUSTER_CORE_MASK_V3 0xff #define HPRE_AM_OOO_SHUTDOWN_ENB 0x301044 #define HPRE_AM_OOO_SHUTDOWN_ENABLE BIT(0) @@ -86,6 +89,11 @@ #define HPRE_QM_PM_FLR BIT(11) #define HPRE_QM_SRIOV_FLR BIT(12) +#define HPRE_CLUSTERS_NUM(qm) \ + (((qm)->ver >= QM_HW_V3) ? HPRE_CLUSTERS_NUM_V3 : HPRE_CLUSTERS_NUM_V2) +#define HPRE_CLUSTER_CORE_MASK(qm) \ + (((qm)->ver >= QM_HW_V3) ? HPRE_CLUSTER_CORE_MASK_V3 :\ + HPRE_CLUSTER_CORE_MASK_V2) #define HPRE_VIA_MSI_DSM 1 #define HPRE_SQE_MASK_OFFSET 8 #define HPRE_SQE_MASK_LEN 24 @@ -129,7 +137,11 @@ static const struct hpre_hw_error hpre_hw_errors[] = { { .int_msk = BIT(9), .msg = "cluster4_shb_timeout_int_set" }, { .int_msk = GENMASK(15, 10), .msg = "ooo_rdrsp_err_int_set" }, { .int_msk = GENMASK(21, 16), .msg = "ooo_wrrsp_err_int_set" }, - { /* sentinel */ } + { .int_msk = BIT(22), .msg = "pt_rng_timeout_int_set"}, + { .int_msk = BIT(23), .msg = "sva_fsm_timeout_int_set"}, + { + /* sentinel */ + } }; static const u64 hpre_cluster_offsets[] = { @@ -178,6 +190,19 @@ static const char *hpre_dfx_files[HPRE_DFX_FILE_NUM] = { "invalid_req_cnt" }; +static const struct kernel_param_ops hpre_uacce_mode_ops = { + .set = uacce_mode_set, + .get = param_get_int, +}; + +/* + * uacce_mode = 0 means hpre only register to crypto, + * uacce_mode = 1 means hpre both register to crypto and uacce. + */ +static u32 uacce_mode = UACCE_MODE_NOUACCE; +module_param_cb(uacce_mode, &hpre_uacce_mode_ops, &uacce_mode, 0444); +MODULE_PARM_DESC(uacce_mode, UACCE_MODE_DESC); + static int pf_q_num_set(const char *val, const struct kernel_param *kp) { return q_num_set(val, kp, HPRE_PCI_DEVICE_ID); @@ -214,6 +239,30 @@ struct hisi_qp *hpre_create_qp(void) return NULL; } +static void hpre_pasid_enable(struct hisi_qm *qm) +{ + u32 val; + + val = readl_relaxed(qm->io_base + HPRE_DATA_RUSER_CFG); + val |= BIT(HPRE_PASID_EN_BIT); + writel_relaxed(val, qm->io_base + HPRE_DATA_RUSER_CFG); + val = readl_relaxed(qm->io_base + HPRE_DATA_WUSER_CFG); + val |= BIT(HPRE_PASID_EN_BIT); + writel_relaxed(val, qm->io_base + HPRE_DATA_WUSER_CFG); +} + +static void hpre_pasid_disable(struct hisi_qm *qm) +{ + u32 val; + + val = readl_relaxed(qm->io_base + HPRE_DATA_RUSER_CFG); + val &= ~BIT(HPRE_PASID_EN_BIT); + writel_relaxed(val, qm->io_base + HPRE_DATA_RUSER_CFG); + val = readl_relaxed(qm->io_base + HPRE_DATA_WUSER_CFG); + val &= ~BIT(HPRE_PASID_EN_BIT); + writel_relaxed(val, qm->io_base + HPRE_DATA_WUSER_CFG); +} + static int hpre_cfg_by_dsm(struct hisi_qm *qm) { struct device *dev = &qm->pdev->dev; @@ -238,8 +287,40 @@ static int hpre_cfg_by_dsm(struct hisi_qm *qm) return 0; } +static int hpre_set_cluster(struct hisi_qm *qm) +{ + u32 cluster_core_mask = HPRE_CLUSTER_CORE_MASK(qm); + u8 clusters_num = HPRE_CLUSTERS_NUM(qm); + struct device *dev = &qm->pdev->dev; + unsigned long offset; + u32 val = 0; + int ret, i; + + for (i = 0; i < clusters_num; i++) { + offset = i * HPRE_CLSTR_ADDR_INTRVL; + + /* clusters initiating */ + writel(cluster_core_mask, + HPRE_ADDR(qm, offset + HPRE_CORE_ENB)); + writel(0x1, HPRE_ADDR(qm, offset + HPRE_CORE_INI_CFG)); + ret = readl_relaxed_poll_timeout(HPRE_ADDR(qm, offset + + HPRE_CORE_INI_STATUS), val, + ((val & cluster_core_mask) == + cluster_core_mask), + HPRE_REG_RD_INTVRL_US, + HPRE_REG_RD_TMOUT_US); + if (ret) { + dev_err(dev, + "cluster %d int st status timeout!\n", i); + return -ETIMEDOUT; + } + } + + return 0; +} + /* - * For Hi1620, we shoul disable FLR triggered by hardware (BME/PM/SRIOV). + * For Kunpeng 920, we shoul disable FLR triggered by hardware (BME/PM/SRIOV). * Or it may stay in D3 state when we bind and unbind hpre quickly, * as it does FLR triggered by hardware. */ @@ -257,9 +338,8 @@ static void disable_flr_of_bme(struct hisi_qm *qm) static int hpre_set_user_domain_and_cache(struct hisi_qm *qm) { struct device *dev = &qm->pdev->dev; - unsigned long offset; - int ret, i; u32 val; + int ret; writel(HPRE_QM_USR_CFG_MASK, HPRE_ADDR(qm, QM_ARUSER_M_CFG_ENABLE)); writel(HPRE_QM_USR_CFG_MASK, HPRE_ADDR(qm, QM_AWUSER_M_CFG_ENABLE)); @@ -270,11 +350,15 @@ static int hpre_set_user_domain_and_cache(struct hisi_qm *qm) val |= BIT(HPRE_TIMEOUT_ABNML_BIT); writel_relaxed(val, HPRE_ADDR(qm, HPRE_QM_ABNML_INT_MASK)); - writel(0x1, HPRE_ADDR(qm, HPRE_TYPES_ENB)); + if (qm->ver >= QM_HW_V3) + writel(HPRE_RSA_ENB | HPRE_ECC_ENB, + HPRE_ADDR(qm, HPRE_TYPES_ENB)); + else + writel(HPRE_RSA_ENB, HPRE_ADDR(qm, HPRE_TYPES_ENB)); + writel(HPRE_QM_VFG_AX_MASK, HPRE_ADDR(qm, HPRE_VFG_AXCACHE)); writel(0x0, HPRE_ADDR(qm, HPRE_BD_ENDIAN)); writel(0x0, HPRE_ADDR(qm, HPRE_INT_MASK)); - writel(0x0, HPRE_ADDR(qm, HPRE_RAS_ECC_1BIT_TH)); writel(0x0, HPRE_ADDR(qm, HPRE_POISON_BYPASS)); writel(0x0, HPRE_ADDR(qm, HPRE_COMM_CNT_CLR_CE)); writel(0x0, HPRE_ADDR(qm, HPRE_ECC_BYPASS)); @@ -291,37 +375,29 @@ static int hpre_set_user_domain_and_cache(struct hisi_qm *qm) return -ETIMEDOUT; } - for (i = 0; i < HPRE_CLUSTERS_NUM; i++) { - offset = i * HPRE_CLSTR_ADDR_INTRVL; + ret = hpre_set_cluster(qm); + if (ret) + return -ETIMEDOUT; - /* clusters initiating */ - writel(HPRE_CLUSTER_CORE_MASK, - HPRE_ADDR(qm, offset + HPRE_CORE_ENB)); - writel(0x1, HPRE_ADDR(qm, offset + HPRE_CORE_INI_CFG)); - ret = readl_relaxed_poll_timeout(HPRE_ADDR(qm, offset + - HPRE_CORE_INI_STATUS), val, - ((val & HPRE_CLUSTER_CORE_MASK) == - HPRE_CLUSTER_CORE_MASK), - HPRE_REG_RD_INTVRL_US, - HPRE_REG_RD_TMOUT_US); - if (ret) { - dev_err(dev, - "cluster %d int st status timeout!\n", i); - return -ETIMEDOUT; - } - } + /* This setting is only needed by Kunpeng 920. */ + if (qm->ver == QM_HW_V2) { + ret = hpre_cfg_by_dsm(qm); + if (ret) + dev_err(dev, "acpi_evaluate_dsm err.\n"); - ret = hpre_cfg_by_dsm(qm); - if (ret) - dev_err(dev, "acpi_evaluate_dsm err.\n"); + disable_flr_of_bme(qm); - disable_flr_of_bme(qm); + /* Enable data buffer pasid */ + if (qm->use_sva) + hpre_pasid_enable(qm); + } return ret; } static void hpre_cnt_regs_clear(struct hisi_qm *qm) { + u8 clusters_num = HPRE_CLUSTERS_NUM(qm); unsigned long offset; int i; @@ -330,7 +406,7 @@ static void hpre_cnt_regs_clear(struct hisi_qm *qm) writel(0x0, qm->io_base + QM_DFX_DB_CNT_VF); /* clear clusterX/cluster_ctrl */ - for (i = 0; i < HPRE_CLUSTERS_NUM; i++) { + for (i = 0; i < clusters_num; i++) { offset = HPRE_CLSTR_BASE + i * HPRE_CLSTR_ADDR_INTRVL; writel(0x0, qm->io_base + offset + HPRE_CLUSTER_INQURY); } @@ -629,13 +705,14 @@ static int hpre_pf_comm_regs_debugfs_init(struct hisi_qm *qm) static int hpre_cluster_debugfs_init(struct hisi_qm *qm) { + u8 clusters_num = HPRE_CLUSTERS_NUM(qm); struct device *dev = &qm->pdev->dev; char buf[HPRE_DBGFS_VAL_MAX_LEN]; struct debugfs_regset32 *regset; struct dentry *tmp_d; int i, ret; - for (i = 0; i < HPRE_CLUSTERS_NUM; i++) { + for (i = 0; i < clusters_num; i++) { ret = snprintf(buf, HPRE_DBGFS_VAL_MAX_LEN, "cluster%d", i); if (ret < 0) return -EINVAL; @@ -734,6 +811,11 @@ static int hpre_qm_init(struct hisi_qm *qm, struct pci_dev *pdev) return -EINVAL; } + if (pdev->revision >= QM_HW_V3) + qm->algs = "rsa\ndh\necdh\nx25519\nx448\necdsa\nsm2\n"; + else + qm->algs = "rsa\ndh\n"; + qm->mode = uacce_mode; qm->pdev = pdev; qm->ver = pdev->revision; qm->sqe_size = HPRE_SQE_SIZE; @@ -799,6 +881,7 @@ static const struct hisi_qm_err_ini hpre_err_ini = { .fe = 0, .ecc_2bits_mask = HPRE_CORE_ECC_2BIT_ERR | HPRE_OOO_ECC_2BIT_ERR, + .dev_ce_mask = HPRE_HAC_RAS_CE_ENABLE, .msi_wr_port = HPRE_WR_MSI_PORT, .acpi_rst = "HRST", } @@ -872,6 +955,14 @@ static int hpre_probe(struct pci_dev *pdev, const struct pci_device_id *id) goto err_with_qm_start; } + if (qm->uacce) { + ret = uacce_register(qm->uacce); + if (ret) { + pci_err(pdev, "failed to register uacce (%d)!\n", ret); + goto err_with_alg_register; + } + } + if (qm->fun_type == QM_HW_PF && vfs_num) { ret = hisi_qm_sriov_enable(pdev, vfs_num); if (ret < 0) @@ -904,20 +995,24 @@ static void hpre_remove(struct pci_dev *pdev) hisi_qm_wait_task_finish(qm, &hpre_devices); hisi_qm_alg_unregister(qm, &hpre_devices); if (qm->fun_type == QM_HW_PF && qm->vfs_num) { - ret = hisi_qm_sriov_disable(pdev, qm->is_frozen); + ret = hisi_qm_sriov_disable(pdev, true); if (ret) { pci_err(pdev, "Disable SRIOV fail!\n"); return; } } + + hpre_debugfs_exit(qm); + hisi_qm_stop(qm, QM_NORMAL); + if (qm->fun_type == QM_HW_PF) { + if (qm->use_sva && qm->ver == QM_HW_V2) + hpre_pasid_disable(qm); hpre_cnt_regs_clear(qm); qm->debug.curr_qm_qp_num = 0; + hisi_qm_dev_err_uninit(qm); } - hpre_debugfs_exit(qm); - hisi_qm_stop(qm, QM_NORMAL); - hisi_qm_dev_err_uninit(qm); hisi_qm_uninit(qm); } diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index f21ccae0e8ea8af257e7a2a68b02874d01040d3f..13cb4216561a08d8e671f58cc14ee113856fe3bd 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -54,6 +54,8 @@ #define QM_SQ_PRIORITY_SHIFT 0 #define QM_SQ_ORDERS_SHIFT 4 #define QM_SQ_TYPE_SHIFT 8 +#define QM_QC_PASID_ENABLE 0x1 +#define QM_QC_PASID_ENABLE_SHIFT 7 #define QM_SQ_TYPE_MASK GENMASK(3, 0) #define QM_SQ_TAIL_IDX(sqc) ((le16_to_cpu((sqc)->w11) >> 6) & 0x1) @@ -120,7 +122,7 @@ #define QM_CQC_VFT_VALID (1ULL << 28) #define QM_SQC_VFT_BASE_SHIFT_V2 28 -#define QM_SQC_VFT_BASE_MASK_V2 GENMASK(5, 0) +#define QM_SQC_VFT_BASE_MASK_V2 GENMASK(15, 0) #define QM_SQC_VFT_NUM_SHIFT_V2 45 #define QM_SQC_VFT_NUM_MASK_v2 GENMASK(9, 0) @@ -147,7 +149,6 @@ #define QM_RAS_CE_TIMES_PER_IRQ 1 #define QM_RAS_MSI_INT_SEL 0x1040f4 -#define QM_DEV_RESET_FLAG 0 #define QM_RESET_WAIT_TIMEOUT 400 #define QM_PEH_VENDOR_ID 0x1000d8 #define ACC_VENDOR_ID_VALUE 0x5a5a @@ -185,6 +186,10 @@ #define QM_SQE_ADDR_MASK GENMASK(7, 0) #define QM_EQ_DEPTH (1024 * 2) +#define QM_DRIVER_REMOVING 0 +#define QM_RST_SCHED 1 +#define QM_RESETTING 2 + #define QM_MK_CQC_DW3_V1(hop_num, pg_sz, buf_sz, cqe_sz) \ (((hop_num) << QM_CQ_HOP_NUM_SHIFT) | \ ((pg_sz) << QM_CQ_PAGE_SIZE_SHIFT) | \ @@ -619,6 +624,9 @@ static void qm_cq_head_update(struct hisi_qp *qp) static void qm_poll_qp(struct hisi_qp *qp, struct hisi_qm *qm) { + if (unlikely(atomic_read(&qp->qp_status.flags) == QP_STOP)) + return; + if (qp->event_cb) { qp->event_cb(qp); return; @@ -717,7 +725,7 @@ static irqreturn_t qm_aeq_irq(int irq, void *data) dev_err(&qm->pdev->dev, "%s overflow\n", qm_fifo_overflow[type]); else - dev_err(&qm->pdev->dev, "unknown error type %d\n", + dev_err(&qm->pdev->dev, "unknown error type %u\n", type); if (qm->status.aeq_head == QM_Q_DEPTH - 1) { @@ -1121,7 +1129,7 @@ static int dump_show(struct hisi_qm *qm, void *info, dev_info(dev, "%s DUMP\n", info_name); for (i = 0; i < info_size; i += BYTE_PER_DW) { - pr_info("DW%d: %02X%02X %02X%02X\n", i / BYTE_PER_DW, + pr_info("DW%u: %02X%02X %02X%02X\n", i / BYTE_PER_DW, info_buf[i], info_buf[i + 1UL], info_buf[i + 2UL], info_buf[i + 3UL]); } @@ -1154,7 +1162,7 @@ static int qm_sqc_dump(struct hisi_qm *qm, const char *s) ret = kstrtou32(s, 0, &qp_id); if (ret || qp_id >= qm->qp_num) { - dev_err(dev, "Please input qp num (0-%d)", qm->qp_num - 1); + dev_err(dev, "Please input qp num (0-%u)", qm->qp_num - 1); return -EINVAL; } @@ -1200,7 +1208,7 @@ static int qm_cqc_dump(struct hisi_qm *qm, const char *s) ret = kstrtou32(s, 0, &qp_id); if (ret || qp_id >= qm->qp_num) { - dev_err(dev, "Please input qp num (0-%d)", qm->qp_num - 1); + dev_err(dev, "Please input qp num (0-%u)", qm->qp_num - 1); return -EINVAL; } @@ -1279,7 +1287,7 @@ static int q_dump_param_parse(struct hisi_qm *qm, char *s, ret = kstrtou32(presult, 0, q_id); if (ret || *q_id >= qp_num) { - dev_err(dev, "Please input qp num (0-%d)", qp_num - 1); + dev_err(dev, "Please input qp num (0-%u)", qp_num - 1); return -EINVAL; } @@ -1604,7 +1612,7 @@ static void qm_log_hw_error(struct hisi_qm *qm, u32 error_status) static enum acc_err_result qm_hw_error_handle_v2(struct hisi_qm *qm) { - u32 error_status, tmp; + u32 error_status, tmp, val; /* read err sts */ tmp = readl(qm->io_base + QM_ABNORMAL_INT_STATUS); @@ -1615,9 +1623,13 @@ static enum acc_err_result qm_hw_error_handle_v2(struct hisi_qm *qm) qm->err_status.is_qm_ecc_mbit = true; qm_log_hw_error(qm, error_status); - if (error_status == QM_DB_RANDOM_INVALID) { + val = error_status | QM_DB_RANDOM_INVALID | QM_BASE_CE; + /* ce error does not need to be reset */ + if (val == (QM_DB_RANDOM_INVALID | QM_BASE_CE)) { writel(error_status, qm->io_base + QM_ABNORMAL_INT_SOURCE); + writel(qm->err_ini->err_info.nfe, + qm->io_base + QM_RAS_NFE_ENABLE); return ACC_ERR_RECOVERED; } @@ -1685,6 +1697,7 @@ static struct hisi_qp *qm_create_qp_nolock(struct hisi_qm *qm, u8 alg_type) qp->req_cb = NULL; qp->qp_id = qp_id; qp->alg_type = alg_type; + qp->is_in_kernel = true; qm->qp_in_used++; atomic_set(&qp->qp_status.flags, QP_INIT); @@ -1747,12 +1760,6 @@ static int qm_sq_ctx_cfg(struct hisi_qp *qp, int qp_id, u32 pasid) sqc = kzalloc(sizeof(struct qm_sqc), GFP_KERNEL); if (!sqc) return -ENOMEM; - sqc_dma = dma_map_single(dev, sqc, sizeof(struct qm_sqc), - DMA_TO_DEVICE); - if (dma_mapping_error(dev, sqc_dma)) { - kfree(sqc); - return -ENOMEM; - } INIT_QC_COMMON(sqc, qp->sqe_dma, pasid); if (ver == QM_HW_V1) { @@ -1765,6 +1772,17 @@ static int qm_sq_ctx_cfg(struct hisi_qp *qp, int qp_id, u32 pasid) sqc->cq_num = cpu_to_le16(qp_id); sqc->w13 = cpu_to_le16(QM_MK_SQC_W13(0, 1, qp->alg_type)); + if (ver >= QM_HW_V3 && qm->use_sva && !qp->is_in_kernel) + sqc->w11 = cpu_to_le16(QM_QC_PASID_ENABLE << + QM_QC_PASID_ENABLE_SHIFT); + + sqc_dma = dma_map_single(dev, sqc, sizeof(struct qm_sqc), + DMA_TO_DEVICE); + if (dma_mapping_error(dev, sqc_dma)) { + kfree(sqc); + return -ENOMEM; + } + ret = qm_mb(qm, QM_MB_CMD_SQC, sqc_dma, qp_id, 0); dma_unmap_single(dev, sqc_dma, sizeof(struct qm_sqc), DMA_TO_DEVICE); kfree(sqc); @@ -1784,12 +1802,6 @@ static int qm_cq_ctx_cfg(struct hisi_qp *qp, int qp_id, u32 pasid) cqc = kzalloc(sizeof(struct qm_cqc), GFP_KERNEL); if (!cqc) return -ENOMEM; - cqc_dma = dma_map_single(dev, cqc, sizeof(struct qm_cqc), - DMA_TO_DEVICE); - if (dma_mapping_error(dev, cqc_dma)) { - kfree(cqc); - return -ENOMEM; - } INIT_QC_COMMON(cqc, qp->cqe_dma, pasid); if (ver == QM_HW_V1) { @@ -1802,6 +1814,16 @@ static int qm_cq_ctx_cfg(struct hisi_qp *qp, int qp_id, u32 pasid) } cqc->dw6 = cpu_to_le32(1 << QM_CQ_PHASE_SHIFT | 1 << QM_CQ_FLAG_SHIFT); + if (ver >= QM_HW_V3 && qm->use_sva && !qp->is_in_kernel) + cqc->w11 = cpu_to_le16(QM_QC_PASID_ENABLE); + + cqc_dma = dma_map_single(dev, cqc, sizeof(struct qm_cqc), + DMA_TO_DEVICE); + if (dma_mapping_error(dev, cqc_dma)) { + kfree(cqc); + return -ENOMEM; + } + ret = qm_mb(qm, QM_MB_CMD_CQC, cqc_dma, qp_id, 0); dma_unmap_single(dev, cqc_dma, sizeof(struct qm_cqc), DMA_TO_DEVICE); kfree(cqc); @@ -1864,6 +1886,28 @@ int hisi_qm_start_qp(struct hisi_qp *qp, unsigned long arg) } EXPORT_SYMBOL_GPL(hisi_qm_start_qp); +/** + * qp_stop_fail_cb() - call request cb. + * @qp: stopped failed qp. + * + * Callback function should be called whether task completed or not. + */ +static void qp_stop_fail_cb(struct hisi_qp *qp) +{ + int qp_used = atomic_read(&qp->qp_status.used); + u16 cur_tail = qp->qp_status.sq_tail; + u16 cur_head = (cur_tail + QM_Q_DEPTH - qp_used) % QM_Q_DEPTH; + struct hisi_qm *qm = qp->qm; + u16 pos; + int i; + + for (i = 0; i < qp_used; i++) { + pos = (i + cur_head) % QM_Q_DEPTH; + qp->req_cb(qp, qp->sqe + (u32)(qm->sqe_size * pos)); + atomic_dec(&qp->qp_status.used); + } +} + /** * qm_drain_qp() - Drain a qp. * @qp: The qp we want to drain. @@ -1959,6 +2003,9 @@ static int qm_stop_qp_nolock(struct hisi_qp *qp) else flush_work(&qp->qm->work); + if (unlikely(qp->is_resetting && atomic_read(&qp->qp_status.used))) + qp_stop_fail_cb(qp); + dev_dbg(dev, "stop queue %u!", qp->qp_id); return 0; @@ -2065,6 +2112,7 @@ static int hisi_qm_uacce_get_queue(struct uacce_device *uacce, qp->uacce_q = q; qp->event_cb = qm_qp_event_notifier; qp->pasid = arg; + qp->is_in_kernel = false; return 0; } @@ -2206,7 +2254,7 @@ static int qm_alloc_uacce(struct hisi_qm *qm) if (IS_ERR(uacce)) return PTR_ERR(uacce); - if (uacce->flags & UACCE_DEV_SVA) { + if (uacce->flags & UACCE_DEV_SVA && qm->mode == UACCE_MODE_SVA) { qm->use_sva = true; } else { /* only consider sva case */ @@ -2248,17 +2296,15 @@ static int qm_alloc_uacce(struct hisi_qm *qm) */ static int qm_frozen(struct hisi_qm *qm) { - down_write(&qm->qps_lock); - - if (qm->is_frozen) { - up_write(&qm->qps_lock); + if (test_bit(QM_DRIVER_REMOVING, &qm->misc_ctl)) return 0; - } + + down_write(&qm->qps_lock); if (!qm->qp_in_used) { qm->qp_in_used = qm->qp_num; - qm->is_frozen = true; up_write(&qm->qps_lock); + set_bit(QM_DRIVER_REMOVING, &qm->misc_ctl); return 0; } @@ -2311,6 +2357,10 @@ void hisi_qm_wait_task_finish(struct hisi_qm *qm, struct hisi_qm_list *qm_list) msleep(WAIT_PERIOD); } + while (test_bit(QM_RST_SCHED, &qm->misc_ctl) || + test_bit(QM_RESETTING, &qm->misc_ctl)) + msleep(WAIT_PERIOD); + udelay(REMOVE_WAIT_DELAY); } EXPORT_SYMBOL_GPL(hisi_qm_wait_task_finish); @@ -2439,7 +2489,7 @@ static void hisi_qm_pre_init(struct hisi_qm *qm) mutex_init(&qm->mailbox_lock); init_rwsem(&qm->qps_lock); qm->qp_in_used = 0; - qm->is_frozen = false; + qm->misc_ctl = false; } static void hisi_qm_pci_uninit(struct hisi_qm *qm) @@ -2558,15 +2608,9 @@ static int qm_eq_ctx_cfg(struct hisi_qm *qm) dma_addr_t eqc_dma; int ret; - eqc = kzalloc(sizeof(struct qm_eqc), GFP_KERNEL); //todo + eqc = kzalloc(sizeof(struct qm_eqc), GFP_KERNEL); if (!eqc) return -ENOMEM; - eqc_dma = dma_map_single(dev, eqc, sizeof(struct qm_eqc), - DMA_TO_DEVICE); - if (dma_mapping_error(dev, eqc_dma)) { - kfree(eqc); - return -ENOMEM; - } eqc->base_l = cpu_to_le32(lower_32_bits(qm->eqe_dma)); eqc->base_h = cpu_to_le32(upper_32_bits(qm->eqe_dma)); @@ -2574,6 +2618,13 @@ static int qm_eq_ctx_cfg(struct hisi_qm *qm) eqc->dw3 = cpu_to_le32(QM_EQE_AEQE_SIZE); eqc->dw6 = cpu_to_le32((QM_EQ_DEPTH - 1) | (1 << QM_EQC_PHASE_SHIFT)); + eqc_dma = dma_map_single(dev, eqc, sizeof(struct qm_eqc), + DMA_TO_DEVICE); + if (dma_mapping_error(dev, eqc_dma)) { + kfree(eqc); + return -ENOMEM; + } + ret = qm_mb(qm, QM_MB_CMD_EQC, eqc_dma, 0, 0); dma_unmap_single(dev, eqc_dma, sizeof(struct qm_eqc), DMA_TO_DEVICE); kfree(eqc); @@ -2591,6 +2642,11 @@ static int qm_aeq_ctx_cfg(struct hisi_qm *qm) aeqc = kzalloc(sizeof(struct qm_aeqc), GFP_KERNEL); if (!aeqc) return -ENOMEM; + + aeqc->base_l = cpu_to_le32(lower_32_bits(qm->aeqe_dma)); + aeqc->base_h = cpu_to_le32(upper_32_bits(qm->aeqe_dma)); + aeqc->dw6 = cpu_to_le32((QM_Q_DEPTH - 1) | (1 << QM_EQC_PHASE_SHIFT)); + aeqc_dma = dma_map_single(dev, aeqc, sizeof(struct qm_aeqc), DMA_TO_DEVICE); if (dma_mapping_error(dev, aeqc_dma)) { @@ -2598,10 +2654,6 @@ static int qm_aeq_ctx_cfg(struct hisi_qm *qm) return -ENOMEM; } - aeqc->base_l = cpu_to_le32(lower_32_bits(qm->aeqe_dma)); - aeqc->base_h = cpu_to_le32(upper_32_bits(qm->aeqe_dma)); - aeqc->dw6 = cpu_to_le32((QM_Q_DEPTH - 1) | (1 << QM_EQC_PHASE_SHIFT)); - ret = qm_mb(qm, QM_MB_CMD_AEQC, aeqc_dma, 0, 0); dma_unmap_single(dev, aeqc_dma, sizeof(struct qm_aeqc), DMA_TO_DEVICE); kfree(aeqc); @@ -2677,7 +2729,7 @@ int hisi_qm_start(struct hisi_qm *qm) return -EPERM; } - dev_dbg(dev, "qm start with %d queue pairs\n", qm->qp_num); + dev_dbg(dev, "qm start with %u queue pairs\n", qm->qp_num); if (!qm->qp_num) { dev_err(dev, "qp_num should not be 0\n"); @@ -3112,7 +3164,7 @@ int hisi_qm_alloc_qps_node(struct hisi_qm_list *qm_list, int qp_num, mutex_unlock(&qm_list->lock); if (ret) - pr_info("Failed to create qps, node[%d], alg[%d], qp[%d]!\n", + pr_info("Failed to create qps, node[%d], alg[%u], qp[%d]!\n", node, alg_type, qp_num); err: @@ -3248,7 +3300,7 @@ EXPORT_SYMBOL_GPL(hisi_qm_sriov_disable); int hisi_qm_sriov_configure(struct pci_dev *pdev, int num_vfs) { if (num_vfs == 0) - return hisi_qm_sriov_disable(pdev, 0); + return hisi_qm_sriov_disable(pdev, false); else return hisi_qm_sriov_enable(pdev, num_vfs); } @@ -3269,12 +3321,19 @@ static enum acc_err_result qm_dev_err_handle(struct hisi_qm *qm) if (err_sts & qm->err_ini->err_info.ecc_2bits_mask) qm->err_status.is_dev_ecc_mbit = true; - if (!qm->err_ini->log_dev_hw_err) { - dev_err(&qm->pdev->dev, "Device doesn't support log hw error!\n"); - return ACC_ERR_NEED_RESET; + if (qm->err_ini->log_dev_hw_err) + qm->err_ini->log_dev_hw_err(qm, err_sts); + + /* ce error does not need to be reset */ + if ((err_sts | qm->err_ini->err_info.dev_ce_mask) == + qm->err_ini->err_info.dev_ce_mask) { + if (qm->err_ini->clear_dev_hw_err_status) + qm->err_ini->clear_dev_hw_err_status(qm, + err_sts); + + return ACC_ERR_RECOVERED; } - qm->err_ini->log_dev_hw_err(qm, err_sts); return ACC_ERR_NEED_RESET; } @@ -3313,7 +3372,7 @@ pci_ers_result_t hisi_qm_dev_err_detected(struct pci_dev *pdev, if (pdev->is_virtfn) return PCI_ERS_RESULT_NONE; - pci_info(pdev, "PCI error detected, state(=%d)!!\n", state); + pci_info(pdev, "PCI error detected, state(=%u)!!\n", state); if (state == pci_channel_io_perm_failure) return PCI_ERS_RESULT_DISCONNECT; @@ -3465,7 +3524,7 @@ static int qm_reset_prepare_ready(struct hisi_qm *qm) int delay = 0; /* All reset requests need to be queued for processing */ - while (test_and_set_bit(QM_DEV_RESET_FLAG, &pf_qm->reset_flag)) { + while (test_and_set_bit(QM_RESETTING, &pf_qm->misc_ctl)) { msleep(++delay); if (delay > QM_RESET_WAIT_TIMEOUT) return -EBUSY; @@ -3489,6 +3548,7 @@ static int qm_controller_reset_prepare(struct hisi_qm *qm) ret = qm_vf_reset_prepare(qm, QM_SOFT_RESET); if (ret) { pci_err(pdev, "Fails to stop VFs!\n"); + clear_bit(QM_RESETTING, &qm->misc_ctl); return ret; } } @@ -3496,9 +3556,12 @@ static int qm_controller_reset_prepare(struct hisi_qm *qm) ret = hisi_qm_stop(qm, QM_SOFT_RESET); if (ret) { pci_err(pdev, "Fails to stop QM!\n"); + clear_bit(QM_RESETTING, &qm->misc_ctl); return ret; } + clear_bit(QM_RST_SCHED, &qm->misc_ctl); + return 0; } @@ -3736,7 +3799,7 @@ static int qm_controller_reset_done(struct hisi_qm *qm) hisi_qm_dev_err_init(qm); qm_restart_done(qm); - clear_bit(QM_DEV_RESET_FLAG, &qm->reset_flag); + clear_bit(QM_RESETTING, &qm->misc_ctl); return 0; } @@ -3749,18 +3812,23 @@ static int qm_controller_reset(struct hisi_qm *qm) pci_info(pdev, "Controller resetting...\n"); ret = qm_controller_reset_prepare(qm); - if (ret) + if (ret) { + clear_bit(QM_RST_SCHED, &qm->misc_ctl); return ret; + } ret = qm_soft_reset(qm); if (ret) { pci_err(pdev, "Controller reset failed (%d)\n", ret); + clear_bit(QM_RESETTING, &qm->misc_ctl); return ret; } ret = qm_controller_reset_done(qm); - if (ret) + if (ret) { + clear_bit(QM_RESETTING, &qm->misc_ctl); return ret; + } pci_info(pdev, "Controller reset complete\n"); @@ -3867,8 +3935,6 @@ static bool qm_flr_reset_complete(struct pci_dev *pdev) return false; } - clear_bit(QM_DEV_RESET_FLAG, &qm->reset_flag); - return true; } @@ -3912,6 +3978,8 @@ void hisi_qm_reset_done(struct pci_dev *pdev) flr_done: if (qm_flr_reset_complete(pdev)) pci_info(pdev, "FLR reset complete\n"); + + clear_bit(QM_RESETTING, &qm->misc_ctl); } EXPORT_SYMBOL_GPL(hisi_qm_reset_done); @@ -3922,7 +3990,9 @@ static irqreturn_t qm_abnormal_irq(int irq, void *data) atomic64_inc(&qm->debug.dfx.abnormal_irq_cnt); ret = qm_process_dev_error(qm); - if (ret == ACC_ERR_NEED_RESET) + if (ret == ACC_ERR_NEED_RESET && + !test_bit(QM_DRIVER_REMOVING, &qm->misc_ctl) && + !test_and_set_bit(QM_RST_SCHED, &qm->misc_ctl)) schedule_work(&qm->rst_work); return IRQ_HANDLED; @@ -3934,21 +4004,20 @@ static int qm_irq_register(struct hisi_qm *qm) int ret; ret = request_irq(pci_irq_vector(pdev, QM_EQ_EVENT_IRQ_VECTOR), - qm_irq, IRQF_SHARED, qm->dev_name, qm); + qm_irq, 0, qm->dev_name, qm); if (ret) return ret; if (qm->ver != QM_HW_V1) { ret = request_irq(pci_irq_vector(pdev, QM_AEQ_EVENT_IRQ_VECTOR), - qm_aeq_irq, IRQF_SHARED, qm->dev_name, qm); + qm_aeq_irq, 0, qm->dev_name, qm); if (ret) goto err_aeq_irq; if (qm->fun_type == QM_HW_PF) { ret = request_irq(pci_irq_vector(pdev, QM_ABNORMAL_EVENT_IRQ_VECTOR), - qm_abnormal_irq, IRQF_SHARED, - qm->dev_name, qm); + qm_abnormal_irq, 0, qm->dev_name, qm); if (ret) goto err_abonormal_irq; } @@ -4004,6 +4073,9 @@ int hisi_qm_alg_register(struct hisi_qm *qm, struct hisi_qm_list *qm_list) { int flag = 0; int ret = 0; + /* HW V2 not support both use uacce sva mode and hardware crypto algs */ + if (qm->ver <= QM_HW_V2 && qm->use_sva) + return 0; mutex_lock(&qm_list->lock); if (list_empty(&qm_list->list)) @@ -4035,6 +4107,9 @@ EXPORT_SYMBOL_GPL(hisi_qm_alg_register); */ void hisi_qm_alg_unregister(struct hisi_qm *qm, struct hisi_qm_list *qm_list) { + if (qm->ver <= QM_HW_V2 && qm->use_sva) + return; + mutex_lock(&qm_list->lock); list_del(&qm->list); mutex_unlock(&qm_list->lock); diff --git a/drivers/crypto/hisilicon/qm.h b/drivers/crypto/hisilicon/qm.h index 8624d1288afe58b2d74198810a9009a32caed1bb..54967c6b9c78874ff1ed18c9131892b812c8f0b8 100644 --- a/drivers/crypto/hisilicon/qm.h +++ b/drivers/crypto/hisilicon/qm.h @@ -85,6 +85,11 @@ /* page number for queue file region */ #define QM_DOORBELL_PAGE_NR 1 +/* uacce mode of the driver */ +#define UACCE_MODE_NOUACCE 0 /* don't use uacce */ +#define UACCE_MODE_SVA 1 /* use uacce sva mode */ +#define UACCE_MODE_DESC "0(default) means only register to crypto, 1 means both register to crypto and uacce" + enum qm_stop_reason { QM_NORMAL, QM_SOFT_RESET, @@ -168,6 +173,7 @@ struct hisi_qm_err_info { char *acpi_rst; u32 msi_wr_port; u32 ecc_2bits_mask; + u32 dev_ce_mask; u32 ce; u32 nfe; u32 fe; @@ -225,7 +231,7 @@ struct hisi_qm { struct hisi_qm_status status; const struct hisi_qm_err_ini *err_ini; struct hisi_qm_err_status err_status; - unsigned long reset_flag; + unsigned long misc_ctl; /* driver removing and reset sched */ struct rw_semaphore qps_lock; struct idr qp_idr; @@ -249,6 +255,7 @@ struct hisi_qm { resource_size_t phys_base; resource_size_t phys_size; struct uacce_device *uacce; + int mode; }; struct hisi_qp_status { @@ -282,6 +289,7 @@ struct hisi_qp { struct hisi_qm *qm; bool is_resetting; + bool is_in_kernel; u16 pasid; struct uacce_queue *uacce_q; }; @@ -299,7 +307,7 @@ static inline int q_num_set(const char *val, const struct kernel_param *kp, if (!pdev) { q_num = min_t(u32, QM_QNUM_V1, QM_QNUM_V2); - pr_info("No device found currently, suppose queue number is %d\n", + pr_info("No device found currently, suppose queue number is %u\n", q_num); } else { if (pdev->revision == QM_HW_V1) @@ -333,6 +341,27 @@ static inline int vfs_num_set(const char *val, const struct kernel_param *kp) return param_set_int(val, kp); } +static inline int mode_set(const char *val, const struct kernel_param *kp) +{ + u32 n; + int ret; + + if (!val) + return -EINVAL; + + ret = kstrtou32(val, 10, &n); + if (ret != 0 || (n != UACCE_MODE_SVA && + n != UACCE_MODE_NOUACCE)) + return -EINVAL; + + return param_set_int(val, kp); +} + +static inline int uacce_mode_set(const char *val, const struct kernel_param *kp) +{ + return mode_set(val, kp); +} + static inline void hisi_qm_init_list(struct hisi_qm_list *qm_list) { INIT_LIST_HEAD(&qm_list->list); diff --git a/drivers/crypto/hisilicon/sec2/sec_main.c b/drivers/crypto/hisilicon/sec2/sec_main.c index b35c1c2271a3b5fb65b486190d6d143cc34dd817..dc68ba76f65e585bddfb8d4301fcf7f3bd54e165 100644 --- a/drivers/crypto/hisilicon/sec2/sec_main.c +++ b/drivers/crypto/hisilicon/sec2/sec_main.c @@ -13,6 +13,7 @@ #include #include #include +#include #include "sec.h" @@ -74,6 +75,16 @@ #define SEC_USER0_SMMU_NORMAL (BIT(23) | BIT(15)) #define SEC_USER1_SMMU_NORMAL (BIT(31) | BIT(23) | BIT(15) | BIT(7)) +#define SEC_USER1_ENABLE_CONTEXT_SSV BIT(24) +#define SEC_USER1_ENABLE_DATA_SSV BIT(16) +#define SEC_USER1_WB_CONTEXT_SSV BIT(8) +#define SEC_USER1_WB_DATA_SSV BIT(0) +#define SEC_USER1_SVA_SET (SEC_USER1_ENABLE_CONTEXT_SSV | \ + SEC_USER1_ENABLE_DATA_SSV | \ + SEC_USER1_WB_CONTEXT_SSV | \ + SEC_USER1_WB_DATA_SSV) +#define SEC_USER1_SMMU_SVA (SEC_USER1_SMMU_NORMAL | SEC_USER1_SVA_SET) +#define SEC_USER1_SMMU_MASK (~SEC_USER1_SVA_SET) #define SEC_CORE_INT_STATUS_M_ECC BIT(2) #define SEC_DELAY_10_US 10 @@ -233,6 +244,18 @@ struct hisi_qp **sec_create_qps(void) return NULL; } +static const struct kernel_param_ops sec_uacce_mode_ops = { + .set = uacce_mode_set, + .get = param_get_int, +}; + +/* + * uacce_mode = 0 means sec only register to crypto, + * uacce_mode = 1 means sec both register to crypto and uacce. + */ +static u32 uacce_mode = UACCE_MODE_NOUACCE; +module_param_cb(uacce_mode, &sec_uacce_mode_ops, &uacce_mode, 0444); +MODULE_PARM_DESC(uacce_mode, UACCE_MODE_DESC); static const struct pci_device_id sec_dev_ids[] = { { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, SEC_PF_PCI_DEVICE_ID) }, @@ -299,7 +322,11 @@ static int sec_engine_init(struct hisi_qm *qm) writel_relaxed(reg, SEC_ADDR(qm, SEC_INTERFACE_USER_CTRL0_REG)); reg = readl_relaxed(SEC_ADDR(qm, SEC_INTERFACE_USER_CTRL1_REG)); - reg |= SEC_USER1_SMMU_NORMAL; + reg &= SEC_USER1_SMMU_MASK; + if (qm->use_sva && qm->ver == QM_HW_V2) + reg |= SEC_USER1_SMMU_SVA; + else + reg |= SEC_USER1_SMMU_NORMAL; writel_relaxed(reg, SEC_ADDR(qm, SEC_INTERFACE_USER_CTRL1_REG)); writel(SEC_SINGLE_PORT_MAX_TRANS, @@ -725,6 +752,7 @@ static const struct hisi_qm_err_ini sec_err_ini = { QM_ACC_WB_NOT_READY_TIMEOUT, .fe = 0, .ecc_2bits_mask = SEC_CORE_INT_STATUS_M_ECC, + .dev_ce_mask = SEC_RAS_CE_ENB_MSK, .msi_wr_port = BIT(0), .acpi_rst = "SRST", } @@ -758,6 +786,8 @@ static int sec_qm_init(struct hisi_qm *qm, struct pci_dev *pdev) qm->pdev = pdev; qm->ver = pdev->revision; + qm->algs = "cipher\ndigest\naead\n"; + qm->mode = uacce_mode; qm->sqe_size = SEC_SQE_SIZE; qm->dev_name = sec_name; @@ -885,6 +915,14 @@ static int sec_probe(struct pci_dev *pdev, const struct pci_device_id *id) goto err_qm_stop; } + if (qm->uacce) { + ret = uacce_register(qm->uacce); + if (ret) { + pci_err(pdev, "failed to register uacce (%d)!\n", ret); + goto err_alg_unregister; + } + } + if (qm->fun_type == QM_HW_PF && vfs_num) { ret = hisi_qm_sriov_enable(pdev, vfs_num); if (ret < 0) @@ -912,7 +950,7 @@ static void sec_remove(struct pci_dev *pdev) hisi_qm_wait_task_finish(qm, &sec_devices); hisi_qm_alg_unregister(qm, &sec_devices); if (qm->fun_type == QM_HW_PF && qm->vfs_num) - hisi_qm_sriov_disable(pdev, qm->is_frozen); + hisi_qm_sriov_disable(pdev, true); sec_debugfs_exit(qm); diff --git a/drivers/crypto/hisilicon/zip/zip_main.c b/drivers/crypto/hisilicon/zip/zip_main.c index 4fb5a32bf830f9d387b5ce32ce98f4b0895bd8ba..02c445722445d6d4ae2e9c9dbbec2bd98630a3b3 100644 --- a/drivers/crypto/hisilicon/zip/zip_main.c +++ b/drivers/crypto/hisilicon/zip/zip_main.c @@ -66,6 +66,7 @@ #define HZIP_CORE_INT_STATUS_M_ECC BIT(1) #define HZIP_CORE_SRAM_ECC_ERR_INFO 0x301148 #define HZIP_CORE_INT_RAS_CE_ENB 0x301160 +#define HZIP_CORE_INT_RAS_CE_ENABLE 0x1 #define HZIP_CORE_INT_RAS_NFE_ENB 0x301164 #define HZIP_CORE_INT_RAS_FE_ENB 0x301168 #define HZIP_CORE_INT_RAS_NFE_ENABLE 0x7FE @@ -211,6 +212,19 @@ static const struct debugfs_reg32 hzip_dfx_regs[] = { {"HZIP_DECOMP_LZ77_CURR_ST ", 0x9cull}, }; +static const struct kernel_param_ops zip_uacce_mode_ops = { + .set = uacce_mode_set, + .get = param_get_int, +}; + +/* + * uacce_mode = 0 means zip only register to crypto, + * uacce_mode = 1 means zip both register to crypto and uacce. + */ +static u32 uacce_mode = UACCE_MODE_NOUACCE; +module_param_cb(uacce_mode, &zip_uacce_mode_ops, &uacce_mode, 0444); +MODULE_PARM_DESC(uacce_mode, UACCE_MODE_DESC); + static int pf_q_num_set(const char *val, const struct kernel_param *kp) { return q_num_set(val, kp, PCI_DEVICE_ID_ZIP_PF); @@ -279,7 +293,7 @@ static int hisi_zip_set_user_domain_and_cache(struct hisi_qm *qm) writel(AXUSER_BASE, base + HZIP_SGL_RUSER_32_63); writel(AXUSER_BASE, base + HZIP_BD_WUSER_32_63); - if (qm->use_sva) { + if (qm->use_sva && qm->ver == QM_HW_V2) { writel(AXUSER_BASE | AXUSER_SSV, base + HZIP_DATA_RUSER_32_63); writel(AXUSER_BASE | AXUSER_SSV, base + HZIP_DATA_WUSER_32_63); } else { @@ -314,7 +328,8 @@ static void hisi_zip_hw_error_enable(struct hisi_qm *qm) writel(HZIP_CORE_INT_MASK_ALL, qm->io_base + HZIP_CORE_INT_SOURCE); /* configure error type */ - writel(0x1, qm->io_base + HZIP_CORE_INT_RAS_CE_ENB); + writel(HZIP_CORE_INT_RAS_CE_ENABLE, + qm->io_base + HZIP_CORE_INT_RAS_CE_ENB); writel(0x0, qm->io_base + HZIP_CORE_INT_RAS_FE_ENB); writel(HZIP_CORE_INT_RAS_NFE_ENABLE, qm->io_base + HZIP_CORE_INT_RAS_NFE_ENB); @@ -714,6 +729,7 @@ static const struct hisi_qm_err_ini hisi_zip_err_ini = { QM_ACC_WB_NOT_READY_TIMEOUT, .fe = 0, .ecc_2bits_mask = HZIP_CORE_INT_STATUS_M_ECC, + .dev_ce_mask = HZIP_CORE_INT_RAS_CE_ENABLE, .msi_wr_port = HZIP_WR_PORT, .acpi_rst = "ZRST", } @@ -752,6 +768,7 @@ static int hisi_zip_qm_init(struct hisi_qm *qm, struct pci_dev *pdev) qm->pdev = pdev; qm->ver = pdev->revision; qm->algs = "zlib\ngzip"; + qm->mode = uacce_mode; qm->sqe_size = HZIP_SQE_SIZE; qm->dev_name = hisi_zip_name; @@ -887,7 +904,7 @@ static void hisi_zip_remove(struct pci_dev *pdev) hisi_qm_alg_unregister(qm, &zip_devices); if (qm->fun_type == QM_HW_PF && qm->vfs_num) - hisi_qm_sriov_disable(pdev, qm->is_frozen); + hisi_qm_sriov_disable(pdev, true); hisi_zip_debugfs_exit(qm); hisi_qm_stop(qm, QM_NORMAL); diff --git a/drivers/crypto/inside-secure/safexcel.c b/drivers/crypto/inside-secure/safexcel.c index 2e1562108a858a9a0451518e558cf43bdfe123be..6364583b88b216fdf5c0e2e8fc3f632061413f07 100644 --- a/drivers/crypto/inside-secure/safexcel.c +++ b/drivers/crypto/inside-secure/safexcel.c @@ -1166,11 +1166,8 @@ static int safexcel_request_ring_irq(void *pdev, int irqid, dev = &plf_pdev->dev; irq = platform_get_irq_byname(plf_pdev, irq_name); - if (irq < 0) { - dev_err(dev, "unable to get IRQ '%s' (err %d)\n", - irq_name, irq); + if (irq < 0) return irq; - } } else { return -ENXIO; } @@ -1999,3 +1996,4 @@ MODULE_AUTHOR("Ofer Heifetz "); MODULE_AUTHOR("Igal Liberman "); MODULE_DESCRIPTION("Support for SafeXcel cryptographic engines: EIP97 & EIP197"); MODULE_LICENSE("GPL v2"); +MODULE_IMPORT_NS(CRYPTO_INTERNAL); diff --git a/drivers/crypto/inside-secure/safexcel_hash.c b/drivers/crypto/inside-secure/safexcel_hash.c index 50fb6d90a2e0a36ca53fa73d9f28f4851cb6feef..bc60b5802256453399c33ebacb5a5b325d915955 100644 --- a/drivers/crypto/inside-secure/safexcel_hash.c +++ b/drivers/crypto/inside-secure/safexcel_hash.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include diff --git a/drivers/crypto/keembay/Kconfig b/drivers/crypto/keembay/Kconfig index f2e17b0c4fa05dc70aade364ba5dd2e9ce6b90d4..00cf8f028cb9f408d5c0f15177335ee0fe8ffc7a 100644 --- a/drivers/crypto/keembay/Kconfig +++ b/drivers/crypto/keembay/Kconfig @@ -38,3 +38,34 @@ config CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS Provides OCS version of cts(cbc(aes)) and cts(cbc(sm4)). Intel does not recommend use of CTS mode with AES/SM4. + +config CRYPTO_DEV_KEEMBAY_OCS_HCU + tristate "Support for Intel Keem Bay OCS HCU HW acceleration" + select CRYPTO_HASH + select CRYPTO_ENGINE + depends on HAS_IOMEM + depends on ARCH_KEEMBAY || COMPILE_TEST + depends on OF || COMPILE_TEST + help + Support for Intel Keem Bay Offload and Crypto Subsystem (OCS) Hash + Control Unit (HCU) hardware acceleration for use with Crypto API. + + Provides OCS HCU hardware acceleration of sha256, sha384, sha512, and + sm3, as well as the HMAC variant of these algorithms. + + Say Y or M if you're building for the Intel Keem Bay SoC. If compiled + as a module, the module will be called keembay-ocs-hcu. + + If unsure, say N. + +config CRYPTO_DEV_KEEMBAY_OCS_HCU_HMAC_SHA224 + bool "Enable sha224 and hmac(sha224) support in Intel Keem Bay OCS HCU" + depends on CRYPTO_DEV_KEEMBAY_OCS_HCU + help + Enables support for sha224 and hmac(sha224) algorithms in the Intel + Keem Bay OCS HCU driver. Intel recommends not to use these + algorithms. + + Provides OCS HCU hardware acceleration of sha224 and hmac(224). + + If unsure, say N. diff --git a/drivers/crypto/keembay/Makefile b/drivers/crypto/keembay/Makefile index f21e2c4ab3b3a833cc01eeb73342589ab5071ecd..aea03d4432c4178a6def29c385d5b5f24537b4dc 100644 --- a/drivers/crypto/keembay/Makefile +++ b/drivers/crypto/keembay/Makefile @@ -3,3 +3,6 @@ # obj-$(CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4) += keembay-ocs-aes.o keembay-ocs-aes-objs := keembay-ocs-aes-core.o ocs-aes.o + +obj-$(CONFIG_CRYPTO_DEV_KEEMBAY_OCS_HCU) += keembay-ocs-hcu.o +keembay-ocs-hcu-objs := keembay-ocs-hcu-core.o ocs-hcu.o diff --git a/drivers/crypto/keembay/keembay-ocs-hcu-core.c b/drivers/crypto/keembay/keembay-ocs-hcu-core.c new file mode 100644 index 0000000000000000000000000000000000000000..c4b97b4160e9b60f2104a0f4059b9cacd1324df2 --- /dev/null +++ b/drivers/crypto/keembay/keembay-ocs-hcu-core.c @@ -0,0 +1,1264 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Intel Keem Bay OCS HCU Crypto Driver. + * + * Copyright (C) 2018-2020 Intel Corporation + */ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "ocs-hcu.h" + +#define DRV_NAME "keembay-ocs-hcu" + +/* Flag marking a final request. */ +#define REQ_FINAL BIT(0) +/* Flag marking a HMAC request. */ +#define REQ_FLAGS_HMAC BIT(1) +/* Flag set when HW HMAC is being used. */ +#define REQ_FLAGS_HMAC_HW BIT(2) +/* Flag set when SW HMAC is being used. */ +#define REQ_FLAGS_HMAC_SW BIT(3) + +/** + * struct ocs_hcu_ctx: OCS HCU Transform context. + * @engine_ctx: Crypto Engine context. + * @hcu_dev: The OCS HCU device used by the transformation. + * @key: The key (used only for HMAC transformations). + * @key_len: The length of the key. + * @is_sm3_tfm: Whether or not this is an SM3 transformation. + * @is_hmac_tfm: Whether or not this is a HMAC transformation. + */ +struct ocs_hcu_ctx { + struct crypto_engine_ctx engine_ctx; + struct ocs_hcu_dev *hcu_dev; + u8 key[SHA512_BLOCK_SIZE]; + size_t key_len; + bool is_sm3_tfm; + bool is_hmac_tfm; +}; + +/** + * struct ocs_hcu_rctx - Context for the request. + * @hcu_dev: OCS HCU device to be used to service the request. + * @flags: Flags tracking request status. + * @algo: Algorithm to use for the request. + * @blk_sz: Block size of the transformation / request. + * @dig_sz: Digest size of the transformation / request. + * @dma_list: OCS DMA linked list. + * @hash_ctx: OCS HCU hashing context. + * @buffer: Buffer to store: partial block of data and SW HMAC + * artifacts (ipad, opad, etc.). + * @buf_cnt: Number of bytes currently stored in the buffer. + * @buf_dma_addr: The DMA address of @buffer (when mapped). + * @buf_dma_count: The number of bytes in @buffer currently DMA-mapped. + * @sg: Head of the scatterlist entries containing data. + * @sg_data_total: Total data in the SG list at any time. + * @sg_data_offset: Offset into the data of the current individual SG node. + * @sg_dma_nents: Number of sg entries mapped in dma_list. + */ +struct ocs_hcu_rctx { + struct ocs_hcu_dev *hcu_dev; + u32 flags; + enum ocs_hcu_algo algo; + size_t blk_sz; + size_t dig_sz; + struct ocs_hcu_dma_list *dma_list; + struct ocs_hcu_hash_ctx hash_ctx; + /* + * Buffer is double the block size because we need space for SW HMAC + * artifacts, i.e: + * - ipad (1 block) + a possible partial block of data. + * - opad (1 block) + digest of H(k ^ ipad || m) + */ + u8 buffer[2 * SHA512_BLOCK_SIZE]; + size_t buf_cnt; + dma_addr_t buf_dma_addr; + size_t buf_dma_count; + struct scatterlist *sg; + unsigned int sg_data_total; + unsigned int sg_data_offset; + unsigned int sg_dma_nents; +}; + +/** + * struct ocs_hcu_drv - Driver data + * @dev_list: The list of HCU devices. + * @lock: The lock protecting dev_list. + */ +struct ocs_hcu_drv { + struct list_head dev_list; + spinlock_t lock; /* Protects dev_list. */ +}; + +static struct ocs_hcu_drv ocs_hcu = { + .dev_list = LIST_HEAD_INIT(ocs_hcu.dev_list), + .lock = __SPIN_LOCK_UNLOCKED(ocs_hcu.lock), +}; + +/* + * Return the total amount of data in the request; that is: the data in the + * request buffer + the data in the sg list. + */ +static inline unsigned int kmb_get_total_data(struct ocs_hcu_rctx *rctx) +{ + return rctx->sg_data_total + rctx->buf_cnt; +} + +/* Move remaining content of scatter-gather list to context buffer. */ +static int flush_sg_to_ocs_buffer(struct ocs_hcu_rctx *rctx) +{ + size_t count; + + if (rctx->sg_data_total > (sizeof(rctx->buffer) - rctx->buf_cnt)) { + WARN(1, "%s: sg data does not fit in buffer\n", __func__); + return -EINVAL; + } + + while (rctx->sg_data_total) { + if (!rctx->sg) { + WARN(1, "%s: unexpected NULL sg\n", __func__); + return -EINVAL; + } + /* + * If current sg has been fully processed, skip to the next + * one. + */ + if (rctx->sg_data_offset == rctx->sg->length) { + rctx->sg = sg_next(rctx->sg); + rctx->sg_data_offset = 0; + continue; + } + /* + * Determine the maximum data available to copy from the node. + * Minimum of the length left in the sg node, or the total data + * in the request. + */ + count = min(rctx->sg->length - rctx->sg_data_offset, + rctx->sg_data_total); + /* Copy from scatter-list entry to context buffer. */ + scatterwalk_map_and_copy(&rctx->buffer[rctx->buf_cnt], + rctx->sg, rctx->sg_data_offset, + count, 0); + + rctx->sg_data_offset += count; + rctx->sg_data_total -= count; + rctx->buf_cnt += count; + } + + return 0; +} + +static struct ocs_hcu_dev *kmb_ocs_hcu_find_dev(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct ocs_hcu_ctx *tctx = crypto_ahash_ctx(tfm); + + /* If the HCU device for the request was previously set, return it. */ + if (tctx->hcu_dev) + return tctx->hcu_dev; + + /* + * Otherwise, get the first HCU device available (there should be one + * and only one device). + */ + spin_lock_bh(&ocs_hcu.lock); + tctx->hcu_dev = list_first_entry_or_null(&ocs_hcu.dev_list, + struct ocs_hcu_dev, + list); + spin_unlock_bh(&ocs_hcu.lock); + + return tctx->hcu_dev; +} + +/* Free OCS DMA linked list and DMA-able context buffer. */ +static void kmb_ocs_hcu_dma_cleanup(struct ahash_request *req, + struct ocs_hcu_rctx *rctx) +{ + struct ocs_hcu_dev *hcu_dev = rctx->hcu_dev; + struct device *dev = hcu_dev->dev; + + /* Unmap rctx->buffer (if mapped). */ + if (rctx->buf_dma_count) { + dma_unmap_single(dev, rctx->buf_dma_addr, rctx->buf_dma_count, + DMA_TO_DEVICE); + rctx->buf_dma_count = 0; + } + + /* Unmap req->src (if mapped). */ + if (rctx->sg_dma_nents) { + dma_unmap_sg(dev, req->src, rctx->sg_dma_nents, DMA_TO_DEVICE); + rctx->sg_dma_nents = 0; + } + + /* Free dma_list (if allocated). */ + if (rctx->dma_list) { + ocs_hcu_dma_list_free(hcu_dev, rctx->dma_list); + rctx->dma_list = NULL; + } +} + +/* + * Prepare for DMA operation: + * - DMA-map request context buffer (if needed) + * - DMA-map SG list (only the entries to be processed, see note below) + * - Allocate OCS HCU DMA linked list (number of elements = SG entries to + * process + context buffer (if not empty)). + * - Add DMA-mapped request context buffer to OCS HCU DMA list. + * - Add SG entries to DMA list. + * + * Note: if this is a final request, we process all the data in the SG list, + * otherwise we can only process up to the maximum amount of block-aligned data + * (the remainder will be put into the context buffer and processed in the next + * request). + */ +static int kmb_ocs_dma_prepare(struct ahash_request *req) +{ + struct ocs_hcu_rctx *rctx = ahash_request_ctx(req); + struct device *dev = rctx->hcu_dev->dev; + unsigned int remainder = 0; + unsigned int total; + size_t nents; + size_t count; + int rc; + int i; + + /* This function should be called only when there is data to process. */ + total = kmb_get_total_data(rctx); + if (!total) + return -EINVAL; + + /* + * If this is not a final DMA (terminated DMA), the data passed to the + * HCU must be aligned to the block size; compute the remainder data to + * be processed in the next request. + */ + if (!(rctx->flags & REQ_FINAL)) + remainder = total % rctx->blk_sz; + + /* Determine the number of scatter gather list entries to process. */ + nents = sg_nents_for_len(req->src, rctx->sg_data_total - remainder); + + /* If there are entries to process, map them. */ + if (nents) { + rctx->sg_dma_nents = dma_map_sg(dev, req->src, nents, + DMA_TO_DEVICE); + if (!rctx->sg_dma_nents) { + dev_err(dev, "Failed to MAP SG\n"); + rc = -ENOMEM; + goto cleanup; + } + /* + * The value returned by dma_map_sg() can be < nents; so update + * nents accordingly. + */ + nents = rctx->sg_dma_nents; + } + + /* + * If context buffer is not empty, map it and add extra DMA entry for + * it. + */ + if (rctx->buf_cnt) { + rctx->buf_dma_addr = dma_map_single(dev, rctx->buffer, + rctx->buf_cnt, + DMA_TO_DEVICE); + if (dma_mapping_error(dev, rctx->buf_dma_addr)) { + dev_err(dev, "Failed to map request context buffer\n"); + rc = -ENOMEM; + goto cleanup; + } + rctx->buf_dma_count = rctx->buf_cnt; + /* Increase number of dma entries. */ + nents++; + } + + /* Allocate OCS HCU DMA list. */ + rctx->dma_list = ocs_hcu_dma_list_alloc(rctx->hcu_dev, nents); + if (!rctx->dma_list) { + rc = -ENOMEM; + goto cleanup; + } + + /* Add request context buffer (if previously DMA-mapped) */ + if (rctx->buf_dma_count) { + rc = ocs_hcu_dma_list_add_tail(rctx->hcu_dev, rctx->dma_list, + rctx->buf_dma_addr, + rctx->buf_dma_count); + if (rc) + goto cleanup; + } + + /* Add the SG nodes to be processed to the DMA linked list. */ + for_each_sg(req->src, rctx->sg, rctx->sg_dma_nents, i) { + /* + * The number of bytes to add to the list entry is the minimum + * between: + * - The DMA length of the SG entry. + * - The data left to be processed. + */ + count = min(rctx->sg_data_total - remainder, + sg_dma_len(rctx->sg) - rctx->sg_data_offset); + /* + * Do not create a zero length DMA descriptor. Check in case of + * zero length SG node. + */ + if (count == 0) + continue; + /* Add sg to HCU DMA list. */ + rc = ocs_hcu_dma_list_add_tail(rctx->hcu_dev, + rctx->dma_list, + rctx->sg->dma_address, + count); + if (rc) + goto cleanup; + + /* Update amount of data remaining in SG list. */ + rctx->sg_data_total -= count; + + /* + * If remaining data is equal to remainder (note: 'less than' + * case should never happen in practice), we are done: update + * offset and exit the loop. + */ + if (rctx->sg_data_total <= remainder) { + WARN_ON(rctx->sg_data_total < remainder); + rctx->sg_data_offset += count; + break; + } + + /* + * If we get here is because we need to process the next sg in + * the list; set offset within the sg to 0. + */ + rctx->sg_data_offset = 0; + } + + return 0; +cleanup: + dev_err(dev, "Failed to prepare DMA.\n"); + kmb_ocs_hcu_dma_cleanup(req, rctx); + + return rc; +} + +static void kmb_ocs_hcu_secure_cleanup(struct ahash_request *req) +{ + struct ocs_hcu_rctx *rctx = ahash_request_ctx(req); + + /* Clear buffer of any data. */ + memzero_explicit(rctx->buffer, sizeof(rctx->buffer)); +} + +static int kmb_ocs_hcu_handle_queue(struct ahash_request *req) +{ + struct ocs_hcu_dev *hcu_dev = kmb_ocs_hcu_find_dev(req); + + if (!hcu_dev) + return -ENOENT; + + return crypto_transfer_hash_request_to_engine(hcu_dev->engine, req); +} + +static int prepare_ipad(struct ahash_request *req) +{ + struct ocs_hcu_rctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct ocs_hcu_ctx *ctx = crypto_ahash_ctx(tfm); + int i; + + WARN(rctx->buf_cnt, "%s: Context buffer is not empty\n", __func__); + WARN(!(rctx->flags & REQ_FLAGS_HMAC_SW), + "%s: HMAC_SW flag is not set\n", __func__); + /* + * Key length must be equal to block size. If key is shorter, + * we pad it with zero (note: key cannot be longer, since + * longer keys are hashed by kmb_ocs_hcu_setkey()). + */ + if (ctx->key_len > rctx->blk_sz) { + WARN(1, "%s: Invalid key length in tfm context\n", __func__); + return -EINVAL; + } + memzero_explicit(&ctx->key[ctx->key_len], + rctx->blk_sz - ctx->key_len); + ctx->key_len = rctx->blk_sz; + /* + * Prepare IPAD for HMAC. Only done for first block. + * HMAC(k,m) = H(k ^ opad || H(k ^ ipad || m)) + * k ^ ipad will be first hashed block. + * k ^ opad will be calculated in the final request. + * Only needed if not using HW HMAC. + */ + for (i = 0; i < rctx->blk_sz; i++) + rctx->buffer[i] = ctx->key[i] ^ HMAC_IPAD_VALUE; + rctx->buf_cnt = rctx->blk_sz; + + return 0; +} + +static int kmb_ocs_hcu_do_one_request(struct crypto_engine *engine, void *areq) +{ + struct ahash_request *req = container_of(areq, struct ahash_request, + base); + struct ocs_hcu_dev *hcu_dev = kmb_ocs_hcu_find_dev(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct ocs_hcu_rctx *rctx = ahash_request_ctx(req); + struct ocs_hcu_ctx *tctx = crypto_ahash_ctx(tfm); + int rc; + int i; + + if (!hcu_dev) { + rc = -ENOENT; + goto error; + } + + /* + * If hardware HMAC flag is set, perform HMAC in hardware. + * + * NOTE: this flag implies REQ_FINAL && kmb_get_total_data(rctx) + */ + if (rctx->flags & REQ_FLAGS_HMAC_HW) { + /* Map input data into the HCU DMA linked list. */ + rc = kmb_ocs_dma_prepare(req); + if (rc) + goto error; + + rc = ocs_hcu_hmac(hcu_dev, rctx->algo, tctx->key, tctx->key_len, + rctx->dma_list, req->result, rctx->dig_sz); + + /* Unmap data and free DMA list regardless of return code. */ + kmb_ocs_hcu_dma_cleanup(req, rctx); + + /* Process previous return code. */ + if (rc) + goto error; + + goto done; + } + + /* Handle update request case. */ + if (!(rctx->flags & REQ_FINAL)) { + /* Update should always have input data. */ + if (!kmb_get_total_data(rctx)) + return -EINVAL; + + /* Map input data into the HCU DMA linked list. */ + rc = kmb_ocs_dma_prepare(req); + if (rc) + goto error; + + /* Do hashing step. */ + rc = ocs_hcu_hash_update(hcu_dev, &rctx->hash_ctx, + rctx->dma_list); + + /* Unmap data and free DMA list regardless of return code. */ + kmb_ocs_hcu_dma_cleanup(req, rctx); + + /* Process previous return code. */ + if (rc) + goto error; + + /* + * Reset request buffer count (data in the buffer was just + * processed). + */ + rctx->buf_cnt = 0; + /* + * Move remaining sg data into the request buffer, so that it + * will be processed during the next request. + * + * NOTE: we have remaining data if kmb_get_total_data() was not + * a multiple of block size. + */ + rc = flush_sg_to_ocs_buffer(rctx); + if (rc) + goto error; + + goto done; + } + + /* If we get here, this is a final request. */ + + /* If there is data to process, use finup. */ + if (kmb_get_total_data(rctx)) { + /* Map input data into the HCU DMA linked list. */ + rc = kmb_ocs_dma_prepare(req); + if (rc) + goto error; + + /* Do hashing step. */ + rc = ocs_hcu_hash_finup(hcu_dev, &rctx->hash_ctx, + rctx->dma_list, + req->result, rctx->dig_sz); + /* Free DMA list regardless of return code. */ + kmb_ocs_hcu_dma_cleanup(req, rctx); + + /* Process previous return code. */ + if (rc) + goto error; + + } else { /* Otherwise (if we have no data), use final. */ + rc = ocs_hcu_hash_final(hcu_dev, &rctx->hash_ctx, req->result, + rctx->dig_sz); + if (rc) + goto error; + } + + /* + * If we are finalizing a SW HMAC request, we just computed the result + * of: H(k ^ ipad || m). + * + * We now need to complete the HMAC calculation with the OPAD step, + * that is, we need to compute H(k ^ opad || digest), where digest is + * the digest we just obtained, i.e., H(k ^ ipad || m). + */ + if (rctx->flags & REQ_FLAGS_HMAC_SW) { + /* + * Compute k ^ opad and store it in the request buffer (which + * is not used anymore at this point). + * Note: key has been padded / hashed already (so keylen == + * blksz) . + */ + WARN_ON(tctx->key_len != rctx->blk_sz); + for (i = 0; i < rctx->blk_sz; i++) + rctx->buffer[i] = tctx->key[i] ^ HMAC_OPAD_VALUE; + /* Now append the digest to the rest of the buffer. */ + for (i = 0; (i < rctx->dig_sz); i++) + rctx->buffer[rctx->blk_sz + i] = req->result[i]; + + /* Now hash the buffer to obtain the final HMAC. */ + rc = ocs_hcu_digest(hcu_dev, rctx->algo, rctx->buffer, + rctx->blk_sz + rctx->dig_sz, req->result, + rctx->dig_sz); + if (rc) + goto error; + } + + /* Perform secure clean-up. */ + kmb_ocs_hcu_secure_cleanup(req); +done: + crypto_finalize_hash_request(hcu_dev->engine, req, 0); + + return 0; + +error: + kmb_ocs_hcu_secure_cleanup(req); + return rc; +} + +static int kmb_ocs_hcu_init(struct ahash_request *req) +{ + struct ocs_hcu_dev *hcu_dev = kmb_ocs_hcu_find_dev(req); + struct ocs_hcu_rctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct ocs_hcu_ctx *ctx = crypto_ahash_ctx(tfm); + + if (!hcu_dev) + return -ENOENT; + + /* Initialize entire request context to zero. */ + memset(rctx, 0, sizeof(*rctx)); + + rctx->hcu_dev = hcu_dev; + rctx->dig_sz = crypto_ahash_digestsize(tfm); + + switch (rctx->dig_sz) { +#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_HCU_HMAC_SHA224 + case SHA224_DIGEST_SIZE: + rctx->blk_sz = SHA224_BLOCK_SIZE; + rctx->algo = OCS_HCU_ALGO_SHA224; + break; +#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_HCU_HMAC_SHA224 */ + case SHA256_DIGEST_SIZE: + rctx->blk_sz = SHA256_BLOCK_SIZE; + /* + * SHA256 and SM3 have the same digest size: use info from tfm + * context to find out which one we should use. + */ + rctx->algo = ctx->is_sm3_tfm ? OCS_HCU_ALGO_SM3 : + OCS_HCU_ALGO_SHA256; + break; + case SHA384_DIGEST_SIZE: + rctx->blk_sz = SHA384_BLOCK_SIZE; + rctx->algo = OCS_HCU_ALGO_SHA384; + break; + case SHA512_DIGEST_SIZE: + rctx->blk_sz = SHA512_BLOCK_SIZE; + rctx->algo = OCS_HCU_ALGO_SHA512; + break; + default: + return -EINVAL; + } + + /* Initialize intermediate data. */ + ocs_hcu_hash_init(&rctx->hash_ctx, rctx->algo); + + /* If this a HMAC request, set HMAC flag. */ + if (ctx->is_hmac_tfm) + rctx->flags |= REQ_FLAGS_HMAC; + + return 0; +} + +static int kmb_ocs_hcu_update(struct ahash_request *req) +{ + struct ocs_hcu_rctx *rctx = ahash_request_ctx(req); + int rc; + + if (!req->nbytes) + return 0; + + rctx->sg_data_total = req->nbytes; + rctx->sg_data_offset = 0; + rctx->sg = req->src; + + /* + * If we are doing HMAC, then we must use SW-assisted HMAC, since HW + * HMAC does not support context switching (there it can only be used + * with finup() or digest()). + */ + if (rctx->flags & REQ_FLAGS_HMAC && + !(rctx->flags & REQ_FLAGS_HMAC_SW)) { + rctx->flags |= REQ_FLAGS_HMAC_SW; + rc = prepare_ipad(req); + if (rc) + return rc; + } + + /* + * If remaining sg_data fits into ctx buffer, just copy it there; we'll + * process it at the next update() or final(). + */ + if (rctx->sg_data_total <= (sizeof(rctx->buffer) - rctx->buf_cnt)) + return flush_sg_to_ocs_buffer(rctx); + + return kmb_ocs_hcu_handle_queue(req); +} + +/* Common logic for kmb_ocs_hcu_final() and kmb_ocs_hcu_finup(). */ +static int kmb_ocs_hcu_fin_common(struct ahash_request *req) +{ + struct ocs_hcu_rctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct ocs_hcu_ctx *ctx = crypto_ahash_ctx(tfm); + int rc; + + rctx->flags |= REQ_FINAL; + + /* + * If this is a HMAC request and, so far, we didn't have to switch to + * SW HMAC, check if we can use HW HMAC. + */ + if (rctx->flags & REQ_FLAGS_HMAC && + !(rctx->flags & REQ_FLAGS_HMAC_SW)) { + /* + * If we are here, it means we never processed any data so far, + * so we can use HW HMAC, but only if there is some data to + * process (since OCS HW MAC does not support zero-length + * messages) and the key length is supported by the hardware + * (OCS HCU HW only supports length <= 64); if HW HMAC cannot + * be used, fall back to SW-assisted HMAC. + */ + if (kmb_get_total_data(rctx) && + ctx->key_len <= OCS_HCU_HW_KEY_LEN) { + rctx->flags |= REQ_FLAGS_HMAC_HW; + } else { + rctx->flags |= REQ_FLAGS_HMAC_SW; + rc = prepare_ipad(req); + if (rc) + return rc; + } + } + + return kmb_ocs_hcu_handle_queue(req); +} + +static int kmb_ocs_hcu_final(struct ahash_request *req) +{ + struct ocs_hcu_rctx *rctx = ahash_request_ctx(req); + + rctx->sg_data_total = 0; + rctx->sg_data_offset = 0; + rctx->sg = NULL; + + return kmb_ocs_hcu_fin_common(req); +} + +static int kmb_ocs_hcu_finup(struct ahash_request *req) +{ + struct ocs_hcu_rctx *rctx = ahash_request_ctx(req); + + rctx->sg_data_total = req->nbytes; + rctx->sg_data_offset = 0; + rctx->sg = req->src; + + return kmb_ocs_hcu_fin_common(req); +} + +static int kmb_ocs_hcu_digest(struct ahash_request *req) +{ + int rc = 0; + struct ocs_hcu_dev *hcu_dev = kmb_ocs_hcu_find_dev(req); + + if (!hcu_dev) + return -ENOENT; + + rc = kmb_ocs_hcu_init(req); + if (rc) + return rc; + + rc = kmb_ocs_hcu_finup(req); + + return rc; +} + +static int kmb_ocs_hcu_export(struct ahash_request *req, void *out) +{ + struct ocs_hcu_rctx *rctx = ahash_request_ctx(req); + + /* Intermediate data is always stored and applied per request. */ + memcpy(out, rctx, sizeof(*rctx)); + + return 0; +} + +static int kmb_ocs_hcu_import(struct ahash_request *req, const void *in) +{ + struct ocs_hcu_rctx *rctx = ahash_request_ctx(req); + + /* Intermediate data is always stored and applied per request. */ + memcpy(rctx, in, sizeof(*rctx)); + + return 0; +} + +static int kmb_ocs_hcu_setkey(struct crypto_ahash *tfm, const u8 *key, + unsigned int keylen) +{ + unsigned int digestsize = crypto_ahash_digestsize(tfm); + struct ocs_hcu_ctx *ctx = crypto_ahash_ctx(tfm); + size_t blk_sz = crypto_ahash_blocksize(tfm); + struct crypto_ahash *ahash_tfm; + struct ahash_request *req; + struct crypto_wait wait; + struct scatterlist sg; + const char *alg_name; + int rc; + + /* + * Key length must be equal to block size: + * - If key is shorter, we are done for now (the key will be padded + * later on); this is to maximize the use of HW HMAC (which works + * only for keys <= 64 bytes). + * - If key is longer, we hash it. + */ + if (keylen <= blk_sz) { + memcpy(ctx->key, key, keylen); + ctx->key_len = keylen; + return 0; + } + + switch (digestsize) { +#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_HCU_HMAC_SHA224 + case SHA224_DIGEST_SIZE: + alg_name = "sha224-keembay-ocs"; + break; +#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_HCU_HMAC_SHA224 */ + case SHA256_DIGEST_SIZE: + alg_name = ctx->is_sm3_tfm ? "sm3-keembay-ocs" : + "sha256-keembay-ocs"; + break; + case SHA384_DIGEST_SIZE: + alg_name = "sha384-keembay-ocs"; + break; + case SHA512_DIGEST_SIZE: + alg_name = "sha512-keembay-ocs"; + break; + default: + return -EINVAL; + } + + ahash_tfm = crypto_alloc_ahash(alg_name, 0, 0); + if (IS_ERR(ahash_tfm)) + return PTR_ERR(ahash_tfm); + + req = ahash_request_alloc(ahash_tfm, GFP_KERNEL); + if (!req) { + rc = -ENOMEM; + goto err_free_ahash; + } + + crypto_init_wait(&wait); + ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, + crypto_req_done, &wait); + crypto_ahash_clear_flags(ahash_tfm, ~0); + + sg_init_one(&sg, key, keylen); + ahash_request_set_crypt(req, &sg, ctx->key, keylen); + + rc = crypto_wait_req(crypto_ahash_digest(req), &wait); + if (rc == 0) + ctx->key_len = digestsize; + + ahash_request_free(req); +err_free_ahash: + crypto_free_ahash(ahash_tfm); + + return rc; +} + +/* Set request size and initialize tfm context. */ +static void __cra_init(struct crypto_tfm *tfm, struct ocs_hcu_ctx *ctx) +{ + crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), + sizeof(struct ocs_hcu_rctx)); + + /* Init context to 0. */ + memzero_explicit(ctx, sizeof(*ctx)); + /* Set engine ops. */ + ctx->engine_ctx.op.do_one_request = kmb_ocs_hcu_do_one_request; +} + +static int kmb_ocs_hcu_sha_cra_init(struct crypto_tfm *tfm) +{ + struct ocs_hcu_ctx *ctx = crypto_tfm_ctx(tfm); + + __cra_init(tfm, ctx); + + return 0; +} + +static int kmb_ocs_hcu_sm3_cra_init(struct crypto_tfm *tfm) +{ + struct ocs_hcu_ctx *ctx = crypto_tfm_ctx(tfm); + + __cra_init(tfm, ctx); + + ctx->is_sm3_tfm = true; + + return 0; +} + +static int kmb_ocs_hcu_hmac_sm3_cra_init(struct crypto_tfm *tfm) +{ + struct ocs_hcu_ctx *ctx = crypto_tfm_ctx(tfm); + + __cra_init(tfm, ctx); + + ctx->is_sm3_tfm = true; + ctx->is_hmac_tfm = true; + + return 0; +} + +static int kmb_ocs_hcu_hmac_cra_init(struct crypto_tfm *tfm) +{ + struct ocs_hcu_ctx *ctx = crypto_tfm_ctx(tfm); + + __cra_init(tfm, ctx); + + ctx->is_hmac_tfm = true; + + return 0; +} + +/* Function called when 'tfm' is de-initialized. */ +static void kmb_ocs_hcu_hmac_cra_exit(struct crypto_tfm *tfm) +{ + struct ocs_hcu_ctx *ctx = crypto_tfm_ctx(tfm); + + /* Clear the key. */ + memzero_explicit(ctx->key, sizeof(ctx->key)); +} + +static struct ahash_alg ocs_hcu_algs[] = { +#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_HCU_HMAC_SHA224 +{ + .init = kmb_ocs_hcu_init, + .update = kmb_ocs_hcu_update, + .final = kmb_ocs_hcu_final, + .finup = kmb_ocs_hcu_finup, + .digest = kmb_ocs_hcu_digest, + .export = kmb_ocs_hcu_export, + .import = kmb_ocs_hcu_import, + .halg = { + .digestsize = SHA224_DIGEST_SIZE, + .statesize = sizeof(struct ocs_hcu_rctx), + .base = { + .cra_name = "sha224", + .cra_driver_name = "sha224-keembay-ocs", + .cra_priority = 255, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = SHA224_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct ocs_hcu_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = kmb_ocs_hcu_sha_cra_init, + } + } +}, +{ + .init = kmb_ocs_hcu_init, + .update = kmb_ocs_hcu_update, + .final = kmb_ocs_hcu_final, + .finup = kmb_ocs_hcu_finup, + .digest = kmb_ocs_hcu_digest, + .export = kmb_ocs_hcu_export, + .import = kmb_ocs_hcu_import, + .setkey = kmb_ocs_hcu_setkey, + .halg = { + .digestsize = SHA224_DIGEST_SIZE, + .statesize = sizeof(struct ocs_hcu_rctx), + .base = { + .cra_name = "hmac(sha224)", + .cra_driver_name = "hmac-sha224-keembay-ocs", + .cra_priority = 255, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = SHA224_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct ocs_hcu_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = kmb_ocs_hcu_hmac_cra_init, + .cra_exit = kmb_ocs_hcu_hmac_cra_exit, + } + } +}, +#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_HCU_HMAC_SHA224 */ +{ + .init = kmb_ocs_hcu_init, + .update = kmb_ocs_hcu_update, + .final = kmb_ocs_hcu_final, + .finup = kmb_ocs_hcu_finup, + .digest = kmb_ocs_hcu_digest, + .export = kmb_ocs_hcu_export, + .import = kmb_ocs_hcu_import, + .halg = { + .digestsize = SHA256_DIGEST_SIZE, + .statesize = sizeof(struct ocs_hcu_rctx), + .base = { + .cra_name = "sha256", + .cra_driver_name = "sha256-keembay-ocs", + .cra_priority = 255, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct ocs_hcu_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = kmb_ocs_hcu_sha_cra_init, + } + } +}, +{ + .init = kmb_ocs_hcu_init, + .update = kmb_ocs_hcu_update, + .final = kmb_ocs_hcu_final, + .finup = kmb_ocs_hcu_finup, + .digest = kmb_ocs_hcu_digest, + .export = kmb_ocs_hcu_export, + .import = kmb_ocs_hcu_import, + .setkey = kmb_ocs_hcu_setkey, + .halg = { + .digestsize = SHA256_DIGEST_SIZE, + .statesize = sizeof(struct ocs_hcu_rctx), + .base = { + .cra_name = "hmac(sha256)", + .cra_driver_name = "hmac-sha256-keembay-ocs", + .cra_priority = 255, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct ocs_hcu_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = kmb_ocs_hcu_hmac_cra_init, + .cra_exit = kmb_ocs_hcu_hmac_cra_exit, + } + } +}, +{ + .init = kmb_ocs_hcu_init, + .update = kmb_ocs_hcu_update, + .final = kmb_ocs_hcu_final, + .finup = kmb_ocs_hcu_finup, + .digest = kmb_ocs_hcu_digest, + .export = kmb_ocs_hcu_export, + .import = kmb_ocs_hcu_import, + .halg = { + .digestsize = SM3_DIGEST_SIZE, + .statesize = sizeof(struct ocs_hcu_rctx), + .base = { + .cra_name = "sm3", + .cra_driver_name = "sm3-keembay-ocs", + .cra_priority = 255, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = SM3_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct ocs_hcu_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = kmb_ocs_hcu_sm3_cra_init, + } + } +}, +{ + .init = kmb_ocs_hcu_init, + .update = kmb_ocs_hcu_update, + .final = kmb_ocs_hcu_final, + .finup = kmb_ocs_hcu_finup, + .digest = kmb_ocs_hcu_digest, + .export = kmb_ocs_hcu_export, + .import = kmb_ocs_hcu_import, + .setkey = kmb_ocs_hcu_setkey, + .halg = { + .digestsize = SM3_DIGEST_SIZE, + .statesize = sizeof(struct ocs_hcu_rctx), + .base = { + .cra_name = "hmac(sm3)", + .cra_driver_name = "hmac-sm3-keembay-ocs", + .cra_priority = 255, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = SM3_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct ocs_hcu_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = kmb_ocs_hcu_hmac_sm3_cra_init, + .cra_exit = kmb_ocs_hcu_hmac_cra_exit, + } + } +}, +{ + .init = kmb_ocs_hcu_init, + .update = kmb_ocs_hcu_update, + .final = kmb_ocs_hcu_final, + .finup = kmb_ocs_hcu_finup, + .digest = kmb_ocs_hcu_digest, + .export = kmb_ocs_hcu_export, + .import = kmb_ocs_hcu_import, + .halg = { + .digestsize = SHA384_DIGEST_SIZE, + .statesize = sizeof(struct ocs_hcu_rctx), + .base = { + .cra_name = "sha384", + .cra_driver_name = "sha384-keembay-ocs", + .cra_priority = 255, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = SHA384_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct ocs_hcu_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = kmb_ocs_hcu_sha_cra_init, + } + } +}, +{ + .init = kmb_ocs_hcu_init, + .update = kmb_ocs_hcu_update, + .final = kmb_ocs_hcu_final, + .finup = kmb_ocs_hcu_finup, + .digest = kmb_ocs_hcu_digest, + .export = kmb_ocs_hcu_export, + .import = kmb_ocs_hcu_import, + .setkey = kmb_ocs_hcu_setkey, + .halg = { + .digestsize = SHA384_DIGEST_SIZE, + .statesize = sizeof(struct ocs_hcu_rctx), + .base = { + .cra_name = "hmac(sha384)", + .cra_driver_name = "hmac-sha384-keembay-ocs", + .cra_priority = 255, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = SHA384_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct ocs_hcu_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = kmb_ocs_hcu_hmac_cra_init, + .cra_exit = kmb_ocs_hcu_hmac_cra_exit, + } + } +}, +{ + .init = kmb_ocs_hcu_init, + .update = kmb_ocs_hcu_update, + .final = kmb_ocs_hcu_final, + .finup = kmb_ocs_hcu_finup, + .digest = kmb_ocs_hcu_digest, + .export = kmb_ocs_hcu_export, + .import = kmb_ocs_hcu_import, + .halg = { + .digestsize = SHA512_DIGEST_SIZE, + .statesize = sizeof(struct ocs_hcu_rctx), + .base = { + .cra_name = "sha512", + .cra_driver_name = "sha512-keembay-ocs", + .cra_priority = 255, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = SHA512_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct ocs_hcu_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = kmb_ocs_hcu_sha_cra_init, + } + } +}, +{ + .init = kmb_ocs_hcu_init, + .update = kmb_ocs_hcu_update, + .final = kmb_ocs_hcu_final, + .finup = kmb_ocs_hcu_finup, + .digest = kmb_ocs_hcu_digest, + .export = kmb_ocs_hcu_export, + .import = kmb_ocs_hcu_import, + .setkey = kmb_ocs_hcu_setkey, + .halg = { + .digestsize = SHA512_DIGEST_SIZE, + .statesize = sizeof(struct ocs_hcu_rctx), + .base = { + .cra_name = "hmac(sha512)", + .cra_driver_name = "hmac-sha512-keembay-ocs", + .cra_priority = 255, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = SHA512_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct ocs_hcu_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = kmb_ocs_hcu_hmac_cra_init, + .cra_exit = kmb_ocs_hcu_hmac_cra_exit, + } + } +}, +}; + +/* Device tree driver match. */ +static const struct of_device_id kmb_ocs_hcu_of_match[] = { + { + .compatible = "intel,keembay-ocs-hcu", + }, + {} +}; + +static int kmb_ocs_hcu_remove(struct platform_device *pdev) +{ + struct ocs_hcu_dev *hcu_dev; + int rc; + + hcu_dev = platform_get_drvdata(pdev); + if (!hcu_dev) + return -ENODEV; + + crypto_unregister_ahashes(ocs_hcu_algs, ARRAY_SIZE(ocs_hcu_algs)); + + rc = crypto_engine_exit(hcu_dev->engine); + + spin_lock_bh(&ocs_hcu.lock); + list_del(&hcu_dev->list); + spin_unlock_bh(&ocs_hcu.lock); + + return rc; +} + +static int kmb_ocs_hcu_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct ocs_hcu_dev *hcu_dev; + struct resource *hcu_mem; + int rc; + + hcu_dev = devm_kzalloc(dev, sizeof(*hcu_dev), GFP_KERNEL); + if (!hcu_dev) + return -ENOMEM; + + hcu_dev->dev = dev; + + platform_set_drvdata(pdev, hcu_dev); + rc = dma_set_mask_and_coherent(&pdev->dev, OCS_HCU_DMA_BIT_MASK); + if (rc) + return rc; + + /* Get the memory address and remap. */ + hcu_mem = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!hcu_mem) { + dev_err(dev, "Could not retrieve io mem resource.\n"); + return -ENODEV; + } + + hcu_dev->io_base = devm_ioremap_resource(dev, hcu_mem); + if (IS_ERR(hcu_dev->io_base)) { + dev_err(dev, "Could not io-remap mem resource.\n"); + return PTR_ERR(hcu_dev->io_base); + } + + init_completion(&hcu_dev->irq_done); + + /* Get and request IRQ. */ + hcu_dev->irq = platform_get_irq(pdev, 0); + if (hcu_dev->irq < 0) + return hcu_dev->irq; + + rc = devm_request_threaded_irq(&pdev->dev, hcu_dev->irq, + ocs_hcu_irq_handler, NULL, 0, + "keembay-ocs-hcu", hcu_dev); + if (rc < 0) { + dev_err(dev, "Could not request IRQ.\n"); + return rc; + } + + INIT_LIST_HEAD(&hcu_dev->list); + + spin_lock_bh(&ocs_hcu.lock); + list_add_tail(&hcu_dev->list, &ocs_hcu.dev_list); + spin_unlock_bh(&ocs_hcu.lock); + + /* Initialize crypto engine */ + hcu_dev->engine = crypto_engine_alloc_init(dev, 1); + if (!hcu_dev->engine) + goto list_del; + + rc = crypto_engine_start(hcu_dev->engine); + if (rc) { + dev_err(dev, "Could not start engine.\n"); + goto cleanup; + } + + /* Security infrastructure guarantees OCS clock is enabled. */ + + rc = crypto_register_ahashes(ocs_hcu_algs, ARRAY_SIZE(ocs_hcu_algs)); + if (rc) { + dev_err(dev, "Could not register algorithms.\n"); + goto cleanup; + } + + return 0; + +cleanup: + crypto_engine_exit(hcu_dev->engine); +list_del: + spin_lock_bh(&ocs_hcu.lock); + list_del(&hcu_dev->list); + spin_unlock_bh(&ocs_hcu.lock); + + return rc; +} + +/* The OCS driver is a platform device. */ +static struct platform_driver kmb_ocs_hcu_driver = { + .probe = kmb_ocs_hcu_probe, + .remove = kmb_ocs_hcu_remove, + .driver = { + .name = DRV_NAME, + .of_match_table = kmb_ocs_hcu_of_match, + }, +}; + +module_platform_driver(kmb_ocs_hcu_driver); + +MODULE_LICENSE("GPL"); diff --git a/drivers/crypto/keembay/ocs-aes.c b/drivers/crypto/keembay/ocs-aes.c index cc286adb1c4a642d92ce3673cb854cd020a97f64..be9f32fc8f42a292956b0503577ea4d6902a11b2 100644 --- a/drivers/crypto/keembay/ocs-aes.c +++ b/drivers/crypto/keembay/ocs-aes.c @@ -958,14 +958,14 @@ int ocs_aes_gcm_op(struct ocs_aes_dev *aes_dev, ocs_aes_write_last_data_blk_len(aes_dev, src_size); /* Write ciphertext bit length */ - bit_len = src_size * 8; + bit_len = (u64)src_size * 8; val = bit_len & 0xFFFFFFFF; iowrite32(val, aes_dev->base_reg + AES_MULTIPURPOSE2_0_OFFSET); val = bit_len >> 32; iowrite32(val, aes_dev->base_reg + AES_MULTIPURPOSE2_1_OFFSET); /* Write aad bit length */ - bit_len = aad_size * 8; + bit_len = (u64)aad_size * 8; val = bit_len & 0xFFFFFFFF; iowrite32(val, aes_dev->base_reg + AES_MULTIPURPOSE2_2_OFFSET); val = bit_len >> 32; @@ -1080,15 +1080,15 @@ static int ocs_aes_ccm_write_b0(const struct ocs_aes_dev *aes_dev, /* * q is the octet length of Q. * q can only be an element of {2, 3, 4, 5, 6, 7, 8} and is encoded as - * q - 1 == iv[0] + * q - 1 == iv[0] & 0x7; */ b0[0] |= iv[0] & 0x7; /* * Copy the Nonce N from IV to B0; N is located in iv[1]..iv[15 - q] * and must be copied to b0[1]..b0[15-q]. - * q == iv[0] + 1 + * q == (iv[0] & 0x7) + 1 */ - q = iv[0] + 1; + q = (iv[0] & 0x7) + 1; for (i = 1; i <= 15 - q; i++) b0[i] = iv[i]; /* diff --git a/drivers/crypto/keembay/ocs-hcu.c b/drivers/crypto/keembay/ocs-hcu.c new file mode 100644 index 0000000000000000000000000000000000000000..81eecacf603adc25bde90770a2dcc3b80d042086 --- /dev/null +++ b/drivers/crypto/keembay/ocs-hcu.c @@ -0,0 +1,840 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Intel Keem Bay OCS HCU Crypto Driver. + * + * Copyright (C) 2018-2020 Intel Corporation + */ + +#include +#include +#include +#include +#include + +#include + +#include "ocs-hcu.h" + +/* Registers. */ +#define OCS_HCU_MODE 0x00 +#define OCS_HCU_CHAIN 0x04 +#define OCS_HCU_OPERATION 0x08 +#define OCS_HCU_KEY_0 0x0C +#define OCS_HCU_ISR 0x50 +#define OCS_HCU_IER 0x54 +#define OCS_HCU_STATUS 0x58 +#define OCS_HCU_MSG_LEN_LO 0x60 +#define OCS_HCU_MSG_LEN_HI 0x64 +#define OCS_HCU_KEY_BYTE_ORDER_CFG 0x80 +#define OCS_HCU_DMA_SRC_ADDR 0x400 +#define OCS_HCU_DMA_SRC_SIZE 0x408 +#define OCS_HCU_DMA_DST_SIZE 0x40C +#define OCS_HCU_DMA_DMA_MODE 0x410 +#define OCS_HCU_DMA_NEXT_SRC_DESCR 0x418 +#define OCS_HCU_DMA_MSI_ISR 0x480 +#define OCS_HCU_DMA_MSI_IER 0x484 +#define OCS_HCU_DMA_MSI_MASK 0x488 + +/* Register bit definitions. */ +#define HCU_MODE_ALGO_SHIFT 16 +#define HCU_MODE_HMAC_SHIFT 22 + +#define HCU_STATUS_BUSY BIT(0) + +#define HCU_BYTE_ORDER_SWAP BIT(0) + +#define HCU_IRQ_HASH_DONE BIT(2) +#define HCU_IRQ_HASH_ERR_MASK (BIT(3) | BIT(1) | BIT(0)) + +#define HCU_DMA_IRQ_SRC_DONE BIT(0) +#define HCU_DMA_IRQ_SAI_ERR BIT(2) +#define HCU_DMA_IRQ_BAD_COMP_ERR BIT(3) +#define HCU_DMA_IRQ_INBUF_RD_ERR BIT(4) +#define HCU_DMA_IRQ_INBUF_WD_ERR BIT(5) +#define HCU_DMA_IRQ_OUTBUF_WR_ERR BIT(6) +#define HCU_DMA_IRQ_OUTBUF_RD_ERR BIT(7) +#define HCU_DMA_IRQ_CRD_ERR BIT(8) +#define HCU_DMA_IRQ_ERR_MASK (HCU_DMA_IRQ_SAI_ERR | \ + HCU_DMA_IRQ_BAD_COMP_ERR | \ + HCU_DMA_IRQ_INBUF_RD_ERR | \ + HCU_DMA_IRQ_INBUF_WD_ERR | \ + HCU_DMA_IRQ_OUTBUF_WR_ERR | \ + HCU_DMA_IRQ_OUTBUF_RD_ERR | \ + HCU_DMA_IRQ_CRD_ERR) + +#define HCU_DMA_SNOOP_MASK (0x7 << 28) +#define HCU_DMA_SRC_LL_EN BIT(25) +#define HCU_DMA_EN BIT(31) + +#define OCS_HCU_ENDIANNESS_VALUE 0x2A + +#define HCU_DMA_MSI_UNMASK BIT(0) +#define HCU_DMA_MSI_DISABLE 0 +#define HCU_IRQ_DISABLE 0 + +#define OCS_HCU_START BIT(0) +#define OCS_HCU_TERMINATE BIT(1) + +#define OCS_LL_DMA_FLAG_TERMINATE BIT(31) + +#define OCS_HCU_HW_KEY_LEN_U32 (OCS_HCU_HW_KEY_LEN / sizeof(u32)) + +#define HCU_DATA_WRITE_ENDIANNESS_OFFSET 26 + +#define OCS_HCU_NUM_CHAINS_SHA256_224_SM3 (SHA256_DIGEST_SIZE / sizeof(u32)) +#define OCS_HCU_NUM_CHAINS_SHA384_512 (SHA512_DIGEST_SIZE / sizeof(u32)) + +/* + * While polling on a busy HCU, wait maximum 200us between one check and the + * other. + */ +#define OCS_HCU_WAIT_BUSY_RETRY_DELAY_US 200 +/* Wait on a busy HCU for maximum 1 second. */ +#define OCS_HCU_WAIT_BUSY_TIMEOUT_US 1000000 + +/** + * struct ocs_hcu_dma_list - An entry in an OCS DMA linked list. + * @src_addr: Source address of the data. + * @src_len: Length of data to be fetched. + * @nxt_desc: Next descriptor to fetch. + * @ll_flags: Flags (Freeze @ terminate) for the DMA engine. + */ +struct ocs_hcu_dma_entry { + u32 src_addr; + u32 src_len; + u32 nxt_desc; + u32 ll_flags; +}; + +/** + * struct ocs_dma_list - OCS-specific DMA linked list. + * @head: The head of the list (points to the array backing the list). + * @tail: The current tail of the list; NULL if the list is empty. + * @dma_addr: The DMA address of @head (i.e., the DMA address of the backing + * array). + * @max_nents: Maximum number of entries in the list (i.e., number of elements + * in the backing array). + * + * The OCS DMA list is an array-backed list of OCS DMA descriptors. The array + * backing the list is allocated with dma_alloc_coherent() and pointed by + * @head. + */ +struct ocs_hcu_dma_list { + struct ocs_hcu_dma_entry *head; + struct ocs_hcu_dma_entry *tail; + dma_addr_t dma_addr; + size_t max_nents; +}; + +static inline u32 ocs_hcu_num_chains(enum ocs_hcu_algo algo) +{ + switch (algo) { + case OCS_HCU_ALGO_SHA224: + case OCS_HCU_ALGO_SHA256: + case OCS_HCU_ALGO_SM3: + return OCS_HCU_NUM_CHAINS_SHA256_224_SM3; + case OCS_HCU_ALGO_SHA384: + case OCS_HCU_ALGO_SHA512: + return OCS_HCU_NUM_CHAINS_SHA384_512; + default: + return 0; + }; +} + +static inline u32 ocs_hcu_digest_size(enum ocs_hcu_algo algo) +{ + switch (algo) { + case OCS_HCU_ALGO_SHA224: + return SHA224_DIGEST_SIZE; + case OCS_HCU_ALGO_SHA256: + case OCS_HCU_ALGO_SM3: + /* SM3 shares the same block size. */ + return SHA256_DIGEST_SIZE; + case OCS_HCU_ALGO_SHA384: + return SHA384_DIGEST_SIZE; + case OCS_HCU_ALGO_SHA512: + return SHA512_DIGEST_SIZE; + default: + return 0; + } +} + +/** + * ocs_hcu_wait_busy() - Wait for HCU OCS hardware to became usable. + * @hcu_dev: OCS HCU device to wait for. + * + * Return: 0 if device free, -ETIMEOUT if device busy and internal timeout has + * expired. + */ +static int ocs_hcu_wait_busy(struct ocs_hcu_dev *hcu_dev) +{ + long val; + + return readl_poll_timeout(hcu_dev->io_base + OCS_HCU_STATUS, val, + !(val & HCU_STATUS_BUSY), + OCS_HCU_WAIT_BUSY_RETRY_DELAY_US, + OCS_HCU_WAIT_BUSY_TIMEOUT_US); +} + +static void ocs_hcu_done_irq_en(struct ocs_hcu_dev *hcu_dev) +{ + /* Clear any pending interrupts. */ + writel(0xFFFFFFFF, hcu_dev->io_base + OCS_HCU_ISR); + hcu_dev->irq_err = false; + /* Enable error and HCU done interrupts. */ + writel(HCU_IRQ_HASH_DONE | HCU_IRQ_HASH_ERR_MASK, + hcu_dev->io_base + OCS_HCU_IER); +} + +static void ocs_hcu_dma_irq_en(struct ocs_hcu_dev *hcu_dev) +{ + /* Clear any pending interrupts. */ + writel(0xFFFFFFFF, hcu_dev->io_base + OCS_HCU_DMA_MSI_ISR); + hcu_dev->irq_err = false; + /* Only operating on DMA source completion and error interrupts. */ + writel(HCU_DMA_IRQ_ERR_MASK | HCU_DMA_IRQ_SRC_DONE, + hcu_dev->io_base + OCS_HCU_DMA_MSI_IER); + /* Unmask */ + writel(HCU_DMA_MSI_UNMASK, hcu_dev->io_base + OCS_HCU_DMA_MSI_MASK); +} + +static void ocs_hcu_irq_dis(struct ocs_hcu_dev *hcu_dev) +{ + writel(HCU_IRQ_DISABLE, hcu_dev->io_base + OCS_HCU_IER); + writel(HCU_DMA_MSI_DISABLE, hcu_dev->io_base + OCS_HCU_DMA_MSI_IER); +} + +static int ocs_hcu_wait_and_disable_irq(struct ocs_hcu_dev *hcu_dev) +{ + int rc; + + rc = wait_for_completion_interruptible(&hcu_dev->irq_done); + if (rc) + goto exit; + + if (hcu_dev->irq_err) { + /* Unset flag and return error. */ + hcu_dev->irq_err = false; + rc = -EIO; + goto exit; + } + +exit: + ocs_hcu_irq_dis(hcu_dev); + + return rc; +} + +/** + * ocs_hcu_get_intermediate_data() - Get intermediate data. + * @hcu_dev: The target HCU device. + * @data: Where to store the intermediate. + * @algo: The algorithm being used. + * + * This function is used to save the current hashing process state in order to + * continue it in the future. + * + * Note: once all data has been processed, the intermediate data actually + * contains the hashing result. So this function is also used to retrieve the + * final result of a hashing process. + * + * Return: 0 on success, negative error code otherwise. + */ +static int ocs_hcu_get_intermediate_data(struct ocs_hcu_dev *hcu_dev, + struct ocs_hcu_idata *data, + enum ocs_hcu_algo algo) +{ + const int n = ocs_hcu_num_chains(algo); + u32 *chain; + int rc; + int i; + + /* Data not requested. */ + if (!data) + return -EINVAL; + + chain = (u32 *)data->digest; + + /* Ensure that the OCS is no longer busy before reading the chains. */ + rc = ocs_hcu_wait_busy(hcu_dev); + if (rc) + return rc; + + /* + * This loops is safe because data->digest is an array of + * SHA512_DIGEST_SIZE bytes and the maximum value returned by + * ocs_hcu_num_chains() is OCS_HCU_NUM_CHAINS_SHA384_512 which is equal + * to SHA512_DIGEST_SIZE / sizeof(u32). + */ + for (i = 0; i < n; i++) + chain[i] = readl(hcu_dev->io_base + OCS_HCU_CHAIN); + + data->msg_len_lo = readl(hcu_dev->io_base + OCS_HCU_MSG_LEN_LO); + data->msg_len_hi = readl(hcu_dev->io_base + OCS_HCU_MSG_LEN_HI); + + return 0; +} + +/** + * ocs_hcu_set_intermediate_data() - Set intermediate data. + * @hcu_dev: The target HCU device. + * @data: The intermediate data to be set. + * @algo: The algorithm being used. + * + * This function is used to continue a previous hashing process. + */ +static void ocs_hcu_set_intermediate_data(struct ocs_hcu_dev *hcu_dev, + const struct ocs_hcu_idata *data, + enum ocs_hcu_algo algo) +{ + const int n = ocs_hcu_num_chains(algo); + u32 *chain = (u32 *)data->digest; + int i; + + /* + * This loops is safe because data->digest is an array of + * SHA512_DIGEST_SIZE bytes and the maximum value returned by + * ocs_hcu_num_chains() is OCS_HCU_NUM_CHAINS_SHA384_512 which is equal + * to SHA512_DIGEST_SIZE / sizeof(u32). + */ + for (i = 0; i < n; i++) + writel(chain[i], hcu_dev->io_base + OCS_HCU_CHAIN); + + writel(data->msg_len_lo, hcu_dev->io_base + OCS_HCU_MSG_LEN_LO); + writel(data->msg_len_hi, hcu_dev->io_base + OCS_HCU_MSG_LEN_HI); +} + +static int ocs_hcu_get_digest(struct ocs_hcu_dev *hcu_dev, + enum ocs_hcu_algo algo, u8 *dgst, size_t dgst_len) +{ + u32 *chain; + int rc; + int i; + + if (!dgst) + return -EINVAL; + + /* Length of the output buffer must match the algo digest size. */ + if (dgst_len != ocs_hcu_digest_size(algo)) + return -EINVAL; + + /* Ensure that the OCS is no longer busy before reading the chains. */ + rc = ocs_hcu_wait_busy(hcu_dev); + if (rc) + return rc; + + chain = (u32 *)dgst; + for (i = 0; i < dgst_len / sizeof(u32); i++) + chain[i] = readl(hcu_dev->io_base + OCS_HCU_CHAIN); + + return 0; +} + +/** + * ocs_hcu_hw_cfg() - Configure the HCU hardware. + * @hcu_dev: The HCU device to configure. + * @algo: The algorithm to be used by the HCU device. + * @use_hmac: Whether or not HW HMAC should be used. + * + * Return: 0 on success, negative error code otherwise. + */ +static int ocs_hcu_hw_cfg(struct ocs_hcu_dev *hcu_dev, enum ocs_hcu_algo algo, + bool use_hmac) +{ + u32 cfg; + int rc; + + if (algo != OCS_HCU_ALGO_SHA256 && algo != OCS_HCU_ALGO_SHA224 && + algo != OCS_HCU_ALGO_SHA384 && algo != OCS_HCU_ALGO_SHA512 && + algo != OCS_HCU_ALGO_SM3) + return -EINVAL; + + rc = ocs_hcu_wait_busy(hcu_dev); + if (rc) + return rc; + + /* Ensure interrupts are disabled. */ + ocs_hcu_irq_dis(hcu_dev); + + /* Configure endianness, hashing algorithm and HW HMAC (if needed) */ + cfg = OCS_HCU_ENDIANNESS_VALUE << HCU_DATA_WRITE_ENDIANNESS_OFFSET; + cfg |= algo << HCU_MODE_ALGO_SHIFT; + if (use_hmac) + cfg |= BIT(HCU_MODE_HMAC_SHIFT); + + writel(cfg, hcu_dev->io_base + OCS_HCU_MODE); + + return 0; +} + +/** + * ocs_hcu_clear_key() - Clear key stored in OCS HMAC KEY registers. + * @hcu_dev: The OCS HCU device whose key registers should be cleared. + */ +static void ocs_hcu_clear_key(struct ocs_hcu_dev *hcu_dev) +{ + int reg_off; + + /* Clear OCS_HCU_KEY_[0..15] */ + for (reg_off = 0; reg_off < OCS_HCU_HW_KEY_LEN; reg_off += sizeof(u32)) + writel(0, hcu_dev->io_base + OCS_HCU_KEY_0 + reg_off); +} + +/** + * ocs_hcu_write_key() - Write key to OCS HMAC KEY registers. + * @hcu_dev: The OCS HCU device the key should be written to. + * @key: The key to be written. + * @len: The size of the key to write. It must be OCS_HCU_HW_KEY_LEN. + * + * Return: 0 on success, negative error code otherwise. + */ +static int ocs_hcu_write_key(struct ocs_hcu_dev *hcu_dev, const u8 *key, size_t len) +{ + u32 key_u32[OCS_HCU_HW_KEY_LEN_U32]; + int i; + + if (len > OCS_HCU_HW_KEY_LEN) + return -EINVAL; + + /* Copy key into temporary u32 array. */ + memcpy(key_u32, key, len); + + /* + * Hardware requires all the bytes of the HW Key vector to be + * written. So pad with zero until we reach OCS_HCU_HW_KEY_LEN. + */ + memzero_explicit((u8 *)key_u32 + len, OCS_HCU_HW_KEY_LEN - len); + + /* + * OCS hardware expects the MSB of the key to be written at the highest + * address of the HCU Key vector; in other word, the key must be + * written in reverse order. + * + * Therefore, we first enable byte swapping for the HCU key vector; + * so that bytes of 32-bit word written to OCS_HCU_KEY_[0..15] will be + * swapped: + * 3 <---> 0, 2 <---> 1. + */ + writel(HCU_BYTE_ORDER_SWAP, + hcu_dev->io_base + OCS_HCU_KEY_BYTE_ORDER_CFG); + /* + * And then we write the 32-bit words composing the key starting from + * the end of the key. + */ + for (i = 0; i < OCS_HCU_HW_KEY_LEN_U32; i++) + writel(key_u32[OCS_HCU_HW_KEY_LEN_U32 - 1 - i], + hcu_dev->io_base + OCS_HCU_KEY_0 + (sizeof(u32) * i)); + + memzero_explicit(key_u32, OCS_HCU_HW_KEY_LEN); + + return 0; +} + +/** + * ocs_hcu_ll_dma_start() - Start OCS HCU hashing via DMA + * @hcu_dev: The OCS HCU device to use. + * @dma_list: The OCS DMA list mapping the data to hash. + * @finalize: Whether or not this is the last hashing operation and therefore + * the final hash should be compute even if data is not + * block-aligned. + * + * Return: 0 on success, negative error code otherwise. + */ +static int ocs_hcu_ll_dma_start(struct ocs_hcu_dev *hcu_dev, + const struct ocs_hcu_dma_list *dma_list, + bool finalize) +{ + u32 cfg = HCU_DMA_SNOOP_MASK | HCU_DMA_SRC_LL_EN | HCU_DMA_EN; + int rc; + + if (!dma_list) + return -EINVAL; + + /* + * For final requests we use HCU_DONE IRQ to be notified when all input + * data has been processed by the HCU; however, we cannot do so for + * non-final requests, because we don't get a HCU_DONE IRQ when we + * don't terminate the operation. + * + * Therefore, for non-final requests, we use the DMA IRQ, which + * triggers when DMA has finishing feeding all the input data to the + * HCU, but the HCU may still be processing it. This is fine, since we + * will wait for the HCU processing to be completed when we try to read + * intermediate results, in ocs_hcu_get_intermediate_data(). + */ + if (finalize) + ocs_hcu_done_irq_en(hcu_dev); + else + ocs_hcu_dma_irq_en(hcu_dev); + + reinit_completion(&hcu_dev->irq_done); + writel(dma_list->dma_addr, hcu_dev->io_base + OCS_HCU_DMA_NEXT_SRC_DESCR); + writel(0, hcu_dev->io_base + OCS_HCU_DMA_SRC_SIZE); + writel(0, hcu_dev->io_base + OCS_HCU_DMA_DST_SIZE); + + writel(OCS_HCU_START, hcu_dev->io_base + OCS_HCU_OPERATION); + + writel(cfg, hcu_dev->io_base + OCS_HCU_DMA_DMA_MODE); + + if (finalize) + writel(OCS_HCU_TERMINATE, hcu_dev->io_base + OCS_HCU_OPERATION); + + rc = ocs_hcu_wait_and_disable_irq(hcu_dev); + if (rc) + return rc; + + return 0; +} + +struct ocs_hcu_dma_list *ocs_hcu_dma_list_alloc(struct ocs_hcu_dev *hcu_dev, + int max_nents) +{ + struct ocs_hcu_dma_list *dma_list; + + dma_list = kmalloc(sizeof(*dma_list), GFP_KERNEL); + if (!dma_list) + return NULL; + + /* Total size of the DMA list to allocate. */ + dma_list->head = dma_alloc_coherent(hcu_dev->dev, + sizeof(*dma_list->head) * max_nents, + &dma_list->dma_addr, GFP_KERNEL); + if (!dma_list->head) { + kfree(dma_list); + return NULL; + } + dma_list->max_nents = max_nents; + dma_list->tail = NULL; + + return dma_list; +} + +void ocs_hcu_dma_list_free(struct ocs_hcu_dev *hcu_dev, + struct ocs_hcu_dma_list *dma_list) +{ + if (!dma_list) + return; + + dma_free_coherent(hcu_dev->dev, + sizeof(*dma_list->head) * dma_list->max_nents, + dma_list->head, dma_list->dma_addr); + + kfree(dma_list); +} + +/* Add a new DMA entry at the end of the OCS DMA list. */ +int ocs_hcu_dma_list_add_tail(struct ocs_hcu_dev *hcu_dev, + struct ocs_hcu_dma_list *dma_list, + dma_addr_t addr, u32 len) +{ + struct device *dev = hcu_dev->dev; + struct ocs_hcu_dma_entry *old_tail; + struct ocs_hcu_dma_entry *new_tail; + + if (!len) + return 0; + + if (!dma_list) + return -EINVAL; + + if (addr & ~OCS_HCU_DMA_BIT_MASK) { + dev_err(dev, + "Unexpected error: Invalid DMA address for OCS HCU\n"); + return -EINVAL; + } + + old_tail = dma_list->tail; + new_tail = old_tail ? old_tail + 1 : dma_list->head; + + /* Check if list is full. */ + if (new_tail - dma_list->head >= dma_list->max_nents) + return -ENOMEM; + + /* + * If there was an old tail (i.e., this is not the first element we are + * adding), un-terminate the old tail and make it point to the new one. + */ + if (old_tail) { + old_tail->ll_flags &= ~OCS_LL_DMA_FLAG_TERMINATE; + /* + * The old tail 'nxt_desc' must point to the DMA address of the + * new tail. + */ + old_tail->nxt_desc = dma_list->dma_addr + + sizeof(*dma_list->tail) * (new_tail - + dma_list->head); + } + + new_tail->src_addr = (u32)addr; + new_tail->src_len = (u32)len; + new_tail->ll_flags = OCS_LL_DMA_FLAG_TERMINATE; + new_tail->nxt_desc = 0; + + /* Update list tail with new tail. */ + dma_list->tail = new_tail; + + return 0; +} + +/** + * ocs_hcu_hash_init() - Initialize hash operation context. + * @ctx: The context to initialize. + * @algo: The hashing algorithm to use. + * + * Return: 0 on success, negative error code otherwise. + */ +int ocs_hcu_hash_init(struct ocs_hcu_hash_ctx *ctx, enum ocs_hcu_algo algo) +{ + if (!ctx) + return -EINVAL; + + ctx->algo = algo; + ctx->idata.msg_len_lo = 0; + ctx->idata.msg_len_hi = 0; + /* No need to set idata.digest to 0. */ + + return 0; +} + +/** + * ocs_hcu_digest() - Perform a hashing iteration. + * @hcu_dev: The OCS HCU device to use. + * @ctx: The OCS HCU hashing context. + * @dma_list: The OCS DMA list mapping the input data to process. + * + * Return: 0 on success; negative error code otherwise. + */ +int ocs_hcu_hash_update(struct ocs_hcu_dev *hcu_dev, + struct ocs_hcu_hash_ctx *ctx, + const struct ocs_hcu_dma_list *dma_list) +{ + int rc; + + if (!hcu_dev || !ctx) + return -EINVAL; + + /* Configure the hardware for the current request. */ + rc = ocs_hcu_hw_cfg(hcu_dev, ctx->algo, false); + if (rc) + return rc; + + /* If we already processed some data, idata needs to be set. */ + if (ctx->idata.msg_len_lo || ctx->idata.msg_len_hi) + ocs_hcu_set_intermediate_data(hcu_dev, &ctx->idata, ctx->algo); + + /* Start linked-list DMA hashing. */ + rc = ocs_hcu_ll_dma_start(hcu_dev, dma_list, false); + if (rc) + return rc; + + /* Update idata and return. */ + return ocs_hcu_get_intermediate_data(hcu_dev, &ctx->idata, ctx->algo); +} + +/** + * ocs_hcu_hash_final() - Update and finalize hash computation. + * @hcu_dev: The OCS HCU device to use. + * @ctx: The OCS HCU hashing context. + * @dma_list: The OCS DMA list mapping the input data to process. + * @dgst: The buffer where to save the computed digest. + * @dgst_len: The length of @dgst. + * + * Return: 0 on success; negative error code otherwise. + */ +int ocs_hcu_hash_finup(struct ocs_hcu_dev *hcu_dev, + const struct ocs_hcu_hash_ctx *ctx, + const struct ocs_hcu_dma_list *dma_list, + u8 *dgst, size_t dgst_len) +{ + int rc; + + if (!hcu_dev || !ctx) + return -EINVAL; + + /* Configure the hardware for the current request. */ + rc = ocs_hcu_hw_cfg(hcu_dev, ctx->algo, false); + if (rc) + return rc; + + /* If we already processed some data, idata needs to be set. */ + if (ctx->idata.msg_len_lo || ctx->idata.msg_len_hi) + ocs_hcu_set_intermediate_data(hcu_dev, &ctx->idata, ctx->algo); + + /* Start linked-list DMA hashing. */ + rc = ocs_hcu_ll_dma_start(hcu_dev, dma_list, true); + if (rc) + return rc; + + /* Get digest and return. */ + return ocs_hcu_get_digest(hcu_dev, ctx->algo, dgst, dgst_len); +} + +/** + * ocs_hcu_hash_final() - Finalize hash computation. + * @hcu_dev: The OCS HCU device to use. + * @ctx: The OCS HCU hashing context. + * @dgst: The buffer where to save the computed digest. + * @dgst_len: The length of @dgst. + * + * Return: 0 on success; negative error code otherwise. + */ +int ocs_hcu_hash_final(struct ocs_hcu_dev *hcu_dev, + const struct ocs_hcu_hash_ctx *ctx, u8 *dgst, + size_t dgst_len) +{ + int rc; + + if (!hcu_dev || !ctx) + return -EINVAL; + + /* Configure the hardware for the current request. */ + rc = ocs_hcu_hw_cfg(hcu_dev, ctx->algo, false); + if (rc) + return rc; + + /* If we already processed some data, idata needs to be set. */ + if (ctx->idata.msg_len_lo || ctx->idata.msg_len_hi) + ocs_hcu_set_intermediate_data(hcu_dev, &ctx->idata, ctx->algo); + + /* + * Enable HCU interrupts, so that HCU_DONE will be triggered once the + * final hash is computed. + */ + ocs_hcu_done_irq_en(hcu_dev); + reinit_completion(&hcu_dev->irq_done); + writel(OCS_HCU_TERMINATE, hcu_dev->io_base + OCS_HCU_OPERATION); + + rc = ocs_hcu_wait_and_disable_irq(hcu_dev); + if (rc) + return rc; + + /* Get digest and return. */ + return ocs_hcu_get_digest(hcu_dev, ctx->algo, dgst, dgst_len); +} + +/** + * ocs_hcu_digest() - Compute hash digest. + * @hcu_dev: The OCS HCU device to use. + * @algo: The hash algorithm to use. + * @data: The input data to process. + * @data_len: The length of @data. + * @dgst: The buffer where to save the computed digest. + * @dgst_len: The length of @dgst. + * + * Return: 0 on success; negative error code otherwise. + */ +int ocs_hcu_digest(struct ocs_hcu_dev *hcu_dev, enum ocs_hcu_algo algo, + void *data, size_t data_len, u8 *dgst, size_t dgst_len) +{ + struct device *dev = hcu_dev->dev; + dma_addr_t dma_handle; + u32 reg; + int rc; + + /* Configure the hardware for the current request. */ + rc = ocs_hcu_hw_cfg(hcu_dev, algo, false); + if (rc) + return rc; + + dma_handle = dma_map_single(dev, data, data_len, DMA_TO_DEVICE); + if (dma_mapping_error(dev, dma_handle)) + return -EIO; + + reg = HCU_DMA_SNOOP_MASK | HCU_DMA_EN; + + ocs_hcu_done_irq_en(hcu_dev); + + reinit_completion(&hcu_dev->irq_done); + + writel(dma_handle, hcu_dev->io_base + OCS_HCU_DMA_SRC_ADDR); + writel(data_len, hcu_dev->io_base + OCS_HCU_DMA_SRC_SIZE); + writel(OCS_HCU_START, hcu_dev->io_base + OCS_HCU_OPERATION); + writel(reg, hcu_dev->io_base + OCS_HCU_DMA_DMA_MODE); + + writel(OCS_HCU_TERMINATE, hcu_dev->io_base + OCS_HCU_OPERATION); + + rc = ocs_hcu_wait_and_disable_irq(hcu_dev); + if (rc) + return rc; + + dma_unmap_single(dev, dma_handle, data_len, DMA_TO_DEVICE); + + return ocs_hcu_get_digest(hcu_dev, algo, dgst, dgst_len); +} + +/** + * ocs_hcu_hmac() - Compute HMAC. + * @hcu_dev: The OCS HCU device to use. + * @algo: The hash algorithm to use with HMAC. + * @key: The key to use. + * @dma_list: The OCS DMA list mapping the input data to process. + * @key_len: The length of @key. + * @dgst: The buffer where to save the computed HMAC. + * @dgst_len: The length of @dgst. + * + * Return: 0 on success; negative error code otherwise. + */ +int ocs_hcu_hmac(struct ocs_hcu_dev *hcu_dev, enum ocs_hcu_algo algo, + const u8 *key, size_t key_len, + const struct ocs_hcu_dma_list *dma_list, + u8 *dgst, size_t dgst_len) +{ + int rc; + + /* Ensure 'key' is not NULL. */ + if (!key || key_len == 0) + return -EINVAL; + + /* Configure the hardware for the current request. */ + rc = ocs_hcu_hw_cfg(hcu_dev, algo, true); + if (rc) + return rc; + + rc = ocs_hcu_write_key(hcu_dev, key, key_len); + if (rc) + return rc; + + rc = ocs_hcu_ll_dma_start(hcu_dev, dma_list, true); + + /* Clear HW key before processing return code. */ + ocs_hcu_clear_key(hcu_dev); + + if (rc) + return rc; + + return ocs_hcu_get_digest(hcu_dev, algo, dgst, dgst_len); +} + +irqreturn_t ocs_hcu_irq_handler(int irq, void *dev_id) +{ + struct ocs_hcu_dev *hcu_dev = dev_id; + u32 hcu_irq; + u32 dma_irq; + + /* Read and clear the HCU interrupt. */ + hcu_irq = readl(hcu_dev->io_base + OCS_HCU_ISR); + writel(hcu_irq, hcu_dev->io_base + OCS_HCU_ISR); + + /* Read and clear the HCU DMA interrupt. */ + dma_irq = readl(hcu_dev->io_base + OCS_HCU_DMA_MSI_ISR); + writel(dma_irq, hcu_dev->io_base + OCS_HCU_DMA_MSI_ISR); + + /* Check for errors. */ + if (hcu_irq & HCU_IRQ_HASH_ERR_MASK || dma_irq & HCU_DMA_IRQ_ERR_MASK) { + hcu_dev->irq_err = true; + goto complete; + } + + /* Check for DONE IRQs. */ + if (hcu_irq & HCU_IRQ_HASH_DONE || dma_irq & HCU_DMA_IRQ_SRC_DONE) + goto complete; + + return IRQ_NONE; + +complete: + complete(&hcu_dev->irq_done); + + return IRQ_HANDLED; +} + +MODULE_LICENSE("GPL"); diff --git a/drivers/crypto/keembay/ocs-hcu.h b/drivers/crypto/keembay/ocs-hcu.h new file mode 100644 index 0000000000000000000000000000000000000000..fbbbb92a05926d3b001dc582ed0fc25779432e09 --- /dev/null +++ b/drivers/crypto/keembay/ocs-hcu.h @@ -0,0 +1,106 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Intel Keem Bay OCS HCU Crypto Driver. + * + * Copyright (C) 2018-2020 Intel Corporation + */ + +#include + +#ifndef _CRYPTO_OCS_HCU_H +#define _CRYPTO_OCS_HCU_H + +#define OCS_HCU_DMA_BIT_MASK DMA_BIT_MASK(32) + +#define OCS_HCU_HW_KEY_LEN 64 + +struct ocs_hcu_dma_list; + +enum ocs_hcu_algo { + OCS_HCU_ALGO_SHA256 = 2, + OCS_HCU_ALGO_SHA224 = 3, + OCS_HCU_ALGO_SHA384 = 4, + OCS_HCU_ALGO_SHA512 = 5, + OCS_HCU_ALGO_SM3 = 6, +}; + +/** + * struct ocs_hcu_dev - OCS HCU device context. + * @list: List of device contexts. + * @dev: OCS HCU device. + * @io_base: Base address of OCS HCU registers. + * @engine: Crypto engine for the device. + * @irq: IRQ number. + * @irq_done: Completion for IRQ. + * @irq_err: Flag indicating an IRQ error has happened. + */ +struct ocs_hcu_dev { + struct list_head list; + struct device *dev; + void __iomem *io_base; + struct crypto_engine *engine; + int irq; + struct completion irq_done; + bool irq_err; +}; + +/** + * struct ocs_hcu_idata - Intermediate data generated by the HCU. + * @msg_len_lo: Length of data the HCU has operated on in bits, low 32b. + * @msg_len_hi: Length of data the HCU has operated on in bits, high 32b. + * @digest: The digest read from the HCU. If the HCU is terminated, it will + * contain the actual hash digest. Otherwise it is the intermediate + * state. + */ +struct ocs_hcu_idata { + u32 msg_len_lo; + u32 msg_len_hi; + u8 digest[SHA512_DIGEST_SIZE]; +}; + +/** + * struct ocs_hcu_hash_ctx - Context for OCS HCU hashing operation. + * @algo: The hashing algorithm being used. + * @idata: The current intermediate data. + */ +struct ocs_hcu_hash_ctx { + enum ocs_hcu_algo algo; + struct ocs_hcu_idata idata; +}; + +irqreturn_t ocs_hcu_irq_handler(int irq, void *dev_id); + +struct ocs_hcu_dma_list *ocs_hcu_dma_list_alloc(struct ocs_hcu_dev *hcu_dev, + int max_nents); + +void ocs_hcu_dma_list_free(struct ocs_hcu_dev *hcu_dev, + struct ocs_hcu_dma_list *dma_list); + +int ocs_hcu_dma_list_add_tail(struct ocs_hcu_dev *hcu_dev, + struct ocs_hcu_dma_list *dma_list, + dma_addr_t addr, u32 len); + +int ocs_hcu_hash_init(struct ocs_hcu_hash_ctx *ctx, enum ocs_hcu_algo algo); + +int ocs_hcu_hash_update(struct ocs_hcu_dev *hcu_dev, + struct ocs_hcu_hash_ctx *ctx, + const struct ocs_hcu_dma_list *dma_list); + +int ocs_hcu_hash_finup(struct ocs_hcu_dev *hcu_dev, + const struct ocs_hcu_hash_ctx *ctx, + const struct ocs_hcu_dma_list *dma_list, + u8 *dgst, size_t dgst_len); + +int ocs_hcu_hash_final(struct ocs_hcu_dev *hcu_dev, + const struct ocs_hcu_hash_ctx *ctx, u8 *dgst, + size_t dgst_len); + +int ocs_hcu_digest(struct ocs_hcu_dev *hcu_dev, enum ocs_hcu_algo algo, + void *data, size_t data_len, u8 *dgst, size_t dgst_len); + +int ocs_hcu_hmac(struct ocs_hcu_dev *hcu_dev, enum ocs_hcu_algo algo, + const u8 *key, size_t key_len, + const struct ocs_hcu_dma_list *dma_list, + u8 *dgst, size_t dgst_len); + +#endif /* _CRYPTO_OCS_HCU_H */ diff --git a/drivers/crypto/marvell/Kconfig b/drivers/crypto/marvell/Kconfig index 13063384f9587c1f0e44d046068c7204437ecead..9125199f1702bbebd22d414cf7bd358417fd9e71 100644 --- a/drivers/crypto/marvell/Kconfig +++ b/drivers/crypto/marvell/Kconfig @@ -35,3 +35,18 @@ config CRYPTO_DEV_OCTEONTX_CPT To compile this driver as module, choose M here: the modules will be called octeontx-cpt and octeontx-cptvf + +config CRYPTO_DEV_OCTEONTX2_CPT + tristate "Marvell OcteonTX2 CPT driver" + depends on ARCH_THUNDER2 || COMPILE_TEST + depends on PCI_MSI && 64BIT + depends on CRYPTO_LIB_AES + depends on NET_VENDOR_MARVELL + select OCTEONTX2_MBOX + select CRYPTO_DEV_MARVELL + select CRYPTO_SKCIPHER + select CRYPTO_HASH + select CRYPTO_AEAD + help + This driver allows you to utilize the Marvell Cryptographic + Accelerator Unit(CPT) found in OcteonTX2 series of processors. diff --git a/drivers/crypto/marvell/Makefile b/drivers/crypto/marvell/Makefile index 6c6a1519b0f130fdbec578ef788edbe3274eec1b..39db6d9c0aaf94bdb4f07f191ce3ede12eae96b3 100644 --- a/drivers/crypto/marvell/Makefile +++ b/drivers/crypto/marvell/Makefile @@ -2,3 +2,4 @@ obj-$(CONFIG_CRYPTO_DEV_MARVELL_CESA) += cesa/ obj-$(CONFIG_CRYPTO_DEV_OCTEONTX_CPT) += octeontx/ +obj-$(CONFIG_CRYPTO_DEV_OCTEONTX2_CPT) += octeontx2/ diff --git a/drivers/crypto/marvell/cesa/cesa.c b/drivers/crypto/marvell/cesa/cesa.c index 06211858bf2e7f7353235c5c7e34ab46e3081350..f14aac532f53dcb398be6912169ca93f2ef6e1cc 100644 --- a/drivers/crypto/marvell/cesa/cesa.c +++ b/drivers/crypto/marvell/cesa/cesa.c @@ -381,10 +381,10 @@ static int mv_cesa_get_sram(struct platform_device *pdev, int idx) engine->pool = of_gen_pool_get(cesa->dev->of_node, "marvell,crypto-srams", idx); if (engine->pool) { - engine->sram = gen_pool_dma_alloc(engine->pool, - cesa->sram_size, - &engine->sram_dma); - if (engine->sram) + engine->sram_pool = gen_pool_dma_alloc(engine->pool, + cesa->sram_size, + &engine->sram_dma); + if (engine->sram_pool) return 0; engine->pool = NULL; @@ -422,7 +422,7 @@ static void mv_cesa_put_sram(struct platform_device *pdev, int idx) struct mv_cesa_engine *engine = &cesa->engines[idx]; if (engine->pool) - gen_pool_free(engine->pool, (unsigned long)engine->sram, + gen_pool_free(engine->pool, (unsigned long)engine->sram_pool, cesa->sram_size); else dma_unmap_resource(cesa->dev, engine->sram_dma, diff --git a/drivers/crypto/marvell/cesa/cesa.h b/drivers/crypto/marvell/cesa/cesa.h index fa56b45620c7962dc45e3efef9792929e1574f38..c1007f2ba79c81f29d72f33596d2995f86c77fca 100644 --- a/drivers/crypto/marvell/cesa/cesa.h +++ b/drivers/crypto/marvell/cesa/cesa.h @@ -428,6 +428,7 @@ struct mv_cesa_dev { * @id: engine id * @regs: engine registers * @sram: SRAM memory region + * @sram_pool: SRAM memory region from pool * @sram_dma: DMA address of the SRAM memory region * @lock: engine lock * @req: current crypto request @@ -448,7 +449,10 @@ struct mv_cesa_dev { struct mv_cesa_engine { int id; void __iomem *regs; - void __iomem *sram; + union { + void __iomem *sram; + void *sram_pool; + }; dma_addr_t sram_dma; spinlock_t lock; struct crypto_async_request *req; @@ -867,6 +871,31 @@ int mv_cesa_dma_add_op_transfers(struct mv_cesa_tdma_chain *chain, struct mv_cesa_sg_dma_iter *sgiter, gfp_t gfp_flags); +size_t mv_cesa_sg_copy(struct mv_cesa_engine *engine, + struct scatterlist *sgl, unsigned int nents, + unsigned int sram_off, size_t buflen, off_t skip, + bool to_sram); + +static inline size_t mv_cesa_sg_copy_to_sram(struct mv_cesa_engine *engine, + struct scatterlist *sgl, + unsigned int nents, + unsigned int sram_off, + size_t buflen, off_t skip) +{ + return mv_cesa_sg_copy(engine, sgl, nents, sram_off, buflen, skip, + true); +} + +static inline size_t mv_cesa_sg_copy_from_sram(struct mv_cesa_engine *engine, + struct scatterlist *sgl, + unsigned int nents, + unsigned int sram_off, + size_t buflen, off_t skip) +{ + return mv_cesa_sg_copy(engine, sgl, nents, sram_off, buflen, skip, + false); +} + /* Algorithm definitions */ extern struct ahash_alg mv_md5_alg; diff --git a/drivers/crypto/marvell/cesa/cipher.c b/drivers/crypto/marvell/cesa/cipher.c index b4a6ff9dd6d5e10a247c5245f552fd71f9c0ea26..b739d3b873dcf28183927586cef43a16374dfd5c 100644 --- a/drivers/crypto/marvell/cesa/cipher.c +++ b/drivers/crypto/marvell/cesa/cipher.c @@ -89,22 +89,29 @@ static void mv_cesa_skcipher_std_step(struct skcipher_request *req) CESA_SA_SRAM_PAYLOAD_SIZE); mv_cesa_adjust_op(engine, &sreq->op); - memcpy_toio(engine->sram, &sreq->op, sizeof(sreq->op)); + if (engine->pool) + memcpy(engine->sram_pool, &sreq->op, sizeof(sreq->op)); + else + memcpy_toio(engine->sram, &sreq->op, sizeof(sreq->op)); - len = sg_pcopy_to_buffer(req->src, creq->src_nents, - engine->sram + CESA_SA_DATA_SRAM_OFFSET, - len, sreq->offset); + len = mv_cesa_sg_copy_to_sram(engine, req->src, creq->src_nents, + CESA_SA_DATA_SRAM_OFFSET, len, + sreq->offset); sreq->size = len; mv_cesa_set_crypt_op_len(&sreq->op, len); /* FIXME: only update enc_len field */ if (!sreq->skip_ctx) { - memcpy_toio(engine->sram, &sreq->op, sizeof(sreq->op)); + if (engine->pool) + memcpy(engine->sram_pool, &sreq->op, sizeof(sreq->op)); + else + memcpy_toio(engine->sram, &sreq->op, sizeof(sreq->op)); sreq->skip_ctx = true; - } else { + } else if (engine->pool) + memcpy(engine->sram_pool, &sreq->op, sizeof(sreq->op.desc)); + else memcpy_toio(engine->sram, &sreq->op, sizeof(sreq->op.desc)); - } mv_cesa_set_int_mask(engine, CESA_SA_INT_ACCEL0_DONE); writel_relaxed(CESA_SA_CFG_PARA_DIS, engine->regs + CESA_SA_CFG); @@ -121,9 +128,9 @@ static int mv_cesa_skcipher_std_process(struct skcipher_request *req, struct mv_cesa_engine *engine = creq->base.engine; size_t len; - len = sg_pcopy_from_buffer(req->dst, creq->dst_nents, - engine->sram + CESA_SA_DATA_SRAM_OFFSET, - sreq->size, sreq->offset); + len = mv_cesa_sg_copy_from_sram(engine, req->dst, creq->dst_nents, + CESA_SA_DATA_SRAM_OFFSET, sreq->size, + sreq->offset); sreq->offset += len; if (sreq->offset < req->cryptlen) @@ -214,11 +221,14 @@ mv_cesa_skcipher_complete(struct crypto_async_request *req) basereq = &creq->base; memcpy(skreq->iv, basereq->chain.last->op->ctx.skcipher.iv, ivsize); - } else { + } else if (engine->pool) + memcpy(skreq->iv, + engine->sram_pool + CESA_SA_CRYPT_IV_SRAM_OFFSET, + ivsize); + else memcpy_fromio(skreq->iv, engine->sram + CESA_SA_CRYPT_IV_SRAM_OFFSET, ivsize); - } } static const struct mv_cesa_req_ops mv_cesa_skcipher_req_ops = { diff --git a/drivers/crypto/marvell/cesa/hash.c b/drivers/crypto/marvell/cesa/hash.c index 8cf9fd518d86ceb4b83ce767c62e6a656f7439fd..c72b0672fc7103a9c5461d830afdbdad9964d378 100644 --- a/drivers/crypto/marvell/cesa/hash.c +++ b/drivers/crypto/marvell/cesa/hash.c @@ -168,7 +168,12 @@ static void mv_cesa_ahash_std_step(struct ahash_request *req) int i; mv_cesa_adjust_op(engine, &creq->op_tmpl); - memcpy_toio(engine->sram, &creq->op_tmpl, sizeof(creq->op_tmpl)); + if (engine->pool) + memcpy(engine->sram_pool, &creq->op_tmpl, + sizeof(creq->op_tmpl)); + else + memcpy_toio(engine->sram, &creq->op_tmpl, + sizeof(creq->op_tmpl)); if (!sreq->offset) { digsize = crypto_ahash_digestsize(crypto_ahash_reqtfm(req)); @@ -177,9 +182,14 @@ static void mv_cesa_ahash_std_step(struct ahash_request *req) engine->regs + CESA_IVDIG(i)); } - if (creq->cache_ptr) - memcpy_toio(engine->sram + CESA_SA_DATA_SRAM_OFFSET, - creq->cache, creq->cache_ptr); + if (creq->cache_ptr) { + if (engine->pool) + memcpy(engine->sram_pool + CESA_SA_DATA_SRAM_OFFSET, + creq->cache, creq->cache_ptr); + else + memcpy_toio(engine->sram + CESA_SA_DATA_SRAM_OFFSET, + creq->cache, creq->cache_ptr); + } len = min_t(size_t, req->nbytes + creq->cache_ptr - sreq->offset, CESA_SA_SRAM_PAYLOAD_SIZE); @@ -190,12 +200,10 @@ static void mv_cesa_ahash_std_step(struct ahash_request *req) } if (len - creq->cache_ptr) - sreq->offset += sg_pcopy_to_buffer(req->src, creq->src_nents, - engine->sram + - CESA_SA_DATA_SRAM_OFFSET + - creq->cache_ptr, - len - creq->cache_ptr, - sreq->offset); + sreq->offset += mv_cesa_sg_copy_to_sram( + engine, req->src, creq->src_nents, + CESA_SA_DATA_SRAM_OFFSET + creq->cache_ptr, + len - creq->cache_ptr, sreq->offset); op = &creq->op_tmpl; @@ -220,16 +228,28 @@ static void mv_cesa_ahash_std_step(struct ahash_request *req) if (len + trailerlen > CESA_SA_SRAM_PAYLOAD_SIZE) { len &= CESA_HASH_BLOCK_SIZE_MSK; new_cache_ptr = 64 - trailerlen; - memcpy_fromio(creq->cache, - engine->sram + - CESA_SA_DATA_SRAM_OFFSET + len, - new_cache_ptr); + if (engine->pool) + memcpy(creq->cache, + engine->sram_pool + + CESA_SA_DATA_SRAM_OFFSET + len, + new_cache_ptr); + else + memcpy_fromio(creq->cache, + engine->sram + + CESA_SA_DATA_SRAM_OFFSET + + len, + new_cache_ptr); } else { i = mv_cesa_ahash_pad_req(creq, creq->cache); len += i; - memcpy_toio(engine->sram + len + - CESA_SA_DATA_SRAM_OFFSET, - creq->cache, i); + if (engine->pool) + memcpy(engine->sram_pool + len + + CESA_SA_DATA_SRAM_OFFSET, + creq->cache, i); + else + memcpy_toio(engine->sram + len + + CESA_SA_DATA_SRAM_OFFSET, + creq->cache, i); } if (frag_mode == CESA_SA_DESC_CFG_LAST_FRAG) @@ -243,7 +263,10 @@ static void mv_cesa_ahash_std_step(struct ahash_request *req) mv_cesa_update_op_cfg(op, frag_mode, CESA_SA_DESC_CFG_FRAG_MSK); /* FIXME: only update enc_len field */ - memcpy_toio(engine->sram, op, sizeof(*op)); + if (engine->pool) + memcpy(engine->sram_pool, op, sizeof(*op)); + else + memcpy_toio(engine->sram, op, sizeof(*op)); if (frag_mode == CESA_SA_DESC_CFG_FIRST_FRAG) mv_cesa_update_op_cfg(op, CESA_SA_DESC_CFG_MID_FRAG, diff --git a/drivers/crypto/marvell/cesa/tdma.c b/drivers/crypto/marvell/cesa/tdma.c index 5d9c48fb72b2c2832dce8714ec35be60ed662656..f0b5537038c2e6b9ff0c78318ec467f840e90e20 100644 --- a/drivers/crypto/marvell/cesa/tdma.c +++ b/drivers/crypto/marvell/cesa/tdma.c @@ -177,7 +177,7 @@ int mv_cesa_tdma_process(struct mv_cesa_engine *engine, u32 status) /* * Save the last request in error to engine->req, so that the core - * knows which request was fautly + * knows which request was faulty */ if (res) { spin_lock_bh(&engine->lock); @@ -350,3 +350,53 @@ int mv_cesa_dma_add_op_transfers(struct mv_cesa_tdma_chain *chain, return 0; } + +size_t mv_cesa_sg_copy(struct mv_cesa_engine *engine, + struct scatterlist *sgl, unsigned int nents, + unsigned int sram_off, size_t buflen, off_t skip, + bool to_sram) +{ + unsigned int sg_flags = SG_MITER_ATOMIC; + struct sg_mapping_iter miter; + unsigned int offset = 0; + + if (to_sram) + sg_flags |= SG_MITER_FROM_SG; + else + sg_flags |= SG_MITER_TO_SG; + + sg_miter_start(&miter, sgl, nents, sg_flags); + + if (!sg_miter_skip(&miter, skip)) + return 0; + + while ((offset < buflen) && sg_miter_next(&miter)) { + unsigned int len; + + len = min(miter.length, buflen - offset); + + if (to_sram) { + if (engine->pool) + memcpy(engine->sram_pool + sram_off + offset, + miter.addr, len); + else + memcpy_toio(engine->sram + sram_off + offset, + miter.addr, len); + } else { + if (engine->pool) + memcpy(miter.addr, + engine->sram_pool + sram_off + offset, + len); + else + memcpy_fromio(miter.addr, + engine->sram + sram_off + offset, + len); + } + + offset += len; + } + + sg_miter_stop(&miter); + + return offset; +} diff --git a/drivers/crypto/marvell/octeontx2/Makefile b/drivers/crypto/marvell/octeontx2/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..b9c6201019e0c4c42e5f93343b7b3c2e86c00cd3 --- /dev/null +++ b/drivers/crypto/marvell/octeontx2/Makefile @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_CRYPTO_DEV_OCTEONTX2_CPT) += octeontx2-cpt.o octeontx2-cptvf.o + +octeontx2-cpt-objs := otx2_cptpf_main.o otx2_cptpf_mbox.o \ + otx2_cpt_mbox_common.o otx2_cptpf_ucode.o otx2_cptlf.o +octeontx2-cptvf-objs := otx2_cptvf_main.o otx2_cptvf_mbox.o otx2_cptlf.o \ + otx2_cpt_mbox_common.o otx2_cptvf_reqmgr.o \ + otx2_cptvf_algs.o + +ccflags-y += -I$(srctree)/drivers/net/ethernet/marvell/octeontx2/af diff --git a/drivers/crypto/marvell/octeontx2/otx2_cpt_common.h b/drivers/crypto/marvell/octeontx2/otx2_cpt_common.h new file mode 100644 index 0000000000000000000000000000000000000000..3518fac2983468b3b36b9460fae15f0808f32f7c --- /dev/null +++ b/drivers/crypto/marvell/octeontx2/otx2_cpt_common.h @@ -0,0 +1,137 @@ +/* SPDX-License-Identifier: GPL-2.0-only + * Copyright (C) 2020 Marvell. + */ + +#ifndef __OTX2_CPT_COMMON_H +#define __OTX2_CPT_COMMON_H + +#include +#include +#include +#include +#include +#include "otx2_cpt_hw_types.h" +#include "rvu.h" +#include "mbox.h" + +#define OTX2_CPT_MAX_VFS_NUM 128 +#define OTX2_CPT_RVU_FUNC_ADDR_S(blk, slot, offs) \ + (((blk) << 20) | ((slot) << 12) | (offs)) +#define OTX2_CPT_RVU_PFFUNC(pf, func) \ + ((((pf) & RVU_PFVF_PF_MASK) << RVU_PFVF_PF_SHIFT) | \ + (((func) & RVU_PFVF_FUNC_MASK) << RVU_PFVF_FUNC_SHIFT)) + +#define OTX2_CPT_INVALID_CRYPTO_ENG_GRP 0xFF +#define OTX2_CPT_NAME_LENGTH 64 +#define OTX2_CPT_DMA_MINALIGN 128 + +#define BAD_OTX2_CPT_ENG_TYPE OTX2_CPT_MAX_ENG_TYPES + +enum otx2_cpt_eng_type { + OTX2_CPT_AE_TYPES = 1, + OTX2_CPT_SE_TYPES = 2, + OTX2_CPT_IE_TYPES = 3, + OTX2_CPT_MAX_ENG_TYPES, +}; + +/* Take mbox id from end of CPT mbox range in AF (range 0xA00 - 0xBFF) */ +#define MBOX_MSG_GET_ENG_GRP_NUM 0xBFF +#define MBOX_MSG_GET_CAPS 0xBFD +#define MBOX_MSG_GET_KVF_LIMITS 0xBFC + +/* + * Message request and response to get engine group number + * which has attached a given type of engines (SE, AE, IE) + * This messages are only used between CPT PF <=> CPT VF + */ +struct otx2_cpt_egrp_num_msg { + struct mbox_msghdr hdr; + u8 eng_type; +}; + +struct otx2_cpt_egrp_num_rsp { + struct mbox_msghdr hdr; + u8 eng_type; + u8 eng_grp_num; +}; + +/* + * Message request and response to get kernel crypto limits + * This messages are only used between CPT PF <-> CPT VF + */ +struct otx2_cpt_kvf_limits_msg { + struct mbox_msghdr hdr; +}; + +struct otx2_cpt_kvf_limits_rsp { + struct mbox_msghdr hdr; + u8 kvf_limits; +}; + +/* CPT HW capabilities */ +union otx2_cpt_eng_caps { + u64 u; + struct { + u64 reserved_0_4:5; + u64 mul:1; + u64 sha1_sha2:1; + u64 chacha20:1; + u64 zuc_snow3g:1; + u64 sha3:1; + u64 aes:1; + u64 kasumi:1; + u64 des:1; + u64 crc:1; + u64 reserved_14_63:50; + }; +}; + +/* + * Message request and response to get HW capabilities for each + * engine type (SE, IE, AE). + * This messages are only used between CPT PF <=> CPT VF + */ +struct otx2_cpt_caps_msg { + struct mbox_msghdr hdr; +}; + +struct otx2_cpt_caps_rsp { + struct mbox_msghdr hdr; + u16 cpt_pf_drv_version; + u8 cpt_revision; + union otx2_cpt_eng_caps eng_caps[OTX2_CPT_MAX_ENG_TYPES]; +}; + +static inline void otx2_cpt_write64(void __iomem *reg_base, u64 blk, u64 slot, + u64 offs, u64 val) +{ + writeq_relaxed(val, reg_base + + OTX2_CPT_RVU_FUNC_ADDR_S(blk, slot, offs)); +} + +static inline u64 otx2_cpt_read64(void __iomem *reg_base, u64 blk, u64 slot, + u64 offs) +{ + return readq_relaxed(reg_base + + OTX2_CPT_RVU_FUNC_ADDR_S(blk, slot, offs)); +} + +int otx2_cpt_send_ready_msg(struct otx2_mbox *mbox, struct pci_dev *pdev); +int otx2_cpt_send_mbox_msg(struct otx2_mbox *mbox, struct pci_dev *pdev); + +int otx2_cpt_send_af_reg_requests(struct otx2_mbox *mbox, + struct pci_dev *pdev); +int otx2_cpt_add_read_af_reg(struct otx2_mbox *mbox, + struct pci_dev *pdev, u64 reg, u64 *val); +int otx2_cpt_add_write_af_reg(struct otx2_mbox *mbox, struct pci_dev *pdev, + u64 reg, u64 val); +int otx2_cpt_read_af_reg(struct otx2_mbox *mbox, struct pci_dev *pdev, + u64 reg, u64 *val); +int otx2_cpt_write_af_reg(struct otx2_mbox *mbox, struct pci_dev *pdev, + u64 reg, u64 val); +struct otx2_cptlfs_info; +int otx2_cpt_attach_rscrs_msg(struct otx2_cptlfs_info *lfs); +int otx2_cpt_detach_rsrcs_msg(struct otx2_cptlfs_info *lfs); +int otx2_cpt_msix_offset_msg(struct otx2_cptlfs_info *lfs); + +#endif /* __OTX2_CPT_COMMON_H */ diff --git a/drivers/crypto/marvell/octeontx2/otx2_cpt_hw_types.h b/drivers/crypto/marvell/octeontx2/otx2_cpt_hw_types.h new file mode 100644 index 0000000000000000000000000000000000000000..ecafc42f37a26d05f6ed9c81bd39e0886d585ed2 --- /dev/null +++ b/drivers/crypto/marvell/octeontx2/otx2_cpt_hw_types.h @@ -0,0 +1,464 @@ +/* SPDX-License-Identifier: GPL-2.0-only + * Copyright (C) 2020 Marvell. + */ + +#ifndef __OTX2_CPT_HW_TYPES_H +#define __OTX2_CPT_HW_TYPES_H + +#include + +/* Device IDs */ +#define OTX2_CPT_PCI_PF_DEVICE_ID 0xA0FD +#define OTX2_CPT_PCI_VF_DEVICE_ID 0xA0FE + +/* Mailbox interrupts offset */ +#define OTX2_CPT_PF_MBOX_INT 6 +#define OTX2_CPT_PF_INT_VEC_E_MBOXX(x, a) ((x) + (a)) + +/* Maximum supported microcode groups */ +#define OTX2_CPT_MAX_ENGINE_GROUPS 8 + +/* CPT instruction size in bytes */ +#define OTX2_CPT_INST_SIZE 64 +/* + * CPT VF MSIX vectors and their offsets + */ +#define OTX2_CPT_VF_MSIX_VECTORS 1 +#define OTX2_CPT_VF_INTR_MBOX_MASK BIT(0) + +/* CPT LF MSIX vectors */ +#define OTX2_CPT_LF_MSIX_VECTORS 2 + +/* OcteonTX2 CPT PF registers */ +#define OTX2_CPT_PF_CONSTANTS (0x0) +#define OTX2_CPT_PF_RESET (0x100) +#define OTX2_CPT_PF_DIAG (0x120) +#define OTX2_CPT_PF_BIST_STATUS (0x160) +#define OTX2_CPT_PF_ECC0_CTL (0x200) +#define OTX2_CPT_PF_ECC0_FLIP (0x210) +#define OTX2_CPT_PF_ECC0_INT (0x220) +#define OTX2_CPT_PF_ECC0_INT_W1S (0x230) +#define OTX2_CPT_PF_ECC0_ENA_W1S (0x240) +#define OTX2_CPT_PF_ECC0_ENA_W1C (0x250) +#define OTX2_CPT_PF_MBOX_INTX(b) (0x400 | (b) << 3) +#define OTX2_CPT_PF_MBOX_INT_W1SX(b) (0x420 | (b) << 3) +#define OTX2_CPT_PF_MBOX_ENA_W1CX(b) (0x440 | (b) << 3) +#define OTX2_CPT_PF_MBOX_ENA_W1SX(b) (0x460 | (b) << 3) +#define OTX2_CPT_PF_EXEC_INT (0x500) +#define OTX2_CPT_PF_EXEC_INT_W1S (0x520) +#define OTX2_CPT_PF_EXEC_ENA_W1C (0x540) +#define OTX2_CPT_PF_EXEC_ENA_W1S (0x560) +#define OTX2_CPT_PF_GX_EN(b) (0x600 | (b) << 3) +#define OTX2_CPT_PF_EXEC_INFO (0x700) +#define OTX2_CPT_PF_EXEC_BUSY (0x800) +#define OTX2_CPT_PF_EXEC_INFO0 (0x900) +#define OTX2_CPT_PF_EXEC_INFO1 (0x910) +#define OTX2_CPT_PF_INST_REQ_PC (0x10000) +#define OTX2_CPT_PF_INST_LATENCY_PC (0x10020) +#define OTX2_CPT_PF_RD_REQ_PC (0x10040) +#define OTX2_CPT_PF_RD_LATENCY_PC (0x10060) +#define OTX2_CPT_PF_RD_UC_PC (0x10080) +#define OTX2_CPT_PF_ACTIVE_CYCLES_PC (0x10100) +#define OTX2_CPT_PF_EXE_CTL (0x4000000) +#define OTX2_CPT_PF_EXE_STATUS (0x4000008) +#define OTX2_CPT_PF_EXE_CLK (0x4000010) +#define OTX2_CPT_PF_EXE_DBG_CTL (0x4000018) +#define OTX2_CPT_PF_EXE_DBG_DATA (0x4000020) +#define OTX2_CPT_PF_EXE_BIST_STATUS (0x4000028) +#define OTX2_CPT_PF_EXE_REQ_TIMER (0x4000030) +#define OTX2_CPT_PF_EXE_MEM_CTL (0x4000038) +#define OTX2_CPT_PF_EXE_PERF_CTL (0x4001000) +#define OTX2_CPT_PF_EXE_DBG_CNTX(b) (0x4001100 | (b) << 3) +#define OTX2_CPT_PF_EXE_PERF_EVENT_CNT (0x4001180) +#define OTX2_CPT_PF_EXE_EPCI_INBX_CNT(b) (0x4001200 | (b) << 3) +#define OTX2_CPT_PF_EXE_EPCI_OUTBX_CNT(b) (0x4001240 | (b) << 3) +#define OTX2_CPT_PF_ENGX_UCODE_BASE(b) (0x4002000 | (b) << 3) +#define OTX2_CPT_PF_QX_CTL(b) (0x8000000 | (b) << 20) +#define OTX2_CPT_PF_QX_GMCTL(b) (0x8000020 | (b) << 20) +#define OTX2_CPT_PF_QX_CTL2(b) (0x8000100 | (b) << 20) +#define OTX2_CPT_PF_VFX_MBOXX(b, c) (0x8001000 | (b) << 20 | \ + (c) << 8) + +/* OcteonTX2 CPT LF registers */ +#define OTX2_CPT_LF_CTL (0x10) +#define OTX2_CPT_LF_DONE_WAIT (0x30) +#define OTX2_CPT_LF_INPROG (0x40) +#define OTX2_CPT_LF_DONE (0x50) +#define OTX2_CPT_LF_DONE_ACK (0x60) +#define OTX2_CPT_LF_DONE_INT_ENA_W1S (0x90) +#define OTX2_CPT_LF_DONE_INT_ENA_W1C (0xa0) +#define OTX2_CPT_LF_MISC_INT (0xb0) +#define OTX2_CPT_LF_MISC_INT_W1S (0xc0) +#define OTX2_CPT_LF_MISC_INT_ENA_W1S (0xd0) +#define OTX2_CPT_LF_MISC_INT_ENA_W1C (0xe0) +#define OTX2_CPT_LF_Q_BASE (0xf0) +#define OTX2_CPT_LF_Q_SIZE (0x100) +#define OTX2_CPT_LF_Q_INST_PTR (0x110) +#define OTX2_CPT_LF_Q_GRP_PTR (0x120) +#define OTX2_CPT_LF_NQX(a) (0x400 | (a) << 3) +#define OTX2_CPT_RVU_FUNC_BLKADDR_SHIFT 20 +/* LMT LF registers */ +#define OTX2_CPT_LMT_LFBASE BIT_ULL(OTX2_CPT_RVU_FUNC_BLKADDR_SHIFT) +#define OTX2_CPT_LMT_LF_LMTLINEX(a) (OTX2_CPT_LMT_LFBASE | 0x000 | \ + (a) << 12) +/* RVU VF registers */ +#define OTX2_RVU_VF_INT (0x20) +#define OTX2_RVU_VF_INT_W1S (0x28) +#define OTX2_RVU_VF_INT_ENA_W1S (0x30) +#define OTX2_RVU_VF_INT_ENA_W1C (0x38) + +/* + * Enumeration otx2_cpt_ucode_error_code_e + * + * Enumerates ucode errors + */ +enum otx2_cpt_ucode_comp_code_e { + OTX2_CPT_UCC_SUCCESS = 0x00, + OTX2_CPT_UCC_INVALID_OPCODE = 0x01, + + /* Scatter gather */ + OTX2_CPT_UCC_SG_WRITE_LENGTH = 0x02, + OTX2_CPT_UCC_SG_LIST = 0x03, + OTX2_CPT_UCC_SG_NOT_SUPPORTED = 0x04, + +}; + +/* + * Enumeration otx2_cpt_comp_e + * + * OcteonTX2 CPT Completion Enumeration + * Enumerates the values of CPT_RES_S[COMPCODE]. + */ +enum otx2_cpt_comp_e { + OTX2_CPT_COMP_E_NOTDONE = 0x00, + OTX2_CPT_COMP_E_GOOD = 0x01, + OTX2_CPT_COMP_E_FAULT = 0x02, + OTX2_CPT_COMP_E_HWERR = 0x04, + OTX2_CPT_COMP_E_INSTERR = 0x05, + OTX2_CPT_COMP_E_LAST_ENTRY = 0x06 +}; + +/* + * Enumeration otx2_cpt_vf_int_vec_e + * + * OcteonTX2 CPT VF MSI-X Vector Enumeration + * Enumerates the MSI-X interrupt vectors. + */ +enum otx2_cpt_vf_int_vec_e { + OTX2_CPT_VF_INT_VEC_E_MBOX = 0x00 +}; + +/* + * Enumeration otx2_cpt_lf_int_vec_e + * + * OcteonTX2 CPT LF MSI-X Vector Enumeration + * Enumerates the MSI-X interrupt vectors. + */ +enum otx2_cpt_lf_int_vec_e { + OTX2_CPT_LF_INT_VEC_E_MISC = 0x00, + OTX2_CPT_LF_INT_VEC_E_DONE = 0x01 +}; + +/* + * Structure otx2_cpt_inst_s + * + * CPT Instruction Structure + * This structure specifies the instruction layout. Instructions are + * stored in memory as little-endian unless CPT()_PF_Q()_CTL[INST_BE] is set. + * cpt_inst_s_s + * Word 0 + * doneint:1 Done interrupt. + * 0 = No interrupts related to this instruction. + * 1 = When the instruction completes, CPT()_VQ()_DONE[DONE] will be + * incremented,and based on the rules described there an interrupt may + * occur. + * Word 1 + * res_addr [127: 64] Result IOVA. + * If nonzero, specifies where to write CPT_RES_S. + * If zero, no result structure will be written. + * Address must be 16-byte aligned. + * Bits <63:49> are ignored by hardware; software should use a + * sign-extended bit <48> for forward compatibility. + * Word 2 + * grp:10 [171:162] If [WQ_PTR] is nonzero, the SSO guest-group to use when + * CPT submits work SSO. + * For the SSO to not discard the add-work request, FPA_PF_MAP() must map + * [GRP] and CPT()_PF_Q()_GMCTL[GMID] as valid. + * tt:2 [161:160] If [WQ_PTR] is nonzero, the SSO tag type to use when CPT + * submits work to SSO + * tag:32 [159:128] If [WQ_PTR] is nonzero, the SSO tag to use when CPT + * submits work to SSO. + * Word 3 + * wq_ptr [255:192] If [WQ_PTR] is nonzero, it is a pointer to a + * work-queue entry that CPT submits work to SSO after all context, + * output data, and result write operations are visible to other + * CNXXXX units and the cores. Bits <2:0> must be zero. + * Bits <63:49> are ignored by hardware; software should + * use a sign-extended bit <48> for forward compatibility. + * Internal: + * Bits <63:49>, <2:0> are ignored by hardware, treated as always 0x0. + * Word 4 + * ei0; [319:256] Engine instruction word 0. Passed to the AE/SE. + * Word 5 + * ei1; [383:320] Engine instruction word 1. Passed to the AE/SE. + * Word 6 + * ei2; [447:384] Engine instruction word 1. Passed to the AE/SE. + * Word 7 + * ei3; [511:448] Engine instruction word 1. Passed to the AE/SE. + * + */ +union otx2_cpt_inst_s { + u64 u[8]; + + struct { + /* Word 0 */ + u64 nixtxl:3; + u64 doneint:1; + u64 nixtx_addr:60; + /* Word 1 */ + u64 res_addr; + /* Word 2 */ + u64 tag:32; + u64 tt:2; + u64 grp:10; + u64 reserved_172_175:4; + u64 rvu_pf_func:16; + /* Word 3 */ + u64 qord:1; + u64 reserved_194_193:2; + u64 wq_ptr:61; + /* Word 4 */ + u64 ei0; + /* Word 5 */ + u64 ei1; + /* Word 6 */ + u64 ei2; + /* Word 7 */ + u64 ei3; + } s; +}; + +/* + * Structure otx2_cpt_res_s + * + * CPT Result Structure + * The CPT coprocessor writes the result structure after it completes a + * CPT_INST_S instruction. The result structure is exactly 16 bytes, and + * each instruction completion produces exactly one result structure. + * + * This structure is stored in memory as little-endian unless + * CPT()_PF_Q()_CTL[INST_BE] is set. + * cpt_res_s_s + * Word 0 + * doneint:1 [16:16] Done interrupt. This bit is copied from the + * corresponding instruction's CPT_INST_S[DONEINT]. + * compcode:8 [7:0] Indicates completion/error status of the CPT coprocessor + * for the associated instruction, as enumerated by CPT_COMP_E. + * Core software may write the memory location containing [COMPCODE] to + * 0x0 before ringing the doorbell, and then poll for completion by + * checking for a nonzero value. + * Once the core observes a nonzero [COMPCODE] value in this case,the CPT + * coprocessor will have also completed L2/DRAM write operations. + * Word 1 + * reserved + * + */ +union otx2_cpt_res_s { + u64 u[2]; + + struct { + u64 compcode:8; + u64 uc_compcode:8; + u64 doneint:1; + u64 reserved_17_63:47; + u64 reserved_64_127; + } s; +}; + +/* + * Register (RVU_PF_BAR0) cpt#_af_constants1 + * + * CPT AF Constants Register + * This register contains implementation-related parameters of CPT. + */ +union otx2_cptx_af_constants1 { + u64 u; + struct otx2_cptx_af_constants1_s { + u64 se:16; + u64 ie:16; + u64 ae:16; + u64 reserved_48_63:16; + } s; +}; + +/* + * RVU_PFVF_BAR2 - cpt_lf_misc_int + * + * This register contain the per-queue miscellaneous interrupts. + * + */ +union otx2_cptx_lf_misc_int { + u64 u; + struct otx2_cptx_lf_misc_int_s { + u64 reserved_0:1; + u64 nqerr:1; + u64 irde:1; + u64 nwrp:1; + u64 reserved_4:1; + u64 hwerr:1; + u64 fault:1; + u64 reserved_7_63:57; + } s; +}; + +/* + * RVU_PFVF_BAR2 - cpt_lf_misc_int_ena_w1s + * + * This register sets interrupt enable bits. + * + */ +union otx2_cptx_lf_misc_int_ena_w1s { + u64 u; + struct otx2_cptx_lf_misc_int_ena_w1s_s { + u64 reserved_0:1; + u64 nqerr:1; + u64 irde:1; + u64 nwrp:1; + u64 reserved_4:1; + u64 hwerr:1; + u64 fault:1; + u64 reserved_7_63:57; + } s; +}; + +/* + * RVU_PFVF_BAR2 - cpt_lf_ctl + * + * This register configures the queue. + * + * When the queue is not execution-quiescent (see CPT_LF_INPROG[EENA,INFLIGHT]), + * software must only write this register with [ENA]=0. + */ +union otx2_cptx_lf_ctl { + u64 u; + struct otx2_cptx_lf_ctl_s { + u64 ena:1; + u64 fc_ena:1; + u64 fc_up_crossing:1; + u64 reserved_3:1; + u64 fc_hyst_bits:4; + u64 reserved_8_63:56; + } s; +}; + +/* + * RVU_PFVF_BAR2 - cpt_lf_done_wait + * + * This register specifies the per-queue interrupt coalescing settings. + */ +union otx2_cptx_lf_done_wait { + u64 u; + struct otx2_cptx_lf_done_wait_s { + u64 num_wait:20; + u64 reserved_20_31:12; + u64 time_wait:16; + u64 reserved_48_63:16; + } s; +}; + +/* + * RVU_PFVF_BAR2 - cpt_lf_done + * + * This register contain the per-queue instruction done count. + */ +union otx2_cptx_lf_done { + u64 u; + struct otx2_cptx_lf_done_s { + u64 done:20; + u64 reserved_20_63:44; + } s; +}; + +/* + * RVU_PFVF_BAR2 - cpt_lf_inprog + * + * These registers contain the per-queue instruction in flight registers. + * + */ +union otx2_cptx_lf_inprog { + u64 u; + struct otx2_cptx_lf_inprog_s { + u64 inflight:9; + u64 reserved_9_15:7; + u64 eena:1; + u64 grp_drp:1; + u64 reserved_18_30:13; + u64 grb_partial:1; + u64 grb_cnt:8; + u64 gwb_cnt:8; + u64 reserved_48_63:16; + } s; +}; + +/* + * RVU_PFVF_BAR2 - cpt_lf_q_base + * + * CPT initializes these CSR fields to these values on any CPT_LF_Q_BASE write: + * _ CPT_LF_Q_INST_PTR[XQ_XOR]=0. + * _ CPT_LF_Q_INST_PTR[NQ_PTR]=2. + * _ CPT_LF_Q_INST_PTR[DQ_PTR]=2. + * _ CPT_LF_Q_GRP_PTR[XQ_XOR]=0. + * _ CPT_LF_Q_GRP_PTR[NQ_PTR]=1. + * _ CPT_LF_Q_GRP_PTR[DQ_PTR]=1. + */ +union otx2_cptx_lf_q_base { + u64 u; + struct otx2_cptx_lf_q_base_s { + u64 fault:1; + u64 reserved_1_6:6; + u64 addr:46; + u64 reserved_53_63:11; + } s; +}; + +/* + * RVU_PFVF_BAR2 - cpt_lf_q_size + * + * CPT initializes these CSR fields to these values on any CPT_LF_Q_SIZE write: + * _ CPT_LF_Q_INST_PTR[XQ_XOR]=0. + * _ CPT_LF_Q_INST_PTR[NQ_PTR]=2. + * _ CPT_LF_Q_INST_PTR[DQ_PTR]=2. + * _ CPT_LF_Q_GRP_PTR[XQ_XOR]=0. + * _ CPT_LF_Q_GRP_PTR[NQ_PTR]=1. + * _ CPT_LF_Q_GRP_PTR[DQ_PTR]=1. + */ +union otx2_cptx_lf_q_size { + u64 u; + struct otx2_cptx_lf_q_size_s { + u64 size_div40:15; + u64 reserved_15_63:49; + } s; +}; + +/* + * RVU_PF_BAR0 - cpt_af_lf_ctl + * + * This register configures queues. This register should be written only + * when the queue is execution-quiescent (see CPT_LF_INPROG[INFLIGHT]). + */ +union otx2_cptx_af_lf_ctrl { + u64 u; + struct otx2_cptx_af_lf_ctrl_s { + u64 pri:1; + u64 reserved_1_8:8; + u64 pf_func_inst:1; + u64 cont_err:1; + u64 reserved_11_15:5; + u64 nixtx_en:1; + u64 reserved_17_47:31; + u64 grp:8; + u64 reserved_56_63:8; + } s; +}; + +#endif /* __OTX2_CPT_HW_TYPES_H */ diff --git a/drivers/crypto/marvell/octeontx2/otx2_cpt_mbox_common.c b/drivers/crypto/marvell/octeontx2/otx2_cpt_mbox_common.c new file mode 100644 index 0000000000000000000000000000000000000000..51cb6404ded7a33e62e90db928c67adf87c86d8c --- /dev/null +++ b/drivers/crypto/marvell/octeontx2/otx2_cpt_mbox_common.c @@ -0,0 +1,202 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (C) 2020 Marvell. */ + +#include "otx2_cpt_common.h" +#include "otx2_cptlf.h" + +int otx2_cpt_send_mbox_msg(struct otx2_mbox *mbox, struct pci_dev *pdev) +{ + int ret; + + otx2_mbox_msg_send(mbox, 0); + ret = otx2_mbox_wait_for_rsp(mbox, 0); + if (ret == -EIO) { + dev_err(&pdev->dev, "RVU MBOX timeout.\n"); + return ret; + } else if (ret) { + dev_err(&pdev->dev, "RVU MBOX error: %d.\n", ret); + return -EFAULT; + } + return ret; +} + +int otx2_cpt_send_ready_msg(struct otx2_mbox *mbox, struct pci_dev *pdev) +{ + struct mbox_msghdr *req; + + req = otx2_mbox_alloc_msg_rsp(mbox, 0, sizeof(*req), + sizeof(struct ready_msg_rsp)); + if (req == NULL) { + dev_err(&pdev->dev, "RVU MBOX failed to get message.\n"); + return -EFAULT; + } + req->id = MBOX_MSG_READY; + req->sig = OTX2_MBOX_REQ_SIG; + req->pcifunc = 0; + + return otx2_cpt_send_mbox_msg(mbox, pdev); +} + +int otx2_cpt_send_af_reg_requests(struct otx2_mbox *mbox, struct pci_dev *pdev) +{ + return otx2_cpt_send_mbox_msg(mbox, pdev); +} + +int otx2_cpt_add_read_af_reg(struct otx2_mbox *mbox, struct pci_dev *pdev, + u64 reg, u64 *val) +{ + struct cpt_rd_wr_reg_msg *reg_msg; + + reg_msg = (struct cpt_rd_wr_reg_msg *) + otx2_mbox_alloc_msg_rsp(mbox, 0, sizeof(*reg_msg), + sizeof(*reg_msg)); + if (reg_msg == NULL) { + dev_err(&pdev->dev, "RVU MBOX failed to get message.\n"); + return -EFAULT; + } + + reg_msg->hdr.id = MBOX_MSG_CPT_RD_WR_REGISTER; + reg_msg->hdr.sig = OTX2_MBOX_REQ_SIG; + reg_msg->hdr.pcifunc = 0; + + reg_msg->is_write = 0; + reg_msg->reg_offset = reg; + reg_msg->ret_val = val; + + return 0; +} + +int otx2_cpt_add_write_af_reg(struct otx2_mbox *mbox, struct pci_dev *pdev, + u64 reg, u64 val) +{ + struct cpt_rd_wr_reg_msg *reg_msg; + + reg_msg = (struct cpt_rd_wr_reg_msg *) + otx2_mbox_alloc_msg_rsp(mbox, 0, sizeof(*reg_msg), + sizeof(*reg_msg)); + if (reg_msg == NULL) { + dev_err(&pdev->dev, "RVU MBOX failed to get message.\n"); + return -EFAULT; + } + + reg_msg->hdr.id = MBOX_MSG_CPT_RD_WR_REGISTER; + reg_msg->hdr.sig = OTX2_MBOX_REQ_SIG; + reg_msg->hdr.pcifunc = 0; + + reg_msg->is_write = 1; + reg_msg->reg_offset = reg; + reg_msg->val = val; + + return 0; +} + +int otx2_cpt_read_af_reg(struct otx2_mbox *mbox, struct pci_dev *pdev, + u64 reg, u64 *val) +{ + int ret; + + ret = otx2_cpt_add_read_af_reg(mbox, pdev, reg, val); + if (ret) + return ret; + + return otx2_cpt_send_mbox_msg(mbox, pdev); +} + +int otx2_cpt_write_af_reg(struct otx2_mbox *mbox, struct pci_dev *pdev, + u64 reg, u64 val) +{ + int ret; + + ret = otx2_cpt_add_write_af_reg(mbox, pdev, reg, val); + if (ret) + return ret; + + return otx2_cpt_send_mbox_msg(mbox, pdev); +} + +int otx2_cpt_attach_rscrs_msg(struct otx2_cptlfs_info *lfs) +{ + struct otx2_mbox *mbox = lfs->mbox; + struct rsrc_attach *req; + int ret; + + req = (struct rsrc_attach *) + otx2_mbox_alloc_msg_rsp(mbox, 0, sizeof(*req), + sizeof(struct msg_rsp)); + if (req == NULL) { + dev_err(&lfs->pdev->dev, "RVU MBOX failed to get message.\n"); + return -EFAULT; + } + + req->hdr.id = MBOX_MSG_ATTACH_RESOURCES; + req->hdr.sig = OTX2_MBOX_REQ_SIG; + req->hdr.pcifunc = 0; + req->cptlfs = lfs->lfs_num; + ret = otx2_cpt_send_mbox_msg(mbox, lfs->pdev); + if (ret) + return ret; + + if (!lfs->are_lfs_attached) + ret = -EINVAL; + + return ret; +} + +int otx2_cpt_detach_rsrcs_msg(struct otx2_cptlfs_info *lfs) +{ + struct otx2_mbox *mbox = lfs->mbox; + struct rsrc_detach *req; + int ret; + + req = (struct rsrc_detach *) + otx2_mbox_alloc_msg_rsp(mbox, 0, sizeof(*req), + sizeof(struct msg_rsp)); + if (req == NULL) { + dev_err(&lfs->pdev->dev, "RVU MBOX failed to get message.\n"); + return -EFAULT; + } + + req->hdr.id = MBOX_MSG_DETACH_RESOURCES; + req->hdr.sig = OTX2_MBOX_REQ_SIG; + req->hdr.pcifunc = 0; + ret = otx2_cpt_send_mbox_msg(mbox, lfs->pdev); + if (ret) + return ret; + + if (lfs->are_lfs_attached) + ret = -EINVAL; + + return ret; +} + +int otx2_cpt_msix_offset_msg(struct otx2_cptlfs_info *lfs) +{ + struct otx2_mbox *mbox = lfs->mbox; + struct pci_dev *pdev = lfs->pdev; + struct mbox_msghdr *req; + int ret, i; + + req = otx2_mbox_alloc_msg_rsp(mbox, 0, sizeof(*req), + sizeof(struct msix_offset_rsp)); + if (req == NULL) { + dev_err(&pdev->dev, "RVU MBOX failed to get message.\n"); + return -EFAULT; + } + + req->id = MBOX_MSG_MSIX_OFFSET; + req->sig = OTX2_MBOX_REQ_SIG; + req->pcifunc = 0; + ret = otx2_cpt_send_mbox_msg(mbox, pdev); + if (ret) + return ret; + + for (i = 0; i < lfs->lfs_num; i++) { + if (lfs->lf[i].msix_offset == MSIX_VECTOR_INVALID) { + dev_err(&pdev->dev, + "Invalid msix offset %d for LF %d\n", + lfs->lf[i].msix_offset, i); + return -EINVAL; + } + } + return ret; +} diff --git a/drivers/crypto/marvell/octeontx2/otx2_cpt_reqmgr.h b/drivers/crypto/marvell/octeontx2/otx2_cpt_reqmgr.h new file mode 100644 index 0000000000000000000000000000000000000000..dbb1ee746f4cb061fd5c56c2b7794dea9d12b84b --- /dev/null +++ b/drivers/crypto/marvell/octeontx2/otx2_cpt_reqmgr.h @@ -0,0 +1,197 @@ +/* SPDX-License-Identifier: GPL-2.0-only + * Copyright (C) 2020 Marvell. + */ + +#ifndef __OTX2_CPT_REQMGR_H +#define __OTX2_CPT_REQMGR_H + +#include "otx2_cpt_common.h" + +/* Completion code size and initial value */ +#define OTX2_CPT_COMPLETION_CODE_SIZE 8 +#define OTX2_CPT_COMPLETION_CODE_INIT OTX2_CPT_COMP_E_NOTDONE +/* + * Maximum total number of SG buffers is 100, we divide it equally + * between input and output + */ +#define OTX2_CPT_MAX_SG_IN_CNT 50 +#define OTX2_CPT_MAX_SG_OUT_CNT 50 + +/* DMA mode direct or SG */ +#define OTX2_CPT_DMA_MODE_DIRECT 0 +#define OTX2_CPT_DMA_MODE_SG 1 + +/* Context source CPTR or DPTR */ +#define OTX2_CPT_FROM_CPTR 0 +#define OTX2_CPT_FROM_DPTR 1 + +#define OTX2_CPT_MAX_REQ_SIZE 65535 + +union otx2_cpt_opcode { + u16 flags; + struct { + u8 major; + u8 minor; + } s; +}; + +struct otx2_cptvf_request { + u32 param1; + u32 param2; + u16 dlen; + union otx2_cpt_opcode opcode; +}; + +/* + * CPT_INST_S software command definitions + * Words EI (0-3) + */ +union otx2_cpt_iq_cmd_word0 { + u64 u; + struct { + __be16 opcode; + __be16 param1; + __be16 param2; + __be16 dlen; + } s; +}; + +union otx2_cpt_iq_cmd_word3 { + u64 u; + struct { + u64 cptr:61; + u64 grp:3; + } s; +}; + +struct otx2_cpt_iq_command { + union otx2_cpt_iq_cmd_word0 cmd; + u64 dptr; + u64 rptr; + union otx2_cpt_iq_cmd_word3 cptr; +}; + +struct otx2_cpt_pending_entry { + void *completion_addr; /* Completion address */ + void *info; + /* Kernel async request callback */ + void (*callback)(int status, void *arg1, void *arg2); + struct crypto_async_request *areq; /* Async request callback arg */ + u8 resume_sender; /* Notify sender to resume sending requests */ + u8 busy; /* Entry status (free/busy) */ +}; + +struct otx2_cpt_pending_queue { + struct otx2_cpt_pending_entry *head; /* Head of the queue */ + u32 front; /* Process work from here */ + u32 rear; /* Append new work here */ + u32 pending_count; /* Pending requests count */ + u32 qlen; /* Queue length */ + spinlock_t lock; /* Queue lock */ +}; + +struct otx2_cpt_buf_ptr { + u8 *vptr; + dma_addr_t dma_addr; + u16 size; +}; + +union otx2_cpt_ctrl_info { + u32 flags; + struct { +#if defined(__BIG_ENDIAN_BITFIELD) + u32 reserved_6_31:26; + u32 grp:3; /* Group bits */ + u32 dma_mode:2; /* DMA mode */ + u32 se_req:1; /* To SE core */ +#else + u32 se_req:1; /* To SE core */ + u32 dma_mode:2; /* DMA mode */ + u32 grp:3; /* Group bits */ + u32 reserved_6_31:26; +#endif + } s; +}; + +struct otx2_cpt_req_info { + /* Kernel async request callback */ + void (*callback)(int status, void *arg1, void *arg2); + struct crypto_async_request *areq; /* Async request callback arg */ + struct otx2_cptvf_request req;/* Request information (core specific) */ + union otx2_cpt_ctrl_info ctrl;/* User control information */ + struct otx2_cpt_buf_ptr in[OTX2_CPT_MAX_SG_IN_CNT]; + struct otx2_cpt_buf_ptr out[OTX2_CPT_MAX_SG_OUT_CNT]; + u8 *iv_out; /* IV to send back */ + u16 rlen; /* Output length */ + u8 in_cnt; /* Number of input buffers */ + u8 out_cnt; /* Number of output buffers */ + u8 req_type; /* Type of request */ + u8 is_enc; /* Is a request an encryption request */ + u8 is_trunc_hmac;/* Is truncated hmac used */ +}; + +struct otx2_cpt_inst_info { + struct otx2_cpt_pending_entry *pentry; + struct otx2_cpt_req_info *req; + struct pci_dev *pdev; + void *completion_addr; + u8 *out_buffer; + u8 *in_buffer; + dma_addr_t dptr_baddr; + dma_addr_t rptr_baddr; + dma_addr_t comp_baddr; + unsigned long time_in; + u32 dlen; + u32 dma_len; + u8 extra_time; +}; + +struct otx2_cpt_sglist_component { + __be16 len0; + __be16 len1; + __be16 len2; + __be16 len3; + __be64 ptr0; + __be64 ptr1; + __be64 ptr2; + __be64 ptr3; +}; + +static inline void otx2_cpt_info_destroy(struct pci_dev *pdev, + struct otx2_cpt_inst_info *info) +{ + struct otx2_cpt_req_info *req; + int i; + + if (info->dptr_baddr) + dma_unmap_single(&pdev->dev, info->dptr_baddr, + info->dma_len, DMA_BIDIRECTIONAL); + + if (info->req) { + req = info->req; + for (i = 0; i < req->out_cnt; i++) { + if (req->out[i].dma_addr) + dma_unmap_single(&pdev->dev, + req->out[i].dma_addr, + req->out[i].size, + DMA_BIDIRECTIONAL); + } + + for (i = 0; i < req->in_cnt; i++) { + if (req->in[i].dma_addr) + dma_unmap_single(&pdev->dev, + req->in[i].dma_addr, + req->in[i].size, + DMA_BIDIRECTIONAL); + } + } + kfree(info); +} + +struct otx2_cptlf_wqe; +int otx2_cpt_do_request(struct pci_dev *pdev, struct otx2_cpt_req_info *req, + int cpu_num); +void otx2_cpt_post_process(struct otx2_cptlf_wqe *wqe); +int otx2_cpt_get_kcrypto_eng_grp_num(struct pci_dev *pdev); + +#endif /* __OTX2_CPT_REQMGR_H */ diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptlf.c b/drivers/crypto/marvell/octeontx2/otx2_cptlf.c new file mode 100644 index 0000000000000000000000000000000000000000..823a4571fd674577a864a472af3c439ba0e907ed --- /dev/null +++ b/drivers/crypto/marvell/octeontx2/otx2_cptlf.c @@ -0,0 +1,428 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (C) 2020 Marvell. */ + +#include "otx2_cpt_common.h" +#include "otx2_cptlf.h" +#include "rvu_reg.h" + +#define CPT_TIMER_HOLD 0x03F +#define CPT_COUNT_HOLD 32 + +static void cptlf_do_set_done_time_wait(struct otx2_cptlf_info *lf, + int time_wait) +{ + union otx2_cptx_lf_done_wait done_wait; + + done_wait.u = otx2_cpt_read64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot, + OTX2_CPT_LF_DONE_WAIT); + done_wait.s.time_wait = time_wait; + otx2_cpt_write64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot, + OTX2_CPT_LF_DONE_WAIT, done_wait.u); +} + +static void cptlf_do_set_done_num_wait(struct otx2_cptlf_info *lf, int num_wait) +{ + union otx2_cptx_lf_done_wait done_wait; + + done_wait.u = otx2_cpt_read64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot, + OTX2_CPT_LF_DONE_WAIT); + done_wait.s.num_wait = num_wait; + otx2_cpt_write64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot, + OTX2_CPT_LF_DONE_WAIT, done_wait.u); +} + +static void cptlf_set_done_time_wait(struct otx2_cptlfs_info *lfs, + int time_wait) +{ + int slot; + + for (slot = 0; slot < lfs->lfs_num; slot++) + cptlf_do_set_done_time_wait(&lfs->lf[slot], time_wait); +} + +static void cptlf_set_done_num_wait(struct otx2_cptlfs_info *lfs, int num_wait) +{ + int slot; + + for (slot = 0; slot < lfs->lfs_num; slot++) + cptlf_do_set_done_num_wait(&lfs->lf[slot], num_wait); +} + +static int cptlf_set_pri(struct otx2_cptlf_info *lf, int pri) +{ + struct otx2_cptlfs_info *lfs = lf->lfs; + union otx2_cptx_af_lf_ctrl lf_ctrl; + int ret; + + ret = otx2_cpt_read_af_reg(lfs->mbox, lfs->pdev, + CPT_AF_LFX_CTL(lf->slot), + &lf_ctrl.u); + if (ret) + return ret; + + lf_ctrl.s.pri = pri ? 1 : 0; + + ret = otx2_cpt_write_af_reg(lfs->mbox, lfs->pdev, + CPT_AF_LFX_CTL(lf->slot), + lf_ctrl.u); + return ret; +} + +static int cptlf_set_eng_grps_mask(struct otx2_cptlf_info *lf, + int eng_grps_mask) +{ + struct otx2_cptlfs_info *lfs = lf->lfs; + union otx2_cptx_af_lf_ctrl lf_ctrl; + int ret; + + ret = otx2_cpt_read_af_reg(lfs->mbox, lfs->pdev, + CPT_AF_LFX_CTL(lf->slot), + &lf_ctrl.u); + if (ret) + return ret; + + lf_ctrl.s.grp = eng_grps_mask; + + ret = otx2_cpt_write_af_reg(lfs->mbox, lfs->pdev, + CPT_AF_LFX_CTL(lf->slot), + lf_ctrl.u); + return ret; +} + +static int cptlf_set_grp_and_pri(struct otx2_cptlfs_info *lfs, + int eng_grp_mask, int pri) +{ + int slot, ret = 0; + + for (slot = 0; slot < lfs->lfs_num; slot++) { + ret = cptlf_set_pri(&lfs->lf[slot], pri); + if (ret) + return ret; + + ret = cptlf_set_eng_grps_mask(&lfs->lf[slot], eng_grp_mask); + if (ret) + return ret; + } + return ret; +} + +static void cptlf_hw_init(struct otx2_cptlfs_info *lfs) +{ + /* Disable instruction queues */ + otx2_cptlf_disable_iqueues(lfs); + + /* Set instruction queues base addresses */ + otx2_cptlf_set_iqueues_base_addr(lfs); + + /* Set instruction queues sizes */ + otx2_cptlf_set_iqueues_size(lfs); + + /* Set done interrupts time wait */ + cptlf_set_done_time_wait(lfs, CPT_TIMER_HOLD); + + /* Set done interrupts num wait */ + cptlf_set_done_num_wait(lfs, CPT_COUNT_HOLD); + + /* Enable instruction queues */ + otx2_cptlf_enable_iqueues(lfs); +} + +static void cptlf_hw_cleanup(struct otx2_cptlfs_info *lfs) +{ + /* Disable instruction queues */ + otx2_cptlf_disable_iqueues(lfs); +} + +static void cptlf_set_misc_intrs(struct otx2_cptlfs_info *lfs, u8 enable) +{ + union otx2_cptx_lf_misc_int_ena_w1s irq_misc = { .u = 0x0 }; + u64 reg = enable ? OTX2_CPT_LF_MISC_INT_ENA_W1S : + OTX2_CPT_LF_MISC_INT_ENA_W1C; + int slot; + + irq_misc.s.fault = 0x1; + irq_misc.s.hwerr = 0x1; + irq_misc.s.irde = 0x1; + irq_misc.s.nqerr = 0x1; + irq_misc.s.nwrp = 0x1; + + for (slot = 0; slot < lfs->lfs_num; slot++) + otx2_cpt_write64(lfs->reg_base, BLKADDR_CPT0, slot, reg, + irq_misc.u); +} + +static void cptlf_enable_intrs(struct otx2_cptlfs_info *lfs) +{ + int slot; + + /* Enable done interrupts */ + for (slot = 0; slot < lfs->lfs_num; slot++) + otx2_cpt_write64(lfs->reg_base, BLKADDR_CPT0, slot, + OTX2_CPT_LF_DONE_INT_ENA_W1S, 0x1); + /* Enable Misc interrupts */ + cptlf_set_misc_intrs(lfs, true); +} + +static void cptlf_disable_intrs(struct otx2_cptlfs_info *lfs) +{ + int slot; + + for (slot = 0; slot < lfs->lfs_num; slot++) + otx2_cpt_write64(lfs->reg_base, BLKADDR_CPT0, slot, + OTX2_CPT_LF_DONE_INT_ENA_W1C, 0x1); + cptlf_set_misc_intrs(lfs, false); +} + +static inline int cptlf_read_done_cnt(struct otx2_cptlf_info *lf) +{ + union otx2_cptx_lf_done irq_cnt; + + irq_cnt.u = otx2_cpt_read64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot, + OTX2_CPT_LF_DONE); + return irq_cnt.s.done; +} + +static irqreturn_t cptlf_misc_intr_handler(int __always_unused irq, void *arg) +{ + union otx2_cptx_lf_misc_int irq_misc, irq_misc_ack; + struct otx2_cptlf_info *lf = arg; + struct device *dev; + + dev = &lf->lfs->pdev->dev; + irq_misc.u = otx2_cpt_read64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot, + OTX2_CPT_LF_MISC_INT); + irq_misc_ack.u = 0x0; + + if (irq_misc.s.fault) { + dev_err(dev, "Memory error detected while executing CPT_INST_S, LF %d.\n", + lf->slot); + irq_misc_ack.s.fault = 0x1; + + } else if (irq_misc.s.hwerr) { + dev_err(dev, "HW error from an engine executing CPT_INST_S, LF %d.", + lf->slot); + irq_misc_ack.s.hwerr = 0x1; + + } else if (irq_misc.s.nwrp) { + dev_err(dev, "SMMU fault while writing CPT_RES_S to CPT_INST_S[RES_ADDR], LF %d.\n", + lf->slot); + irq_misc_ack.s.nwrp = 0x1; + + } else if (irq_misc.s.irde) { + dev_err(dev, "Memory error when accessing instruction memory queue CPT_LF_Q_BASE[ADDR].\n"); + irq_misc_ack.s.irde = 0x1; + + } else if (irq_misc.s.nqerr) { + dev_err(dev, "Error enqueuing an instruction received at CPT_LF_NQ.\n"); + irq_misc_ack.s.nqerr = 0x1; + + } else { + dev_err(dev, "Unhandled interrupt in CPT LF %d\n", lf->slot); + return IRQ_NONE; + } + + /* Acknowledge interrupts */ + otx2_cpt_write64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot, + OTX2_CPT_LF_MISC_INT, irq_misc_ack.u); + + return IRQ_HANDLED; +} + +static irqreturn_t cptlf_done_intr_handler(int irq, void *arg) +{ + union otx2_cptx_lf_done_wait done_wait; + struct otx2_cptlf_info *lf = arg; + int irq_cnt; + + /* Read the number of completed requests */ + irq_cnt = cptlf_read_done_cnt(lf); + if (irq_cnt) { + done_wait.u = otx2_cpt_read64(lf->lfs->reg_base, BLKADDR_CPT0, + lf->slot, OTX2_CPT_LF_DONE_WAIT); + /* Acknowledge the number of completed requests */ + otx2_cpt_write64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot, + OTX2_CPT_LF_DONE_ACK, irq_cnt); + + otx2_cpt_write64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot, + OTX2_CPT_LF_DONE_WAIT, done_wait.u); + if (unlikely(!lf->wqe)) { + dev_err(&lf->lfs->pdev->dev, "No work for LF %d\n", + lf->slot); + return IRQ_NONE; + } + + /* Schedule processing of completed requests */ + tasklet_hi_schedule(&lf->wqe->work); + } + return IRQ_HANDLED; +} + +void otx2_cptlf_unregister_interrupts(struct otx2_cptlfs_info *lfs) +{ + int i, offs, vector; + + for (i = 0; i < lfs->lfs_num; i++) { + for (offs = 0; offs < OTX2_CPT_LF_MSIX_VECTORS; offs++) { + if (!lfs->lf[i].is_irq_reg[offs]) + continue; + + vector = pci_irq_vector(lfs->pdev, + lfs->lf[i].msix_offset + offs); + free_irq(vector, &lfs->lf[i]); + lfs->lf[i].is_irq_reg[offs] = false; + } + } + cptlf_disable_intrs(lfs); +} + +static int cptlf_do_register_interrrupts(struct otx2_cptlfs_info *lfs, + int lf_num, int irq_offset, + irq_handler_t handler) +{ + int ret, vector; + + vector = pci_irq_vector(lfs->pdev, lfs->lf[lf_num].msix_offset + + irq_offset); + ret = request_irq(vector, handler, 0, + lfs->lf[lf_num].irq_name[irq_offset], + &lfs->lf[lf_num]); + if (ret) + return ret; + + lfs->lf[lf_num].is_irq_reg[irq_offset] = true; + + return ret; +} + +int otx2_cptlf_register_interrupts(struct otx2_cptlfs_info *lfs) +{ + int irq_offs, ret, i; + + for (i = 0; i < lfs->lfs_num; i++) { + irq_offs = OTX2_CPT_LF_INT_VEC_E_MISC; + snprintf(lfs->lf[i].irq_name[irq_offs], 32, "CPTLF Misc%d", i); + ret = cptlf_do_register_interrrupts(lfs, i, irq_offs, + cptlf_misc_intr_handler); + if (ret) + goto free_irq; + + irq_offs = OTX2_CPT_LF_INT_VEC_E_DONE; + snprintf(lfs->lf[i].irq_name[irq_offs], 32, "OTX2_CPTLF Done%d", + i); + ret = cptlf_do_register_interrrupts(lfs, i, irq_offs, + cptlf_done_intr_handler); + if (ret) + goto free_irq; + } + cptlf_enable_intrs(lfs); + return 0; + +free_irq: + otx2_cptlf_unregister_interrupts(lfs); + return ret; +} + +void otx2_cptlf_free_irqs_affinity(struct otx2_cptlfs_info *lfs) +{ + int slot, offs; + + for (slot = 0; slot < lfs->lfs_num; slot++) { + for (offs = 0; offs < OTX2_CPT_LF_MSIX_VECTORS; offs++) + irq_set_affinity_hint(pci_irq_vector(lfs->pdev, + lfs->lf[slot].msix_offset + + offs), NULL); + free_cpumask_var(lfs->lf[slot].affinity_mask); + } +} + +int otx2_cptlf_set_irqs_affinity(struct otx2_cptlfs_info *lfs) +{ + struct otx2_cptlf_info *lf = lfs->lf; + int slot, offs, ret; + + for (slot = 0; slot < lfs->lfs_num; slot++) { + if (!zalloc_cpumask_var(&lf[slot].affinity_mask, GFP_KERNEL)) { + dev_err(&lfs->pdev->dev, + "cpumask allocation failed for LF %d", slot); + ret = -ENOMEM; + goto free_affinity_mask; + } + + cpumask_set_cpu(cpumask_local_spread(slot, + dev_to_node(&lfs->pdev->dev)), + lf[slot].affinity_mask); + + for (offs = 0; offs < OTX2_CPT_LF_MSIX_VECTORS; offs++) { + ret = irq_set_affinity_hint(pci_irq_vector(lfs->pdev, + lf[slot].msix_offset + offs), + lf[slot].affinity_mask); + if (ret) + goto free_affinity_mask; + } + } + return 0; + +free_affinity_mask: + otx2_cptlf_free_irqs_affinity(lfs); + return ret; +} + +int otx2_cptlf_init(struct otx2_cptlfs_info *lfs, u8 eng_grp_mask, int pri, + int lfs_num) +{ + int slot, ret; + + if (!lfs->pdev || !lfs->reg_base) + return -EINVAL; + + lfs->lfs_num = lfs_num; + for (slot = 0; slot < lfs->lfs_num; slot++) { + lfs->lf[slot].lfs = lfs; + lfs->lf[slot].slot = slot; + lfs->lf[slot].lmtline = lfs->reg_base + + OTX2_CPT_RVU_FUNC_ADDR_S(BLKADDR_LMT, slot, + OTX2_CPT_LMT_LF_LMTLINEX(0)); + lfs->lf[slot].ioreg = lfs->reg_base + + OTX2_CPT_RVU_FUNC_ADDR_S(BLKADDR_CPT0, slot, + OTX2_CPT_LF_NQX(0)); + } + /* Send request to attach LFs */ + ret = otx2_cpt_attach_rscrs_msg(lfs); + if (ret) + goto clear_lfs_num; + + ret = otx2_cpt_alloc_instruction_queues(lfs); + if (ret) { + dev_err(&lfs->pdev->dev, + "Allocating instruction queues failed\n"); + goto detach_rsrcs; + } + cptlf_hw_init(lfs); + /* + * Allow each LF to execute requests destined to any of 8 engine + * groups and set queue priority of each LF to high + */ + ret = cptlf_set_grp_and_pri(lfs, eng_grp_mask, pri); + if (ret) + goto free_iq; + + return 0; + +free_iq: + otx2_cpt_free_instruction_queues(lfs); + cptlf_hw_cleanup(lfs); +detach_rsrcs: + otx2_cpt_detach_rsrcs_msg(lfs); +clear_lfs_num: + lfs->lfs_num = 0; + return ret; +} + +void otx2_cptlf_shutdown(struct otx2_cptlfs_info *lfs) +{ + lfs->lfs_num = 0; + /* Cleanup LFs hardware side */ + cptlf_hw_cleanup(lfs); + /* Send request to detach LFs */ + otx2_cpt_detach_rsrcs_msg(lfs); +} diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptlf.h b/drivers/crypto/marvell/octeontx2/otx2_cptlf.h new file mode 100644 index 0000000000000000000000000000000000000000..314e973541004fe43b41ce7d7b039df32decb0b6 --- /dev/null +++ b/drivers/crypto/marvell/octeontx2/otx2_cptlf.h @@ -0,0 +1,353 @@ +/* SPDX-License-Identifier: GPL-2.0-only + * Copyright (C) 2020 Marvell. + */ +#ifndef __OTX2_CPTLF_H +#define __OTX2_CPTLF_H + +#include +#include +#include +#include "otx2_cpt_common.h" +#include "otx2_cpt_reqmgr.h" + +/* + * CPT instruction and pending queues user requested length in CPT_INST_S msgs + */ +#define OTX2_CPT_USER_REQUESTED_QLEN_MSGS 8200 + +/* + * CPT instruction queue size passed to HW is in units of 40*CPT_INST_S + * messages. + */ +#define OTX2_CPT_SIZE_DIV40 (OTX2_CPT_USER_REQUESTED_QLEN_MSGS/40) + +/* + * CPT instruction and pending queues length in CPT_INST_S messages + */ +#define OTX2_CPT_INST_QLEN_MSGS ((OTX2_CPT_SIZE_DIV40 - 1) * 40) + +/* CPT instruction queue length in bytes */ +#define OTX2_CPT_INST_QLEN_BYTES (OTX2_CPT_SIZE_DIV40 * 40 * \ + OTX2_CPT_INST_SIZE) + +/* CPT instruction group queue length in bytes */ +#define OTX2_CPT_INST_GRP_QLEN_BYTES (OTX2_CPT_SIZE_DIV40 * 16) + +/* CPT FC length in bytes */ +#define OTX2_CPT_Q_FC_LEN 128 + +/* CPT instruction queue alignment */ +#define OTX2_CPT_INST_Q_ALIGNMENT 128 + +/* Mask which selects all engine groups */ +#define OTX2_CPT_ALL_ENG_GRPS_MASK 0xFF + +/* Maximum LFs supported in OcteonTX2 for CPT */ +#define OTX2_CPT_MAX_LFS_NUM 64 + +/* Queue priority */ +#define OTX2_CPT_QUEUE_HI_PRIO 0x1 +#define OTX2_CPT_QUEUE_LOW_PRIO 0x0 + +enum otx2_cptlf_state { + OTX2_CPTLF_IN_RESET, + OTX2_CPTLF_STARTED, +}; + +struct otx2_cpt_inst_queue { + u8 *vaddr; + u8 *real_vaddr; + dma_addr_t dma_addr; + dma_addr_t real_dma_addr; + u32 size; +}; + +struct otx2_cptlfs_info; +struct otx2_cptlf_wqe { + struct tasklet_struct work; + struct otx2_cptlfs_info *lfs; + u8 lf_num; +}; + +struct otx2_cptlf_info { + struct otx2_cptlfs_info *lfs; /* Ptr to cptlfs_info struct */ + void __iomem *lmtline; /* Address of LMTLINE */ + void __iomem *ioreg; /* LMTLINE send register */ + int msix_offset; /* MSI-X interrupts offset */ + cpumask_var_t affinity_mask; /* IRQs affinity mask */ + u8 irq_name[OTX2_CPT_LF_MSIX_VECTORS][32];/* Interrupts name */ + u8 is_irq_reg[OTX2_CPT_LF_MSIX_VECTORS]; /* Is interrupt registered */ + u8 slot; /* Slot number of this LF */ + + struct otx2_cpt_inst_queue iqueue;/* Instruction queue */ + struct otx2_cpt_pending_queue pqueue; /* Pending queue */ + struct otx2_cptlf_wqe *wqe; /* Tasklet work info */ +}; + +struct otx2_cptlfs_info { + /* Registers start address of VF/PF LFs are attached to */ + void __iomem *reg_base; + struct pci_dev *pdev; /* Device LFs are attached to */ + struct otx2_cptlf_info lf[OTX2_CPT_MAX_LFS_NUM]; + struct otx2_mbox *mbox; + u8 are_lfs_attached; /* Whether CPT LFs are attached */ + u8 lfs_num; /* Number of CPT LFs */ + u8 kcrypto_eng_grp_num; /* Kernel crypto engine group number */ + u8 kvf_limits; /* Kernel crypto limits */ + atomic_t state; /* LF's state. started/reset */ +}; + +static inline void otx2_cpt_free_instruction_queues( + struct otx2_cptlfs_info *lfs) +{ + struct otx2_cpt_inst_queue *iq; + int i; + + for (i = 0; i < lfs->lfs_num; i++) { + iq = &lfs->lf[i].iqueue; + if (iq->real_vaddr) + dma_free_coherent(&lfs->pdev->dev, + iq->size, + iq->real_vaddr, + iq->real_dma_addr); + iq->real_vaddr = NULL; + iq->vaddr = NULL; + } +} + +static inline int otx2_cpt_alloc_instruction_queues( + struct otx2_cptlfs_info *lfs) +{ + struct otx2_cpt_inst_queue *iq; + int ret = 0, i; + + if (!lfs->lfs_num) + return -EINVAL; + + for (i = 0; i < lfs->lfs_num; i++) { + iq = &lfs->lf[i].iqueue; + iq->size = OTX2_CPT_INST_QLEN_BYTES + + OTX2_CPT_Q_FC_LEN + + OTX2_CPT_INST_GRP_QLEN_BYTES + + OTX2_CPT_INST_Q_ALIGNMENT; + iq->real_vaddr = dma_alloc_coherent(&lfs->pdev->dev, iq->size, + &iq->real_dma_addr, GFP_KERNEL); + if (!iq->real_vaddr) { + ret = -ENOMEM; + goto error; + } + iq->vaddr = iq->real_vaddr + OTX2_CPT_INST_GRP_QLEN_BYTES; + iq->dma_addr = iq->real_dma_addr + OTX2_CPT_INST_GRP_QLEN_BYTES; + + /* Align pointers */ + iq->vaddr = PTR_ALIGN(iq->vaddr, OTX2_CPT_INST_Q_ALIGNMENT); + iq->dma_addr = PTR_ALIGN(iq->dma_addr, + OTX2_CPT_INST_Q_ALIGNMENT); + } + return 0; + +error: + otx2_cpt_free_instruction_queues(lfs); + return ret; +} + +static inline void otx2_cptlf_set_iqueues_base_addr( + struct otx2_cptlfs_info *lfs) +{ + union otx2_cptx_lf_q_base lf_q_base; + int slot; + + for (slot = 0; slot < lfs->lfs_num; slot++) { + lf_q_base.u = lfs->lf[slot].iqueue.dma_addr; + otx2_cpt_write64(lfs->reg_base, BLKADDR_CPT0, slot, + OTX2_CPT_LF_Q_BASE, lf_q_base.u); + } +} + +static inline void otx2_cptlf_do_set_iqueue_size(struct otx2_cptlf_info *lf) +{ + union otx2_cptx_lf_q_size lf_q_size = { .u = 0x0 }; + + lf_q_size.s.size_div40 = OTX2_CPT_SIZE_DIV40; + otx2_cpt_write64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot, + OTX2_CPT_LF_Q_SIZE, lf_q_size.u); +} + +static inline void otx2_cptlf_set_iqueues_size(struct otx2_cptlfs_info *lfs) +{ + int slot; + + for (slot = 0; slot < lfs->lfs_num; slot++) + otx2_cptlf_do_set_iqueue_size(&lfs->lf[slot]); +} + +static inline void otx2_cptlf_do_disable_iqueue(struct otx2_cptlf_info *lf) +{ + union otx2_cptx_lf_ctl lf_ctl = { .u = 0x0 }; + union otx2_cptx_lf_inprog lf_inprog; + int timeout = 20; + + /* Disable instructions enqueuing */ + otx2_cpt_write64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot, + OTX2_CPT_LF_CTL, lf_ctl.u); + + /* Wait for instruction queue to become empty */ + do { + lf_inprog.u = otx2_cpt_read64(lf->lfs->reg_base, BLKADDR_CPT0, + lf->slot, OTX2_CPT_LF_INPROG); + if (!lf_inprog.s.inflight) + break; + + usleep_range(10000, 20000); + if (timeout-- < 0) { + dev_err(&lf->lfs->pdev->dev, + "Error LF %d is still busy.\n", lf->slot); + break; + } + + } while (1); + + /* + * Disable executions in the LF's queue, + * the queue should be empty at this point + */ + lf_inprog.s.eena = 0x0; + otx2_cpt_write64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot, + OTX2_CPT_LF_INPROG, lf_inprog.u); +} + +static inline void otx2_cptlf_disable_iqueues(struct otx2_cptlfs_info *lfs) +{ + int slot; + + for (slot = 0; slot < lfs->lfs_num; slot++) + otx2_cptlf_do_disable_iqueue(&lfs->lf[slot]); +} + +static inline void otx2_cptlf_set_iqueue_enq(struct otx2_cptlf_info *lf, + bool enable) +{ + union otx2_cptx_lf_ctl lf_ctl; + + lf_ctl.u = otx2_cpt_read64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot, + OTX2_CPT_LF_CTL); + + /* Set iqueue's enqueuing */ + lf_ctl.s.ena = enable ? 0x1 : 0x0; + otx2_cpt_write64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot, + OTX2_CPT_LF_CTL, lf_ctl.u); +} + +static inline void otx2_cptlf_enable_iqueue_enq(struct otx2_cptlf_info *lf) +{ + otx2_cptlf_set_iqueue_enq(lf, true); +} + +static inline void otx2_cptlf_set_iqueue_exec(struct otx2_cptlf_info *lf, + bool enable) +{ + union otx2_cptx_lf_inprog lf_inprog; + + lf_inprog.u = otx2_cpt_read64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot, + OTX2_CPT_LF_INPROG); + + /* Set iqueue's execution */ + lf_inprog.s.eena = enable ? 0x1 : 0x0; + otx2_cpt_write64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot, + OTX2_CPT_LF_INPROG, lf_inprog.u); +} + +static inline void otx2_cptlf_enable_iqueue_exec(struct otx2_cptlf_info *lf) +{ + otx2_cptlf_set_iqueue_exec(lf, true); +} + +static inline void otx2_cptlf_disable_iqueue_exec(struct otx2_cptlf_info *lf) +{ + otx2_cptlf_set_iqueue_exec(lf, false); +} + +static inline void otx2_cptlf_enable_iqueues(struct otx2_cptlfs_info *lfs) +{ + int slot; + + for (slot = 0; slot < lfs->lfs_num; slot++) { + otx2_cptlf_enable_iqueue_exec(&lfs->lf[slot]); + otx2_cptlf_enable_iqueue_enq(&lfs->lf[slot]); + } +} + +static inline void otx2_cpt_fill_inst(union otx2_cpt_inst_s *cptinst, + struct otx2_cpt_iq_command *iq_cmd, + u64 comp_baddr) +{ + cptinst->u[0] = 0x0; + cptinst->s.doneint = true; + cptinst->s.res_addr = comp_baddr; + cptinst->u[2] = 0x0; + cptinst->u[3] = 0x0; + cptinst->s.ei0 = iq_cmd->cmd.u; + cptinst->s.ei1 = iq_cmd->dptr; + cptinst->s.ei2 = iq_cmd->rptr; + cptinst->s.ei3 = iq_cmd->cptr.u; +} + +/* + * On OcteonTX2 platform the parameter insts_num is used as a count of + * instructions to be enqueued. The valid values for insts_num are: + * 1 - 1 CPT instruction will be enqueued during LMTST operation + * 2 - 2 CPT instructions will be enqueued during LMTST operation + */ +static inline void otx2_cpt_send_cmd(union otx2_cpt_inst_s *cptinst, + u32 insts_num, struct otx2_cptlf_info *lf) +{ + void __iomem *lmtline = lf->lmtline; + long ret; + + /* + * Make sure memory areas pointed in CPT_INST_S + * are flushed before the instruction is sent to CPT + */ + dma_wmb(); + + do { + /* Copy CPT command to LMTLINE */ + memcpy_toio(lmtline, cptinst, insts_num * OTX2_CPT_INST_SIZE); + + /* + * LDEOR initiates atomic transfer to I/O device + * The following will cause the LMTST to fail (the LDEOR + * returns zero): + * - No stores have been performed to the LMTLINE since it was + * last invalidated. + * - The bytes which have been stored to LMTLINE since it was + * last invalidated form a pattern that is non-contiguous, does + * not start at byte 0, or does not end on a 8-byte boundary. + * (i.e.comprises a formation of other than 1–16 8-byte + * words.) + * + * These rules are designed such that an operating system + * context switch or hypervisor guest switch need have no + * knowledge of the LMTST operations; the switch code does not + * need to store to LMTCANCEL. Also note as LMTLINE data cannot + * be read, there is no information leakage between processes. + */ + ret = otx2_lmt_flush(lf->ioreg); + + } while (!ret); +} + +static inline bool otx2_cptlf_started(struct otx2_cptlfs_info *lfs) +{ + return atomic_read(&lfs->state) == OTX2_CPTLF_STARTED; +} + +int otx2_cptlf_init(struct otx2_cptlfs_info *lfs, u8 eng_grp_msk, int pri, + int lfs_num); +void otx2_cptlf_shutdown(struct otx2_cptlfs_info *lfs); +int otx2_cptlf_register_interrupts(struct otx2_cptlfs_info *lfs); +void otx2_cptlf_unregister_interrupts(struct otx2_cptlfs_info *lfs); +void otx2_cptlf_free_irqs_affinity(struct otx2_cptlfs_info *lfs); +int otx2_cptlf_set_irqs_affinity(struct otx2_cptlfs_info *lfs); + +#endif /* __OTX2_CPTLF_H */ diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptpf.h b/drivers/crypto/marvell/octeontx2/otx2_cptpf.h new file mode 100644 index 0000000000000000000000000000000000000000..8c899ad531a5c868a3fe031be625746268b4ead8 --- /dev/null +++ b/drivers/crypto/marvell/octeontx2/otx2_cptpf.h @@ -0,0 +1,61 @@ +/* SPDX-License-Identifier: GPL-2.0-only + * Copyright (C) 2020 Marvell. + */ + +#ifndef __OTX2_CPTPF_H +#define __OTX2_CPTPF_H + +#include "otx2_cpt_common.h" +#include "otx2_cptpf_ucode.h" +#include "otx2_cptlf.h" + +struct otx2_cptpf_dev; +struct otx2_cptvf_info { + struct otx2_cptpf_dev *cptpf; /* PF pointer this VF belongs to */ + struct work_struct vfpf_mbox_work; + struct pci_dev *vf_dev; + int vf_id; + int intr_idx; +}; + +struct cptpf_flr_work { + struct work_struct work; + struct otx2_cptpf_dev *pf; +}; + +struct otx2_cptpf_dev { + void __iomem *reg_base; /* CPT PF registers start address */ + void __iomem *afpf_mbox_base; /* PF-AF mbox start address */ + void __iomem *vfpf_mbox_base; /* VF-PF mbox start address */ + struct pci_dev *pdev; /* PCI device handle */ + struct otx2_cptvf_info vf[OTX2_CPT_MAX_VFS_NUM]; + struct otx2_cpt_eng_grps eng_grps;/* Engine groups information */ + struct otx2_cptlfs_info lfs; /* CPT LFs attached to this PF */ + /* HW capabilities for each engine type */ + union otx2_cpt_eng_caps eng_caps[OTX2_CPT_MAX_ENG_TYPES]; + bool is_eng_caps_discovered; + + /* AF <=> PF mbox */ + struct otx2_mbox afpf_mbox; + struct work_struct afpf_mbox_work; + struct workqueue_struct *afpf_mbox_wq; + + /* VF <=> PF mbox */ + struct otx2_mbox vfpf_mbox; + struct workqueue_struct *vfpf_mbox_wq; + + struct workqueue_struct *flr_wq; + struct cptpf_flr_work *flr_work; + + u8 pf_id; /* RVU PF number */ + u8 max_vfs; /* Maximum number of VFs supported by CPT */ + u8 enabled_vfs; /* Number of enabled VFs */ + u8 kvf_limits; /* Kernel crypto limits */ +}; + +irqreturn_t otx2_cptpf_afpf_mbox_intr(int irq, void *arg); +void otx2_cptpf_afpf_mbox_handler(struct work_struct *work); +irqreturn_t otx2_cptpf_vfpf_mbox_intr(int irq, void *arg); +void otx2_cptpf_vfpf_mbox_handler(struct work_struct *work); + +#endif /* __OTX2_CPTPF_H */ diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptpf_main.c b/drivers/crypto/marvell/octeontx2/otx2_cptpf_main.c new file mode 100644 index 0000000000000000000000000000000000000000..5277e04badd9abd527995c55553f229375bec766 --- /dev/null +++ b/drivers/crypto/marvell/octeontx2/otx2_cptpf_main.c @@ -0,0 +1,713 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (C) 2020 Marvell. */ + +#include +#include "otx2_cpt_hw_types.h" +#include "otx2_cpt_common.h" +#include "otx2_cptpf_ucode.h" +#include "otx2_cptpf.h" +#include "rvu_reg.h" + +#define OTX2_CPT_DRV_NAME "octeontx2-cpt" +#define OTX2_CPT_DRV_STRING "Marvell OcteonTX2 CPT Physical Function Driver" + +static void cptpf_enable_vfpf_mbox_intr(struct otx2_cptpf_dev *cptpf, + int num_vfs) +{ + int ena_bits; + + /* Clear any pending interrupts */ + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, + RVU_PF_VFPF_MBOX_INTX(0), ~0x0ULL); + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, + RVU_PF_VFPF_MBOX_INTX(1), ~0x0ULL); + + /* Enable VF interrupts for VFs from 0 to 63 */ + ena_bits = ((num_vfs - 1) % 64); + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, + RVU_PF_VFPF_MBOX_INT_ENA_W1SX(0), + GENMASK_ULL(ena_bits, 0)); + + if (num_vfs > 64) { + /* Enable VF interrupts for VFs from 64 to 127 */ + ena_bits = num_vfs - 64 - 1; + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, + RVU_PF_VFPF_MBOX_INT_ENA_W1SX(1), + GENMASK_ULL(ena_bits, 0)); + } +} + +static void cptpf_disable_vfpf_mbox_intr(struct otx2_cptpf_dev *cptpf, + int num_vfs) +{ + int vector; + + /* Disable VF-PF interrupts */ + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, + RVU_PF_VFPF_MBOX_INT_ENA_W1CX(0), ~0ULL); + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, + RVU_PF_VFPF_MBOX_INT_ENA_W1CX(1), ~0ULL); + /* Clear any pending interrupts */ + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, + RVU_PF_VFPF_MBOX_INTX(0), ~0ULL); + + vector = pci_irq_vector(cptpf->pdev, RVU_PF_INT_VEC_VFPF_MBOX0); + free_irq(vector, cptpf); + + if (num_vfs > 64) { + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, + RVU_PF_VFPF_MBOX_INTX(1), ~0ULL); + vector = pci_irq_vector(cptpf->pdev, RVU_PF_INT_VEC_VFPF_MBOX1); + free_irq(vector, cptpf); + } +} + +static void cptpf_enable_vf_flr_intrs(struct otx2_cptpf_dev *cptpf) +{ + /* Clear interrupt if any */ + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, RVU_PF_VFFLR_INTX(0), + ~0x0ULL); + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, RVU_PF_VFFLR_INTX(1), + ~0x0ULL); + + /* Enable VF FLR interrupts */ + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, + RVU_PF_VFFLR_INT_ENA_W1SX(0), ~0x0ULL); + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, + RVU_PF_VFFLR_INT_ENA_W1SX(1), ~0x0ULL); +} + +static void cptpf_disable_vf_flr_intrs(struct otx2_cptpf_dev *cptpf, + int num_vfs) +{ + int vector; + + /* Disable VF FLR interrupts */ + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, + RVU_PF_VFFLR_INT_ENA_W1CX(0), ~0x0ULL); + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, + RVU_PF_VFFLR_INT_ENA_W1CX(1), ~0x0ULL); + + /* Clear interrupt if any */ + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, RVU_PF_VFFLR_INTX(0), + ~0x0ULL); + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, RVU_PF_VFFLR_INTX(1), + ~0x0ULL); + + vector = pci_irq_vector(cptpf->pdev, RVU_PF_INT_VEC_VFFLR0); + free_irq(vector, cptpf); + + if (num_vfs > 64) { + vector = pci_irq_vector(cptpf->pdev, RVU_PF_INT_VEC_VFFLR1); + free_irq(vector, cptpf); + } +} + +static void cptpf_flr_wq_handler(struct work_struct *work) +{ + struct cptpf_flr_work *flr_work; + struct otx2_cptpf_dev *pf; + struct mbox_msghdr *req; + struct otx2_mbox *mbox; + int vf, reg = 0; + + flr_work = container_of(work, struct cptpf_flr_work, work); + pf = flr_work->pf; + mbox = &pf->afpf_mbox; + + vf = flr_work - pf->flr_work; + + req = otx2_mbox_alloc_msg_rsp(mbox, 0, sizeof(*req), + sizeof(struct msg_rsp)); + if (!req) + return; + + req->sig = OTX2_MBOX_REQ_SIG; + req->id = MBOX_MSG_VF_FLR; + req->pcifunc &= RVU_PFVF_FUNC_MASK; + req->pcifunc |= (vf + 1) & RVU_PFVF_FUNC_MASK; + + otx2_cpt_send_mbox_msg(mbox, pf->pdev); + + if (vf >= 64) { + reg = 1; + vf = vf - 64; + } + /* Clear transaction pending register */ + otx2_cpt_write64(pf->reg_base, BLKADDR_RVUM, 0, + RVU_PF_VFTRPENDX(reg), BIT_ULL(vf)); + otx2_cpt_write64(pf->reg_base, BLKADDR_RVUM, 0, + RVU_PF_VFFLR_INT_ENA_W1SX(reg), BIT_ULL(vf)); +} + +static irqreturn_t cptpf_vf_flr_intr(int __always_unused irq, void *arg) +{ + int reg, dev, vf, start_vf, num_reg = 1; + struct otx2_cptpf_dev *cptpf = arg; + u64 intr; + + if (cptpf->max_vfs > 64) + num_reg = 2; + + for (reg = 0; reg < num_reg; reg++) { + intr = otx2_cpt_read64(cptpf->reg_base, BLKADDR_RVUM, 0, + RVU_PF_VFFLR_INTX(reg)); + if (!intr) + continue; + start_vf = 64 * reg; + for (vf = 0; vf < 64; vf++) { + if (!(intr & BIT_ULL(vf))) + continue; + dev = vf + start_vf; + queue_work(cptpf->flr_wq, &cptpf->flr_work[dev].work); + /* Clear interrupt */ + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, + RVU_PF_VFFLR_INTX(reg), BIT_ULL(vf)); + /* Disable the interrupt */ + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, + RVU_PF_VFFLR_INT_ENA_W1CX(reg), + BIT_ULL(vf)); + } + } + return IRQ_HANDLED; +} + +static void cptpf_unregister_vfpf_intr(struct otx2_cptpf_dev *cptpf, + int num_vfs) +{ + cptpf_disable_vfpf_mbox_intr(cptpf, num_vfs); + cptpf_disable_vf_flr_intrs(cptpf, num_vfs); +} + +static int cptpf_register_vfpf_intr(struct otx2_cptpf_dev *cptpf, int num_vfs) +{ + struct pci_dev *pdev = cptpf->pdev; + struct device *dev = &pdev->dev; + int ret, vector; + + vector = pci_irq_vector(pdev, RVU_PF_INT_VEC_VFPF_MBOX0); + /* Register VF-PF mailbox interrupt handler */ + ret = request_irq(vector, otx2_cptpf_vfpf_mbox_intr, 0, "CPTVFPF Mbox0", + cptpf); + if (ret) { + dev_err(dev, + "IRQ registration failed for PFVF mbox0 irq\n"); + return ret; + } + vector = pci_irq_vector(pdev, RVU_PF_INT_VEC_VFFLR0); + /* Register VF FLR interrupt handler */ + ret = request_irq(vector, cptpf_vf_flr_intr, 0, "CPTPF FLR0", cptpf); + if (ret) { + dev_err(dev, + "IRQ registration failed for VFFLR0 irq\n"); + goto free_mbox0_irq; + } + if (num_vfs > 64) { + vector = pci_irq_vector(pdev, RVU_PF_INT_VEC_VFPF_MBOX1); + ret = request_irq(vector, otx2_cptpf_vfpf_mbox_intr, 0, + "CPTVFPF Mbox1", cptpf); + if (ret) { + dev_err(dev, + "IRQ registration failed for PFVF mbox1 irq\n"); + goto free_flr0_irq; + } + vector = pci_irq_vector(pdev, RVU_PF_INT_VEC_VFFLR1); + /* Register VF FLR interrupt handler */ + ret = request_irq(vector, cptpf_vf_flr_intr, 0, "CPTPF FLR1", + cptpf); + if (ret) { + dev_err(dev, + "IRQ registration failed for VFFLR1 irq\n"); + goto free_mbox1_irq; + } + } + cptpf_enable_vfpf_mbox_intr(cptpf, num_vfs); + cptpf_enable_vf_flr_intrs(cptpf); + + return 0; + +free_mbox1_irq: + vector = pci_irq_vector(pdev, RVU_PF_INT_VEC_VFPF_MBOX1); + free_irq(vector, cptpf); +free_flr0_irq: + vector = pci_irq_vector(pdev, RVU_PF_INT_VEC_VFFLR0); + free_irq(vector, cptpf); +free_mbox0_irq: + vector = pci_irq_vector(pdev, RVU_PF_INT_VEC_VFPF_MBOX0); + free_irq(vector, cptpf); + return ret; +} + +static void cptpf_flr_wq_destroy(struct otx2_cptpf_dev *pf) +{ + if (!pf->flr_wq) + return; + destroy_workqueue(pf->flr_wq); + pf->flr_wq = NULL; + kfree(pf->flr_work); +} + +static int cptpf_flr_wq_init(struct otx2_cptpf_dev *cptpf, int num_vfs) +{ + int vf; + + cptpf->flr_wq = alloc_ordered_workqueue("cptpf_flr_wq", 0); + if (!cptpf->flr_wq) + return -ENOMEM; + + cptpf->flr_work = kcalloc(num_vfs, sizeof(struct cptpf_flr_work), + GFP_KERNEL); + if (!cptpf->flr_work) + goto destroy_wq; + + for (vf = 0; vf < num_vfs; vf++) { + cptpf->flr_work[vf].pf = cptpf; + INIT_WORK(&cptpf->flr_work[vf].work, cptpf_flr_wq_handler); + } + return 0; + +destroy_wq: + destroy_workqueue(cptpf->flr_wq); + return -ENOMEM; +} + +static int cptpf_vfpf_mbox_init(struct otx2_cptpf_dev *cptpf, int num_vfs) +{ + struct device *dev = &cptpf->pdev->dev; + u64 vfpf_mbox_base; + int err, i; + + cptpf->vfpf_mbox_wq = alloc_workqueue("cpt_vfpf_mailbox", + WQ_UNBOUND | WQ_HIGHPRI | + WQ_MEM_RECLAIM, 1); + if (!cptpf->vfpf_mbox_wq) + return -ENOMEM; + + /* Map VF-PF mailbox memory */ + vfpf_mbox_base = readq(cptpf->reg_base + RVU_PF_VF_BAR4_ADDR); + if (!vfpf_mbox_base) { + dev_err(dev, "VF-PF mailbox address not configured\n"); + err = -ENOMEM; + goto free_wqe; + } + cptpf->vfpf_mbox_base = devm_ioremap_wc(dev, vfpf_mbox_base, + MBOX_SIZE * cptpf->max_vfs); + if (!cptpf->vfpf_mbox_base) { + dev_err(dev, "Mapping of VF-PF mailbox address failed\n"); + err = -ENOMEM; + goto free_wqe; + } + err = otx2_mbox_init(&cptpf->vfpf_mbox, cptpf->vfpf_mbox_base, + cptpf->pdev, cptpf->reg_base, MBOX_DIR_PFVF, + num_vfs); + if (err) + goto free_wqe; + + for (i = 0; i < num_vfs; i++) { + cptpf->vf[i].vf_id = i; + cptpf->vf[i].cptpf = cptpf; + cptpf->vf[i].intr_idx = i % 64; + INIT_WORK(&cptpf->vf[i].vfpf_mbox_work, + otx2_cptpf_vfpf_mbox_handler); + } + return 0; + +free_wqe: + destroy_workqueue(cptpf->vfpf_mbox_wq); + return err; +} + +static void cptpf_vfpf_mbox_destroy(struct otx2_cptpf_dev *cptpf) +{ + destroy_workqueue(cptpf->vfpf_mbox_wq); + otx2_mbox_destroy(&cptpf->vfpf_mbox); +} + +static void cptpf_disable_afpf_mbox_intr(struct otx2_cptpf_dev *cptpf) +{ + /* Disable AF-PF interrupt */ + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, RVU_PF_INT_ENA_W1C, + 0x1ULL); + /* Clear interrupt if any */ + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, RVU_PF_INT, 0x1ULL); +} + +static int cptpf_register_afpf_mbox_intr(struct otx2_cptpf_dev *cptpf) +{ + struct pci_dev *pdev = cptpf->pdev; + struct device *dev = &pdev->dev; + int ret, irq; + + irq = pci_irq_vector(pdev, RVU_PF_INT_VEC_AFPF_MBOX); + /* Register AF-PF mailbox interrupt handler */ + ret = devm_request_irq(dev, irq, otx2_cptpf_afpf_mbox_intr, 0, + "CPTAFPF Mbox", cptpf); + if (ret) { + dev_err(dev, + "IRQ registration failed for PFAF mbox irq\n"); + return ret; + } + /* Clear interrupt if any, to avoid spurious interrupts */ + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, RVU_PF_INT, 0x1ULL); + /* Enable AF-PF interrupt */ + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, RVU_PF_INT_ENA_W1S, + 0x1ULL); + + ret = otx2_cpt_send_ready_msg(&cptpf->afpf_mbox, cptpf->pdev); + if (ret) { + dev_warn(dev, + "AF not responding to mailbox, deferring probe\n"); + cptpf_disable_afpf_mbox_intr(cptpf); + return -EPROBE_DEFER; + } + return 0; +} + +static int cptpf_afpf_mbox_init(struct otx2_cptpf_dev *cptpf) +{ + int err; + + cptpf->afpf_mbox_wq = alloc_workqueue("cpt_afpf_mailbox", + WQ_UNBOUND | WQ_HIGHPRI | + WQ_MEM_RECLAIM, 1); + if (!cptpf->afpf_mbox_wq) + return -ENOMEM; + + err = otx2_mbox_init(&cptpf->afpf_mbox, cptpf->afpf_mbox_base, + cptpf->pdev, cptpf->reg_base, MBOX_DIR_PFAF, 1); + if (err) + goto error; + + INIT_WORK(&cptpf->afpf_mbox_work, otx2_cptpf_afpf_mbox_handler); + return 0; + +error: + destroy_workqueue(cptpf->afpf_mbox_wq); + return err; +} + +static void cptpf_afpf_mbox_destroy(struct otx2_cptpf_dev *cptpf) +{ + destroy_workqueue(cptpf->afpf_mbox_wq); + otx2_mbox_destroy(&cptpf->afpf_mbox); +} + +static ssize_t kvf_limits_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct otx2_cptpf_dev *cptpf = dev_get_drvdata(dev); + + return sprintf(buf, "%d\n", cptpf->kvf_limits); +} + +static ssize_t kvf_limits_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct otx2_cptpf_dev *cptpf = dev_get_drvdata(dev); + int lfs_num; + + if (kstrtoint(buf, 0, &lfs_num)) { + dev_err(dev, "lfs count %d must be in range [1 - %d]\n", + lfs_num, num_online_cpus()); + return -EINVAL; + } + if (lfs_num < 1 || lfs_num > num_online_cpus()) { + dev_err(dev, "lfs count %d must be in range [1 - %d]\n", + lfs_num, num_online_cpus()); + return -EINVAL; + } + cptpf->kvf_limits = lfs_num; + + return count; +} + +static DEVICE_ATTR_RW(kvf_limits); +static struct attribute *cptpf_attrs[] = { + &dev_attr_kvf_limits.attr, + NULL +}; + +static const struct attribute_group cptpf_sysfs_group = { + .attrs = cptpf_attrs, +}; + +static int cpt_is_pf_usable(struct otx2_cptpf_dev *cptpf) +{ + u64 rev; + + rev = otx2_cpt_read64(cptpf->reg_base, BLKADDR_RVUM, 0, + RVU_PF_BLOCK_ADDRX_DISC(BLKADDR_RVUM)); + rev = (rev >> 12) & 0xFF; + /* + * Check if AF has setup revision for RVUM block, otherwise + * driver probe should be deferred until AF driver comes up + */ + if (!rev) { + dev_warn(&cptpf->pdev->dev, + "AF is not initialized, deferring probe\n"); + return -EPROBE_DEFER; + } + return 0; +} + +static int cptpf_device_reset(struct otx2_cptpf_dev *cptpf) +{ + int timeout = 10, ret; + u64 reg = 0; + + ret = otx2_cpt_write_af_reg(&cptpf->afpf_mbox, cptpf->pdev, + CPT_AF_BLK_RST, 0x1); + if (ret) + return ret; + + do { + ret = otx2_cpt_read_af_reg(&cptpf->afpf_mbox, cptpf->pdev, + CPT_AF_BLK_RST, ®); + if (ret) + return ret; + + if (!((reg >> 63) & 0x1)) + break; + + usleep_range(10000, 20000); + if (timeout-- < 0) + return -EBUSY; + } while (1); + + return ret; +} + +static int cptpf_device_init(struct otx2_cptpf_dev *cptpf) +{ + union otx2_cptx_af_constants1 af_cnsts1 = {0}; + int ret = 0; + + /* Reset the CPT PF device */ + ret = cptpf_device_reset(cptpf); + if (ret) + return ret; + + /* Get number of SE, IE and AE engines */ + ret = otx2_cpt_read_af_reg(&cptpf->afpf_mbox, cptpf->pdev, + CPT_AF_CONSTANTS1, &af_cnsts1.u); + if (ret) + return ret; + + cptpf->eng_grps.avail.max_se_cnt = af_cnsts1.s.se; + cptpf->eng_grps.avail.max_ie_cnt = af_cnsts1.s.ie; + cptpf->eng_grps.avail.max_ae_cnt = af_cnsts1.s.ae; + + /* Disable all cores */ + ret = otx2_cpt_disable_all_cores(cptpf); + + return ret; +} + +static int cptpf_sriov_disable(struct pci_dev *pdev) +{ + struct otx2_cptpf_dev *cptpf = pci_get_drvdata(pdev); + int num_vfs = pci_num_vf(pdev); + + if (!num_vfs) + return 0; + + pci_disable_sriov(pdev); + cptpf_unregister_vfpf_intr(cptpf, num_vfs); + cptpf_flr_wq_destroy(cptpf); + cptpf_vfpf_mbox_destroy(cptpf); + module_put(THIS_MODULE); + cptpf->enabled_vfs = 0; + + return 0; +} + +static int cptpf_sriov_enable(struct pci_dev *pdev, int num_vfs) +{ + struct otx2_cptpf_dev *cptpf = pci_get_drvdata(pdev); + int ret; + + /* Initialize VF<=>PF mailbox */ + ret = cptpf_vfpf_mbox_init(cptpf, num_vfs); + if (ret) + return ret; + + ret = cptpf_flr_wq_init(cptpf, num_vfs); + if (ret) + goto destroy_mbox; + /* Register VF<=>PF mailbox interrupt */ + ret = cptpf_register_vfpf_intr(cptpf, num_vfs); + if (ret) + goto destroy_flr; + + /* Get CPT HW capabilities using LOAD_FVC operation. */ + ret = otx2_cpt_discover_eng_capabilities(cptpf); + if (ret) + goto disable_intr; + + ret = otx2_cpt_create_eng_grps(cptpf->pdev, &cptpf->eng_grps); + if (ret) + goto disable_intr; + + cptpf->enabled_vfs = num_vfs; + ret = pci_enable_sriov(pdev, num_vfs); + if (ret) + goto disable_intr; + + dev_notice(&cptpf->pdev->dev, "VFs enabled: %d\n", num_vfs); + + try_module_get(THIS_MODULE); + return num_vfs; + +disable_intr: + cptpf_unregister_vfpf_intr(cptpf, num_vfs); + cptpf->enabled_vfs = 0; +destroy_flr: + cptpf_flr_wq_destroy(cptpf); +destroy_mbox: + cptpf_vfpf_mbox_destroy(cptpf); + return ret; +} + +static int otx2_cptpf_sriov_configure(struct pci_dev *pdev, int num_vfs) +{ + if (num_vfs > 0) { + return cptpf_sriov_enable(pdev, num_vfs); + } else { + return cptpf_sriov_disable(pdev); + } +} + +static int otx2_cptpf_probe(struct pci_dev *pdev, + const struct pci_device_id *ent) +{ + struct device *dev = &pdev->dev; + resource_size_t offset, size; + struct otx2_cptpf_dev *cptpf; + int err; + + cptpf = devm_kzalloc(dev, sizeof(*cptpf), GFP_KERNEL); + if (!cptpf) + return -ENOMEM; + + err = pcim_enable_device(pdev); + if (err) { + dev_err(dev, "Failed to enable PCI device\n"); + goto clear_drvdata; + } + + err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(48)); + if (err) { + dev_err(dev, "Unable to get usable DMA configuration\n"); + goto clear_drvdata; + } + /* Map PF's configuration registers */ + err = pcim_iomap_regions_request_all(pdev, 1 << PCI_PF_REG_BAR_NUM, + OTX2_CPT_DRV_NAME); + if (err) { + dev_err(dev, "Couldn't get PCI resources 0x%x\n", err); + goto clear_drvdata; + } + pci_set_master(pdev); + pci_set_drvdata(pdev, cptpf); + cptpf->pdev = pdev; + + cptpf->reg_base = pcim_iomap_table(pdev)[PCI_PF_REG_BAR_NUM]; + + /* Check if AF driver is up, otherwise defer probe */ + err = cpt_is_pf_usable(cptpf); + if (err) + goto clear_drvdata; + + offset = pci_resource_start(pdev, PCI_MBOX_BAR_NUM); + size = pci_resource_len(pdev, PCI_MBOX_BAR_NUM); + /* Map AF-PF mailbox memory */ + cptpf->afpf_mbox_base = devm_ioremap_wc(dev, offset, size); + if (!cptpf->afpf_mbox_base) { + dev_err(&pdev->dev, "Unable to map BAR4\n"); + err = -ENODEV; + goto clear_drvdata; + } + err = pci_alloc_irq_vectors(pdev, RVU_PF_INT_VEC_CNT, + RVU_PF_INT_VEC_CNT, PCI_IRQ_MSIX); + if (err < 0) { + dev_err(dev, "Request for %d msix vectors failed\n", + RVU_PF_INT_VEC_CNT); + goto clear_drvdata; + } + /* Initialize AF-PF mailbox */ + err = cptpf_afpf_mbox_init(cptpf); + if (err) + goto clear_drvdata; + /* Register mailbox interrupt */ + err = cptpf_register_afpf_mbox_intr(cptpf); + if (err) + goto destroy_afpf_mbox; + + cptpf->max_vfs = pci_sriov_get_totalvfs(pdev); + + /* Initialize CPT PF device */ + err = cptpf_device_init(cptpf); + if (err) + goto unregister_intr; + + /* Initialize engine groups */ + err = otx2_cpt_init_eng_grps(pdev, &cptpf->eng_grps); + if (err) + goto unregister_intr; + + err = sysfs_create_group(&dev->kobj, &cptpf_sysfs_group); + if (err) + goto cleanup_eng_grps; + return 0; + +cleanup_eng_grps: + otx2_cpt_cleanup_eng_grps(pdev, &cptpf->eng_grps); +unregister_intr: + cptpf_disable_afpf_mbox_intr(cptpf); +destroy_afpf_mbox: + cptpf_afpf_mbox_destroy(cptpf); +clear_drvdata: + pci_set_drvdata(pdev, NULL); + return err; +} + +static void otx2_cptpf_remove(struct pci_dev *pdev) +{ + struct otx2_cptpf_dev *cptpf = pci_get_drvdata(pdev); + + if (!cptpf) + return; + + cptpf_sriov_disable(pdev); + /* Delete sysfs entry created for kernel VF limits */ + sysfs_remove_group(&pdev->dev.kobj, &cptpf_sysfs_group); + /* Cleanup engine groups */ + otx2_cpt_cleanup_eng_grps(pdev, &cptpf->eng_grps); + /* Disable AF-PF mailbox interrupt */ + cptpf_disable_afpf_mbox_intr(cptpf); + /* Destroy AF-PF mbox */ + cptpf_afpf_mbox_destroy(cptpf); + pci_set_drvdata(pdev, NULL); +} + +/* Supported devices */ +static const struct pci_device_id otx2_cpt_id_table[] = { + { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OTX2_CPT_PCI_PF_DEVICE_ID) }, + { 0, } /* end of table */ +}; + +static struct pci_driver otx2_cpt_pci_driver = { + .name = OTX2_CPT_DRV_NAME, + .id_table = otx2_cpt_id_table, + .probe = otx2_cptpf_probe, + .remove = otx2_cptpf_remove, + .sriov_configure = otx2_cptpf_sriov_configure +}; + +module_pci_driver(otx2_cpt_pci_driver); + +MODULE_AUTHOR("Marvell"); +MODULE_DESCRIPTION(OTX2_CPT_DRV_STRING); +MODULE_LICENSE("GPL v2"); +MODULE_DEVICE_TABLE(pci, otx2_cpt_id_table); diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptpf_mbox.c b/drivers/crypto/marvell/octeontx2/otx2_cptpf_mbox.c new file mode 100644 index 0000000000000000000000000000000000000000..186f1c1190c1a520869124bdd3d3f411492f52f4 --- /dev/null +++ b/drivers/crypto/marvell/octeontx2/otx2_cptpf_mbox.c @@ -0,0 +1,356 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (C) 2020 Marvell. */ + +#include "otx2_cpt_common.h" +#include "otx2_cptpf.h" +#include "rvu_reg.h" + +/* + * CPT PF driver version, It will be incremented by 1 for every feature + * addition in CPT mailbox messages. + */ +#define OTX2_CPT_PF_DRV_VERSION 0x1 + +static int forward_to_af(struct otx2_cptpf_dev *cptpf, + struct otx2_cptvf_info *vf, + struct mbox_msghdr *req, int size) +{ + struct mbox_msghdr *msg; + int ret; + + msg = otx2_mbox_alloc_msg(&cptpf->afpf_mbox, 0, size); + if (msg == NULL) + return -ENOMEM; + + memcpy((uint8_t *)msg + sizeof(struct mbox_msghdr), + (uint8_t *)req + sizeof(struct mbox_msghdr), size); + msg->id = req->id; + msg->pcifunc = req->pcifunc; + msg->sig = req->sig; + msg->ver = req->ver; + + otx2_mbox_msg_send(&cptpf->afpf_mbox, 0); + ret = otx2_mbox_wait_for_rsp(&cptpf->afpf_mbox, 0); + if (ret == -EIO) { + dev_err(&cptpf->pdev->dev, "RVU MBOX timeout.\n"); + return ret; + } else if (ret) { + dev_err(&cptpf->pdev->dev, "RVU MBOX error: %d.\n", ret); + return -EFAULT; + } + return 0; +} + +static int handle_msg_get_caps(struct otx2_cptpf_dev *cptpf, + struct otx2_cptvf_info *vf, + struct mbox_msghdr *req) +{ + struct otx2_cpt_caps_rsp *rsp; + + rsp = (struct otx2_cpt_caps_rsp *) + otx2_mbox_alloc_msg(&cptpf->vfpf_mbox, vf->vf_id, + sizeof(*rsp)); + if (!rsp) + return -ENOMEM; + + rsp->hdr.id = MBOX_MSG_GET_CAPS; + rsp->hdr.sig = OTX2_MBOX_RSP_SIG; + rsp->hdr.pcifunc = req->pcifunc; + rsp->cpt_pf_drv_version = OTX2_CPT_PF_DRV_VERSION; + rsp->cpt_revision = cptpf->pdev->revision; + memcpy(&rsp->eng_caps, &cptpf->eng_caps, sizeof(rsp->eng_caps)); + + return 0; +} + +static int handle_msg_get_eng_grp_num(struct otx2_cptpf_dev *cptpf, + struct otx2_cptvf_info *vf, + struct mbox_msghdr *req) +{ + struct otx2_cpt_egrp_num_msg *grp_req; + struct otx2_cpt_egrp_num_rsp *rsp; + + grp_req = (struct otx2_cpt_egrp_num_msg *)req; + rsp = (struct otx2_cpt_egrp_num_rsp *) + otx2_mbox_alloc_msg(&cptpf->vfpf_mbox, vf->vf_id, sizeof(*rsp)); + if (!rsp) + return -ENOMEM; + + rsp->hdr.id = MBOX_MSG_GET_ENG_GRP_NUM; + rsp->hdr.sig = OTX2_MBOX_RSP_SIG; + rsp->hdr.pcifunc = req->pcifunc; + rsp->eng_type = grp_req->eng_type; + rsp->eng_grp_num = otx2_cpt_get_eng_grp(&cptpf->eng_grps, + grp_req->eng_type); + + return 0; +} + +static int handle_msg_kvf_limits(struct otx2_cptpf_dev *cptpf, + struct otx2_cptvf_info *vf, + struct mbox_msghdr *req) +{ + struct otx2_cpt_kvf_limits_rsp *rsp; + + rsp = (struct otx2_cpt_kvf_limits_rsp *) + otx2_mbox_alloc_msg(&cptpf->vfpf_mbox, vf->vf_id, sizeof(*rsp)); + if (!rsp) + return -ENOMEM; + + rsp->hdr.id = MBOX_MSG_GET_KVF_LIMITS; + rsp->hdr.sig = OTX2_MBOX_RSP_SIG; + rsp->hdr.pcifunc = req->pcifunc; + rsp->kvf_limits = cptpf->kvf_limits; + + return 0; +} + +static int cptpf_handle_vf_req(struct otx2_cptpf_dev *cptpf, + struct otx2_cptvf_info *vf, + struct mbox_msghdr *req, int size) +{ + int err = 0; + + /* Check if msg is valid, if not reply with an invalid msg */ + if (req->sig != OTX2_MBOX_REQ_SIG) + goto inval_msg; + + switch (req->id) { + case MBOX_MSG_GET_ENG_GRP_NUM: + err = handle_msg_get_eng_grp_num(cptpf, vf, req); + break; + case MBOX_MSG_GET_CAPS: + err = handle_msg_get_caps(cptpf, vf, req); + break; + case MBOX_MSG_GET_KVF_LIMITS: + err = handle_msg_kvf_limits(cptpf, vf, req); + break; + default: + err = forward_to_af(cptpf, vf, req, size); + break; + } + return err; + +inval_msg: + otx2_reply_invalid_msg(&cptpf->vfpf_mbox, vf->vf_id, 0, req->id); + otx2_mbox_msg_send(&cptpf->vfpf_mbox, vf->vf_id); + return err; +} + +irqreturn_t otx2_cptpf_vfpf_mbox_intr(int __always_unused irq, void *arg) +{ + struct otx2_cptpf_dev *cptpf = arg; + struct otx2_cptvf_info *vf; + int i, vf_idx; + u64 intr; + + /* + * Check which VF has raised an interrupt and schedule + * corresponding work queue to process the messages + */ + for (i = 0; i < 2; i++) { + /* Read the interrupt bits */ + intr = otx2_cpt_read64(cptpf->reg_base, BLKADDR_RVUM, 0, + RVU_PF_VFPF_MBOX_INTX(i)); + + for (vf_idx = i * 64; vf_idx < cptpf->enabled_vfs; vf_idx++) { + vf = &cptpf->vf[vf_idx]; + if (intr & (1ULL << vf->intr_idx)) { + queue_work(cptpf->vfpf_mbox_wq, + &vf->vfpf_mbox_work); + /* Clear the interrupt */ + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, + 0, RVU_PF_VFPF_MBOX_INTX(i), + BIT_ULL(vf->intr_idx)); + } + } + } + return IRQ_HANDLED; +} + +void otx2_cptpf_vfpf_mbox_handler(struct work_struct *work) +{ + struct otx2_cptpf_dev *cptpf; + struct otx2_cptvf_info *vf; + struct otx2_mbox_dev *mdev; + struct mbox_hdr *req_hdr; + struct mbox_msghdr *msg; + struct otx2_mbox *mbox; + int offset, i, err; + + vf = container_of(work, struct otx2_cptvf_info, vfpf_mbox_work); + cptpf = vf->cptpf; + mbox = &cptpf->vfpf_mbox; + /* sync with mbox memory region */ + smp_rmb(); + mdev = &mbox->dev[vf->vf_id]; + /* Process received mbox messages */ + req_hdr = (struct mbox_hdr *)(mdev->mbase + mbox->rx_start); + offset = mbox->rx_start + ALIGN(sizeof(*req_hdr), MBOX_MSG_ALIGN); + + for (i = 0; i < req_hdr->num_msgs; i++) { + msg = (struct mbox_msghdr *)(mdev->mbase + offset); + + /* Set which VF sent this message based on mbox IRQ */ + msg->pcifunc = ((u16)cptpf->pf_id << RVU_PFVF_PF_SHIFT) | + ((vf->vf_id + 1) & RVU_PFVF_FUNC_MASK); + + err = cptpf_handle_vf_req(cptpf, vf, msg, + msg->next_msgoff - offset); + /* + * Behave as the AF, drop the msg if there is + * no memory, timeout handling also goes here + */ + if (err == -ENOMEM || err == -EIO) + break; + offset = msg->next_msgoff; + } + /* Send mbox responses to VF */ + if (mdev->num_msgs) + otx2_mbox_msg_send(mbox, vf->vf_id); +} + +irqreturn_t otx2_cptpf_afpf_mbox_intr(int __always_unused irq, void *arg) +{ + struct otx2_cptpf_dev *cptpf = arg; + u64 intr; + + /* Read the interrupt bits */ + intr = otx2_cpt_read64(cptpf->reg_base, BLKADDR_RVUM, 0, RVU_PF_INT); + + if (intr & 0x1ULL) { + /* Schedule work queue function to process the MBOX request */ + queue_work(cptpf->afpf_mbox_wq, &cptpf->afpf_mbox_work); + /* Clear and ack the interrupt */ + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, RVU_PF_INT, + 0x1ULL); + } + return IRQ_HANDLED; +} + +static void process_afpf_mbox_msg(struct otx2_cptpf_dev *cptpf, + struct mbox_msghdr *msg) +{ + struct device *dev = &cptpf->pdev->dev; + struct cpt_rd_wr_reg_msg *rsp_rd_wr; + + if (msg->id >= MBOX_MSG_MAX) { + dev_err(dev, "MBOX msg with unknown ID %d\n", msg->id); + return; + } + if (msg->sig != OTX2_MBOX_RSP_SIG) { + dev_err(dev, "MBOX msg with wrong signature %x, ID %d\n", + msg->sig, msg->id); + return; + } + + switch (msg->id) { + case MBOX_MSG_READY: + cptpf->pf_id = (msg->pcifunc >> RVU_PFVF_PF_SHIFT) & + RVU_PFVF_PF_MASK; + break; + case MBOX_MSG_CPT_RD_WR_REGISTER: + rsp_rd_wr = (struct cpt_rd_wr_reg_msg *)msg; + if (msg->rc) { + dev_err(dev, "Reg %llx rd/wr(%d) failed %d\n", + rsp_rd_wr->reg_offset, rsp_rd_wr->is_write, + msg->rc); + return; + } + if (!rsp_rd_wr->is_write) + *rsp_rd_wr->ret_val = rsp_rd_wr->val; + break; + case MBOX_MSG_ATTACH_RESOURCES: + if (!msg->rc) + cptpf->lfs.are_lfs_attached = 1; + break; + case MBOX_MSG_DETACH_RESOURCES: + if (!msg->rc) + cptpf->lfs.are_lfs_attached = 0; + break; + + default: + dev_err(dev, + "Unsupported msg %d received.\n", msg->id); + break; + } +} + +static void forward_to_vf(struct otx2_cptpf_dev *cptpf, struct mbox_msghdr *msg, + int vf_id, int size) +{ + struct otx2_mbox *vfpf_mbox; + struct mbox_msghdr *fwd; + + if (msg->id >= MBOX_MSG_MAX) { + dev_err(&cptpf->pdev->dev, + "MBOX msg with unknown ID %d\n", msg->id); + return; + } + if (msg->sig != OTX2_MBOX_RSP_SIG) { + dev_err(&cptpf->pdev->dev, + "MBOX msg with wrong signature %x, ID %d\n", + msg->sig, msg->id); + return; + } + vfpf_mbox = &cptpf->vfpf_mbox; + vf_id--; + if (vf_id >= cptpf->enabled_vfs) { + dev_err(&cptpf->pdev->dev, + "MBOX msg to unknown VF: %d >= %d\n", + vf_id, cptpf->enabled_vfs); + return; + } + if (msg->id == MBOX_MSG_VF_FLR) + return; + + fwd = otx2_mbox_alloc_msg(vfpf_mbox, vf_id, size); + if (!fwd) { + dev_err(&cptpf->pdev->dev, + "Forwarding to VF%d failed.\n", vf_id); + return; + } + memcpy((uint8_t *)fwd + sizeof(struct mbox_msghdr), + (uint8_t *)msg + sizeof(struct mbox_msghdr), size); + fwd->id = msg->id; + fwd->pcifunc = msg->pcifunc; + fwd->sig = msg->sig; + fwd->ver = msg->ver; + fwd->rc = msg->rc; +} + +/* Handle mailbox messages received from AF */ +void otx2_cptpf_afpf_mbox_handler(struct work_struct *work) +{ + struct otx2_cptpf_dev *cptpf; + struct otx2_mbox *afpf_mbox; + struct otx2_mbox_dev *mdev; + struct mbox_hdr *rsp_hdr; + struct mbox_msghdr *msg; + int offset, vf_id, i; + + cptpf = container_of(work, struct otx2_cptpf_dev, afpf_mbox_work); + afpf_mbox = &cptpf->afpf_mbox; + mdev = &afpf_mbox->dev[0]; + /* Sync mbox data into memory */ + smp_wmb(); + + rsp_hdr = (struct mbox_hdr *)(mdev->mbase + afpf_mbox->rx_start); + offset = ALIGN(sizeof(*rsp_hdr), MBOX_MSG_ALIGN); + + for (i = 0; i < rsp_hdr->num_msgs; i++) { + msg = (struct mbox_msghdr *)(mdev->mbase + afpf_mbox->rx_start + + offset); + vf_id = (msg->pcifunc >> RVU_PFVF_FUNC_SHIFT) & + RVU_PFVF_FUNC_MASK; + if (vf_id > 0) + forward_to_vf(cptpf, msg, vf_id, + msg->next_msgoff - offset); + else + process_afpf_mbox_msg(cptpf, msg); + + offset = msg->next_msgoff; + mdev->msgs_acked++; + } + otx2_mbox_reset(afpf_mbox, 0); +} diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c b/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c new file mode 100644 index 0000000000000000000000000000000000000000..1dc3ba298139ff2c5337cf63df564f4cf8796c69 --- /dev/null +++ b/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c @@ -0,0 +1,1415 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (C) 2020 Marvell. */ + +#include +#include +#include "otx2_cptpf_ucode.h" +#include "otx2_cpt_common.h" +#include "otx2_cptpf.h" +#include "otx2_cptlf.h" +#include "otx2_cpt_reqmgr.h" +#include "rvu_reg.h" + +#define CSR_DELAY 30 + +#define LOADFVC_RLEN 8 +#define LOADFVC_MAJOR_OP 0x01 +#define LOADFVC_MINOR_OP 0x08 + +struct fw_info_t { + struct list_head ucodes; +}; + +static struct otx2_cpt_bitmap get_cores_bmap(struct device *dev, + struct otx2_cpt_eng_grp_info *eng_grp) +{ + struct otx2_cpt_bitmap bmap = { {0} }; + bool found = false; + int i; + + if (eng_grp->g->engs_num > OTX2_CPT_MAX_ENGINES) { + dev_err(dev, "unsupported number of engines %d on octeontx2\n", + eng_grp->g->engs_num); + return bmap; + } + + for (i = 0; i < OTX2_CPT_MAX_ETYPES_PER_GRP; i++) { + if (eng_grp->engs[i].type) { + bitmap_or(bmap.bits, bmap.bits, + eng_grp->engs[i].bmap, + eng_grp->g->engs_num); + bmap.size = eng_grp->g->engs_num; + found = true; + } + } + + if (!found) + dev_err(dev, "No engines reserved for engine group %d\n", + eng_grp->idx); + return bmap; +} + +static int is_eng_type(int val, int eng_type) +{ + return val & (1 << eng_type); +} + +static int is_2nd_ucode_used(struct otx2_cpt_eng_grp_info *eng_grp) +{ + if (eng_grp->ucode[1].type) + return true; + else + return false; +} + +static void set_ucode_filename(struct otx2_cpt_ucode *ucode, + const char *filename) +{ + strlcpy(ucode->filename, filename, OTX2_CPT_NAME_LENGTH); +} + +static char *get_eng_type_str(int eng_type) +{ + char *str = "unknown"; + + switch (eng_type) { + case OTX2_CPT_SE_TYPES: + str = "SE"; + break; + + case OTX2_CPT_IE_TYPES: + str = "IE"; + break; + + case OTX2_CPT_AE_TYPES: + str = "AE"; + break; + } + return str; +} + +static char *get_ucode_type_str(int ucode_type) +{ + char *str = "unknown"; + + switch (ucode_type) { + case (1 << OTX2_CPT_SE_TYPES): + str = "SE"; + break; + + case (1 << OTX2_CPT_IE_TYPES): + str = "IE"; + break; + + case (1 << OTX2_CPT_AE_TYPES): + str = "AE"; + break; + + case (1 << OTX2_CPT_SE_TYPES | 1 << OTX2_CPT_IE_TYPES): + str = "SE+IPSEC"; + break; + } + return str; +} + +static int get_ucode_type(struct device *dev, + struct otx2_cpt_ucode_hdr *ucode_hdr, + int *ucode_type) +{ + struct otx2_cptpf_dev *cptpf = dev_get_drvdata(dev); + char ver_str_prefix[OTX2_CPT_UCODE_VER_STR_SZ]; + char tmp_ver_str[OTX2_CPT_UCODE_VER_STR_SZ]; + struct pci_dev *pdev = cptpf->pdev; + int i, val = 0; + u8 nn; + + strlcpy(tmp_ver_str, ucode_hdr->ver_str, OTX2_CPT_UCODE_VER_STR_SZ); + for (i = 0; i < strlen(tmp_ver_str); i++) + tmp_ver_str[i] = tolower(tmp_ver_str[i]); + + sprintf(ver_str_prefix, "ocpt-%02d", pdev->revision); + if (!strnstr(tmp_ver_str, ver_str_prefix, OTX2_CPT_UCODE_VER_STR_SZ)) + return -EINVAL; + + nn = ucode_hdr->ver_num.nn; + if (strnstr(tmp_ver_str, "se-", OTX2_CPT_UCODE_VER_STR_SZ) && + (nn == OTX2_CPT_SE_UC_TYPE1 || nn == OTX2_CPT_SE_UC_TYPE2 || + nn == OTX2_CPT_SE_UC_TYPE3)) + val |= 1 << OTX2_CPT_SE_TYPES; + if (strnstr(tmp_ver_str, "ie-", OTX2_CPT_UCODE_VER_STR_SZ) && + (nn == OTX2_CPT_IE_UC_TYPE1 || nn == OTX2_CPT_IE_UC_TYPE2 || + nn == OTX2_CPT_IE_UC_TYPE3)) + val |= 1 << OTX2_CPT_IE_TYPES; + if (strnstr(tmp_ver_str, "ae", OTX2_CPT_UCODE_VER_STR_SZ) && + nn == OTX2_CPT_AE_UC_TYPE) + val |= 1 << OTX2_CPT_AE_TYPES; + + *ucode_type = val; + + if (!val) + return -EINVAL; + + return 0; +} + +static int __write_ucode_base(struct otx2_cptpf_dev *cptpf, int eng, + dma_addr_t dma_addr) +{ + return otx2_cpt_write_af_reg(&cptpf->afpf_mbox, cptpf->pdev, + CPT_AF_EXEX_UCODE_BASE(eng), + (u64)dma_addr); +} + +static int cpt_set_ucode_base(struct otx2_cpt_eng_grp_info *eng_grp, void *obj) +{ + struct otx2_cptpf_dev *cptpf = obj; + struct otx2_cpt_engs_rsvd *engs; + dma_addr_t dma_addr; + int i, bit, ret; + + /* Set PF number for microcode fetches */ + ret = otx2_cpt_write_af_reg(&cptpf->afpf_mbox, cptpf->pdev, + CPT_AF_PF_FUNC, + cptpf->pf_id << RVU_PFVF_PF_SHIFT); + if (ret) + return ret; + + for (i = 0; i < OTX2_CPT_MAX_ETYPES_PER_GRP; i++) { + engs = &eng_grp->engs[i]; + if (!engs->type) + continue; + + dma_addr = engs->ucode->dma; + + /* + * Set UCODE_BASE only for the cores which are not used, + * other cores should have already valid UCODE_BASE set + */ + for_each_set_bit(bit, engs->bmap, eng_grp->g->engs_num) + if (!eng_grp->g->eng_ref_cnt[bit]) { + ret = __write_ucode_base(cptpf, bit, dma_addr); + if (ret) + return ret; + } + } + return 0; +} + +static int cpt_detach_and_disable_cores(struct otx2_cpt_eng_grp_info *eng_grp, + void *obj) +{ + struct otx2_cptpf_dev *cptpf = obj; + struct otx2_cpt_bitmap bmap; + int i, timeout = 10; + int busy, ret; + u64 reg = 0; + + bmap = get_cores_bmap(&cptpf->pdev->dev, eng_grp); + if (!bmap.size) + return -EINVAL; + + /* Detach the cores from group */ + for_each_set_bit(i, bmap.bits, bmap.size) { + ret = otx2_cpt_read_af_reg(&cptpf->afpf_mbox, cptpf->pdev, + CPT_AF_EXEX_CTL2(i), ®); + if (ret) + return ret; + + if (reg & (1ull << eng_grp->idx)) { + eng_grp->g->eng_ref_cnt[i]--; + reg &= ~(1ull << eng_grp->idx); + + ret = otx2_cpt_write_af_reg(&cptpf->afpf_mbox, + cptpf->pdev, + CPT_AF_EXEX_CTL2(i), reg); + if (ret) + return ret; + } + } + + /* Wait for cores to become idle */ + do { + busy = 0; + usleep_range(10000, 20000); + if (timeout-- < 0) + return -EBUSY; + + for_each_set_bit(i, bmap.bits, bmap.size) { + ret = otx2_cpt_read_af_reg(&cptpf->afpf_mbox, + cptpf->pdev, + CPT_AF_EXEX_STS(i), ®); + if (ret) + return ret; + + if (reg & 0x1) { + busy = 1; + break; + } + } + } while (busy); + + /* Disable the cores only if they are not used anymore */ + for_each_set_bit(i, bmap.bits, bmap.size) { + if (!eng_grp->g->eng_ref_cnt[i]) { + ret = otx2_cpt_write_af_reg(&cptpf->afpf_mbox, + cptpf->pdev, + CPT_AF_EXEX_CTL(i), 0x0); + if (ret) + return ret; + } + } + + return 0; +} + +static int cpt_attach_and_enable_cores(struct otx2_cpt_eng_grp_info *eng_grp, + void *obj) +{ + struct otx2_cptpf_dev *cptpf = obj; + struct otx2_cpt_bitmap bmap; + u64 reg = 0; + int i, ret; + + bmap = get_cores_bmap(&cptpf->pdev->dev, eng_grp); + if (!bmap.size) + return -EINVAL; + + /* Attach the cores to the group */ + for_each_set_bit(i, bmap.bits, bmap.size) { + ret = otx2_cpt_read_af_reg(&cptpf->afpf_mbox, cptpf->pdev, + CPT_AF_EXEX_CTL2(i), ®); + if (ret) + return ret; + + if (!(reg & (1ull << eng_grp->idx))) { + eng_grp->g->eng_ref_cnt[i]++; + reg |= 1ull << eng_grp->idx; + + ret = otx2_cpt_write_af_reg(&cptpf->afpf_mbox, + cptpf->pdev, + CPT_AF_EXEX_CTL2(i), reg); + if (ret) + return ret; + } + } + + /* Enable the cores */ + for_each_set_bit(i, bmap.bits, bmap.size) { + ret = otx2_cpt_add_write_af_reg(&cptpf->afpf_mbox, + cptpf->pdev, + CPT_AF_EXEX_CTL(i), 0x1); + if (ret) + return ret; + } + ret = otx2_cpt_send_af_reg_requests(&cptpf->afpf_mbox, cptpf->pdev); + + return ret; +} + +static int load_fw(struct device *dev, struct fw_info_t *fw_info, + char *filename) +{ + struct otx2_cpt_ucode_hdr *ucode_hdr; + struct otx2_cpt_uc_info_t *uc_info; + int ucode_type, ucode_size; + int ret; + + uc_info = kzalloc(sizeof(*uc_info), GFP_KERNEL); + if (!uc_info) + return -ENOMEM; + + ret = request_firmware(&uc_info->fw, filename, dev); + if (ret) + goto free_uc_info; + + ucode_hdr = (struct otx2_cpt_ucode_hdr *)uc_info->fw->data; + ret = get_ucode_type(dev, ucode_hdr, &ucode_type); + if (ret) + goto release_fw; + + ucode_size = ntohl(ucode_hdr->code_length) * 2; + if (!ucode_size) { + dev_err(dev, "Ucode %s invalid size\n", filename); + ret = -EINVAL; + goto release_fw; + } + + set_ucode_filename(&uc_info->ucode, filename); + memcpy(uc_info->ucode.ver_str, ucode_hdr->ver_str, + OTX2_CPT_UCODE_VER_STR_SZ); + uc_info->ucode.ver_num = ucode_hdr->ver_num; + uc_info->ucode.type = ucode_type; + uc_info->ucode.size = ucode_size; + list_add_tail(&uc_info->list, &fw_info->ucodes); + + return 0; + +release_fw: + release_firmware(uc_info->fw); +free_uc_info: + kfree(uc_info); + return ret; +} + +static void cpt_ucode_release_fw(struct fw_info_t *fw_info) +{ + struct otx2_cpt_uc_info_t *curr, *temp; + + if (!fw_info) + return; + + list_for_each_entry_safe(curr, temp, &fw_info->ucodes, list) { + list_del(&curr->list); + release_firmware(curr->fw); + kfree(curr); + } +} + +static struct otx2_cpt_uc_info_t *get_ucode(struct fw_info_t *fw_info, + int ucode_type) +{ + struct otx2_cpt_uc_info_t *curr; + + list_for_each_entry(curr, &fw_info->ucodes, list) { + if (!is_eng_type(curr->ucode.type, ucode_type)) + continue; + + return curr; + } + return NULL; +} + +static void print_uc_info(struct fw_info_t *fw_info) +{ + struct otx2_cpt_uc_info_t *curr; + + list_for_each_entry(curr, &fw_info->ucodes, list) { + pr_debug("Ucode filename %s\n", curr->ucode.filename); + pr_debug("Ucode version string %s\n", curr->ucode.ver_str); + pr_debug("Ucode version %d.%d.%d.%d\n", + curr->ucode.ver_num.nn, curr->ucode.ver_num.xx, + curr->ucode.ver_num.yy, curr->ucode.ver_num.zz); + pr_debug("Ucode type (%d) %s\n", curr->ucode.type, + get_ucode_type_str(curr->ucode.type)); + pr_debug("Ucode size %d\n", curr->ucode.size); + pr_debug("Ucode ptr %p\n", curr->fw->data); + } +} + +static int cpt_ucode_load_fw(struct pci_dev *pdev, struct fw_info_t *fw_info) +{ + char filename[OTX2_CPT_NAME_LENGTH]; + char eng_type[8] = {0}; + int ret, e, i; + + INIT_LIST_HEAD(&fw_info->ucodes); + + for (e = 1; e < OTX2_CPT_MAX_ENG_TYPES; e++) { + strcpy(eng_type, get_eng_type_str(e)); + for (i = 0; i < strlen(eng_type); i++) + eng_type[i] = tolower(eng_type[i]); + + snprintf(filename, sizeof(filename), "mrvl/cpt%02d/%s.out", + pdev->revision, eng_type); + /* Request firmware for each engine type */ + ret = load_fw(&pdev->dev, fw_info, filename); + if (ret) + goto release_fw; + } + print_uc_info(fw_info); + return 0; + +release_fw: + cpt_ucode_release_fw(fw_info); + return ret; +} + +static struct otx2_cpt_engs_rsvd *find_engines_by_type( + struct otx2_cpt_eng_grp_info *eng_grp, + int eng_type) +{ + int i; + + for (i = 0; i < OTX2_CPT_MAX_ETYPES_PER_GRP; i++) { + if (!eng_grp->engs[i].type) + continue; + + if (eng_grp->engs[i].type == eng_type) + return &eng_grp->engs[i]; + } + return NULL; +} + +static int eng_grp_has_eng_type(struct otx2_cpt_eng_grp_info *eng_grp, + int eng_type) +{ + struct otx2_cpt_engs_rsvd *engs; + + engs = find_engines_by_type(eng_grp, eng_type); + + return (engs != NULL ? 1 : 0); +} + +static int update_engines_avail_count(struct device *dev, + struct otx2_cpt_engs_available *avail, + struct otx2_cpt_engs_rsvd *engs, int val) +{ + switch (engs->type) { + case OTX2_CPT_SE_TYPES: + avail->se_cnt += val; + break; + + case OTX2_CPT_IE_TYPES: + avail->ie_cnt += val; + break; + + case OTX2_CPT_AE_TYPES: + avail->ae_cnt += val; + break; + + default: + dev_err(dev, "Invalid engine type %d\n", engs->type); + return -EINVAL; + } + return 0; +} + +static int update_engines_offset(struct device *dev, + struct otx2_cpt_engs_available *avail, + struct otx2_cpt_engs_rsvd *engs) +{ + switch (engs->type) { + case OTX2_CPT_SE_TYPES: + engs->offset = 0; + break; + + case OTX2_CPT_IE_TYPES: + engs->offset = avail->max_se_cnt; + break; + + case OTX2_CPT_AE_TYPES: + engs->offset = avail->max_se_cnt + avail->max_ie_cnt; + break; + + default: + dev_err(dev, "Invalid engine type %d\n", engs->type); + return -EINVAL; + } + return 0; +} + +static int release_engines(struct device *dev, + struct otx2_cpt_eng_grp_info *grp) +{ + int i, ret = 0; + + for (i = 0; i < OTX2_CPT_MAX_ETYPES_PER_GRP; i++) { + if (!grp->engs[i].type) + continue; + + if (grp->engs[i].count > 0) { + ret = update_engines_avail_count(dev, &grp->g->avail, + &grp->engs[i], + grp->engs[i].count); + if (ret) + return ret; + } + + grp->engs[i].type = 0; + grp->engs[i].count = 0; + grp->engs[i].offset = 0; + grp->engs[i].ucode = NULL; + bitmap_zero(grp->engs[i].bmap, grp->g->engs_num); + } + return 0; +} + +static int do_reserve_engines(struct device *dev, + struct otx2_cpt_eng_grp_info *grp, + struct otx2_cpt_engines *req_engs) +{ + struct otx2_cpt_engs_rsvd *engs = NULL; + int i, ret; + + for (i = 0; i < OTX2_CPT_MAX_ETYPES_PER_GRP; i++) { + if (!grp->engs[i].type) { + engs = &grp->engs[i]; + break; + } + } + + if (!engs) + return -ENOMEM; + + engs->type = req_engs->type; + engs->count = req_engs->count; + + ret = update_engines_offset(dev, &grp->g->avail, engs); + if (ret) + return ret; + + if (engs->count > 0) { + ret = update_engines_avail_count(dev, &grp->g->avail, engs, + -engs->count); + if (ret) + return ret; + } + + return 0; +} + +static int check_engines_availability(struct device *dev, + struct otx2_cpt_eng_grp_info *grp, + struct otx2_cpt_engines *req_eng) +{ + int avail_cnt = 0; + + switch (req_eng->type) { + case OTX2_CPT_SE_TYPES: + avail_cnt = grp->g->avail.se_cnt; + break; + + case OTX2_CPT_IE_TYPES: + avail_cnt = grp->g->avail.ie_cnt; + break; + + case OTX2_CPT_AE_TYPES: + avail_cnt = grp->g->avail.ae_cnt; + break; + + default: + dev_err(dev, "Invalid engine type %d\n", req_eng->type); + return -EINVAL; + } + + if (avail_cnt < req_eng->count) { + dev_err(dev, + "Error available %s engines %d < than requested %d\n", + get_eng_type_str(req_eng->type), + avail_cnt, req_eng->count); + return -EBUSY; + } + return 0; +} + +static int reserve_engines(struct device *dev, + struct otx2_cpt_eng_grp_info *grp, + struct otx2_cpt_engines *req_engs, int ucodes_cnt) +{ + int i, ret = 0; + + /* Validate if a number of requested engines are available */ + for (i = 0; i < ucodes_cnt; i++) { + ret = check_engines_availability(dev, grp, &req_engs[i]); + if (ret) + return ret; + } + + /* Reserve requested engines for this engine group */ + for (i = 0; i < ucodes_cnt; i++) { + ret = do_reserve_engines(dev, grp, &req_engs[i]); + if (ret) + return ret; + } + return 0; +} + +static void ucode_unload(struct device *dev, struct otx2_cpt_ucode *ucode) +{ + if (ucode->va) { + dma_free_coherent(dev, ucode->size, ucode->va, ucode->dma); + ucode->va = NULL; + ucode->dma = 0; + ucode->size = 0; + } + + memset(&ucode->ver_str, 0, OTX2_CPT_UCODE_VER_STR_SZ); + memset(&ucode->ver_num, 0, sizeof(struct otx2_cpt_ucode_ver_num)); + set_ucode_filename(ucode, ""); + ucode->type = 0; +} + +static int copy_ucode_to_dma_mem(struct device *dev, + struct otx2_cpt_ucode *ucode, + const u8 *ucode_data) +{ + u32 i; + + /* Allocate DMAable space */ + ucode->va = dma_alloc_coherent(dev, ucode->size, &ucode->dma, + GFP_KERNEL); + if (!ucode->va) + return -ENOMEM; + + memcpy(ucode->va, ucode_data + sizeof(struct otx2_cpt_ucode_hdr), + ucode->size); + + /* Byte swap 64-bit */ + for (i = 0; i < (ucode->size / 8); i++) + cpu_to_be64s(&((u64 *)ucode->va)[i]); + /* Ucode needs 16-bit swap */ + for (i = 0; i < (ucode->size / 2); i++) + cpu_to_be16s(&((u16 *)ucode->va)[i]); + return 0; +} + +static int enable_eng_grp(struct otx2_cpt_eng_grp_info *eng_grp, + void *obj) +{ + int ret; + + /* Point microcode to each core of the group */ + ret = cpt_set_ucode_base(eng_grp, obj); + if (ret) + return ret; + + /* Attach the cores to the group and enable them */ + ret = cpt_attach_and_enable_cores(eng_grp, obj); + + return ret; +} + +static int disable_eng_grp(struct device *dev, + struct otx2_cpt_eng_grp_info *eng_grp, + void *obj) +{ + int i, ret; + + /* Disable all engines used by this group */ + ret = cpt_detach_and_disable_cores(eng_grp, obj); + if (ret) + return ret; + + /* Unload ucode used by this engine group */ + ucode_unload(dev, &eng_grp->ucode[0]); + ucode_unload(dev, &eng_grp->ucode[1]); + + for (i = 0; i < OTX2_CPT_MAX_ETYPES_PER_GRP; i++) { + if (!eng_grp->engs[i].type) + continue; + + eng_grp->engs[i].ucode = &eng_grp->ucode[0]; + } + + /* Clear UCODE_BASE register for each engine used by this group */ + ret = cpt_set_ucode_base(eng_grp, obj); + + return ret; +} + +static void setup_eng_grp_mirroring(struct otx2_cpt_eng_grp_info *dst_grp, + struct otx2_cpt_eng_grp_info *src_grp) +{ + /* Setup fields for engine group which is mirrored */ + src_grp->mirror.is_ena = false; + src_grp->mirror.idx = 0; + src_grp->mirror.ref_count++; + + /* Setup fields for mirroring engine group */ + dst_grp->mirror.is_ena = true; + dst_grp->mirror.idx = src_grp->idx; + dst_grp->mirror.ref_count = 0; +} + +static void remove_eng_grp_mirroring(struct otx2_cpt_eng_grp_info *dst_grp) +{ + struct otx2_cpt_eng_grp_info *src_grp; + + if (!dst_grp->mirror.is_ena) + return; + + src_grp = &dst_grp->g->grp[dst_grp->mirror.idx]; + + src_grp->mirror.ref_count--; + dst_grp->mirror.is_ena = false; + dst_grp->mirror.idx = 0; + dst_grp->mirror.ref_count = 0; +} + +static void update_requested_engs(struct otx2_cpt_eng_grp_info *mirror_eng_grp, + struct otx2_cpt_engines *engs, int engs_cnt) +{ + struct otx2_cpt_engs_rsvd *mirrored_engs; + int i; + + for (i = 0; i < engs_cnt; i++) { + mirrored_engs = find_engines_by_type(mirror_eng_grp, + engs[i].type); + if (!mirrored_engs) + continue; + + /* + * If mirrored group has this type of engines attached then + * there are 3 scenarios possible: + * 1) mirrored_engs.count == engs[i].count then all engines + * from mirrored engine group will be shared with this engine + * group + * 2) mirrored_engs.count > engs[i].count then only a subset of + * engines from mirrored engine group will be shared with this + * engine group + * 3) mirrored_engs.count < engs[i].count then all engines + * from mirrored engine group will be shared with this group + * and additional engines will be reserved for exclusively use + * by this engine group + */ + engs[i].count -= mirrored_engs->count; + } +} + +static struct otx2_cpt_eng_grp_info *find_mirrored_eng_grp( + struct otx2_cpt_eng_grp_info *grp) +{ + struct otx2_cpt_eng_grps *eng_grps = grp->g; + int i; + + for (i = 0; i < OTX2_CPT_MAX_ENGINE_GROUPS; i++) { + if (!eng_grps->grp[i].is_enabled) + continue; + if (eng_grps->grp[i].ucode[0].type && + eng_grps->grp[i].ucode[1].type) + continue; + if (grp->idx == i) + continue; + if (!strncasecmp(eng_grps->grp[i].ucode[0].ver_str, + grp->ucode[0].ver_str, + OTX2_CPT_UCODE_VER_STR_SZ)) + return &eng_grps->grp[i]; + } + + return NULL; +} + +static struct otx2_cpt_eng_grp_info *find_unused_eng_grp( + struct otx2_cpt_eng_grps *eng_grps) +{ + int i; + + for (i = 0; i < OTX2_CPT_MAX_ENGINE_GROUPS; i++) { + if (!eng_grps->grp[i].is_enabled) + return &eng_grps->grp[i]; + } + return NULL; +} + +static int eng_grp_update_masks(struct device *dev, + struct otx2_cpt_eng_grp_info *eng_grp) +{ + struct otx2_cpt_engs_rsvd *engs, *mirrored_engs; + struct otx2_cpt_bitmap tmp_bmap = { {0} }; + int i, j, cnt, max_cnt; + int bit; + + for (i = 0; i < OTX2_CPT_MAX_ETYPES_PER_GRP; i++) { + engs = &eng_grp->engs[i]; + if (!engs->type) + continue; + if (engs->count <= 0) + continue; + + switch (engs->type) { + case OTX2_CPT_SE_TYPES: + max_cnt = eng_grp->g->avail.max_se_cnt; + break; + + case OTX2_CPT_IE_TYPES: + max_cnt = eng_grp->g->avail.max_ie_cnt; + break; + + case OTX2_CPT_AE_TYPES: + max_cnt = eng_grp->g->avail.max_ae_cnt; + break; + + default: + dev_err(dev, "Invalid engine type %d\n", engs->type); + return -EINVAL; + } + + cnt = engs->count; + WARN_ON(engs->offset + max_cnt > OTX2_CPT_MAX_ENGINES); + bitmap_zero(tmp_bmap.bits, eng_grp->g->engs_num); + for (j = engs->offset; j < engs->offset + max_cnt; j++) { + if (!eng_grp->g->eng_ref_cnt[j]) { + bitmap_set(tmp_bmap.bits, j, 1); + cnt--; + if (!cnt) + break; + } + } + + if (cnt) + return -ENOSPC; + + bitmap_copy(engs->bmap, tmp_bmap.bits, eng_grp->g->engs_num); + } + + if (!eng_grp->mirror.is_ena) + return 0; + + for (i = 0; i < OTX2_CPT_MAX_ETYPES_PER_GRP; i++) { + engs = &eng_grp->engs[i]; + if (!engs->type) + continue; + + mirrored_engs = find_engines_by_type( + &eng_grp->g->grp[eng_grp->mirror.idx], + engs->type); + WARN_ON(!mirrored_engs && engs->count <= 0); + if (!mirrored_engs) + continue; + + bitmap_copy(tmp_bmap.bits, mirrored_engs->bmap, + eng_grp->g->engs_num); + if (engs->count < 0) { + bit = find_first_bit(mirrored_engs->bmap, + eng_grp->g->engs_num); + bitmap_clear(tmp_bmap.bits, bit, -engs->count); + } + bitmap_or(engs->bmap, engs->bmap, tmp_bmap.bits, + eng_grp->g->engs_num); + } + return 0; +} + +static int delete_engine_group(struct device *dev, + struct otx2_cpt_eng_grp_info *eng_grp) +{ + int ret; + + if (!eng_grp->is_enabled) + return 0; + + if (eng_grp->mirror.ref_count) + return -EINVAL; + + /* Removing engine group mirroring if enabled */ + remove_eng_grp_mirroring(eng_grp); + + /* Disable engine group */ + ret = disable_eng_grp(dev, eng_grp, eng_grp->g->obj); + if (ret) + return ret; + + /* Release all engines held by this engine group */ + ret = release_engines(dev, eng_grp); + if (ret) + return ret; + + eng_grp->is_enabled = false; + + return 0; +} + +static void update_ucode_ptrs(struct otx2_cpt_eng_grp_info *eng_grp) +{ + struct otx2_cpt_ucode *ucode; + + if (eng_grp->mirror.is_ena) + ucode = &eng_grp->g->grp[eng_grp->mirror.idx].ucode[0]; + else + ucode = &eng_grp->ucode[0]; + WARN_ON(!eng_grp->engs[0].type); + eng_grp->engs[0].ucode = ucode; + + if (eng_grp->engs[1].type) { + if (is_2nd_ucode_used(eng_grp)) + eng_grp->engs[1].ucode = &eng_grp->ucode[1]; + else + eng_grp->engs[1].ucode = ucode; + } +} + +static int create_engine_group(struct device *dev, + struct otx2_cpt_eng_grps *eng_grps, + struct otx2_cpt_engines *engs, int ucodes_cnt, + void *ucode_data[], int is_print) +{ + struct otx2_cpt_eng_grp_info *mirrored_eng_grp; + struct otx2_cpt_eng_grp_info *eng_grp; + struct otx2_cpt_uc_info_t *uc_info; + int i, ret = 0; + + /* Find engine group which is not used */ + eng_grp = find_unused_eng_grp(eng_grps); + if (!eng_grp) { + dev_err(dev, "Error all engine groups are being used\n"); + return -ENOSPC; + } + /* Load ucode */ + for (i = 0; i < ucodes_cnt; i++) { + uc_info = (struct otx2_cpt_uc_info_t *) ucode_data[i]; + eng_grp->ucode[i] = uc_info->ucode; + ret = copy_ucode_to_dma_mem(dev, &eng_grp->ucode[i], + uc_info->fw->data); + if (ret) + goto unload_ucode; + } + + /* Check if this group mirrors another existing engine group */ + mirrored_eng_grp = find_mirrored_eng_grp(eng_grp); + if (mirrored_eng_grp) { + /* Setup mirroring */ + setup_eng_grp_mirroring(eng_grp, mirrored_eng_grp); + + /* + * Update count of requested engines because some + * of them might be shared with mirrored group + */ + update_requested_engs(mirrored_eng_grp, engs, ucodes_cnt); + } + ret = reserve_engines(dev, eng_grp, engs, ucodes_cnt); + if (ret) + goto unload_ucode; + + /* Update ucode pointers used by engines */ + update_ucode_ptrs(eng_grp); + + /* Update engine masks used by this group */ + ret = eng_grp_update_masks(dev, eng_grp); + if (ret) + goto release_engs; + + /* Enable engine group */ + ret = enable_eng_grp(eng_grp, eng_grps->obj); + if (ret) + goto release_engs; + + /* + * If this engine group mirrors another engine group + * then we need to unload ucode as we will use ucode + * from mirrored engine group + */ + if (eng_grp->mirror.is_ena) + ucode_unload(dev, &eng_grp->ucode[0]); + + eng_grp->is_enabled = true; + + if (!is_print) + return 0; + + if (mirrored_eng_grp) + dev_info(dev, + "Engine_group%d: reuse microcode %s from group %d\n", + eng_grp->idx, mirrored_eng_grp->ucode[0].ver_str, + mirrored_eng_grp->idx); + else + dev_info(dev, "Engine_group%d: microcode loaded %s\n", + eng_grp->idx, eng_grp->ucode[0].ver_str); + if (is_2nd_ucode_used(eng_grp)) + dev_info(dev, "Engine_group%d: microcode loaded %s\n", + eng_grp->idx, eng_grp->ucode[1].ver_str); + + return 0; + +release_engs: + release_engines(dev, eng_grp); +unload_ucode: + ucode_unload(dev, &eng_grp->ucode[0]); + ucode_unload(dev, &eng_grp->ucode[1]); + return ret; +} + +static void delete_engine_grps(struct pci_dev *pdev, + struct otx2_cpt_eng_grps *eng_grps) +{ + int i; + + /* First delete all mirroring engine groups */ + for (i = 0; i < OTX2_CPT_MAX_ENGINE_GROUPS; i++) + if (eng_grps->grp[i].mirror.is_ena) + delete_engine_group(&pdev->dev, &eng_grps->grp[i]); + + /* Delete remaining engine groups */ + for (i = 0; i < OTX2_CPT_MAX_ENGINE_GROUPS; i++) + delete_engine_group(&pdev->dev, &eng_grps->grp[i]); +} + +int otx2_cpt_get_eng_grp(struct otx2_cpt_eng_grps *eng_grps, int eng_type) +{ + + int eng_grp_num = OTX2_CPT_INVALID_CRYPTO_ENG_GRP; + struct otx2_cpt_eng_grp_info *grp; + int i; + + for (i = 0; i < OTX2_CPT_MAX_ENGINE_GROUPS; i++) { + grp = &eng_grps->grp[i]; + if (!grp->is_enabled) + continue; + + if (eng_type == OTX2_CPT_SE_TYPES) { + if (eng_grp_has_eng_type(grp, eng_type) && + !eng_grp_has_eng_type(grp, OTX2_CPT_IE_TYPES)) { + eng_grp_num = i; + break; + } + } else { + if (eng_grp_has_eng_type(grp, eng_type)) { + eng_grp_num = i; + break; + } + } + } + return eng_grp_num; +} + +int otx2_cpt_create_eng_grps(struct pci_dev *pdev, + struct otx2_cpt_eng_grps *eng_grps) +{ + struct otx2_cpt_uc_info_t *uc_info[OTX2_CPT_MAX_ETYPES_PER_GRP] = { }; + struct otx2_cpt_engines engs[OTX2_CPT_MAX_ETYPES_PER_GRP] = { {0} }; + struct fw_info_t fw_info; + int ret; + + /* + * We don't create engine groups if it was already + * made (when user enabled VFs for the first time) + */ + if (eng_grps->is_grps_created) + return 0; + + ret = cpt_ucode_load_fw(pdev, &fw_info); + if (ret) + return ret; + + /* + * Create engine group with SE engines for kernel + * crypto functionality (symmetric crypto) + */ + uc_info[0] = get_ucode(&fw_info, OTX2_CPT_SE_TYPES); + if (uc_info[0] == NULL) { + dev_err(&pdev->dev, "Unable to find firmware for SE\n"); + ret = -EINVAL; + goto release_fw; + } + engs[0].type = OTX2_CPT_SE_TYPES; + engs[0].count = eng_grps->avail.max_se_cnt; + + ret = create_engine_group(&pdev->dev, eng_grps, engs, 1, + (void **) uc_info, 1); + if (ret) + goto release_fw; + + /* + * Create engine group with SE+IE engines for IPSec. + * All SE engines will be shared with engine group 0. + */ + uc_info[0] = get_ucode(&fw_info, OTX2_CPT_SE_TYPES); + uc_info[1] = get_ucode(&fw_info, OTX2_CPT_IE_TYPES); + + if (uc_info[1] == NULL) { + dev_err(&pdev->dev, "Unable to find firmware for IE"); + ret = -EINVAL; + goto delete_eng_grp; + } + engs[0].type = OTX2_CPT_SE_TYPES; + engs[0].count = eng_grps->avail.max_se_cnt; + engs[1].type = OTX2_CPT_IE_TYPES; + engs[1].count = eng_grps->avail.max_ie_cnt; + + ret = create_engine_group(&pdev->dev, eng_grps, engs, 2, + (void **) uc_info, 1); + if (ret) + goto delete_eng_grp; + + /* + * Create engine group with AE engines for asymmetric + * crypto functionality. + */ + uc_info[0] = get_ucode(&fw_info, OTX2_CPT_AE_TYPES); + if (uc_info[0] == NULL) { + dev_err(&pdev->dev, "Unable to find firmware for AE"); + ret = -EINVAL; + goto delete_eng_grp; + } + engs[0].type = OTX2_CPT_AE_TYPES; + engs[0].count = eng_grps->avail.max_ae_cnt; + + ret = create_engine_group(&pdev->dev, eng_grps, engs, 1, + (void **) uc_info, 1); + if (ret) + goto delete_eng_grp; + + eng_grps->is_grps_created = true; + + cpt_ucode_release_fw(&fw_info); + return 0; + +delete_eng_grp: + delete_engine_grps(pdev, eng_grps); +release_fw: + cpt_ucode_release_fw(&fw_info); + return ret; +} + +int otx2_cpt_disable_all_cores(struct otx2_cptpf_dev *cptpf) +{ + int i, ret, busy, total_cores; + int timeout = 10; + u64 reg = 0; + + total_cores = cptpf->eng_grps.avail.max_se_cnt + + cptpf->eng_grps.avail.max_ie_cnt + + cptpf->eng_grps.avail.max_ae_cnt; + + /* Disengage the cores from groups */ + for (i = 0; i < total_cores; i++) { + ret = otx2_cpt_add_write_af_reg(&cptpf->afpf_mbox, cptpf->pdev, + CPT_AF_EXEX_CTL2(i), 0x0); + if (ret) + return ret; + + cptpf->eng_grps.eng_ref_cnt[i] = 0; + } + ret = otx2_cpt_send_af_reg_requests(&cptpf->afpf_mbox, cptpf->pdev); + if (ret) + return ret; + + /* Wait for cores to become idle */ + do { + busy = 0; + usleep_range(10000, 20000); + if (timeout-- < 0) + return -EBUSY; + + for (i = 0; i < total_cores; i++) { + ret = otx2_cpt_read_af_reg(&cptpf->afpf_mbox, + cptpf->pdev, + CPT_AF_EXEX_STS(i), ®); + if (ret) + return ret; + + if (reg & 0x1) { + busy = 1; + break; + } + } + } while (busy); + + /* Disable the cores */ + for (i = 0; i < total_cores; i++) { + ret = otx2_cpt_add_write_af_reg(&cptpf->afpf_mbox, cptpf->pdev, + CPT_AF_EXEX_CTL(i), 0x0); + if (ret) + return ret; + } + return otx2_cpt_send_af_reg_requests(&cptpf->afpf_mbox, cptpf->pdev); +} + +void otx2_cpt_cleanup_eng_grps(struct pci_dev *pdev, + struct otx2_cpt_eng_grps *eng_grps) +{ + struct otx2_cpt_eng_grp_info *grp; + int i, j; + + delete_engine_grps(pdev, eng_grps); + /* Release memory */ + for (i = 0; i < OTX2_CPT_MAX_ENGINE_GROUPS; i++) { + grp = &eng_grps->grp[i]; + for (j = 0; j < OTX2_CPT_MAX_ETYPES_PER_GRP; j++) { + kfree(grp->engs[j].bmap); + grp->engs[j].bmap = NULL; + } + } +} + +int otx2_cpt_init_eng_grps(struct pci_dev *pdev, + struct otx2_cpt_eng_grps *eng_grps) +{ + struct otx2_cpt_eng_grp_info *grp; + int i, j, ret; + + eng_grps->obj = pci_get_drvdata(pdev); + eng_grps->avail.se_cnt = eng_grps->avail.max_se_cnt; + eng_grps->avail.ie_cnt = eng_grps->avail.max_ie_cnt; + eng_grps->avail.ae_cnt = eng_grps->avail.max_ae_cnt; + + eng_grps->engs_num = eng_grps->avail.max_se_cnt + + eng_grps->avail.max_ie_cnt + + eng_grps->avail.max_ae_cnt; + if (eng_grps->engs_num > OTX2_CPT_MAX_ENGINES) { + dev_err(&pdev->dev, + "Number of engines %d > than max supported %d\n", + eng_grps->engs_num, OTX2_CPT_MAX_ENGINES); + ret = -EINVAL; + goto cleanup_eng_grps; + } + + for (i = 0; i < OTX2_CPT_MAX_ENGINE_GROUPS; i++) { + grp = &eng_grps->grp[i]; + grp->g = eng_grps; + grp->idx = i; + + for (j = 0; j < OTX2_CPT_MAX_ETYPES_PER_GRP; j++) { + grp->engs[j].bmap = + kcalloc(BITS_TO_LONGS(eng_grps->engs_num), + sizeof(long), GFP_KERNEL); + if (!grp->engs[j].bmap) { + ret = -ENOMEM; + goto cleanup_eng_grps; + } + } + } + return 0; + +cleanup_eng_grps: + otx2_cpt_cleanup_eng_grps(pdev, eng_grps); + return ret; +} + +static int create_eng_caps_discovery_grps(struct pci_dev *pdev, + struct otx2_cpt_eng_grps *eng_grps) +{ + struct otx2_cpt_uc_info_t *uc_info[OTX2_CPT_MAX_ETYPES_PER_GRP] = { }; + struct otx2_cpt_engines engs[OTX2_CPT_MAX_ETYPES_PER_GRP] = { {0} }; + struct fw_info_t fw_info; + int ret; + + ret = cpt_ucode_load_fw(pdev, &fw_info); + if (ret) + return ret; + + uc_info[0] = get_ucode(&fw_info, OTX2_CPT_SE_TYPES); + if (uc_info[0] == NULL) { + dev_err(&pdev->dev, "Unable to find firmware for AE\n"); + ret = -EINVAL; + goto release_fw; + } + engs[0].type = OTX2_CPT_AE_TYPES; + engs[0].count = 2; + + ret = create_engine_group(&pdev->dev, eng_grps, engs, 1, + (void **) uc_info, 0); + if (ret) + goto release_fw; + + uc_info[0] = get_ucode(&fw_info, OTX2_CPT_SE_TYPES); + if (uc_info[0] == NULL) { + dev_err(&pdev->dev, "Unable to find firmware for SE\n"); + ret = -EINVAL; + goto delete_eng_grp; + } + engs[0].type = OTX2_CPT_SE_TYPES; + engs[0].count = 2; + + ret = create_engine_group(&pdev->dev, eng_grps, engs, 1, + (void **) uc_info, 0); + if (ret) + goto delete_eng_grp; + + uc_info[0] = get_ucode(&fw_info, OTX2_CPT_IE_TYPES); + if (uc_info[0] == NULL) { + dev_err(&pdev->dev, "Unable to find firmware for IE\n"); + ret = -EINVAL; + goto delete_eng_grp; + } + engs[0].type = OTX2_CPT_IE_TYPES; + engs[0].count = 2; + + ret = create_engine_group(&pdev->dev, eng_grps, engs, 1, + (void **) uc_info, 0); + if (ret) + goto delete_eng_grp; + + cpt_ucode_release_fw(&fw_info); + return 0; + +delete_eng_grp: + delete_engine_grps(pdev, eng_grps); +release_fw: + cpt_ucode_release_fw(&fw_info); + return ret; +} + +/* + * Get CPT HW capabilities using LOAD_FVC operation. + */ +int otx2_cpt_discover_eng_capabilities(struct otx2_cptpf_dev *cptpf) +{ + struct otx2_cptlfs_info *lfs = &cptpf->lfs; + struct otx2_cpt_iq_command iq_cmd; + union otx2_cpt_opcode opcode; + union otx2_cpt_res_s *result; + union otx2_cpt_inst_s inst; + dma_addr_t rptr_baddr; + struct pci_dev *pdev; + u32 len, compl_rlen; + int ret, etype; + void *rptr; + + /* + * We don't get capabilities if it was already done + * (when user enabled VFs for the first time) + */ + if (cptpf->is_eng_caps_discovered) + return 0; + + pdev = cptpf->pdev; + /* + * Create engine groups for each type to submit LOAD_FVC op and + * get engine's capabilities. + */ + ret = create_eng_caps_discovery_grps(pdev, &cptpf->eng_grps); + if (ret) + goto delete_grps; + + lfs->pdev = pdev; + lfs->reg_base = cptpf->reg_base; + lfs->mbox = &cptpf->afpf_mbox; + ret = otx2_cptlf_init(&cptpf->lfs, OTX2_CPT_ALL_ENG_GRPS_MASK, + OTX2_CPT_QUEUE_HI_PRIO, 1); + if (ret) + goto delete_grps; + + compl_rlen = ALIGN(sizeof(union otx2_cpt_res_s), OTX2_CPT_DMA_MINALIGN); + len = compl_rlen + LOADFVC_RLEN; + + result = kzalloc(len, GFP_KERNEL); + if (!result) { + ret = -ENOMEM; + goto lf_cleanup; + } + rptr_baddr = dma_map_single(&pdev->dev, (void *)result, len, + DMA_BIDIRECTIONAL); + if (dma_mapping_error(&pdev->dev, rptr_baddr)) { + dev_err(&pdev->dev, "DMA mapping failed\n"); + ret = -EFAULT; + goto free_result; + } + rptr = (u8 *)result + compl_rlen; + + /* Fill in the command */ + opcode.s.major = LOADFVC_MAJOR_OP; + opcode.s.minor = LOADFVC_MINOR_OP; + + iq_cmd.cmd.u = 0; + iq_cmd.cmd.s.opcode = cpu_to_be16(opcode.flags); + + /* 64-bit swap for microcode data reads, not needed for addresses */ + cpu_to_be64s(&iq_cmd.cmd.u); + iq_cmd.dptr = 0; + iq_cmd.rptr = rptr_baddr + compl_rlen; + iq_cmd.cptr.u = 0; + + for (etype = 1; etype < OTX2_CPT_MAX_ENG_TYPES; etype++) { + result->s.compcode = OTX2_CPT_COMPLETION_CODE_INIT; + iq_cmd.cptr.s.grp = otx2_cpt_get_eng_grp(&cptpf->eng_grps, + etype); + otx2_cpt_fill_inst(&inst, &iq_cmd, rptr_baddr); + otx2_cpt_send_cmd(&inst, 1, &cptpf->lfs.lf[0]); + + while (result->s.compcode == OTX2_CPT_COMPLETION_CODE_INIT) + cpu_relax(); + + cptpf->eng_caps[etype].u = be64_to_cpup(rptr); + } + dma_unmap_single(&pdev->dev, rptr_baddr, len, DMA_BIDIRECTIONAL); + cptpf->is_eng_caps_discovered = true; + +free_result: + kfree(result); +lf_cleanup: + otx2_cptlf_shutdown(&cptpf->lfs); +delete_grps: + delete_engine_grps(pdev, &cptpf->eng_grps); + + return ret; +} diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.h b/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.h new file mode 100644 index 0000000000000000000000000000000000000000..6b0d432de0afa04587c1234147c955eb0bed02d6 --- /dev/null +++ b/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.h @@ -0,0 +1,162 @@ +/* SPDX-License-Identifier: GPL-2.0-only + * Copyright (C) 2020 Marvell. + */ + +#ifndef __OTX2_CPTPF_UCODE_H +#define __OTX2_CPTPF_UCODE_H + +#include +#include +#include +#include "otx2_cpt_hw_types.h" +#include "otx2_cpt_common.h" + +/* + * On OcteonTX2 platform IPSec ucode can use both IE and SE engines therefore + * IE and SE engines can be attached to the same engine group. + */ +#define OTX2_CPT_MAX_ETYPES_PER_GRP 2 + +/* CPT ucode signature size */ +#define OTX2_CPT_UCODE_SIGN_LEN 256 + +/* Microcode version string length */ +#define OTX2_CPT_UCODE_VER_STR_SZ 44 + +/* Maximum number of supported engines/cores on OcteonTX2 platform */ +#define OTX2_CPT_MAX_ENGINES 128 + +#define OTX2_CPT_ENGS_BITMASK_LEN BITS_TO_LONGS(OTX2_CPT_MAX_ENGINES) + +/* Microcode types */ +enum otx2_cpt_ucode_type { + OTX2_CPT_AE_UC_TYPE = 1, /* AE-MAIN */ + OTX2_CPT_SE_UC_TYPE1 = 20,/* SE-MAIN - combination of 21 and 22 */ + OTX2_CPT_SE_UC_TYPE2 = 21,/* Fast Path IPSec + AirCrypto */ + OTX2_CPT_SE_UC_TYPE3 = 22,/* + * Hash + HMAC + FlexiCrypto + RNG + + * Full Feature IPSec + AirCrypto + Kasumi + */ + OTX2_CPT_IE_UC_TYPE1 = 30, /* IE-MAIN - combination of 31 and 32 */ + OTX2_CPT_IE_UC_TYPE2 = 31, /* Fast Path IPSec */ + OTX2_CPT_IE_UC_TYPE3 = 32, /* + * Hash + HMAC + FlexiCrypto + RNG + + * Full Future IPSec + */ +}; + +struct otx2_cpt_bitmap { + unsigned long bits[OTX2_CPT_ENGS_BITMASK_LEN]; + int size; +}; + +struct otx2_cpt_engines { + int type; + int count; +}; + +/* Microcode version number */ +struct otx2_cpt_ucode_ver_num { + u8 nn; + u8 xx; + u8 yy; + u8 zz; +}; + +struct otx2_cpt_ucode_hdr { + struct otx2_cpt_ucode_ver_num ver_num; + u8 ver_str[OTX2_CPT_UCODE_VER_STR_SZ]; + __be32 code_length; + u32 padding[3]; +}; + +struct otx2_cpt_ucode { + u8 ver_str[OTX2_CPT_UCODE_VER_STR_SZ];/* + * ucode version in readable + * format + */ + struct otx2_cpt_ucode_ver_num ver_num;/* ucode version number */ + char filename[OTX2_CPT_NAME_LENGTH];/* ucode filename */ + dma_addr_t dma; /* phys address of ucode image */ + void *va; /* virt address of ucode image */ + u32 size; /* ucode image size */ + int type; /* ucode image type SE, IE, AE or SE+IE */ +}; + +struct otx2_cpt_uc_info_t { + struct list_head list; + struct otx2_cpt_ucode ucode;/* microcode information */ + const struct firmware *fw; +}; + +/* Maximum and current number of engines available for all engine groups */ +struct otx2_cpt_engs_available { + int max_se_cnt; + int max_ie_cnt; + int max_ae_cnt; + int se_cnt; + int ie_cnt; + int ae_cnt; +}; + +/* Engines reserved to an engine group */ +struct otx2_cpt_engs_rsvd { + int type; /* engine type */ + int count; /* number of engines attached */ + int offset; /* constant offset of engine type in the bitmap */ + unsigned long *bmap; /* attached engines bitmap */ + struct otx2_cpt_ucode *ucode; /* ucode used by these engines */ +}; + +struct otx2_cpt_mirror_info { + int is_ena; /* + * is mirroring enabled, it is set only for engine + * group which mirrors another engine group + */ + int idx; /* + * index of engine group which is mirrored by this + * group, set only for engine group which mirrors + * another group + */ + int ref_count; /* + * number of times this engine group is mirrored by + * other groups, this is set only for engine group + * which is mirrored by other group(s) + */ +}; + +struct otx2_cpt_eng_grp_info { + struct otx2_cpt_eng_grps *g; /* pointer to engine_groups structure */ + /* engines attached */ + struct otx2_cpt_engs_rsvd engs[OTX2_CPT_MAX_ETYPES_PER_GRP]; + /* ucodes information */ + struct otx2_cpt_ucode ucode[OTX2_CPT_MAX_ETYPES_PER_GRP]; + /* engine group mirroring information */ + struct otx2_cpt_mirror_info mirror; + int idx; /* engine group index */ + bool is_enabled; /* + * is engine group enabled, engine group is enabled + * when it has engines attached and ucode loaded + */ +}; + +struct otx2_cpt_eng_grps { + struct otx2_cpt_eng_grp_info grp[OTX2_CPT_MAX_ENGINE_GROUPS]; + struct otx2_cpt_engs_available avail; + void *obj; /* device specific data */ + int engs_num; /* total number of engines supported */ + u8 eng_ref_cnt[OTX2_CPT_MAX_ENGINES];/* engines reference count */ + bool is_grps_created; /* Is the engine groups are already created */ +}; +struct otx2_cptpf_dev; +int otx2_cpt_init_eng_grps(struct pci_dev *pdev, + struct otx2_cpt_eng_grps *eng_grps); +void otx2_cpt_cleanup_eng_grps(struct pci_dev *pdev, + struct otx2_cpt_eng_grps *eng_grps); +int otx2_cpt_create_eng_grps(struct pci_dev *pdev, + struct otx2_cpt_eng_grps *eng_grps); +int otx2_cpt_disable_all_cores(struct otx2_cptpf_dev *cptpf); +int otx2_cpt_get_eng_grp(struct otx2_cpt_eng_grps *eng_grps, int eng_type); +int otx2_cpt_discover_eng_capabilities(struct otx2_cptpf_dev *cptpf); + +#endif /* __OTX2_CPTPF_UCODE_H */ diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptvf.h b/drivers/crypto/marvell/octeontx2/otx2_cptvf.h new file mode 100644 index 0000000000000000000000000000000000000000..4f0a169fddbd0f297740cb95e35905265315c5f6 --- /dev/null +++ b/drivers/crypto/marvell/octeontx2/otx2_cptvf.h @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: GPL-2.0-only + * Copyright (C) 2020 Marvell. + */ + +#ifndef __OTX2_CPTVF_H +#define __OTX2_CPTVF_H + +#include "mbox.h" +#include "otx2_cptlf.h" + +struct otx2_cptvf_dev { + void __iomem *reg_base; /* Register start address */ + void __iomem *pfvf_mbox_base; /* PF-VF mbox start address */ + struct pci_dev *pdev; /* PCI device handle */ + struct otx2_cptlfs_info lfs; /* CPT LFs attached to this VF */ + u8 vf_id; /* Virtual function index */ + + /* PF <=> VF mbox */ + struct otx2_mbox pfvf_mbox; + struct work_struct pfvf_mbox_work; + struct workqueue_struct *pfvf_mbox_wq; +}; + +irqreturn_t otx2_cptvf_pfvf_mbox_intr(int irq, void *arg); +void otx2_cptvf_pfvf_mbox_handler(struct work_struct *work); +int otx2_cptvf_send_eng_grp_num_msg(struct otx2_cptvf_dev *cptvf, int eng_type); +int otx2_cptvf_send_kvf_limits_msg(struct otx2_cptvf_dev *cptvf); + +#endif /* __OTX2_CPTVF_H */ diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.c b/drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.c new file mode 100644 index 0000000000000000000000000000000000000000..a72723455df724abe8dc38f39f111063086e3b57 --- /dev/null +++ b/drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.c @@ -0,0 +1,1758 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (C) 2020 Marvell. */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "otx2_cptvf.h" +#include "otx2_cptvf_algs.h" +#include "otx2_cpt_reqmgr.h" + +/* Size of salt in AES GCM mode */ +#define AES_GCM_SALT_SIZE 4 +/* Size of IV in AES GCM mode */ +#define AES_GCM_IV_SIZE 8 +/* Size of ICV (Integrity Check Value) in AES GCM mode */ +#define AES_GCM_ICV_SIZE 16 +/* Offset of IV in AES GCM mode */ +#define AES_GCM_IV_OFFSET 8 +#define CONTROL_WORD_LEN 8 +#define KEY2_OFFSET 48 +#define DMA_MODE_FLAG(dma_mode) \ + (((dma_mode) == OTX2_CPT_DMA_MODE_SG) ? (1 << 7) : 0) + +/* Truncated SHA digest size */ +#define SHA1_TRUNC_DIGEST_SIZE 12 +#define SHA256_TRUNC_DIGEST_SIZE 16 +#define SHA384_TRUNC_DIGEST_SIZE 24 +#define SHA512_TRUNC_DIGEST_SIZE 32 + +static DEFINE_MUTEX(mutex); +static int is_crypto_registered; + +struct cpt_device_desc { + struct pci_dev *dev; + int num_queues; +}; + +struct cpt_device_table { + atomic_t count; + struct cpt_device_desc desc[OTX2_CPT_MAX_LFS_NUM]; +}; + +static struct cpt_device_table se_devices = { + .count = ATOMIC_INIT(0) +}; + +static inline int get_se_device(struct pci_dev **pdev, int *cpu_num) +{ + int count; + + count = atomic_read(&se_devices.count); + if (count < 1) + return -ENODEV; + + *cpu_num = get_cpu(); + /* + * On OcteonTX2 platform CPT instruction queue is bound to each + * local function LF, in turn LFs can be attached to PF + * or VF therefore we always use first device. We get maximum + * performance if one CPT queue is available for each cpu + * otherwise CPT queues need to be shared between cpus. + */ + if (*cpu_num >= se_devices.desc[0].num_queues) + *cpu_num %= se_devices.desc[0].num_queues; + *pdev = se_devices.desc[0].dev; + + put_cpu(); + + return 0; +} + +static inline int validate_hmac_cipher_null(struct otx2_cpt_req_info *cpt_req) +{ + struct otx2_cpt_req_ctx *rctx; + struct aead_request *req; + struct crypto_aead *tfm; + + req = container_of(cpt_req->areq, struct aead_request, base); + tfm = crypto_aead_reqtfm(req); + rctx = aead_request_ctx(req); + if (memcmp(rctx->fctx.hmac.s.hmac_calc, + rctx->fctx.hmac.s.hmac_recv, + crypto_aead_authsize(tfm)) != 0) + return -EBADMSG; + + return 0; +} + +static void otx2_cpt_aead_callback(int status, void *arg1, void *arg2) +{ + struct otx2_cpt_inst_info *inst_info = arg2; + struct crypto_async_request *areq = arg1; + struct otx2_cpt_req_info *cpt_req; + struct pci_dev *pdev; + + if (inst_info) { + cpt_req = inst_info->req; + if (!status) { + /* + * When selected cipher is NULL we need to manually + * verify whether calculated hmac value matches + * received hmac value + */ + if (cpt_req->req_type == + OTX2_CPT_AEAD_ENC_DEC_NULL_REQ && + !cpt_req->is_enc) + status = validate_hmac_cipher_null(cpt_req); + } + pdev = inst_info->pdev; + otx2_cpt_info_destroy(pdev, inst_info); + } + if (areq) + areq->complete(areq, status); +} + +static void output_iv_copyback(struct crypto_async_request *areq) +{ + struct otx2_cpt_req_info *req_info; + struct otx2_cpt_req_ctx *rctx; + struct skcipher_request *sreq; + struct crypto_skcipher *stfm; + struct otx2_cpt_enc_ctx *ctx; + u32 start, ivsize; + + sreq = container_of(areq, struct skcipher_request, base); + stfm = crypto_skcipher_reqtfm(sreq); + ctx = crypto_skcipher_ctx(stfm); + if (ctx->cipher_type == OTX2_CPT_AES_CBC || + ctx->cipher_type == OTX2_CPT_DES3_CBC) { + rctx = skcipher_request_ctx(sreq); + req_info = &rctx->cpt_req; + ivsize = crypto_skcipher_ivsize(stfm); + start = sreq->cryptlen - ivsize; + + if (req_info->is_enc) { + scatterwalk_map_and_copy(sreq->iv, sreq->dst, start, + ivsize, 0); + } else { + if (sreq->src != sreq->dst) { + scatterwalk_map_and_copy(sreq->iv, sreq->src, + start, ivsize, 0); + } else { + memcpy(sreq->iv, req_info->iv_out, ivsize); + kfree(req_info->iv_out); + } + } + } +} + +static void otx2_cpt_skcipher_callback(int status, void *arg1, void *arg2) +{ + struct otx2_cpt_inst_info *inst_info = arg2; + struct crypto_async_request *areq = arg1; + struct pci_dev *pdev; + + if (areq) { + if (!status) + output_iv_copyback(areq); + if (inst_info) { + pdev = inst_info->pdev; + otx2_cpt_info_destroy(pdev, inst_info); + } + areq->complete(areq, status); + } +} + +static inline void update_input_data(struct otx2_cpt_req_info *req_info, + struct scatterlist *inp_sg, + u32 nbytes, u32 *argcnt) +{ + req_info->req.dlen += nbytes; + + while (nbytes) { + u32 len = (nbytes < inp_sg->length) ? nbytes : inp_sg->length; + u8 *ptr = sg_virt(inp_sg); + + req_info->in[*argcnt].vptr = (void *)ptr; + req_info->in[*argcnt].size = len; + nbytes -= len; + ++(*argcnt); + inp_sg = sg_next(inp_sg); + } +} + +static inline void update_output_data(struct otx2_cpt_req_info *req_info, + struct scatterlist *outp_sg, + u32 offset, u32 nbytes, u32 *argcnt) +{ + u32 len, sg_len; + u8 *ptr; + + req_info->rlen += nbytes; + + while (nbytes) { + sg_len = outp_sg->length - offset; + len = (nbytes < sg_len) ? nbytes : sg_len; + ptr = sg_virt(outp_sg); + + req_info->out[*argcnt].vptr = (void *) (ptr + offset); + req_info->out[*argcnt].size = len; + nbytes -= len; + ++(*argcnt); + offset = 0; + outp_sg = sg_next(outp_sg); + } +} + +static inline int create_ctx_hdr(struct skcipher_request *req, u32 enc, + u32 *argcnt) +{ + struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req); + struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx(req); + struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(stfm); + struct otx2_cpt_req_info *req_info = &rctx->cpt_req; + struct otx2_cpt_fc_ctx *fctx = &rctx->fctx; + int ivsize = crypto_skcipher_ivsize(stfm); + u32 start = req->cryptlen - ivsize; + gfp_t flags; + + flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC; + req_info->ctrl.s.dma_mode = OTX2_CPT_DMA_MODE_SG; + req_info->ctrl.s.se_req = 1; + + req_info->req.opcode.s.major = OTX2_CPT_MAJOR_OP_FC | + DMA_MODE_FLAG(OTX2_CPT_DMA_MODE_SG); + if (enc) { + req_info->req.opcode.s.minor = 2; + } else { + req_info->req.opcode.s.minor = 3; + if ((ctx->cipher_type == OTX2_CPT_AES_CBC || + ctx->cipher_type == OTX2_CPT_DES3_CBC) && + req->src == req->dst) { + req_info->iv_out = kmalloc(ivsize, flags); + if (!req_info->iv_out) + return -ENOMEM; + + scatterwalk_map_and_copy(req_info->iv_out, req->src, + start, ivsize, 0); + } + } + /* Encryption data length */ + req_info->req.param1 = req->cryptlen; + /* Authentication data length */ + req_info->req.param2 = 0; + + fctx->enc.enc_ctrl.e.enc_cipher = ctx->cipher_type; + fctx->enc.enc_ctrl.e.aes_key = ctx->key_type; + fctx->enc.enc_ctrl.e.iv_source = OTX2_CPT_FROM_CPTR; + + if (ctx->cipher_type == OTX2_CPT_AES_XTS) + memcpy(fctx->enc.encr_key, ctx->enc_key, ctx->key_len * 2); + else + memcpy(fctx->enc.encr_key, ctx->enc_key, ctx->key_len); + + memcpy(fctx->enc.encr_iv, req->iv, crypto_skcipher_ivsize(stfm)); + + cpu_to_be64s(&fctx->enc.enc_ctrl.u); + + /* + * Storing Packet Data Information in offset + * Control Word First 8 bytes + */ + req_info->in[*argcnt].vptr = (u8 *)&rctx->ctrl_word; + req_info->in[*argcnt].size = CONTROL_WORD_LEN; + req_info->req.dlen += CONTROL_WORD_LEN; + ++(*argcnt); + + req_info->in[*argcnt].vptr = (u8 *)fctx; + req_info->in[*argcnt].size = sizeof(struct otx2_cpt_fc_ctx); + req_info->req.dlen += sizeof(struct otx2_cpt_fc_ctx); + + ++(*argcnt); + + return 0; +} + +static inline int create_input_list(struct skcipher_request *req, u32 enc, + u32 enc_iv_len) +{ + struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx(req); + struct otx2_cpt_req_info *req_info = &rctx->cpt_req; + u32 argcnt = 0; + int ret; + + ret = create_ctx_hdr(req, enc, &argcnt); + if (ret) + return ret; + + update_input_data(req_info, req->src, req->cryptlen, &argcnt); + req_info->in_cnt = argcnt; + + return 0; +} + +static inline void create_output_list(struct skcipher_request *req, + u32 enc_iv_len) +{ + struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx(req); + struct otx2_cpt_req_info *req_info = &rctx->cpt_req; + u32 argcnt = 0; + + /* + * OUTPUT Buffer Processing + * AES encryption/decryption output would be + * received in the following format + * + * ------IV--------|------ENCRYPTED/DECRYPTED DATA-----| + * [ 16 Bytes/ [ Request Enc/Dec/ DATA Len AES CBC ] + */ + update_output_data(req_info, req->dst, 0, req->cryptlen, &argcnt); + req_info->out_cnt = argcnt; +} + +static int skcipher_do_fallback(struct skcipher_request *req, bool is_enc) +{ + struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req); + struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx(req); + struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(stfm); + int ret; + + if (ctx->fbk_cipher) { + skcipher_request_set_tfm(&rctx->sk_fbk_req, ctx->fbk_cipher); + skcipher_request_set_callback(&rctx->sk_fbk_req, + req->base.flags, + req->base.complete, + req->base.data); + skcipher_request_set_crypt(&rctx->sk_fbk_req, req->src, + req->dst, req->cryptlen, req->iv); + ret = is_enc ? crypto_skcipher_encrypt(&rctx->sk_fbk_req) : + crypto_skcipher_decrypt(&rctx->sk_fbk_req); + } else { + ret = -EINVAL; + } + return ret; +} + +static inline int cpt_enc_dec(struct skcipher_request *req, u32 enc) +{ + struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req); + struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx(req); + struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(stfm); + struct otx2_cpt_req_info *req_info = &rctx->cpt_req; + u32 enc_iv_len = crypto_skcipher_ivsize(stfm); + struct pci_dev *pdev; + int status, cpu_num; + + if (req->cryptlen == 0) + return 0; + + if (!IS_ALIGNED(req->cryptlen, ctx->enc_align_len)) + return -EINVAL; + + if (req->cryptlen > OTX2_CPT_MAX_REQ_SIZE) + return skcipher_do_fallback(req, enc); + + /* Clear control words */ + rctx->ctrl_word.flags = 0; + rctx->fctx.enc.enc_ctrl.u = 0; + + status = create_input_list(req, enc, enc_iv_len); + if (status) + return status; + create_output_list(req, enc_iv_len); + + status = get_se_device(&pdev, &cpu_num); + if (status) + return status; + + req_info->callback = otx2_cpt_skcipher_callback; + req_info->areq = &req->base; + req_info->req_type = OTX2_CPT_ENC_DEC_REQ; + req_info->is_enc = enc; + req_info->is_trunc_hmac = false; + req_info->ctrl.s.grp = otx2_cpt_get_kcrypto_eng_grp_num(pdev); + + /* + * We perform an asynchronous send and once + * the request is completed the driver would + * intimate through registered call back functions + */ + status = otx2_cpt_do_request(pdev, req_info, cpu_num); + + return status; +} + +static int otx2_cpt_skcipher_encrypt(struct skcipher_request *req) +{ + return cpt_enc_dec(req, true); +} + +static int otx2_cpt_skcipher_decrypt(struct skcipher_request *req) +{ + return cpt_enc_dec(req, false); +} + +static int otx2_cpt_skcipher_xts_setkey(struct crypto_skcipher *tfm, + const u8 *key, u32 keylen) +{ + struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(tfm); + const u8 *key2 = key + (keylen / 2); + const u8 *key1 = key; + int ret; + + ret = xts_check_key(crypto_skcipher_tfm(tfm), key, keylen); + if (ret) + return ret; + ctx->key_len = keylen; + ctx->enc_align_len = 1; + memcpy(ctx->enc_key, key1, keylen / 2); + memcpy(ctx->enc_key + KEY2_OFFSET, key2, keylen / 2); + ctx->cipher_type = OTX2_CPT_AES_XTS; + switch (ctx->key_len) { + case 2 * AES_KEYSIZE_128: + ctx->key_type = OTX2_CPT_AES_128_BIT; + break; + case 2 * AES_KEYSIZE_192: + ctx->key_type = OTX2_CPT_AES_192_BIT; + break; + case 2 * AES_KEYSIZE_256: + ctx->key_type = OTX2_CPT_AES_256_BIT; + break; + default: + return -EINVAL; + } + return crypto_skcipher_setkey(ctx->fbk_cipher, key, keylen); +} + +static int cpt_des_setkey(struct crypto_skcipher *tfm, const u8 *key, + u32 keylen, u8 cipher_type) +{ + struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(tfm); + + if (keylen != DES3_EDE_KEY_SIZE) + return -EINVAL; + + ctx->key_len = keylen; + ctx->cipher_type = cipher_type; + ctx->enc_align_len = 8; + + memcpy(ctx->enc_key, key, keylen); + + return crypto_skcipher_setkey(ctx->fbk_cipher, key, keylen); +} + +static int cpt_aes_setkey(struct crypto_skcipher *tfm, const u8 *key, + u32 keylen, u8 cipher_type) +{ + struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(tfm); + + switch (keylen) { + case AES_KEYSIZE_128: + ctx->key_type = OTX2_CPT_AES_128_BIT; + break; + case AES_KEYSIZE_192: + ctx->key_type = OTX2_CPT_AES_192_BIT; + break; + case AES_KEYSIZE_256: + ctx->key_type = OTX2_CPT_AES_256_BIT; + break; + default: + return -EINVAL; + } + if (cipher_type == OTX2_CPT_AES_CBC || cipher_type == OTX2_CPT_AES_ECB) + ctx->enc_align_len = 16; + else + ctx->enc_align_len = 1; + + ctx->key_len = keylen; + ctx->cipher_type = cipher_type; + + memcpy(ctx->enc_key, key, keylen); + + return crypto_skcipher_setkey(ctx->fbk_cipher, key, keylen); +} + +static int otx2_cpt_skcipher_cbc_aes_setkey(struct crypto_skcipher *tfm, + const u8 *key, u32 keylen) +{ + return cpt_aes_setkey(tfm, key, keylen, OTX2_CPT_AES_CBC); +} + +static int otx2_cpt_skcipher_ecb_aes_setkey(struct crypto_skcipher *tfm, + const u8 *key, u32 keylen) +{ + return cpt_aes_setkey(tfm, key, keylen, OTX2_CPT_AES_ECB); +} + +static int otx2_cpt_skcipher_cbc_des3_setkey(struct crypto_skcipher *tfm, + const u8 *key, u32 keylen) +{ + return cpt_des_setkey(tfm, key, keylen, OTX2_CPT_DES3_CBC); +} + +static int otx2_cpt_skcipher_ecb_des3_setkey(struct crypto_skcipher *tfm, + const u8 *key, u32 keylen) +{ + return cpt_des_setkey(tfm, key, keylen, OTX2_CPT_DES3_ECB); +} + +static int cpt_skcipher_fallback_init(struct otx2_cpt_enc_ctx *ctx, + struct crypto_alg *alg) +{ + if (alg->cra_flags & CRYPTO_ALG_NEED_FALLBACK) { + ctx->fbk_cipher = + crypto_alloc_skcipher(alg->cra_name, 0, + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(ctx->fbk_cipher)) { + pr_err("%s() failed to allocate fallback for %s\n", + __func__, alg->cra_name); + return PTR_ERR(ctx->fbk_cipher); + } + } + return 0; +} + +static int otx2_cpt_enc_dec_init(struct crypto_skcipher *stfm) +{ + struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(stfm); + struct crypto_tfm *tfm = crypto_skcipher_tfm(stfm); + struct crypto_alg *alg = tfm->__crt_alg; + + memset(ctx, 0, sizeof(*ctx)); + /* + * Additional memory for skcipher_request is + * allocated since the cryptd daemon uses + * this memory for request_ctx information + */ + crypto_skcipher_set_reqsize(stfm, sizeof(struct otx2_cpt_req_ctx) + + sizeof(struct skcipher_request)); + + return cpt_skcipher_fallback_init(ctx, alg); +} + +static void otx2_cpt_skcipher_exit(struct crypto_skcipher *tfm) +{ + struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(tfm); + + if (ctx->fbk_cipher) { + crypto_free_skcipher(ctx->fbk_cipher); + ctx->fbk_cipher = NULL; + } +} + +static int cpt_aead_fallback_init(struct otx2_cpt_aead_ctx *ctx, + struct crypto_alg *alg) +{ + if (alg->cra_flags & CRYPTO_ALG_NEED_FALLBACK) { + ctx->fbk_cipher = + crypto_alloc_aead(alg->cra_name, 0, + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(ctx->fbk_cipher)) { + pr_err("%s() failed to allocate fallback for %s\n", + __func__, alg->cra_name); + return PTR_ERR(ctx->fbk_cipher); + } + } + return 0; +} + +static int cpt_aead_init(struct crypto_aead *atfm, u8 cipher_type, u8 mac_type) +{ + struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(atfm); + struct crypto_tfm *tfm = crypto_aead_tfm(atfm); + struct crypto_alg *alg = tfm->__crt_alg; + + ctx->cipher_type = cipher_type; + ctx->mac_type = mac_type; + + /* + * When selected cipher is NULL we use HMAC opcode instead of + * FLEXICRYPTO opcode therefore we don't need to use HASH algorithms + * for calculating ipad and opad + */ + if (ctx->cipher_type != OTX2_CPT_CIPHER_NULL) { + switch (ctx->mac_type) { + case OTX2_CPT_SHA1: + ctx->hashalg = crypto_alloc_shash("sha1", 0, + CRYPTO_ALG_ASYNC); + if (IS_ERR(ctx->hashalg)) + return PTR_ERR(ctx->hashalg); + break; + + case OTX2_CPT_SHA256: + ctx->hashalg = crypto_alloc_shash("sha256", 0, + CRYPTO_ALG_ASYNC); + if (IS_ERR(ctx->hashalg)) + return PTR_ERR(ctx->hashalg); + break; + + case OTX2_CPT_SHA384: + ctx->hashalg = crypto_alloc_shash("sha384", 0, + CRYPTO_ALG_ASYNC); + if (IS_ERR(ctx->hashalg)) + return PTR_ERR(ctx->hashalg); + break; + + case OTX2_CPT_SHA512: + ctx->hashalg = crypto_alloc_shash("sha512", 0, + CRYPTO_ALG_ASYNC); + if (IS_ERR(ctx->hashalg)) + return PTR_ERR(ctx->hashalg); + break; + } + } + switch (ctx->cipher_type) { + case OTX2_CPT_AES_CBC: + case OTX2_CPT_AES_ECB: + ctx->enc_align_len = 16; + break; + case OTX2_CPT_DES3_CBC: + case OTX2_CPT_DES3_ECB: + ctx->enc_align_len = 8; + break; + case OTX2_CPT_AES_GCM: + case OTX2_CPT_CIPHER_NULL: + ctx->enc_align_len = 1; + break; + } + crypto_aead_set_reqsize(atfm, sizeof(struct otx2_cpt_req_ctx)); + + return cpt_aead_fallback_init(ctx, alg); +} + +static int otx2_cpt_aead_cbc_aes_sha1_init(struct crypto_aead *tfm) +{ + return cpt_aead_init(tfm, OTX2_CPT_AES_CBC, OTX2_CPT_SHA1); +} + +static int otx2_cpt_aead_cbc_aes_sha256_init(struct crypto_aead *tfm) +{ + return cpt_aead_init(tfm, OTX2_CPT_AES_CBC, OTX2_CPT_SHA256); +} + +static int otx2_cpt_aead_cbc_aes_sha384_init(struct crypto_aead *tfm) +{ + return cpt_aead_init(tfm, OTX2_CPT_AES_CBC, OTX2_CPT_SHA384); +} + +static int otx2_cpt_aead_cbc_aes_sha512_init(struct crypto_aead *tfm) +{ + return cpt_aead_init(tfm, OTX2_CPT_AES_CBC, OTX2_CPT_SHA512); +} + +static int otx2_cpt_aead_ecb_null_sha1_init(struct crypto_aead *tfm) +{ + return cpt_aead_init(tfm, OTX2_CPT_CIPHER_NULL, OTX2_CPT_SHA1); +} + +static int otx2_cpt_aead_ecb_null_sha256_init(struct crypto_aead *tfm) +{ + return cpt_aead_init(tfm, OTX2_CPT_CIPHER_NULL, OTX2_CPT_SHA256); +} + +static int otx2_cpt_aead_ecb_null_sha384_init(struct crypto_aead *tfm) +{ + return cpt_aead_init(tfm, OTX2_CPT_CIPHER_NULL, OTX2_CPT_SHA384); +} + +static int otx2_cpt_aead_ecb_null_sha512_init(struct crypto_aead *tfm) +{ + return cpt_aead_init(tfm, OTX2_CPT_CIPHER_NULL, OTX2_CPT_SHA512); +} + +static int otx2_cpt_aead_gcm_aes_init(struct crypto_aead *tfm) +{ + return cpt_aead_init(tfm, OTX2_CPT_AES_GCM, OTX2_CPT_MAC_NULL); +} + +static void otx2_cpt_aead_exit(struct crypto_aead *tfm) +{ + struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(tfm); + + kfree(ctx->ipad); + kfree(ctx->opad); + if (ctx->hashalg) + crypto_free_shash(ctx->hashalg); + kfree(ctx->sdesc); + + if (ctx->fbk_cipher) { + crypto_free_aead(ctx->fbk_cipher); + ctx->fbk_cipher = NULL; + } +} + +static int otx2_cpt_aead_gcm_set_authsize(struct crypto_aead *tfm, + unsigned int authsize) +{ + struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(tfm); + + if (crypto_rfc4106_check_authsize(authsize)) + return -EINVAL; + + tfm->authsize = authsize; + /* Set authsize for fallback case */ + if (ctx->fbk_cipher) + ctx->fbk_cipher->authsize = authsize; + + return 0; +} + +static int otx2_cpt_aead_set_authsize(struct crypto_aead *tfm, + unsigned int authsize) +{ + tfm->authsize = authsize; + + return 0; +} + +static int otx2_cpt_aead_null_set_authsize(struct crypto_aead *tfm, + unsigned int authsize) +{ + struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(tfm); + + ctx->is_trunc_hmac = true; + tfm->authsize = authsize; + + return 0; +} + +static struct otx2_cpt_sdesc *alloc_sdesc(struct crypto_shash *alg) +{ + struct otx2_cpt_sdesc *sdesc; + int size; + + size = sizeof(struct shash_desc) + crypto_shash_descsize(alg); + sdesc = kmalloc(size, GFP_KERNEL); + if (!sdesc) + return NULL; + + sdesc->shash.tfm = alg; + + return sdesc; +} + +static inline void swap_data32(void *buf, u32 len) +{ + cpu_to_be32_array(buf, buf, len / 4); +} + +static inline void swap_data64(void *buf, u32 len) +{ + u64 *src = buf; + int i = 0; + + for (i = 0 ; i < len / 8; i++, src++) + cpu_to_be64s(src); +} + +static int copy_pad(u8 mac_type, u8 *out_pad, u8 *in_pad) +{ + struct sha512_state *sha512; + struct sha256_state *sha256; + struct sha1_state *sha1; + + switch (mac_type) { + case OTX2_CPT_SHA1: + sha1 = (struct sha1_state *) in_pad; + swap_data32(sha1->state, SHA1_DIGEST_SIZE); + memcpy(out_pad, &sha1->state, SHA1_DIGEST_SIZE); + break; + + case OTX2_CPT_SHA256: + sha256 = (struct sha256_state *) in_pad; + swap_data32(sha256->state, SHA256_DIGEST_SIZE); + memcpy(out_pad, &sha256->state, SHA256_DIGEST_SIZE); + break; + + case OTX2_CPT_SHA384: + case OTX2_CPT_SHA512: + sha512 = (struct sha512_state *) in_pad; + swap_data64(sha512->state, SHA512_DIGEST_SIZE); + memcpy(out_pad, &sha512->state, SHA512_DIGEST_SIZE); + break; + + default: + return -EINVAL; + } + + return 0; +} + +static int aead_hmac_init(struct crypto_aead *cipher) +{ + struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(cipher); + int state_size = crypto_shash_statesize(ctx->hashalg); + int ds = crypto_shash_digestsize(ctx->hashalg); + int bs = crypto_shash_blocksize(ctx->hashalg); + int authkeylen = ctx->auth_key_len; + u8 *ipad = NULL, *opad = NULL; + int ret = 0, icount = 0; + + ctx->sdesc = alloc_sdesc(ctx->hashalg); + if (!ctx->sdesc) + return -ENOMEM; + + ctx->ipad = kzalloc(bs, GFP_KERNEL); + if (!ctx->ipad) { + ret = -ENOMEM; + goto calc_fail; + } + + ctx->opad = kzalloc(bs, GFP_KERNEL); + if (!ctx->opad) { + ret = -ENOMEM; + goto calc_fail; + } + + ipad = kzalloc(state_size, GFP_KERNEL); + if (!ipad) { + ret = -ENOMEM; + goto calc_fail; + } + + opad = kzalloc(state_size, GFP_KERNEL); + if (!opad) { + ret = -ENOMEM; + goto calc_fail; + } + + if (authkeylen > bs) { + ret = crypto_shash_digest(&ctx->sdesc->shash, ctx->key, + authkeylen, ipad); + if (ret) + goto calc_fail; + + authkeylen = ds; + } else { + memcpy(ipad, ctx->key, authkeylen); + } + + memset(ipad + authkeylen, 0, bs - authkeylen); + memcpy(opad, ipad, bs); + + for (icount = 0; icount < bs; icount++) { + ipad[icount] ^= 0x36; + opad[icount] ^= 0x5c; + } + + /* + * Partial Hash calculated from the software + * algorithm is retrieved for IPAD & OPAD + */ + + /* IPAD Calculation */ + crypto_shash_init(&ctx->sdesc->shash); + crypto_shash_update(&ctx->sdesc->shash, ipad, bs); + crypto_shash_export(&ctx->sdesc->shash, ipad); + ret = copy_pad(ctx->mac_type, ctx->ipad, ipad); + if (ret) + goto calc_fail; + + /* OPAD Calculation */ + crypto_shash_init(&ctx->sdesc->shash); + crypto_shash_update(&ctx->sdesc->shash, opad, bs); + crypto_shash_export(&ctx->sdesc->shash, opad); + ret = copy_pad(ctx->mac_type, ctx->opad, opad); + if (ret) + goto calc_fail; + + kfree(ipad); + kfree(opad); + + return 0; + +calc_fail: + kfree(ctx->ipad); + ctx->ipad = NULL; + kfree(ctx->opad); + ctx->opad = NULL; + kfree(ipad); + kfree(opad); + kfree(ctx->sdesc); + ctx->sdesc = NULL; + + return ret; +} + +static int otx2_cpt_aead_cbc_aes_sha_setkey(struct crypto_aead *cipher, + const unsigned char *key, + unsigned int keylen) +{ + struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(cipher); + struct crypto_authenc_key_param *param; + int enckeylen = 0, authkeylen = 0; + struct rtattr *rta = (void *)key; + int status; + + if (!RTA_OK(rta, keylen)) + return -EINVAL; + + if (rta->rta_type != CRYPTO_AUTHENC_KEYA_PARAM) + return -EINVAL; + + if (RTA_PAYLOAD(rta) < sizeof(*param)) + return -EINVAL; + + param = RTA_DATA(rta); + enckeylen = be32_to_cpu(param->enckeylen); + key += RTA_ALIGN(rta->rta_len); + keylen -= RTA_ALIGN(rta->rta_len); + if (keylen < enckeylen) + return -EINVAL; + + if (keylen > OTX2_CPT_MAX_KEY_SIZE) + return -EINVAL; + + authkeylen = keylen - enckeylen; + memcpy(ctx->key, key, keylen); + + switch (enckeylen) { + case AES_KEYSIZE_128: + ctx->key_type = OTX2_CPT_AES_128_BIT; + break; + case AES_KEYSIZE_192: + ctx->key_type = OTX2_CPT_AES_192_BIT; + break; + case AES_KEYSIZE_256: + ctx->key_type = OTX2_CPT_AES_256_BIT; + break; + default: + /* Invalid key length */ + return -EINVAL; + } + + ctx->enc_key_len = enckeylen; + ctx->auth_key_len = authkeylen; + + status = aead_hmac_init(cipher); + if (status) + return status; + + return 0; +} + +static int otx2_cpt_aead_ecb_null_sha_setkey(struct crypto_aead *cipher, + const unsigned char *key, + unsigned int keylen) +{ + struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(cipher); + struct crypto_authenc_key_param *param; + struct rtattr *rta = (void *)key; + int enckeylen = 0; + + if (!RTA_OK(rta, keylen)) + return -EINVAL; + + if (rta->rta_type != CRYPTO_AUTHENC_KEYA_PARAM) + return -EINVAL; + + if (RTA_PAYLOAD(rta) < sizeof(*param)) + return -EINVAL; + + param = RTA_DATA(rta); + enckeylen = be32_to_cpu(param->enckeylen); + key += RTA_ALIGN(rta->rta_len); + keylen -= RTA_ALIGN(rta->rta_len); + if (enckeylen != 0) + return -EINVAL; + + if (keylen > OTX2_CPT_MAX_KEY_SIZE) + return -EINVAL; + + memcpy(ctx->key, key, keylen); + ctx->enc_key_len = enckeylen; + ctx->auth_key_len = keylen; + + return 0; +} + +static int otx2_cpt_aead_gcm_aes_setkey(struct crypto_aead *cipher, + const unsigned char *key, + unsigned int keylen) +{ + struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(cipher); + + /* + * For aes gcm we expect to get encryption key (16, 24, 32 bytes) + * and salt (4 bytes) + */ + switch (keylen) { + case AES_KEYSIZE_128 + AES_GCM_SALT_SIZE: + ctx->key_type = OTX2_CPT_AES_128_BIT; + ctx->enc_key_len = AES_KEYSIZE_128; + break; + case AES_KEYSIZE_192 + AES_GCM_SALT_SIZE: + ctx->key_type = OTX2_CPT_AES_192_BIT; + ctx->enc_key_len = AES_KEYSIZE_192; + break; + case AES_KEYSIZE_256 + AES_GCM_SALT_SIZE: + ctx->key_type = OTX2_CPT_AES_256_BIT; + ctx->enc_key_len = AES_KEYSIZE_256; + break; + default: + /* Invalid key and salt length */ + return -EINVAL; + } + + /* Store encryption key and salt */ + memcpy(ctx->key, key, keylen); + + return crypto_aead_setkey(ctx->fbk_cipher, key, keylen); +} + +static inline int create_aead_ctx_hdr(struct aead_request *req, u32 enc, + u32 *argcnt) +{ + struct otx2_cpt_req_ctx *rctx = aead_request_ctx(req); + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(tfm); + struct otx2_cpt_req_info *req_info = &rctx->cpt_req; + struct otx2_cpt_fc_ctx *fctx = &rctx->fctx; + int mac_len = crypto_aead_authsize(tfm); + int ds; + + rctx->ctrl_word.e.enc_data_offset = req->assoclen; + + switch (ctx->cipher_type) { + case OTX2_CPT_AES_CBC: + if (req->assoclen > 248 || !IS_ALIGNED(req->assoclen, 8)) + return -EINVAL; + + fctx->enc.enc_ctrl.e.iv_source = OTX2_CPT_FROM_CPTR; + /* Copy encryption key to context */ + memcpy(fctx->enc.encr_key, ctx->key + ctx->auth_key_len, + ctx->enc_key_len); + /* Copy IV to context */ + memcpy(fctx->enc.encr_iv, req->iv, crypto_aead_ivsize(tfm)); + + ds = crypto_shash_digestsize(ctx->hashalg); + if (ctx->mac_type == OTX2_CPT_SHA384) + ds = SHA512_DIGEST_SIZE; + if (ctx->ipad) + memcpy(fctx->hmac.e.ipad, ctx->ipad, ds); + if (ctx->opad) + memcpy(fctx->hmac.e.opad, ctx->opad, ds); + break; + + case OTX2_CPT_AES_GCM: + if (crypto_ipsec_check_assoclen(req->assoclen)) + return -EINVAL; + + fctx->enc.enc_ctrl.e.iv_source = OTX2_CPT_FROM_DPTR; + /* Copy encryption key to context */ + memcpy(fctx->enc.encr_key, ctx->key, ctx->enc_key_len); + /* Copy salt to context */ + memcpy(fctx->enc.encr_iv, ctx->key + ctx->enc_key_len, + AES_GCM_SALT_SIZE); + + rctx->ctrl_word.e.iv_offset = req->assoclen - AES_GCM_IV_OFFSET; + break; + + default: + /* Unknown cipher type */ + return -EINVAL; + } + cpu_to_be64s(&rctx->ctrl_word.flags); + + req_info->ctrl.s.dma_mode = OTX2_CPT_DMA_MODE_SG; + req_info->ctrl.s.se_req = 1; + req_info->req.opcode.s.major = OTX2_CPT_MAJOR_OP_FC | + DMA_MODE_FLAG(OTX2_CPT_DMA_MODE_SG); + if (enc) { + req_info->req.opcode.s.minor = 2; + req_info->req.param1 = req->cryptlen; + req_info->req.param2 = req->cryptlen + req->assoclen; + } else { + req_info->req.opcode.s.minor = 3; + req_info->req.param1 = req->cryptlen - mac_len; + req_info->req.param2 = req->cryptlen + req->assoclen - mac_len; + } + + fctx->enc.enc_ctrl.e.enc_cipher = ctx->cipher_type; + fctx->enc.enc_ctrl.e.aes_key = ctx->key_type; + fctx->enc.enc_ctrl.e.mac_type = ctx->mac_type; + fctx->enc.enc_ctrl.e.mac_len = mac_len; + cpu_to_be64s(&fctx->enc.enc_ctrl.u); + + /* + * Storing Packet Data Information in offset + * Control Word First 8 bytes + */ + req_info->in[*argcnt].vptr = (u8 *)&rctx->ctrl_word; + req_info->in[*argcnt].size = CONTROL_WORD_LEN; + req_info->req.dlen += CONTROL_WORD_LEN; + ++(*argcnt); + + req_info->in[*argcnt].vptr = (u8 *)fctx; + req_info->in[*argcnt].size = sizeof(struct otx2_cpt_fc_ctx); + req_info->req.dlen += sizeof(struct otx2_cpt_fc_ctx); + ++(*argcnt); + + return 0; +} + +static inline void create_hmac_ctx_hdr(struct aead_request *req, u32 *argcnt, + u32 enc) +{ + struct otx2_cpt_req_ctx *rctx = aead_request_ctx(req); + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(tfm); + struct otx2_cpt_req_info *req_info = &rctx->cpt_req; + + req_info->ctrl.s.dma_mode = OTX2_CPT_DMA_MODE_SG; + req_info->ctrl.s.se_req = 1; + req_info->req.opcode.s.major = OTX2_CPT_MAJOR_OP_HMAC | + DMA_MODE_FLAG(OTX2_CPT_DMA_MODE_SG); + req_info->is_trunc_hmac = ctx->is_trunc_hmac; + + req_info->req.opcode.s.minor = 0; + req_info->req.param1 = ctx->auth_key_len; + req_info->req.param2 = ctx->mac_type << 8; + + /* Add authentication key */ + req_info->in[*argcnt].vptr = ctx->key; + req_info->in[*argcnt].size = round_up(ctx->auth_key_len, 8); + req_info->req.dlen += round_up(ctx->auth_key_len, 8); + ++(*argcnt); +} + +static inline int create_aead_input_list(struct aead_request *req, u32 enc) +{ + struct otx2_cpt_req_ctx *rctx = aead_request_ctx(req); + struct otx2_cpt_req_info *req_info = &rctx->cpt_req; + u32 inputlen = req->cryptlen + req->assoclen; + u32 status, argcnt = 0; + + status = create_aead_ctx_hdr(req, enc, &argcnt); + if (status) + return status; + update_input_data(req_info, req->src, inputlen, &argcnt); + req_info->in_cnt = argcnt; + + return 0; +} + +static inline void create_aead_output_list(struct aead_request *req, u32 enc, + u32 mac_len) +{ + struct otx2_cpt_req_ctx *rctx = aead_request_ctx(req); + struct otx2_cpt_req_info *req_info = &rctx->cpt_req; + u32 argcnt = 0, outputlen = 0; + + if (enc) + outputlen = req->cryptlen + req->assoclen + mac_len; + else + outputlen = req->cryptlen + req->assoclen - mac_len; + + update_output_data(req_info, req->dst, 0, outputlen, &argcnt); + req_info->out_cnt = argcnt; +} + +static inline void create_aead_null_input_list(struct aead_request *req, + u32 enc, u32 mac_len) +{ + struct otx2_cpt_req_ctx *rctx = aead_request_ctx(req); + struct otx2_cpt_req_info *req_info = &rctx->cpt_req; + u32 inputlen, argcnt = 0; + + if (enc) + inputlen = req->cryptlen + req->assoclen; + else + inputlen = req->cryptlen + req->assoclen - mac_len; + + create_hmac_ctx_hdr(req, &argcnt, enc); + update_input_data(req_info, req->src, inputlen, &argcnt); + req_info->in_cnt = argcnt; +} + +static inline int create_aead_null_output_list(struct aead_request *req, + u32 enc, u32 mac_len) +{ + struct otx2_cpt_req_ctx *rctx = aead_request_ctx(req); + struct otx2_cpt_req_info *req_info = &rctx->cpt_req; + struct scatterlist *dst; + u8 *ptr = NULL; + int argcnt = 0, status, offset; + u32 inputlen; + + if (enc) + inputlen = req->cryptlen + req->assoclen; + else + inputlen = req->cryptlen + req->assoclen - mac_len; + + /* + * If source and destination are different + * then copy payload to destination + */ + if (req->src != req->dst) { + + ptr = kmalloc(inputlen, (req_info->areq->flags & + CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC); + if (!ptr) + return -ENOMEM; + + status = sg_copy_to_buffer(req->src, sg_nents(req->src), ptr, + inputlen); + if (status != inputlen) { + status = -EINVAL; + goto error_free; + } + status = sg_copy_from_buffer(req->dst, sg_nents(req->dst), ptr, + inputlen); + if (status != inputlen) { + status = -EINVAL; + goto error_free; + } + kfree(ptr); + } + + if (enc) { + /* + * In an encryption scenario hmac needs + * to be appended after payload + */ + dst = req->dst; + offset = inputlen; + while (offset >= dst->length) { + offset -= dst->length; + dst = sg_next(dst); + if (!dst) + return -ENOENT; + } + + update_output_data(req_info, dst, offset, mac_len, &argcnt); + } else { + /* + * In a decryption scenario calculated hmac for received + * payload needs to be compare with hmac received + */ + status = sg_copy_buffer(req->src, sg_nents(req->src), + rctx->fctx.hmac.s.hmac_recv, mac_len, + inputlen, true); + if (status != mac_len) + return -EINVAL; + + req_info->out[argcnt].vptr = rctx->fctx.hmac.s.hmac_calc; + req_info->out[argcnt].size = mac_len; + argcnt++; + } + + req_info->out_cnt = argcnt; + return 0; + +error_free: + kfree(ptr); + return status; +} + +static int aead_do_fallback(struct aead_request *req, bool is_enc) +{ + struct otx2_cpt_req_ctx *rctx = aead_request_ctx(req); + struct crypto_aead *aead = crypto_aead_reqtfm(req); + struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(aead); + int ret; + + if (ctx->fbk_cipher) { + /* Store the cipher tfm and then use the fallback tfm */ + aead_request_set_tfm(&rctx->fbk_req, ctx->fbk_cipher); + aead_request_set_callback(&rctx->fbk_req, req->base.flags, + req->base.complete, req->base.data); + aead_request_set_crypt(&rctx->fbk_req, req->src, + req->dst, req->cryptlen, req->iv); + ret = is_enc ? crypto_aead_encrypt(&rctx->fbk_req) : + crypto_aead_decrypt(&rctx->fbk_req); + } else { + ret = -EINVAL; + } + + return ret; +} + +static int cpt_aead_enc_dec(struct aead_request *req, u8 reg_type, u8 enc) +{ + struct otx2_cpt_req_ctx *rctx = aead_request_ctx(req); + struct otx2_cpt_req_info *req_info = &rctx->cpt_req; + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx(tfm); + struct pci_dev *pdev; + int status, cpu_num; + + /* Clear control words */ + rctx->ctrl_word.flags = 0; + rctx->fctx.enc.enc_ctrl.u = 0; + + req_info->callback = otx2_cpt_aead_callback; + req_info->areq = &req->base; + req_info->req_type = reg_type; + req_info->is_enc = enc; + req_info->is_trunc_hmac = false; + + switch (reg_type) { + case OTX2_CPT_AEAD_ENC_DEC_REQ: + status = create_aead_input_list(req, enc); + if (status) + return status; + create_aead_output_list(req, enc, crypto_aead_authsize(tfm)); + break; + + case OTX2_CPT_AEAD_ENC_DEC_NULL_REQ: + create_aead_null_input_list(req, enc, + crypto_aead_authsize(tfm)); + status = create_aead_null_output_list(req, enc, + crypto_aead_authsize(tfm)); + if (status) + return status; + break; + + default: + return -EINVAL; + } + if (!IS_ALIGNED(req_info->req.param1, ctx->enc_align_len)) + return -EINVAL; + + if (!req_info->req.param2 || + (req_info->req.param1 > OTX2_CPT_MAX_REQ_SIZE) || + (req_info->req.param2 > OTX2_CPT_MAX_REQ_SIZE)) + return aead_do_fallback(req, enc); + + status = get_se_device(&pdev, &cpu_num); + if (status) + return status; + + req_info->ctrl.s.grp = otx2_cpt_get_kcrypto_eng_grp_num(pdev); + + /* + * We perform an asynchronous send and once + * the request is completed the driver would + * intimate through registered call back functions + */ + return otx2_cpt_do_request(pdev, req_info, cpu_num); +} + +static int otx2_cpt_aead_encrypt(struct aead_request *req) +{ + return cpt_aead_enc_dec(req, OTX2_CPT_AEAD_ENC_DEC_REQ, true); +} + +static int otx2_cpt_aead_decrypt(struct aead_request *req) +{ + return cpt_aead_enc_dec(req, OTX2_CPT_AEAD_ENC_DEC_REQ, false); +} + +static int otx2_cpt_aead_null_encrypt(struct aead_request *req) +{ + return cpt_aead_enc_dec(req, OTX2_CPT_AEAD_ENC_DEC_NULL_REQ, true); +} + +static int otx2_cpt_aead_null_decrypt(struct aead_request *req) +{ + return cpt_aead_enc_dec(req, OTX2_CPT_AEAD_ENC_DEC_NULL_REQ, false); +} + +static struct skcipher_alg otx2_cpt_skciphers[] = { { + .base.cra_name = "xts(aes)", + .base.cra_driver_name = "cpt_xts_aes", + .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct otx2_cpt_enc_ctx), + .base.cra_alignmask = 7, + .base.cra_priority = 4001, + .base.cra_module = THIS_MODULE, + + .init = otx2_cpt_enc_dec_init, + .exit = otx2_cpt_skcipher_exit, + .ivsize = AES_BLOCK_SIZE, + .min_keysize = 2 * AES_MIN_KEY_SIZE, + .max_keysize = 2 * AES_MAX_KEY_SIZE, + .setkey = otx2_cpt_skcipher_xts_setkey, + .encrypt = otx2_cpt_skcipher_encrypt, + .decrypt = otx2_cpt_skcipher_decrypt, +}, { + .base.cra_name = "cbc(aes)", + .base.cra_driver_name = "cpt_cbc_aes", + .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct otx2_cpt_enc_ctx), + .base.cra_alignmask = 7, + .base.cra_priority = 4001, + .base.cra_module = THIS_MODULE, + + .init = otx2_cpt_enc_dec_init, + .exit = otx2_cpt_skcipher_exit, + .ivsize = AES_BLOCK_SIZE, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .setkey = otx2_cpt_skcipher_cbc_aes_setkey, + .encrypt = otx2_cpt_skcipher_encrypt, + .decrypt = otx2_cpt_skcipher_decrypt, +}, { + .base.cra_name = "ecb(aes)", + .base.cra_driver_name = "cpt_ecb_aes", + .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct otx2_cpt_enc_ctx), + .base.cra_alignmask = 7, + .base.cra_priority = 4001, + .base.cra_module = THIS_MODULE, + + .init = otx2_cpt_enc_dec_init, + .exit = otx2_cpt_skcipher_exit, + .ivsize = 0, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .setkey = otx2_cpt_skcipher_ecb_aes_setkey, + .encrypt = otx2_cpt_skcipher_encrypt, + .decrypt = otx2_cpt_skcipher_decrypt, +}, { + .base.cra_name = "cbc(des3_ede)", + .base.cra_driver_name = "cpt_cbc_des3_ede", + .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, + .base.cra_blocksize = DES3_EDE_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct otx2_cpt_enc_ctx), + .base.cra_alignmask = 7, + .base.cra_priority = 4001, + .base.cra_module = THIS_MODULE, + + .init = otx2_cpt_enc_dec_init, + .exit = otx2_cpt_skcipher_exit, + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .ivsize = DES_BLOCK_SIZE, + .setkey = otx2_cpt_skcipher_cbc_des3_setkey, + .encrypt = otx2_cpt_skcipher_encrypt, + .decrypt = otx2_cpt_skcipher_decrypt, +}, { + .base.cra_name = "ecb(des3_ede)", + .base.cra_driver_name = "cpt_ecb_des3_ede", + .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, + .base.cra_blocksize = DES3_EDE_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct otx2_cpt_enc_ctx), + .base.cra_alignmask = 7, + .base.cra_priority = 4001, + .base.cra_module = THIS_MODULE, + + .init = otx2_cpt_enc_dec_init, + .exit = otx2_cpt_skcipher_exit, + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .ivsize = 0, + .setkey = otx2_cpt_skcipher_ecb_des3_setkey, + .encrypt = otx2_cpt_skcipher_encrypt, + .decrypt = otx2_cpt_skcipher_decrypt, +} }; + +static struct aead_alg otx2_cpt_aeads[] = { { + .base = { + .cra_name = "authenc(hmac(sha1),cbc(aes))", + .cra_driver_name = "cpt_hmac_sha1_cbc_aes", + .cra_blocksize = AES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, + .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx), + .cra_priority = 4001, + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .init = otx2_cpt_aead_cbc_aes_sha1_init, + .exit = otx2_cpt_aead_exit, + .setkey = otx2_cpt_aead_cbc_aes_sha_setkey, + .setauthsize = otx2_cpt_aead_set_authsize, + .encrypt = otx2_cpt_aead_encrypt, + .decrypt = otx2_cpt_aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, +}, { + .base = { + .cra_name = "authenc(hmac(sha256),cbc(aes))", + .cra_driver_name = "cpt_hmac_sha256_cbc_aes", + .cra_blocksize = AES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, + .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx), + .cra_priority = 4001, + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .init = otx2_cpt_aead_cbc_aes_sha256_init, + .exit = otx2_cpt_aead_exit, + .setkey = otx2_cpt_aead_cbc_aes_sha_setkey, + .setauthsize = otx2_cpt_aead_set_authsize, + .encrypt = otx2_cpt_aead_encrypt, + .decrypt = otx2_cpt_aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, +}, { + .base = { + .cra_name = "authenc(hmac(sha384),cbc(aes))", + .cra_driver_name = "cpt_hmac_sha384_cbc_aes", + .cra_blocksize = AES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, + .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx), + .cra_priority = 4001, + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .init = otx2_cpt_aead_cbc_aes_sha384_init, + .exit = otx2_cpt_aead_exit, + .setkey = otx2_cpt_aead_cbc_aes_sha_setkey, + .setauthsize = otx2_cpt_aead_set_authsize, + .encrypt = otx2_cpt_aead_encrypt, + .decrypt = otx2_cpt_aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA384_DIGEST_SIZE, +}, { + .base = { + .cra_name = "authenc(hmac(sha512),cbc(aes))", + .cra_driver_name = "cpt_hmac_sha512_cbc_aes", + .cra_blocksize = AES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, + .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx), + .cra_priority = 4001, + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .init = otx2_cpt_aead_cbc_aes_sha512_init, + .exit = otx2_cpt_aead_exit, + .setkey = otx2_cpt_aead_cbc_aes_sha_setkey, + .setauthsize = otx2_cpt_aead_set_authsize, + .encrypt = otx2_cpt_aead_encrypt, + .decrypt = otx2_cpt_aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, +}, { + .base = { + .cra_name = "authenc(hmac(sha1),ecb(cipher_null))", + .cra_driver_name = "cpt_hmac_sha1_ecb_null", + .cra_blocksize = 1, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx), + .cra_priority = 4001, + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .init = otx2_cpt_aead_ecb_null_sha1_init, + .exit = otx2_cpt_aead_exit, + .setkey = otx2_cpt_aead_ecb_null_sha_setkey, + .setauthsize = otx2_cpt_aead_null_set_authsize, + .encrypt = otx2_cpt_aead_null_encrypt, + .decrypt = otx2_cpt_aead_null_decrypt, + .ivsize = 0, + .maxauthsize = SHA1_DIGEST_SIZE, +}, { + .base = { + .cra_name = "authenc(hmac(sha256),ecb(cipher_null))", + .cra_driver_name = "cpt_hmac_sha256_ecb_null", + .cra_blocksize = 1, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx), + .cra_priority = 4001, + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .init = otx2_cpt_aead_ecb_null_sha256_init, + .exit = otx2_cpt_aead_exit, + .setkey = otx2_cpt_aead_ecb_null_sha_setkey, + .setauthsize = otx2_cpt_aead_null_set_authsize, + .encrypt = otx2_cpt_aead_null_encrypt, + .decrypt = otx2_cpt_aead_null_decrypt, + .ivsize = 0, + .maxauthsize = SHA256_DIGEST_SIZE, +}, { + .base = { + .cra_name = "authenc(hmac(sha384),ecb(cipher_null))", + .cra_driver_name = "cpt_hmac_sha384_ecb_null", + .cra_blocksize = 1, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx), + .cra_priority = 4001, + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .init = otx2_cpt_aead_ecb_null_sha384_init, + .exit = otx2_cpt_aead_exit, + .setkey = otx2_cpt_aead_ecb_null_sha_setkey, + .setauthsize = otx2_cpt_aead_null_set_authsize, + .encrypt = otx2_cpt_aead_null_encrypt, + .decrypt = otx2_cpt_aead_null_decrypt, + .ivsize = 0, + .maxauthsize = SHA384_DIGEST_SIZE, +}, { + .base = { + .cra_name = "authenc(hmac(sha512),ecb(cipher_null))", + .cra_driver_name = "cpt_hmac_sha512_ecb_null", + .cra_blocksize = 1, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx), + .cra_priority = 4001, + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .init = otx2_cpt_aead_ecb_null_sha512_init, + .exit = otx2_cpt_aead_exit, + .setkey = otx2_cpt_aead_ecb_null_sha_setkey, + .setauthsize = otx2_cpt_aead_null_set_authsize, + .encrypt = otx2_cpt_aead_null_encrypt, + .decrypt = otx2_cpt_aead_null_decrypt, + .ivsize = 0, + .maxauthsize = SHA512_DIGEST_SIZE, +}, { + .base = { + .cra_name = "rfc4106(gcm(aes))", + .cra_driver_name = "cpt_rfc4106_gcm_aes", + .cra_blocksize = 1, + .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, + .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx), + .cra_priority = 4001, + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .init = otx2_cpt_aead_gcm_aes_init, + .exit = otx2_cpt_aead_exit, + .setkey = otx2_cpt_aead_gcm_aes_setkey, + .setauthsize = otx2_cpt_aead_gcm_set_authsize, + .encrypt = otx2_cpt_aead_encrypt, + .decrypt = otx2_cpt_aead_decrypt, + .ivsize = AES_GCM_IV_SIZE, + .maxauthsize = AES_GCM_ICV_SIZE, +} }; + +static inline int cpt_register_algs(void) +{ + int i, err = 0; + + if (!IS_ENABLED(CONFIG_DM_CRYPT)) { + for (i = 0; i < ARRAY_SIZE(otx2_cpt_skciphers); i++) + otx2_cpt_skciphers[i].base.cra_flags &= + ~CRYPTO_ALG_DEAD; + + err = crypto_register_skciphers(otx2_cpt_skciphers, + ARRAY_SIZE(otx2_cpt_skciphers)); + if (err) + return err; + } + + for (i = 0; i < ARRAY_SIZE(otx2_cpt_aeads); i++) + otx2_cpt_aeads[i].base.cra_flags &= ~CRYPTO_ALG_DEAD; + + err = crypto_register_aeads(otx2_cpt_aeads, + ARRAY_SIZE(otx2_cpt_aeads)); + if (err) { + crypto_unregister_skciphers(otx2_cpt_skciphers, + ARRAY_SIZE(otx2_cpt_skciphers)); + return err; + } + + return 0; +} + +static inline void cpt_unregister_algs(void) +{ + crypto_unregister_skciphers(otx2_cpt_skciphers, + ARRAY_SIZE(otx2_cpt_skciphers)); + crypto_unregister_aeads(otx2_cpt_aeads, ARRAY_SIZE(otx2_cpt_aeads)); +} + +static int compare_func(const void *lptr, const void *rptr) +{ + const struct cpt_device_desc *ldesc = (struct cpt_device_desc *) lptr; + const struct cpt_device_desc *rdesc = (struct cpt_device_desc *) rptr; + + if (ldesc->dev->devfn < rdesc->dev->devfn) + return -1; + if (ldesc->dev->devfn > rdesc->dev->devfn) + return 1; + return 0; +} + +static void swap_func(void *lptr, void *rptr, int size) +{ + struct cpt_device_desc *ldesc = lptr; + struct cpt_device_desc *rdesc = rptr; + struct cpt_device_desc desc; + + desc = *ldesc; + *ldesc = *rdesc; + *rdesc = desc; +} + +int otx2_cpt_crypto_init(struct pci_dev *pdev, struct module *mod, + int num_queues, int num_devices) +{ + int ret = 0; + int count; + + mutex_lock(&mutex); + count = atomic_read(&se_devices.count); + if (count >= OTX2_CPT_MAX_LFS_NUM) { + dev_err(&pdev->dev, "No space to add a new device\n"); + ret = -ENOSPC; + goto unlock; + } + se_devices.desc[count].num_queues = num_queues; + se_devices.desc[count++].dev = pdev; + atomic_inc(&se_devices.count); + + if (atomic_read(&se_devices.count) == num_devices && + is_crypto_registered == false) { + if (cpt_register_algs()) { + dev_err(&pdev->dev, + "Error in registering crypto algorithms\n"); + ret = -EINVAL; + goto unlock; + } + try_module_get(mod); + is_crypto_registered = true; + } + sort(se_devices.desc, count, sizeof(struct cpt_device_desc), + compare_func, swap_func); + +unlock: + mutex_unlock(&mutex); + return ret; +} + +void otx2_cpt_crypto_exit(struct pci_dev *pdev, struct module *mod) +{ + struct cpt_device_table *dev_tbl; + bool dev_found = false; + int i, j, count; + + mutex_lock(&mutex); + + dev_tbl = &se_devices; + count = atomic_read(&dev_tbl->count); + for (i = 0; i < count; i++) { + if (pdev == dev_tbl->desc[i].dev) { + for (j = i; j < count-1; j++) + dev_tbl->desc[j] = dev_tbl->desc[j+1]; + dev_found = true; + break; + } + } + + if (!dev_found) { + dev_err(&pdev->dev, "%s device not found\n", __func__); + goto unlock; + } + if (atomic_dec_and_test(&se_devices.count)) { + cpt_unregister_algs(); + module_put(mod); + is_crypto_registered = false; + } + +unlock: + mutex_unlock(&mutex); +} diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.h b/drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.h new file mode 100644 index 0000000000000000000000000000000000000000..f04184bd174470e20bb45139bc1f7ff0683594e7 --- /dev/null +++ b/drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.h @@ -0,0 +1,178 @@ +/* SPDX-License-Identifier: GPL-2.0-only + * Copyright (C) 2020 Marvell. + */ + +#ifndef __OTX2_CPT_ALGS_H +#define __OTX2_CPT_ALGS_H + +#include +#include +#include +#include "otx2_cpt_common.h" + +#define OTX2_CPT_MAX_ENC_KEY_SIZE 32 +#define OTX2_CPT_MAX_HASH_KEY_SIZE 64 +#define OTX2_CPT_MAX_KEY_SIZE (OTX2_CPT_MAX_ENC_KEY_SIZE + \ + OTX2_CPT_MAX_HASH_KEY_SIZE) +enum otx2_cpt_request_type { + OTX2_CPT_ENC_DEC_REQ = 0x1, + OTX2_CPT_AEAD_ENC_DEC_REQ = 0x2, + OTX2_CPT_AEAD_ENC_DEC_NULL_REQ = 0x3, + OTX2_CPT_PASSTHROUGH_REQ = 0x4 +}; + +enum otx2_cpt_major_opcodes { + OTX2_CPT_MAJOR_OP_MISC = 0x01, + OTX2_CPT_MAJOR_OP_FC = 0x33, + OTX2_CPT_MAJOR_OP_HMAC = 0x35, +}; + +enum otx2_cpt_cipher_type { + OTX2_CPT_CIPHER_NULL = 0x0, + OTX2_CPT_DES3_CBC = 0x1, + OTX2_CPT_DES3_ECB = 0x2, + OTX2_CPT_AES_CBC = 0x3, + OTX2_CPT_AES_ECB = 0x4, + OTX2_CPT_AES_CFB = 0x5, + OTX2_CPT_AES_CTR = 0x6, + OTX2_CPT_AES_GCM = 0x7, + OTX2_CPT_AES_XTS = 0x8 +}; + +enum otx2_cpt_mac_type { + OTX2_CPT_MAC_NULL = 0x0, + OTX2_CPT_MD5 = 0x1, + OTX2_CPT_SHA1 = 0x2, + OTX2_CPT_SHA224 = 0x3, + OTX2_CPT_SHA256 = 0x4, + OTX2_CPT_SHA384 = 0x5, + OTX2_CPT_SHA512 = 0x6, + OTX2_CPT_GMAC = 0x7 +}; + +enum otx2_cpt_aes_key_len { + OTX2_CPT_AES_128_BIT = 0x1, + OTX2_CPT_AES_192_BIT = 0x2, + OTX2_CPT_AES_256_BIT = 0x3 +}; + +union otx2_cpt_encr_ctrl { + u64 u; + struct { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 enc_cipher:4; + u64 reserved_59:1; + u64 aes_key:2; + u64 iv_source:1; + u64 mac_type:4; + u64 reserved_49_51:3; + u64 auth_input_type:1; + u64 mac_len:8; + u64 reserved_32_39:8; + u64 encr_offset:16; + u64 iv_offset:8; + u64 auth_offset:8; +#else + u64 auth_offset:8; + u64 iv_offset:8; + u64 encr_offset:16; + u64 reserved_32_39:8; + u64 mac_len:8; + u64 auth_input_type:1; + u64 reserved_49_51:3; + u64 mac_type:4; + u64 iv_source:1; + u64 aes_key:2; + u64 reserved_59:1; + u64 enc_cipher:4; +#endif + } e; +}; + +struct otx2_cpt_cipher { + const char *name; + u8 value; +}; + +struct otx2_cpt_fc_enc_ctx { + union otx2_cpt_encr_ctrl enc_ctrl; + u8 encr_key[32]; + u8 encr_iv[16]; +}; + +union otx2_cpt_fc_hmac_ctx { + struct { + u8 ipad[64]; + u8 opad[64]; + } e; + struct { + u8 hmac_calc[64]; /* HMAC calculated */ + u8 hmac_recv[64]; /* HMAC received */ + } s; +}; + +struct otx2_cpt_fc_ctx { + struct otx2_cpt_fc_enc_ctx enc; + union otx2_cpt_fc_hmac_ctx hmac; +}; + +struct otx2_cpt_enc_ctx { + u32 key_len; + u8 enc_key[OTX2_CPT_MAX_KEY_SIZE]; + u8 cipher_type; + u8 key_type; + u8 enc_align_len; + struct crypto_skcipher *fbk_cipher; +}; + +union otx2_cpt_offset_ctrl { + u64 flags; + struct { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 reserved:32; + u64 enc_data_offset:16; + u64 iv_offset:8; + u64 auth_offset:8; +#else + u64 auth_offset:8; + u64 iv_offset:8; + u64 enc_data_offset:16; + u64 reserved:32; +#endif + } e; +}; + +struct otx2_cpt_req_ctx { + struct otx2_cpt_req_info cpt_req; + union otx2_cpt_offset_ctrl ctrl_word; + struct otx2_cpt_fc_ctx fctx; + union { + struct skcipher_request sk_fbk_req; + struct aead_request fbk_req; + }; +}; + +struct otx2_cpt_sdesc { + struct shash_desc shash; +}; + +struct otx2_cpt_aead_ctx { + u8 key[OTX2_CPT_MAX_KEY_SIZE]; + struct crypto_shash *hashalg; + struct otx2_cpt_sdesc *sdesc; + struct crypto_aead *fbk_cipher; + u8 *ipad; + u8 *opad; + u32 enc_key_len; + u32 auth_key_len; + u8 cipher_type; + u8 mac_type; + u8 key_type; + u8 is_trunc_hmac; + u8 enc_align_len; +}; +int otx2_cpt_crypto_init(struct pci_dev *pdev, struct module *mod, + int num_queues, int num_devices); +void otx2_cpt_crypto_exit(struct pci_dev *pdev, struct module *mod); + +#endif /* __OTX2_CPT_ALGS_H */ diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptvf_main.c b/drivers/crypto/marvell/octeontx2/otx2_cptvf_main.c new file mode 100644 index 0000000000000000000000000000000000000000..47f378731024321d2d50ab9b25429984765a52dc --- /dev/null +++ b/drivers/crypto/marvell/octeontx2/otx2_cptvf_main.c @@ -0,0 +1,410 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (C) 2020 Marvell. */ + +#include "otx2_cpt_common.h" +#include "otx2_cptvf.h" +#include "otx2_cptlf.h" +#include "otx2_cptvf_algs.h" +#include + +#define OTX2_CPTVF_DRV_NAME "octeontx2-cptvf" + +static void cptvf_enable_pfvf_mbox_intrs(struct otx2_cptvf_dev *cptvf) +{ + /* Clear interrupt if any */ + otx2_cpt_write64(cptvf->reg_base, BLKADDR_RVUM, 0, OTX2_RVU_VF_INT, + 0x1ULL); + + /* Enable PF-VF interrupt */ + otx2_cpt_write64(cptvf->reg_base, BLKADDR_RVUM, 0, + OTX2_RVU_VF_INT_ENA_W1S, 0x1ULL); +} + +static void cptvf_disable_pfvf_mbox_intrs(struct otx2_cptvf_dev *cptvf) +{ + /* Disable PF-VF interrupt */ + otx2_cpt_write64(cptvf->reg_base, BLKADDR_RVUM, 0, + OTX2_RVU_VF_INT_ENA_W1C, 0x1ULL); + + /* Clear interrupt if any */ + otx2_cpt_write64(cptvf->reg_base, BLKADDR_RVUM, 0, OTX2_RVU_VF_INT, + 0x1ULL); +} + +static int cptvf_register_interrupts(struct otx2_cptvf_dev *cptvf) +{ + int ret, irq; + int num_vec; + + num_vec = pci_msix_vec_count(cptvf->pdev); + if (num_vec <= 0) + return -EINVAL; + + /* Enable MSI-X */ + ret = pci_alloc_irq_vectors(cptvf->pdev, num_vec, num_vec, + PCI_IRQ_MSIX); + if (ret < 0) { + dev_err(&cptvf->pdev->dev, + "Request for %d msix vectors failed\n", num_vec); + return ret; + } + irq = pci_irq_vector(cptvf->pdev, OTX2_CPT_VF_INT_VEC_E_MBOX); + /* Register VF<=>PF mailbox interrupt handler */ + ret = devm_request_irq(&cptvf->pdev->dev, irq, + otx2_cptvf_pfvf_mbox_intr, 0, + "CPTPFVF Mbox", cptvf); + if (ret) + return ret; + /* Enable PF-VF mailbox interrupts */ + cptvf_enable_pfvf_mbox_intrs(cptvf); + + ret = otx2_cpt_send_ready_msg(&cptvf->pfvf_mbox, cptvf->pdev); + if (ret) { + dev_warn(&cptvf->pdev->dev, + "PF not responding to mailbox, deferring probe\n"); + cptvf_disable_pfvf_mbox_intrs(cptvf); + return -EPROBE_DEFER; + } + return 0; +} + +static int cptvf_pfvf_mbox_init(struct otx2_cptvf_dev *cptvf) +{ + int ret; + + cptvf->pfvf_mbox_wq = alloc_workqueue("cpt_pfvf_mailbox", + WQ_UNBOUND | WQ_HIGHPRI | + WQ_MEM_RECLAIM, 1); + if (!cptvf->pfvf_mbox_wq) + return -ENOMEM; + + ret = otx2_mbox_init(&cptvf->pfvf_mbox, cptvf->pfvf_mbox_base, + cptvf->pdev, cptvf->reg_base, MBOX_DIR_VFPF, 1); + if (ret) + goto free_wqe; + + INIT_WORK(&cptvf->pfvf_mbox_work, otx2_cptvf_pfvf_mbox_handler); + return 0; + +free_wqe: + destroy_workqueue(cptvf->pfvf_mbox_wq); + return ret; +} + +static void cptvf_pfvf_mbox_destroy(struct otx2_cptvf_dev *cptvf) +{ + destroy_workqueue(cptvf->pfvf_mbox_wq); + otx2_mbox_destroy(&cptvf->pfvf_mbox); +} + +static void cptlf_work_handler(unsigned long data) +{ + otx2_cpt_post_process((struct otx2_cptlf_wqe *) data); +} + +static void cleanup_tasklet_work(struct otx2_cptlfs_info *lfs) +{ + int i; + + for (i = 0; i < lfs->lfs_num; i++) { + if (!lfs->lf[i].wqe) + continue; + + tasklet_kill(&lfs->lf[i].wqe->work); + kfree(lfs->lf[i].wqe); + lfs->lf[i].wqe = NULL; + } +} + +static int init_tasklet_work(struct otx2_cptlfs_info *lfs) +{ + struct otx2_cptlf_wqe *wqe; + int i, ret = 0; + + for (i = 0; i < lfs->lfs_num; i++) { + wqe = kzalloc(sizeof(struct otx2_cptlf_wqe), GFP_KERNEL); + if (!wqe) { + ret = -ENOMEM; + goto cleanup_tasklet; + } + + tasklet_init(&wqe->work, cptlf_work_handler, (u64) wqe); + wqe->lfs = lfs; + wqe->lf_num = i; + lfs->lf[i].wqe = wqe; + } + return 0; + +cleanup_tasklet: + cleanup_tasklet_work(lfs); + return ret; +} + +static void free_pending_queues(struct otx2_cptlfs_info *lfs) +{ + int i; + + for (i = 0; i < lfs->lfs_num; i++) { + kfree(lfs->lf[i].pqueue.head); + lfs->lf[i].pqueue.head = NULL; + } +} + +static int alloc_pending_queues(struct otx2_cptlfs_info *lfs) +{ + int size, ret, i; + + if (!lfs->lfs_num) + return -EINVAL; + + for (i = 0; i < lfs->lfs_num; i++) { + lfs->lf[i].pqueue.qlen = OTX2_CPT_INST_QLEN_MSGS; + size = lfs->lf[i].pqueue.qlen * + sizeof(struct otx2_cpt_pending_entry); + + lfs->lf[i].pqueue.head = kzalloc(size, GFP_KERNEL); + if (!lfs->lf[i].pqueue.head) { + ret = -ENOMEM; + goto error; + } + + /* Initialize spin lock */ + spin_lock_init(&lfs->lf[i].pqueue.lock); + } + return 0; + +error: + free_pending_queues(lfs); + return ret; +} + +static void lf_sw_cleanup(struct otx2_cptlfs_info *lfs) +{ + cleanup_tasklet_work(lfs); + free_pending_queues(lfs); +} + +static int lf_sw_init(struct otx2_cptlfs_info *lfs) +{ + int ret; + + ret = alloc_pending_queues(lfs); + if (ret) { + dev_err(&lfs->pdev->dev, + "Allocating pending queues failed\n"); + return ret; + } + ret = init_tasklet_work(lfs); + if (ret) { + dev_err(&lfs->pdev->dev, + "Tasklet work init failed\n"); + goto pending_queues_free; + } + return 0; + +pending_queues_free: + free_pending_queues(lfs); + return ret; +} + +static void cptvf_lf_shutdown(struct otx2_cptlfs_info *lfs) +{ + atomic_set(&lfs->state, OTX2_CPTLF_IN_RESET); + + /* Remove interrupts affinity */ + otx2_cptlf_free_irqs_affinity(lfs); + /* Disable instruction queue */ + otx2_cptlf_disable_iqueues(lfs); + /* Unregister crypto algorithms */ + otx2_cpt_crypto_exit(lfs->pdev, THIS_MODULE); + /* Unregister LFs interrupts */ + otx2_cptlf_unregister_interrupts(lfs); + /* Cleanup LFs software side */ + lf_sw_cleanup(lfs); + /* Send request to detach LFs */ + otx2_cpt_detach_rsrcs_msg(lfs); +} + +static int cptvf_lf_init(struct otx2_cptvf_dev *cptvf) +{ + struct otx2_cptlfs_info *lfs = &cptvf->lfs; + struct device *dev = &cptvf->pdev->dev; + int ret, lfs_num; + u8 eng_grp_msk; + + /* Get engine group number for symmetric crypto */ + cptvf->lfs.kcrypto_eng_grp_num = OTX2_CPT_INVALID_CRYPTO_ENG_GRP; + ret = otx2_cptvf_send_eng_grp_num_msg(cptvf, OTX2_CPT_SE_TYPES); + if (ret) + return ret; + + if (cptvf->lfs.kcrypto_eng_grp_num == OTX2_CPT_INVALID_CRYPTO_ENG_GRP) { + dev_err(dev, "Engine group for kernel crypto not available\n"); + ret = -ENOENT; + return ret; + } + eng_grp_msk = 1 << cptvf->lfs.kcrypto_eng_grp_num; + + ret = otx2_cptvf_send_kvf_limits_msg(cptvf); + if (ret) + return ret; + + lfs->reg_base = cptvf->reg_base; + lfs->pdev = cptvf->pdev; + lfs->mbox = &cptvf->pfvf_mbox; + + lfs_num = cptvf->lfs.kvf_limits ? cptvf->lfs.kvf_limits : + num_online_cpus(); + ret = otx2_cptlf_init(lfs, eng_grp_msk, OTX2_CPT_QUEUE_HI_PRIO, + lfs_num); + if (ret) + return ret; + + /* Get msix offsets for attached LFs */ + ret = otx2_cpt_msix_offset_msg(lfs); + if (ret) + goto cleanup_lf; + + /* Initialize LFs software side */ + ret = lf_sw_init(lfs); + if (ret) + goto cleanup_lf; + + /* Register LFs interrupts */ + ret = otx2_cptlf_register_interrupts(lfs); + if (ret) + goto cleanup_lf_sw; + + /* Set interrupts affinity */ + ret = otx2_cptlf_set_irqs_affinity(lfs); + if (ret) + goto unregister_intr; + + atomic_set(&lfs->state, OTX2_CPTLF_STARTED); + /* Register crypto algorithms */ + ret = otx2_cpt_crypto_init(lfs->pdev, THIS_MODULE, lfs_num, 1); + if (ret) { + dev_err(&lfs->pdev->dev, "algorithms registration failed\n"); + goto disable_irqs; + } + return 0; + +disable_irqs: + otx2_cptlf_free_irqs_affinity(lfs); +unregister_intr: + otx2_cptlf_unregister_interrupts(lfs); +cleanup_lf_sw: + lf_sw_cleanup(lfs); +cleanup_lf: + otx2_cptlf_shutdown(lfs); + + return ret; +} + +static int otx2_cptvf_probe(struct pci_dev *pdev, + const struct pci_device_id *ent) +{ + struct device *dev = &pdev->dev; + resource_size_t offset, size; + struct otx2_cptvf_dev *cptvf; + int ret; + + cptvf = devm_kzalloc(dev, sizeof(*cptvf), GFP_KERNEL); + if (!cptvf) + return -ENOMEM; + + ret = pcim_enable_device(pdev); + if (ret) { + dev_err(dev, "Failed to enable PCI device\n"); + goto clear_drvdata; + } + + ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(48)); + if (ret) { + dev_err(dev, "Unable to get usable DMA configuration\n"); + goto clear_drvdata; + } + /* Map VF's configuration registers */ + ret = pcim_iomap_regions_request_all(pdev, 1 << PCI_PF_REG_BAR_NUM, + OTX2_CPTVF_DRV_NAME); + if (ret) { + dev_err(dev, "Couldn't get PCI resources 0x%x\n", ret); + goto clear_drvdata; + } + pci_set_master(pdev); + pci_set_drvdata(pdev, cptvf); + cptvf->pdev = pdev; + + cptvf->reg_base = pcim_iomap_table(pdev)[PCI_PF_REG_BAR_NUM]; + + offset = pci_resource_start(pdev, PCI_MBOX_BAR_NUM); + size = pci_resource_len(pdev, PCI_MBOX_BAR_NUM); + /* Map PF-VF mailbox memory */ + cptvf->pfvf_mbox_base = devm_ioremap_wc(dev, offset, size); + if (!cptvf->pfvf_mbox_base) { + dev_err(&pdev->dev, "Unable to map BAR4\n"); + ret = -ENODEV; + goto clear_drvdata; + } + /* Initialize PF<=>VF mailbox */ + ret = cptvf_pfvf_mbox_init(cptvf); + if (ret) + goto clear_drvdata; + + /* Register interrupts */ + ret = cptvf_register_interrupts(cptvf); + if (ret) + goto destroy_pfvf_mbox; + + /* Initialize CPT LFs */ + ret = cptvf_lf_init(cptvf); + if (ret) + goto unregister_interrupts; + + return 0; + +unregister_interrupts: + cptvf_disable_pfvf_mbox_intrs(cptvf); +destroy_pfvf_mbox: + cptvf_pfvf_mbox_destroy(cptvf); +clear_drvdata: + pci_set_drvdata(pdev, NULL); + + return ret; +} + +static void otx2_cptvf_remove(struct pci_dev *pdev) +{ + struct otx2_cptvf_dev *cptvf = pci_get_drvdata(pdev); + + if (!cptvf) { + dev_err(&pdev->dev, "Invalid CPT VF device.\n"); + return; + } + cptvf_lf_shutdown(&cptvf->lfs); + /* Disable PF-VF mailbox interrupt */ + cptvf_disable_pfvf_mbox_intrs(cptvf); + /* Destroy PF-VF mbox */ + cptvf_pfvf_mbox_destroy(cptvf); + pci_set_drvdata(pdev, NULL); +} + +/* Supported devices */ +static const struct pci_device_id otx2_cptvf_id_table[] = { + {PCI_VDEVICE(CAVIUM, OTX2_CPT_PCI_VF_DEVICE_ID), 0}, + { 0, } /* end of table */ +}; + +static struct pci_driver otx2_cptvf_pci_driver = { + .name = OTX2_CPTVF_DRV_NAME, + .id_table = otx2_cptvf_id_table, + .probe = otx2_cptvf_probe, + .remove = otx2_cptvf_remove, +}; + +module_pci_driver(otx2_cptvf_pci_driver); + +MODULE_AUTHOR("Marvell"); +MODULE_DESCRIPTION("Marvell OcteonTX2 CPT Virtual Function Driver"); +MODULE_LICENSE("GPL v2"); +MODULE_DEVICE_TABLE(pci, otx2_cptvf_id_table); diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptvf_mbox.c b/drivers/crypto/marvell/octeontx2/otx2_cptvf_mbox.c new file mode 100644 index 0000000000000000000000000000000000000000..5d73b711cba6140eaa9c347d93740e6c040beca6 --- /dev/null +++ b/drivers/crypto/marvell/octeontx2/otx2_cptvf_mbox.c @@ -0,0 +1,167 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (C) 2020 Marvell. */ + +#include "otx2_cpt_common.h" +#include "otx2_cptvf.h" +#include + +irqreturn_t otx2_cptvf_pfvf_mbox_intr(int __always_unused irq, void *arg) +{ + struct otx2_cptvf_dev *cptvf = arg; + u64 intr; + + /* Read the interrupt bits */ + intr = otx2_cpt_read64(cptvf->reg_base, BLKADDR_RVUM, 0, + OTX2_RVU_VF_INT); + + if (intr & 0x1ULL) { + /* Schedule work queue function to process the MBOX request */ + queue_work(cptvf->pfvf_mbox_wq, &cptvf->pfvf_mbox_work); + /* Clear and ack the interrupt */ + otx2_cpt_write64(cptvf->reg_base, BLKADDR_RVUM, 0, + OTX2_RVU_VF_INT, 0x1ULL); + } + return IRQ_HANDLED; +} + +static void process_pfvf_mbox_mbox_msg(struct otx2_cptvf_dev *cptvf, + struct mbox_msghdr *msg) +{ + struct otx2_cptlfs_info *lfs = &cptvf->lfs; + struct otx2_cpt_kvf_limits_rsp *rsp_limits; + struct otx2_cpt_egrp_num_rsp *rsp_grp; + struct cpt_rd_wr_reg_msg *rsp_reg; + struct msix_offset_rsp *rsp_msix; + int i; + + if (msg->id >= MBOX_MSG_MAX) { + dev_err(&cptvf->pdev->dev, + "MBOX msg with unknown ID %d\n", msg->id); + return; + } + if (msg->sig != OTX2_MBOX_RSP_SIG) { + dev_err(&cptvf->pdev->dev, + "MBOX msg with wrong signature %x, ID %d\n", + msg->sig, msg->id); + return; + } + switch (msg->id) { + case MBOX_MSG_READY: + cptvf->vf_id = ((msg->pcifunc >> RVU_PFVF_FUNC_SHIFT) + & RVU_PFVF_FUNC_MASK) - 1; + break; + case MBOX_MSG_ATTACH_RESOURCES: + /* Check if resources were successfully attached */ + if (!msg->rc) + lfs->are_lfs_attached = 1; + break; + case MBOX_MSG_DETACH_RESOURCES: + /* Check if resources were successfully detached */ + if (!msg->rc) + lfs->are_lfs_attached = 0; + break; + case MBOX_MSG_MSIX_OFFSET: + rsp_msix = (struct msix_offset_rsp *) msg; + for (i = 0; i < rsp_msix->cptlfs; i++) + lfs->lf[i].msix_offset = rsp_msix->cptlf_msixoff[i]; + break; + case MBOX_MSG_CPT_RD_WR_REGISTER: + rsp_reg = (struct cpt_rd_wr_reg_msg *) msg; + if (msg->rc) { + dev_err(&cptvf->pdev->dev, + "Reg %llx rd/wr(%d) failed %d\n", + rsp_reg->reg_offset, rsp_reg->is_write, + msg->rc); + return; + } + if (!rsp_reg->is_write) + *rsp_reg->ret_val = rsp_reg->val; + break; + case MBOX_MSG_GET_ENG_GRP_NUM: + rsp_grp = (struct otx2_cpt_egrp_num_rsp *) msg; + cptvf->lfs.kcrypto_eng_grp_num = rsp_grp->eng_grp_num; + break; + case MBOX_MSG_GET_KVF_LIMITS: + rsp_limits = (struct otx2_cpt_kvf_limits_rsp *) msg; + cptvf->lfs.kvf_limits = rsp_limits->kvf_limits; + break; + default: + dev_err(&cptvf->pdev->dev, "Unsupported msg %d received.\n", + msg->id); + break; + } +} + +void otx2_cptvf_pfvf_mbox_handler(struct work_struct *work) +{ + struct otx2_cptvf_dev *cptvf; + struct otx2_mbox *pfvf_mbox; + struct otx2_mbox_dev *mdev; + struct mbox_hdr *rsp_hdr; + struct mbox_msghdr *msg; + int offset, i; + + /* sync with mbox memory region */ + smp_rmb(); + + cptvf = container_of(work, struct otx2_cptvf_dev, pfvf_mbox_work); + pfvf_mbox = &cptvf->pfvf_mbox; + mdev = &pfvf_mbox->dev[0]; + rsp_hdr = (struct mbox_hdr *)(mdev->mbase + pfvf_mbox->rx_start); + if (rsp_hdr->num_msgs == 0) + return; + offset = ALIGN(sizeof(struct mbox_hdr), MBOX_MSG_ALIGN); + + for (i = 0; i < rsp_hdr->num_msgs; i++) { + msg = (struct mbox_msghdr *)(mdev->mbase + pfvf_mbox->rx_start + + offset); + process_pfvf_mbox_mbox_msg(cptvf, msg); + offset = msg->next_msgoff; + mdev->msgs_acked++; + } + otx2_mbox_reset(pfvf_mbox, 0); +} + +int otx2_cptvf_send_eng_grp_num_msg(struct otx2_cptvf_dev *cptvf, int eng_type) +{ + struct otx2_mbox *mbox = &cptvf->pfvf_mbox; + struct pci_dev *pdev = cptvf->pdev; + struct otx2_cpt_egrp_num_msg *req; + + req = (struct otx2_cpt_egrp_num_msg *) + otx2_mbox_alloc_msg_rsp(mbox, 0, sizeof(*req), + sizeof(struct otx2_cpt_egrp_num_rsp)); + if (req == NULL) { + dev_err(&pdev->dev, "RVU MBOX failed to get message.\n"); + return -EFAULT; + } + req->hdr.id = MBOX_MSG_GET_ENG_GRP_NUM; + req->hdr.sig = OTX2_MBOX_REQ_SIG; + req->hdr.pcifunc = OTX2_CPT_RVU_PFFUNC(cptvf->vf_id, 0); + req->eng_type = eng_type; + + return otx2_cpt_send_mbox_msg(mbox, pdev); +} + +int otx2_cptvf_send_kvf_limits_msg(struct otx2_cptvf_dev *cptvf) +{ + struct otx2_mbox *mbox = &cptvf->pfvf_mbox; + struct pci_dev *pdev = cptvf->pdev; + struct mbox_msghdr *req; + int ret; + + req = (struct mbox_msghdr *) + otx2_mbox_alloc_msg_rsp(mbox, 0, sizeof(*req), + sizeof(struct otx2_cpt_kvf_limits_rsp)); + if (req == NULL) { + dev_err(&pdev->dev, "RVU MBOX failed to get message.\n"); + return -EFAULT; + } + req->id = MBOX_MSG_GET_KVF_LIMITS; + req->sig = OTX2_MBOX_REQ_SIG; + req->pcifunc = OTX2_CPT_RVU_PFFUNC(cptvf->vf_id, 0); + + ret = otx2_cpt_send_mbox_msg(mbox, pdev); + + return ret; +} diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptvf_reqmgr.c b/drivers/crypto/marvell/octeontx2/otx2_cptvf_reqmgr.c new file mode 100644 index 0000000000000000000000000000000000000000..d5c1c1b7c7e4b8a79752b5fe5f85b08cc006f819 --- /dev/null +++ b/drivers/crypto/marvell/octeontx2/otx2_cptvf_reqmgr.c @@ -0,0 +1,541 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (C) 2020 Marvell. */ + +#include "otx2_cptvf.h" +#include "otx2_cpt_common.h" + +/* SG list header size in bytes */ +#define SG_LIST_HDR_SIZE 8 + +/* Default timeout when waiting for free pending entry in us */ +#define CPT_PENTRY_TIMEOUT 1000 +#define CPT_PENTRY_STEP 50 + +/* Default threshold for stopping and resuming sender requests */ +#define CPT_IQ_STOP_MARGIN 128 +#define CPT_IQ_RESUME_MARGIN 512 + +/* Default command timeout in seconds */ +#define CPT_COMMAND_TIMEOUT 4 +#define CPT_TIME_IN_RESET_COUNT 5 + +static void otx2_cpt_dump_sg_list(struct pci_dev *pdev, + struct otx2_cpt_req_info *req) +{ + int i; + + pr_debug("Gather list size %d\n", req->in_cnt); + for (i = 0; i < req->in_cnt; i++) { + pr_debug("Buffer %d size %d, vptr 0x%p, dmaptr 0x%p\n", i, + req->in[i].size, req->in[i].vptr, + (void *) req->in[i].dma_addr); + pr_debug("Buffer hexdump (%d bytes)\n", + req->in[i].size); + print_hex_dump_debug("", DUMP_PREFIX_NONE, 16, 1, + req->in[i].vptr, req->in[i].size, false); + } + pr_debug("Scatter list size %d\n", req->out_cnt); + for (i = 0; i < req->out_cnt; i++) { + pr_debug("Buffer %d size %d, vptr 0x%p, dmaptr 0x%p\n", i, + req->out[i].size, req->out[i].vptr, + (void *) req->out[i].dma_addr); + pr_debug("Buffer hexdump (%d bytes)\n", req->out[i].size); + print_hex_dump_debug("", DUMP_PREFIX_NONE, 16, 1, + req->out[i].vptr, req->out[i].size, false); + } +} + +static inline struct otx2_cpt_pending_entry *get_free_pending_entry( + struct otx2_cpt_pending_queue *q, + int qlen) +{ + struct otx2_cpt_pending_entry *ent = NULL; + + ent = &q->head[q->rear]; + if (unlikely(ent->busy)) + return NULL; + + q->rear++; + if (unlikely(q->rear == qlen)) + q->rear = 0; + + return ent; +} + +static inline u32 modulo_inc(u32 index, u32 length, u32 inc) +{ + if (WARN_ON(inc > length)) + inc = length; + + index += inc; + if (unlikely(index >= length)) + index -= length; + + return index; +} + +static inline void free_pentry(struct otx2_cpt_pending_entry *pentry) +{ + pentry->completion_addr = NULL; + pentry->info = NULL; + pentry->callback = NULL; + pentry->areq = NULL; + pentry->resume_sender = false; + pentry->busy = false; +} + +static inline int setup_sgio_components(struct pci_dev *pdev, + struct otx2_cpt_buf_ptr *list, + int buf_count, u8 *buffer) +{ + struct otx2_cpt_sglist_component *sg_ptr = NULL; + int ret = 0, i, j; + int components; + + if (unlikely(!list)) { + dev_err(&pdev->dev, "Input list pointer is NULL\n"); + return -EFAULT; + } + + for (i = 0; i < buf_count; i++) { + if (unlikely(!list[i].vptr)) + continue; + list[i].dma_addr = dma_map_single(&pdev->dev, list[i].vptr, + list[i].size, + DMA_BIDIRECTIONAL); + if (unlikely(dma_mapping_error(&pdev->dev, list[i].dma_addr))) { + dev_err(&pdev->dev, "Dma mapping failed\n"); + ret = -EIO; + goto sg_cleanup; + } + } + components = buf_count / 4; + sg_ptr = (struct otx2_cpt_sglist_component *)buffer; + for (i = 0; i < components; i++) { + sg_ptr->len0 = cpu_to_be16(list[i * 4 + 0].size); + sg_ptr->len1 = cpu_to_be16(list[i * 4 + 1].size); + sg_ptr->len2 = cpu_to_be16(list[i * 4 + 2].size); + sg_ptr->len3 = cpu_to_be16(list[i * 4 + 3].size); + sg_ptr->ptr0 = cpu_to_be64(list[i * 4 + 0].dma_addr); + sg_ptr->ptr1 = cpu_to_be64(list[i * 4 + 1].dma_addr); + sg_ptr->ptr2 = cpu_to_be64(list[i * 4 + 2].dma_addr); + sg_ptr->ptr3 = cpu_to_be64(list[i * 4 + 3].dma_addr); + sg_ptr++; + } + components = buf_count % 4; + + switch (components) { + case 3: + sg_ptr->len2 = cpu_to_be16(list[i * 4 + 2].size); + sg_ptr->ptr2 = cpu_to_be64(list[i * 4 + 2].dma_addr); + fallthrough; + case 2: + sg_ptr->len1 = cpu_to_be16(list[i * 4 + 1].size); + sg_ptr->ptr1 = cpu_to_be64(list[i * 4 + 1].dma_addr); + fallthrough; + case 1: + sg_ptr->len0 = cpu_to_be16(list[i * 4 + 0].size); + sg_ptr->ptr0 = cpu_to_be64(list[i * 4 + 0].dma_addr); + break; + default: + break; + } + return ret; + +sg_cleanup: + for (j = 0; j < i; j++) { + if (list[j].dma_addr) { + dma_unmap_single(&pdev->dev, list[j].dma_addr, + list[j].size, DMA_BIDIRECTIONAL); + } + + list[j].dma_addr = 0; + } + return ret; +} + +static inline struct otx2_cpt_inst_info *info_create(struct pci_dev *pdev, + struct otx2_cpt_req_info *req, + gfp_t gfp) +{ + int align = OTX2_CPT_DMA_MINALIGN; + struct otx2_cpt_inst_info *info; + u32 dlen, align_dlen, info_len; + u16 g_sz_bytes, s_sz_bytes; + u32 total_mem_len; + + if (unlikely(req->in_cnt > OTX2_CPT_MAX_SG_IN_CNT || + req->out_cnt > OTX2_CPT_MAX_SG_OUT_CNT)) { + dev_err(&pdev->dev, "Error too many sg components\n"); + return NULL; + } + + g_sz_bytes = ((req->in_cnt + 3) / 4) * + sizeof(struct otx2_cpt_sglist_component); + s_sz_bytes = ((req->out_cnt + 3) / 4) * + sizeof(struct otx2_cpt_sglist_component); + + dlen = g_sz_bytes + s_sz_bytes + SG_LIST_HDR_SIZE; + align_dlen = ALIGN(dlen, align); + info_len = ALIGN(sizeof(*info), align); + total_mem_len = align_dlen + info_len + sizeof(union otx2_cpt_res_s); + + info = kzalloc(total_mem_len, gfp); + if (unlikely(!info)) + return NULL; + + info->dlen = dlen; + info->in_buffer = (u8 *)info + info_len; + + ((u16 *)info->in_buffer)[0] = req->out_cnt; + ((u16 *)info->in_buffer)[1] = req->in_cnt; + ((u16 *)info->in_buffer)[2] = 0; + ((u16 *)info->in_buffer)[3] = 0; + cpu_to_be64s((u64 *)info->in_buffer); + + /* Setup gather (input) components */ + if (setup_sgio_components(pdev, req->in, req->in_cnt, + &info->in_buffer[8])) { + dev_err(&pdev->dev, "Failed to setup gather list\n"); + goto destroy_info; + } + + if (setup_sgio_components(pdev, req->out, req->out_cnt, + &info->in_buffer[8 + g_sz_bytes])) { + dev_err(&pdev->dev, "Failed to setup scatter list\n"); + goto destroy_info; + } + + info->dma_len = total_mem_len - info_len; + info->dptr_baddr = dma_map_single(&pdev->dev, info->in_buffer, + info->dma_len, DMA_BIDIRECTIONAL); + if (unlikely(dma_mapping_error(&pdev->dev, info->dptr_baddr))) { + dev_err(&pdev->dev, "DMA Mapping failed for cpt req\n"); + goto destroy_info; + } + /* + * Get buffer for union otx2_cpt_res_s response + * structure and its physical address + */ + info->completion_addr = info->in_buffer + align_dlen; + info->comp_baddr = info->dptr_baddr + align_dlen; + + return info; + +destroy_info: + otx2_cpt_info_destroy(pdev, info); + return NULL; +} + +static int process_request(struct pci_dev *pdev, struct otx2_cpt_req_info *req, + struct otx2_cpt_pending_queue *pqueue, + struct otx2_cptlf_info *lf) +{ + struct otx2_cptvf_request *cpt_req = &req->req; + struct otx2_cpt_pending_entry *pentry = NULL; + union otx2_cpt_ctrl_info *ctrl = &req->ctrl; + struct otx2_cpt_inst_info *info = NULL; + union otx2_cpt_res_s *result = NULL; + struct otx2_cpt_iq_command iq_cmd; + union otx2_cpt_inst_s cptinst; + int retry, ret = 0; + u8 resume_sender; + gfp_t gfp; + + gfp = (req->areq->flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? GFP_KERNEL : + GFP_ATOMIC; + if (unlikely(!otx2_cptlf_started(lf->lfs))) + return -ENODEV; + + info = info_create(pdev, req, gfp); + if (unlikely(!info)) { + dev_err(&pdev->dev, "Setting up cpt inst info failed"); + return -ENOMEM; + } + cpt_req->dlen = info->dlen; + + result = info->completion_addr; + result->s.compcode = OTX2_CPT_COMPLETION_CODE_INIT; + + spin_lock_bh(&pqueue->lock); + pentry = get_free_pending_entry(pqueue, pqueue->qlen); + retry = CPT_PENTRY_TIMEOUT / CPT_PENTRY_STEP; + while (unlikely(!pentry) && retry--) { + spin_unlock_bh(&pqueue->lock); + udelay(CPT_PENTRY_STEP); + spin_lock_bh(&pqueue->lock); + pentry = get_free_pending_entry(pqueue, pqueue->qlen); + } + + if (unlikely(!pentry)) { + ret = -ENOSPC; + goto destroy_info; + } + + /* + * Check if we are close to filling in entire pending queue, + * if so then tell the sender to stop/sleep by returning -EBUSY + * We do it only for context which can sleep (GFP_KERNEL) + */ + if (gfp == GFP_KERNEL && + pqueue->pending_count > (pqueue->qlen - CPT_IQ_STOP_MARGIN)) { + pentry->resume_sender = true; + } else + pentry->resume_sender = false; + resume_sender = pentry->resume_sender; + pqueue->pending_count++; + + pentry->completion_addr = info->completion_addr; + pentry->info = info; + pentry->callback = req->callback; + pentry->areq = req->areq; + pentry->busy = true; + info->pentry = pentry; + info->time_in = jiffies; + info->req = req; + + /* Fill in the command */ + iq_cmd.cmd.u = 0; + iq_cmd.cmd.s.opcode = cpu_to_be16(cpt_req->opcode.flags); + iq_cmd.cmd.s.param1 = cpu_to_be16(cpt_req->param1); + iq_cmd.cmd.s.param2 = cpu_to_be16(cpt_req->param2); + iq_cmd.cmd.s.dlen = cpu_to_be16(cpt_req->dlen); + + /* 64-bit swap for microcode data reads, not needed for addresses*/ + cpu_to_be64s(&iq_cmd.cmd.u); + iq_cmd.dptr = info->dptr_baddr; + iq_cmd.rptr = 0; + iq_cmd.cptr.u = 0; + iq_cmd.cptr.s.grp = ctrl->s.grp; + + /* Fill in the CPT_INST_S type command for HW interpretation */ + otx2_cpt_fill_inst(&cptinst, &iq_cmd, info->comp_baddr); + + /* Print debug info if enabled */ + otx2_cpt_dump_sg_list(pdev, req); + pr_debug("Cpt_inst_s hexdump (%d bytes)\n", OTX2_CPT_INST_SIZE); + print_hex_dump_debug("", 0, 16, 1, &cptinst, OTX2_CPT_INST_SIZE, false); + pr_debug("Dptr hexdump (%d bytes)\n", cpt_req->dlen); + print_hex_dump_debug("", 0, 16, 1, info->in_buffer, + cpt_req->dlen, false); + + /* Send CPT command */ + otx2_cpt_send_cmd(&cptinst, 1, lf); + + /* + * We allocate and prepare pending queue entry in critical section + * together with submitting CPT instruction to CPT instruction queue + * to make sure that order of CPT requests is the same in both + * pending and instruction queues + */ + spin_unlock_bh(&pqueue->lock); + + ret = resume_sender ? -EBUSY : -EINPROGRESS; + return ret; + +destroy_info: + spin_unlock_bh(&pqueue->lock); + otx2_cpt_info_destroy(pdev, info); + return ret; +} + +int otx2_cpt_do_request(struct pci_dev *pdev, struct otx2_cpt_req_info *req, + int cpu_num) +{ + struct otx2_cptvf_dev *cptvf = pci_get_drvdata(pdev); + struct otx2_cptlfs_info *lfs = &cptvf->lfs; + + return process_request(lfs->pdev, req, &lfs->lf[cpu_num].pqueue, + &lfs->lf[cpu_num]); +} + +static int cpt_process_ccode(struct pci_dev *pdev, + union otx2_cpt_res_s *cpt_status, + struct otx2_cpt_inst_info *info, + u32 *res_code) +{ + u8 uc_ccode = cpt_status->s.uc_compcode; + u8 ccode = cpt_status->s.compcode; + + switch (ccode) { + case OTX2_CPT_COMP_E_FAULT: + dev_err(&pdev->dev, + "Request failed with DMA fault\n"); + otx2_cpt_dump_sg_list(pdev, info->req); + break; + + case OTX2_CPT_COMP_E_HWERR: + dev_err(&pdev->dev, + "Request failed with hardware error\n"); + otx2_cpt_dump_sg_list(pdev, info->req); + break; + + case OTX2_CPT_COMP_E_INSTERR: + dev_err(&pdev->dev, + "Request failed with instruction error\n"); + otx2_cpt_dump_sg_list(pdev, info->req); + break; + + case OTX2_CPT_COMP_E_NOTDONE: + /* check for timeout */ + if (time_after_eq(jiffies, info->time_in + + CPT_COMMAND_TIMEOUT * HZ)) + dev_warn(&pdev->dev, + "Request timed out 0x%p", info->req); + else if (info->extra_time < CPT_TIME_IN_RESET_COUNT) { + info->time_in = jiffies; + info->extra_time++; + } + return 1; + + case OTX2_CPT_COMP_E_GOOD: + /* + * Check microcode completion code, it is only valid + * when completion code is CPT_COMP_E::GOOD + */ + if (uc_ccode != OTX2_CPT_UCC_SUCCESS) { + /* + * If requested hmac is truncated and ucode returns + * s/g write length error then we report success + * because ucode writes as many bytes of calculated + * hmac as available in gather buffer and reports + * s/g write length error if number of bytes in gather + * buffer is less than full hmac size. + */ + if (info->req->is_trunc_hmac && + uc_ccode == OTX2_CPT_UCC_SG_WRITE_LENGTH) { + *res_code = 0; + break; + } + + dev_err(&pdev->dev, + "Request failed with software error code 0x%x\n", + cpt_status->s.uc_compcode); + otx2_cpt_dump_sg_list(pdev, info->req); + break; + } + /* Request has been processed with success */ + *res_code = 0; + break; + + default: + dev_err(&pdev->dev, + "Request returned invalid status %d\n", ccode); + break; + } + return 0; +} + +static inline void process_pending_queue(struct pci_dev *pdev, + struct otx2_cpt_pending_queue *pqueue) +{ + struct otx2_cpt_pending_entry *resume_pentry = NULL; + void (*callback)(int status, void *arg, void *req); + struct otx2_cpt_pending_entry *pentry = NULL; + union otx2_cpt_res_s *cpt_status = NULL; + struct otx2_cpt_inst_info *info = NULL; + struct otx2_cpt_req_info *req = NULL; + struct crypto_async_request *areq; + u32 res_code, resume_index; + + while (1) { + spin_lock_bh(&pqueue->lock); + pentry = &pqueue->head[pqueue->front]; + + if (WARN_ON(!pentry)) { + spin_unlock_bh(&pqueue->lock); + break; + } + + res_code = -EINVAL; + if (unlikely(!pentry->busy)) { + spin_unlock_bh(&pqueue->lock); + break; + } + + if (unlikely(!pentry->callback)) { + dev_err(&pdev->dev, "Callback NULL\n"); + goto process_pentry; + } + + info = pentry->info; + if (unlikely(!info)) { + dev_err(&pdev->dev, "Pending entry post arg NULL\n"); + goto process_pentry; + } + + req = info->req; + if (unlikely(!req)) { + dev_err(&pdev->dev, "Request NULL\n"); + goto process_pentry; + } + + cpt_status = pentry->completion_addr; + if (unlikely(!cpt_status)) { + dev_err(&pdev->dev, "Completion address NULL\n"); + goto process_pentry; + } + + if (cpt_process_ccode(pdev, cpt_status, info, &res_code)) { + spin_unlock_bh(&pqueue->lock); + return; + } + info->pdev = pdev; + +process_pentry: + /* + * Check if we should inform sending side to resume + * We do it CPT_IQ_RESUME_MARGIN elements in advance before + * pending queue becomes empty + */ + resume_index = modulo_inc(pqueue->front, pqueue->qlen, + CPT_IQ_RESUME_MARGIN); + resume_pentry = &pqueue->head[resume_index]; + if (resume_pentry && + resume_pentry->resume_sender) { + resume_pentry->resume_sender = false; + callback = resume_pentry->callback; + areq = resume_pentry->areq; + + if (callback) { + spin_unlock_bh(&pqueue->lock); + + /* + * EINPROGRESS is an indication for sending + * side that it can resume sending requests + */ + callback(-EINPROGRESS, areq, info); + spin_lock_bh(&pqueue->lock); + } + } + + callback = pentry->callback; + areq = pentry->areq; + free_pentry(pentry); + + pqueue->pending_count--; + pqueue->front = modulo_inc(pqueue->front, pqueue->qlen, 1); + spin_unlock_bh(&pqueue->lock); + + /* + * Call callback after current pending entry has been + * processed, we don't do it if the callback pointer is + * invalid. + */ + if (callback) + callback(res_code, areq, info); + } +} + +void otx2_cpt_post_process(struct otx2_cptlf_wqe *wqe) +{ + process_pending_queue(wqe->lfs->pdev, + &wqe->lfs->lf[wqe->lf_num].pqueue); +} + +int otx2_cpt_get_kcrypto_eng_grp_num(struct pci_dev *pdev) +{ + struct otx2_cptvf_dev *cptvf = pci_get_drvdata(pdev); + + return cptvf->lfs.kcrypto_eng_grp_num; +} diff --git a/drivers/crypto/mediatek/Makefile b/drivers/crypto/mediatek/Makefile deleted file mode 100644 index 196a4653974ed4e53a18dedf1afdc4acfe17e9ee..0000000000000000000000000000000000000000 --- a/drivers/crypto/mediatek/Makefile +++ /dev/null @@ -1,3 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -obj-$(CONFIG_CRYPTO_DEV_MEDIATEK) += mtk-crypto.o -mtk-crypto-objs:= mtk-platform.o mtk-aes.o mtk-sha.o diff --git a/drivers/crypto/mediatek/mtk-aes.c b/drivers/crypto/mediatek/mtk-aes.c deleted file mode 100644 index 7323066724c3b94958e32167b160017b8359b131..0000000000000000000000000000000000000000 --- a/drivers/crypto/mediatek/mtk-aes.c +++ /dev/null @@ -1,1271 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Cryptographic API. - * - * Driver for EIP97 AES acceleration. - * - * Copyright (c) 2016 Ryder Lee - * - * Some ideas are from atmel-aes.c drivers. - */ - -#include -#include -#include -#include "mtk-platform.h" - -#define AES_QUEUE_SIZE 512 -#define AES_BUF_ORDER 2 -#define AES_BUF_SIZE ((PAGE_SIZE << AES_BUF_ORDER) \ - & ~(AES_BLOCK_SIZE - 1)) -#define AES_MAX_STATE_BUF_SIZE SIZE_IN_WORDS(AES_KEYSIZE_256 + \ - AES_BLOCK_SIZE * 2) -#define AES_MAX_CT_SIZE 6 - -#define AES_CT_CTRL_HDR cpu_to_le32(0x00220000) - -/* AES-CBC/ECB/CTR/OFB/CFB command token */ -#define AES_CMD0 cpu_to_le32(0x05000000) -#define AES_CMD1 cpu_to_le32(0x2d060000) -#define AES_CMD2 cpu_to_le32(0xe4a63806) -/* AES-GCM command token */ -#define AES_GCM_CMD0 cpu_to_le32(0x0b000000) -#define AES_GCM_CMD1 cpu_to_le32(0xa0800000) -#define AES_GCM_CMD2 cpu_to_le32(0x25000010) -#define AES_GCM_CMD3 cpu_to_le32(0x0f020000) -#define AES_GCM_CMD4 cpu_to_le32(0x21e60000) -#define AES_GCM_CMD5 cpu_to_le32(0x40e60000) -#define AES_GCM_CMD6 cpu_to_le32(0xd0070000) - -/* AES transform information word 0 fields */ -#define AES_TFM_BASIC_OUT cpu_to_le32(0x4 << 0) -#define AES_TFM_BASIC_IN cpu_to_le32(0x5 << 0) -#define AES_TFM_GCM_OUT cpu_to_le32(0x6 << 0) -#define AES_TFM_GCM_IN cpu_to_le32(0xf << 0) -#define AES_TFM_SIZE(x) cpu_to_le32((x) << 8) -#define AES_TFM_128BITS cpu_to_le32(0xb << 16) -#define AES_TFM_192BITS cpu_to_le32(0xd << 16) -#define AES_TFM_256BITS cpu_to_le32(0xf << 16) -#define AES_TFM_GHASH_DIGEST cpu_to_le32(0x2 << 21) -#define AES_TFM_GHASH cpu_to_le32(0x4 << 23) -/* AES transform information word 1 fields */ -#define AES_TFM_ECB cpu_to_le32(0x0 << 0) -#define AES_TFM_CBC cpu_to_le32(0x1 << 0) -#define AES_TFM_OFB cpu_to_le32(0x4 << 0) -#define AES_TFM_CFB128 cpu_to_le32(0x5 << 0) -#define AES_TFM_CTR_INIT cpu_to_le32(0x2 << 0) /* init counter to 1 */ -#define AES_TFM_CTR_LOAD cpu_to_le32(0x6 << 0) /* load/reuse counter */ -#define AES_TFM_3IV cpu_to_le32(0x7 << 5) /* using IV 0-2 */ -#define AES_TFM_FULL_IV cpu_to_le32(0xf << 5) /* using IV 0-3 */ -#define AES_TFM_IV_CTR_MODE cpu_to_le32(0x1 << 10) -#define AES_TFM_ENC_HASH cpu_to_le32(0x1 << 17) - -/* AES flags */ -#define AES_FLAGS_CIPHER_MSK GENMASK(4, 0) -#define AES_FLAGS_ECB BIT(0) -#define AES_FLAGS_CBC BIT(1) -#define AES_FLAGS_CTR BIT(2) -#define AES_FLAGS_OFB BIT(3) -#define AES_FLAGS_CFB128 BIT(4) -#define AES_FLAGS_GCM BIT(5) -#define AES_FLAGS_ENCRYPT BIT(6) -#define AES_FLAGS_BUSY BIT(7) - -#define AES_AUTH_TAG_ERR cpu_to_le32(BIT(26)) - -/** - * mtk_aes_info - hardware information of AES - * @cmd: command token, hardware instruction - * @tfm: transform state of cipher algorithm. - * @state: contains keys and initial vectors. - * - * Memory layout of GCM buffer: - * /-----------\ - * | AES KEY | 128/196/256 bits - * |-----------| - * | HASH KEY | a string 128 zero bits encrypted using the block cipher - * |-----------| - * | IVs | 4 * 4 bytes - * \-----------/ - * - * The engine requires all these info to do: - * - Commands decoding and control of the engine's data path. - * - Coordinating hardware data fetch and store operations. - * - Result token construction and output. - */ -struct mtk_aes_info { - __le32 cmd[AES_MAX_CT_SIZE]; - __le32 tfm[2]; - __le32 state[AES_MAX_STATE_BUF_SIZE]; -}; - -struct mtk_aes_reqctx { - u64 mode; -}; - -struct mtk_aes_base_ctx { - struct mtk_cryp *cryp; - u32 keylen; - __le32 key[12]; - __le32 keymode; - - mtk_aes_fn start; - - struct mtk_aes_info info; - dma_addr_t ct_dma; - dma_addr_t tfm_dma; - - __le32 ct_hdr; - u32 ct_size; -}; - -struct mtk_aes_ctx { - struct mtk_aes_base_ctx base; -}; - -struct mtk_aes_ctr_ctx { - struct mtk_aes_base_ctx base; - - __be32 iv[AES_BLOCK_SIZE / sizeof(u32)]; - size_t offset; - struct scatterlist src[2]; - struct scatterlist dst[2]; -}; - -struct mtk_aes_gcm_ctx { - struct mtk_aes_base_ctx base; - - u32 authsize; - size_t textlen; -}; - -struct mtk_aes_drv { - struct list_head dev_list; - /* Device list lock */ - spinlock_t lock; -}; - -static struct mtk_aes_drv mtk_aes = { - .dev_list = LIST_HEAD_INIT(mtk_aes.dev_list), - .lock = __SPIN_LOCK_UNLOCKED(mtk_aes.lock), -}; - -static inline u32 mtk_aes_read(struct mtk_cryp *cryp, u32 offset) -{ - return readl_relaxed(cryp->base + offset); -} - -static inline void mtk_aes_write(struct mtk_cryp *cryp, - u32 offset, u32 value) -{ - writel_relaxed(value, cryp->base + offset); -} - -static struct mtk_cryp *mtk_aes_find_dev(struct mtk_aes_base_ctx *ctx) -{ - struct mtk_cryp *cryp = NULL; - struct mtk_cryp *tmp; - - spin_lock_bh(&mtk_aes.lock); - if (!ctx->cryp) { - list_for_each_entry(tmp, &mtk_aes.dev_list, aes_list) { - cryp = tmp; - break; - } - ctx->cryp = cryp; - } else { - cryp = ctx->cryp; - } - spin_unlock_bh(&mtk_aes.lock); - - return cryp; -} - -static inline size_t mtk_aes_padlen(size_t len) -{ - len &= AES_BLOCK_SIZE - 1; - return len ? AES_BLOCK_SIZE - len : 0; -} - -static bool mtk_aes_check_aligned(struct scatterlist *sg, size_t len, - struct mtk_aes_dma *dma) -{ - int nents; - - if (!IS_ALIGNED(len, AES_BLOCK_SIZE)) - return false; - - for (nents = 0; sg; sg = sg_next(sg), ++nents) { - if (!IS_ALIGNED(sg->offset, sizeof(u32))) - return false; - - if (len <= sg->length) { - if (!IS_ALIGNED(len, AES_BLOCK_SIZE)) - return false; - - dma->nents = nents + 1; - dma->remainder = sg->length - len; - sg->length = len; - return true; - } - - if (!IS_ALIGNED(sg->length, AES_BLOCK_SIZE)) - return false; - - len -= sg->length; - } - - return false; -} - -static inline void mtk_aes_set_mode(struct mtk_aes_rec *aes, - const struct mtk_aes_reqctx *rctx) -{ - /* Clear all but persistent flags and set request flags. */ - aes->flags = (aes->flags & AES_FLAGS_BUSY) | rctx->mode; -} - -static inline void mtk_aes_restore_sg(const struct mtk_aes_dma *dma) -{ - struct scatterlist *sg = dma->sg; - int nents = dma->nents; - - if (!dma->remainder) - return; - - while (--nents > 0 && sg) - sg = sg_next(sg); - - if (!sg) - return; - - sg->length += dma->remainder; -} - -static inline int mtk_aes_complete(struct mtk_cryp *cryp, - struct mtk_aes_rec *aes, - int err) -{ - aes->flags &= ~AES_FLAGS_BUSY; - aes->areq->complete(aes->areq, err); - /* Handle new request */ - tasklet_schedule(&aes->queue_task); - return err; -} - -/* - * Write descriptors for processing. This will configure the engine, load - * the transform information and then start the packet processing. - */ -static int mtk_aes_xmit(struct mtk_cryp *cryp, struct mtk_aes_rec *aes) -{ - struct mtk_ring *ring = cryp->ring[aes->id]; - struct mtk_desc *cmd = NULL, *res = NULL; - struct scatterlist *ssg = aes->src.sg, *dsg = aes->dst.sg; - u32 slen = aes->src.sg_len, dlen = aes->dst.sg_len; - int nents; - - /* Write command descriptors */ - for (nents = 0; nents < slen; ++nents, ssg = sg_next(ssg)) { - cmd = ring->cmd_next; - cmd->hdr = MTK_DESC_BUF_LEN(ssg->length); - cmd->buf = cpu_to_le32(sg_dma_address(ssg)); - - if (nents == 0) { - cmd->hdr |= MTK_DESC_FIRST | - MTK_DESC_CT_LEN(aes->ctx->ct_size); - cmd->ct = cpu_to_le32(aes->ctx->ct_dma); - cmd->ct_hdr = aes->ctx->ct_hdr; - cmd->tfm = cpu_to_le32(aes->ctx->tfm_dma); - } - - /* Shift ring buffer and check boundary */ - if (++ring->cmd_next == ring->cmd_base + MTK_DESC_NUM) - ring->cmd_next = ring->cmd_base; - } - cmd->hdr |= MTK_DESC_LAST; - - /* Prepare result descriptors */ - for (nents = 0; nents < dlen; ++nents, dsg = sg_next(dsg)) { - res = ring->res_next; - res->hdr = MTK_DESC_BUF_LEN(dsg->length); - res->buf = cpu_to_le32(sg_dma_address(dsg)); - - if (nents == 0) - res->hdr |= MTK_DESC_FIRST; - - /* Shift ring buffer and check boundary */ - if (++ring->res_next == ring->res_base + MTK_DESC_NUM) - ring->res_next = ring->res_base; - } - res->hdr |= MTK_DESC_LAST; - - /* Pointer to current result descriptor */ - ring->res_prev = res; - - /* Prepare enough space for authenticated tag */ - if (aes->flags & AES_FLAGS_GCM) - le32_add_cpu(&res->hdr, AES_BLOCK_SIZE); - - /* - * Make sure that all changes to the DMA ring are done before we - * start engine. - */ - wmb(); - /* Start DMA transfer */ - mtk_aes_write(cryp, RDR_PREP_COUNT(aes->id), MTK_DESC_CNT(dlen)); - mtk_aes_write(cryp, CDR_PREP_COUNT(aes->id), MTK_DESC_CNT(slen)); - - return -EINPROGRESS; -} - -static void mtk_aes_unmap(struct mtk_cryp *cryp, struct mtk_aes_rec *aes) -{ - struct mtk_aes_base_ctx *ctx = aes->ctx; - - dma_unmap_single(cryp->dev, ctx->ct_dma, sizeof(ctx->info), - DMA_TO_DEVICE); - - if (aes->src.sg == aes->dst.sg) { - dma_unmap_sg(cryp->dev, aes->src.sg, aes->src.nents, - DMA_BIDIRECTIONAL); - - if (aes->src.sg != &aes->aligned_sg) - mtk_aes_restore_sg(&aes->src); - } else { - dma_unmap_sg(cryp->dev, aes->dst.sg, aes->dst.nents, - DMA_FROM_DEVICE); - - if (aes->dst.sg != &aes->aligned_sg) - mtk_aes_restore_sg(&aes->dst); - - dma_unmap_sg(cryp->dev, aes->src.sg, aes->src.nents, - DMA_TO_DEVICE); - - if (aes->src.sg != &aes->aligned_sg) - mtk_aes_restore_sg(&aes->src); - } - - if (aes->dst.sg == &aes->aligned_sg) - sg_copy_from_buffer(aes->real_dst, sg_nents(aes->real_dst), - aes->buf, aes->total); -} - -static int mtk_aes_map(struct mtk_cryp *cryp, struct mtk_aes_rec *aes) -{ - struct mtk_aes_base_ctx *ctx = aes->ctx; - struct mtk_aes_info *info = &ctx->info; - - ctx->ct_dma = dma_map_single(cryp->dev, info, sizeof(*info), - DMA_TO_DEVICE); - if (unlikely(dma_mapping_error(cryp->dev, ctx->ct_dma))) - goto exit; - - ctx->tfm_dma = ctx->ct_dma + sizeof(info->cmd); - - if (aes->src.sg == aes->dst.sg) { - aes->src.sg_len = dma_map_sg(cryp->dev, aes->src.sg, - aes->src.nents, - DMA_BIDIRECTIONAL); - aes->dst.sg_len = aes->src.sg_len; - if (unlikely(!aes->src.sg_len)) - goto sg_map_err; - } else { - aes->src.sg_len = dma_map_sg(cryp->dev, aes->src.sg, - aes->src.nents, DMA_TO_DEVICE); - if (unlikely(!aes->src.sg_len)) - goto sg_map_err; - - aes->dst.sg_len = dma_map_sg(cryp->dev, aes->dst.sg, - aes->dst.nents, DMA_FROM_DEVICE); - if (unlikely(!aes->dst.sg_len)) { - dma_unmap_sg(cryp->dev, aes->src.sg, aes->src.nents, - DMA_TO_DEVICE); - goto sg_map_err; - } - } - - return mtk_aes_xmit(cryp, aes); - -sg_map_err: - dma_unmap_single(cryp->dev, ctx->ct_dma, sizeof(*info), DMA_TO_DEVICE); -exit: - return mtk_aes_complete(cryp, aes, -EINVAL); -} - -/* Initialize transform information of CBC/ECB/CTR/OFB/CFB mode */ -static void mtk_aes_info_init(struct mtk_cryp *cryp, struct mtk_aes_rec *aes, - size_t len) -{ - struct skcipher_request *req = skcipher_request_cast(aes->areq); - struct mtk_aes_base_ctx *ctx = aes->ctx; - struct mtk_aes_info *info = &ctx->info; - u32 cnt = 0; - - ctx->ct_hdr = AES_CT_CTRL_HDR | cpu_to_le32(len); - info->cmd[cnt++] = AES_CMD0 | cpu_to_le32(len); - info->cmd[cnt++] = AES_CMD1; - - info->tfm[0] = AES_TFM_SIZE(ctx->keylen) | ctx->keymode; - if (aes->flags & AES_FLAGS_ENCRYPT) - info->tfm[0] |= AES_TFM_BASIC_OUT; - else - info->tfm[0] |= AES_TFM_BASIC_IN; - - switch (aes->flags & AES_FLAGS_CIPHER_MSK) { - case AES_FLAGS_CBC: - info->tfm[1] = AES_TFM_CBC; - break; - case AES_FLAGS_ECB: - info->tfm[1] = AES_TFM_ECB; - goto ecb; - case AES_FLAGS_CTR: - info->tfm[1] = AES_TFM_CTR_LOAD; - goto ctr; - case AES_FLAGS_OFB: - info->tfm[1] = AES_TFM_OFB; - break; - case AES_FLAGS_CFB128: - info->tfm[1] = AES_TFM_CFB128; - break; - default: - /* Should not happen... */ - return; - } - - memcpy(info->state + ctx->keylen, req->iv, AES_BLOCK_SIZE); -ctr: - le32_add_cpu(&info->tfm[0], - le32_to_cpu(AES_TFM_SIZE(SIZE_IN_WORDS(AES_BLOCK_SIZE)))); - info->tfm[1] |= AES_TFM_FULL_IV; - info->cmd[cnt++] = AES_CMD2; -ecb: - ctx->ct_size = cnt; -} - -static int mtk_aes_dma(struct mtk_cryp *cryp, struct mtk_aes_rec *aes, - struct scatterlist *src, struct scatterlist *dst, - size_t len) -{ - size_t padlen = 0; - bool src_aligned, dst_aligned; - - aes->total = len; - aes->src.sg = src; - aes->dst.sg = dst; - aes->real_dst = dst; - - src_aligned = mtk_aes_check_aligned(src, len, &aes->src); - if (src == dst) - dst_aligned = src_aligned; - else - dst_aligned = mtk_aes_check_aligned(dst, len, &aes->dst); - - if (!src_aligned || !dst_aligned) { - padlen = mtk_aes_padlen(len); - - if (len + padlen > AES_BUF_SIZE) - return mtk_aes_complete(cryp, aes, -ENOMEM); - - if (!src_aligned) { - sg_copy_to_buffer(src, sg_nents(src), aes->buf, len); - aes->src.sg = &aes->aligned_sg; - aes->src.nents = 1; - aes->src.remainder = 0; - } - - if (!dst_aligned) { - aes->dst.sg = &aes->aligned_sg; - aes->dst.nents = 1; - aes->dst.remainder = 0; - } - - sg_init_table(&aes->aligned_sg, 1); - sg_set_buf(&aes->aligned_sg, aes->buf, len + padlen); - } - - mtk_aes_info_init(cryp, aes, len + padlen); - - return mtk_aes_map(cryp, aes); -} - -static int mtk_aes_handle_queue(struct mtk_cryp *cryp, u8 id, - struct crypto_async_request *new_areq) -{ - struct mtk_aes_rec *aes = cryp->aes[id]; - struct crypto_async_request *areq, *backlog; - struct mtk_aes_base_ctx *ctx; - unsigned long flags; - int ret = 0; - - spin_lock_irqsave(&aes->lock, flags); - if (new_areq) - ret = crypto_enqueue_request(&aes->queue, new_areq); - if (aes->flags & AES_FLAGS_BUSY) { - spin_unlock_irqrestore(&aes->lock, flags); - return ret; - } - backlog = crypto_get_backlog(&aes->queue); - areq = crypto_dequeue_request(&aes->queue); - if (areq) - aes->flags |= AES_FLAGS_BUSY; - spin_unlock_irqrestore(&aes->lock, flags); - - if (!areq) - return ret; - - if (backlog) - backlog->complete(backlog, -EINPROGRESS); - - ctx = crypto_tfm_ctx(areq->tfm); - /* Write key into state buffer */ - memcpy(ctx->info.state, ctx->key, sizeof(ctx->key)); - - aes->areq = areq; - aes->ctx = ctx; - - return ctx->start(cryp, aes); -} - -static int mtk_aes_transfer_complete(struct mtk_cryp *cryp, - struct mtk_aes_rec *aes) -{ - return mtk_aes_complete(cryp, aes, 0); -} - -static int mtk_aes_start(struct mtk_cryp *cryp, struct mtk_aes_rec *aes) -{ - struct skcipher_request *req = skcipher_request_cast(aes->areq); - struct mtk_aes_reqctx *rctx = skcipher_request_ctx(req); - - mtk_aes_set_mode(aes, rctx); - aes->resume = mtk_aes_transfer_complete; - - return mtk_aes_dma(cryp, aes, req->src, req->dst, req->cryptlen); -} - -static inline struct mtk_aes_ctr_ctx * -mtk_aes_ctr_ctx_cast(struct mtk_aes_base_ctx *ctx) -{ - return container_of(ctx, struct mtk_aes_ctr_ctx, base); -} - -static int mtk_aes_ctr_transfer(struct mtk_cryp *cryp, struct mtk_aes_rec *aes) -{ - struct mtk_aes_base_ctx *ctx = aes->ctx; - struct mtk_aes_ctr_ctx *cctx = mtk_aes_ctr_ctx_cast(ctx); - struct skcipher_request *req = skcipher_request_cast(aes->areq); - struct scatterlist *src, *dst; - u32 start, end, ctr, blocks; - size_t datalen; - bool fragmented = false; - - /* Check for transfer completion. */ - cctx->offset += aes->total; - if (cctx->offset >= req->cryptlen) - return mtk_aes_transfer_complete(cryp, aes); - - /* Compute data length. */ - datalen = req->cryptlen - cctx->offset; - blocks = DIV_ROUND_UP(datalen, AES_BLOCK_SIZE); - ctr = be32_to_cpu(cctx->iv[3]); - - /* Check 32bit counter overflow. */ - start = ctr; - end = start + blocks - 1; - if (end < start) { - ctr = 0xffffffff; - datalen = AES_BLOCK_SIZE * -start; - fragmented = true; - } - - /* Jump to offset. */ - src = scatterwalk_ffwd(cctx->src, req->src, cctx->offset); - dst = ((req->src == req->dst) ? src : - scatterwalk_ffwd(cctx->dst, req->dst, cctx->offset)); - - /* Write IVs into transform state buffer. */ - memcpy(ctx->info.state + ctx->keylen, cctx->iv, AES_BLOCK_SIZE); - - if (unlikely(fragmented)) { - /* - * Increment the counter manually to cope with the hardware - * counter overflow. - */ - cctx->iv[3] = cpu_to_be32(ctr); - crypto_inc((u8 *)cctx->iv, AES_BLOCK_SIZE); - } - - return mtk_aes_dma(cryp, aes, src, dst, datalen); -} - -static int mtk_aes_ctr_start(struct mtk_cryp *cryp, struct mtk_aes_rec *aes) -{ - struct mtk_aes_ctr_ctx *cctx = mtk_aes_ctr_ctx_cast(aes->ctx); - struct skcipher_request *req = skcipher_request_cast(aes->areq); - struct mtk_aes_reqctx *rctx = skcipher_request_ctx(req); - - mtk_aes_set_mode(aes, rctx); - - memcpy(cctx->iv, req->iv, AES_BLOCK_SIZE); - cctx->offset = 0; - aes->total = 0; - aes->resume = mtk_aes_ctr_transfer; - - return mtk_aes_ctr_transfer(cryp, aes); -} - -/* Check and set the AES key to transform state buffer */ -static int mtk_aes_setkey(struct crypto_skcipher *tfm, - const u8 *key, u32 keylen) -{ - struct mtk_aes_base_ctx *ctx = crypto_skcipher_ctx(tfm); - - switch (keylen) { - case AES_KEYSIZE_128: - ctx->keymode = AES_TFM_128BITS; - break; - case AES_KEYSIZE_192: - ctx->keymode = AES_TFM_192BITS; - break; - case AES_KEYSIZE_256: - ctx->keymode = AES_TFM_256BITS; - break; - - default: - return -EINVAL; - } - - ctx->keylen = SIZE_IN_WORDS(keylen); - memcpy(ctx->key, key, keylen); - - return 0; -} - -static int mtk_aes_crypt(struct skcipher_request *req, u64 mode) -{ - struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); - struct mtk_aes_base_ctx *ctx = crypto_skcipher_ctx(skcipher); - struct mtk_aes_reqctx *rctx; - struct mtk_cryp *cryp; - - cryp = mtk_aes_find_dev(ctx); - if (!cryp) - return -ENODEV; - - rctx = skcipher_request_ctx(req); - rctx->mode = mode; - - return mtk_aes_handle_queue(cryp, !(mode & AES_FLAGS_ENCRYPT), - &req->base); -} - -static int mtk_aes_ecb_encrypt(struct skcipher_request *req) -{ - return mtk_aes_crypt(req, AES_FLAGS_ENCRYPT | AES_FLAGS_ECB); -} - -static int mtk_aes_ecb_decrypt(struct skcipher_request *req) -{ - return mtk_aes_crypt(req, AES_FLAGS_ECB); -} - -static int mtk_aes_cbc_encrypt(struct skcipher_request *req) -{ - return mtk_aes_crypt(req, AES_FLAGS_ENCRYPT | AES_FLAGS_CBC); -} - -static int mtk_aes_cbc_decrypt(struct skcipher_request *req) -{ - return mtk_aes_crypt(req, AES_FLAGS_CBC); -} - -static int mtk_aes_ctr_encrypt(struct skcipher_request *req) -{ - return mtk_aes_crypt(req, AES_FLAGS_ENCRYPT | AES_FLAGS_CTR); -} - -static int mtk_aes_ctr_decrypt(struct skcipher_request *req) -{ - return mtk_aes_crypt(req, AES_FLAGS_CTR); -} - -static int mtk_aes_ofb_encrypt(struct skcipher_request *req) -{ - return mtk_aes_crypt(req, AES_FLAGS_ENCRYPT | AES_FLAGS_OFB); -} - -static int mtk_aes_ofb_decrypt(struct skcipher_request *req) -{ - return mtk_aes_crypt(req, AES_FLAGS_OFB); -} - -static int mtk_aes_cfb_encrypt(struct skcipher_request *req) -{ - return mtk_aes_crypt(req, AES_FLAGS_ENCRYPT | AES_FLAGS_CFB128); -} - -static int mtk_aes_cfb_decrypt(struct skcipher_request *req) -{ - return mtk_aes_crypt(req, AES_FLAGS_CFB128); -} - -static int mtk_aes_init_tfm(struct crypto_skcipher *tfm) -{ - struct mtk_aes_ctx *ctx = crypto_skcipher_ctx(tfm); - - crypto_skcipher_set_reqsize(tfm, sizeof(struct mtk_aes_reqctx)); - ctx->base.start = mtk_aes_start; - return 0; -} - -static int mtk_aes_ctr_init_tfm(struct crypto_skcipher *tfm) -{ - struct mtk_aes_ctx *ctx = crypto_skcipher_ctx(tfm); - - crypto_skcipher_set_reqsize(tfm, sizeof(struct mtk_aes_reqctx)); - ctx->base.start = mtk_aes_ctr_start; - return 0; -} - -static struct skcipher_alg aes_algs[] = { -{ - .base.cra_name = "cbc(aes)", - .base.cra_driver_name = "cbc-aes-mtk", - .base.cra_priority = 400, - .base.cra_flags = CRYPTO_ALG_ASYNC, - .base.cra_blocksize = AES_BLOCK_SIZE, - .base.cra_ctxsize = sizeof(struct mtk_aes_ctx), - .base.cra_alignmask = 0xf, - .base.cra_module = THIS_MODULE, - - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .setkey = mtk_aes_setkey, - .encrypt = mtk_aes_cbc_encrypt, - .decrypt = mtk_aes_cbc_decrypt, - .ivsize = AES_BLOCK_SIZE, - .init = mtk_aes_init_tfm, -}, -{ - .base.cra_name = "ecb(aes)", - .base.cra_driver_name = "ecb-aes-mtk", - .base.cra_priority = 400, - .base.cra_flags = CRYPTO_ALG_ASYNC, - .base.cra_blocksize = AES_BLOCK_SIZE, - .base.cra_ctxsize = sizeof(struct mtk_aes_ctx), - .base.cra_alignmask = 0xf, - .base.cra_module = THIS_MODULE, - - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .setkey = mtk_aes_setkey, - .encrypt = mtk_aes_ecb_encrypt, - .decrypt = mtk_aes_ecb_decrypt, - .init = mtk_aes_init_tfm, -}, -{ - .base.cra_name = "ctr(aes)", - .base.cra_driver_name = "ctr-aes-mtk", - .base.cra_priority = 400, - .base.cra_flags = CRYPTO_ALG_ASYNC, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct mtk_aes_ctx), - .base.cra_alignmask = 0xf, - .base.cra_module = THIS_MODULE, - - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .ivsize = AES_BLOCK_SIZE, - .setkey = mtk_aes_setkey, - .encrypt = mtk_aes_ctr_encrypt, - .decrypt = mtk_aes_ctr_decrypt, - .init = mtk_aes_ctr_init_tfm, -}, -{ - .base.cra_name = "ofb(aes)", - .base.cra_driver_name = "ofb-aes-mtk", - .base.cra_priority = 400, - .base.cra_flags = CRYPTO_ALG_ASYNC, - .base.cra_blocksize = AES_BLOCK_SIZE, - .base.cra_ctxsize = sizeof(struct mtk_aes_ctx), - .base.cra_alignmask = 0xf, - .base.cra_module = THIS_MODULE, - - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .ivsize = AES_BLOCK_SIZE, - .setkey = mtk_aes_setkey, - .encrypt = mtk_aes_ofb_encrypt, - .decrypt = mtk_aes_ofb_decrypt, -}, -{ - .base.cra_name = "cfb(aes)", - .base.cra_driver_name = "cfb-aes-mtk", - .base.cra_priority = 400, - .base.cra_flags = CRYPTO_ALG_ASYNC, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct mtk_aes_ctx), - .base.cra_alignmask = 0xf, - .base.cra_module = THIS_MODULE, - - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .ivsize = AES_BLOCK_SIZE, - .setkey = mtk_aes_setkey, - .encrypt = mtk_aes_cfb_encrypt, - .decrypt = mtk_aes_cfb_decrypt, -}, -}; - -static inline struct mtk_aes_gcm_ctx * -mtk_aes_gcm_ctx_cast(struct mtk_aes_base_ctx *ctx) -{ - return container_of(ctx, struct mtk_aes_gcm_ctx, base); -} - -/* - * Engine will verify and compare tag automatically, so we just need - * to check returned status which stored in the result descriptor. - */ -static int mtk_aes_gcm_tag_verify(struct mtk_cryp *cryp, - struct mtk_aes_rec *aes) -{ - __le32 status = cryp->ring[aes->id]->res_prev->ct; - - return mtk_aes_complete(cryp, aes, (status & AES_AUTH_TAG_ERR) ? - -EBADMSG : 0); -} - -/* Initialize transform information of GCM mode */ -static void mtk_aes_gcm_info_init(struct mtk_cryp *cryp, - struct mtk_aes_rec *aes, - size_t len) -{ - struct aead_request *req = aead_request_cast(aes->areq); - struct mtk_aes_base_ctx *ctx = aes->ctx; - struct mtk_aes_gcm_ctx *gctx = mtk_aes_gcm_ctx_cast(ctx); - struct mtk_aes_info *info = &ctx->info; - u32 ivsize = crypto_aead_ivsize(crypto_aead_reqtfm(req)); - u32 cnt = 0; - - ctx->ct_hdr = AES_CT_CTRL_HDR | cpu_to_le32(len); - - info->cmd[cnt++] = AES_GCM_CMD0 | cpu_to_le32(req->assoclen); - info->cmd[cnt++] = AES_GCM_CMD1 | cpu_to_le32(req->assoclen); - info->cmd[cnt++] = AES_GCM_CMD2; - info->cmd[cnt++] = AES_GCM_CMD3 | cpu_to_le32(gctx->textlen); - - if (aes->flags & AES_FLAGS_ENCRYPT) { - info->cmd[cnt++] = AES_GCM_CMD4 | cpu_to_le32(gctx->authsize); - info->tfm[0] = AES_TFM_GCM_OUT; - } else { - info->cmd[cnt++] = AES_GCM_CMD5 | cpu_to_le32(gctx->authsize); - info->cmd[cnt++] = AES_GCM_CMD6 | cpu_to_le32(gctx->authsize); - info->tfm[0] = AES_TFM_GCM_IN; - } - ctx->ct_size = cnt; - - info->tfm[0] |= AES_TFM_GHASH_DIGEST | AES_TFM_GHASH | AES_TFM_SIZE( - ctx->keylen + SIZE_IN_WORDS(AES_BLOCK_SIZE + ivsize)) | - ctx->keymode; - info->tfm[1] = AES_TFM_CTR_INIT | AES_TFM_IV_CTR_MODE | AES_TFM_3IV | - AES_TFM_ENC_HASH; - - memcpy(info->state + ctx->keylen + SIZE_IN_WORDS(AES_BLOCK_SIZE), - req->iv, ivsize); -} - -static int mtk_aes_gcm_dma(struct mtk_cryp *cryp, struct mtk_aes_rec *aes, - struct scatterlist *src, struct scatterlist *dst, - size_t len) -{ - bool src_aligned, dst_aligned; - - aes->src.sg = src; - aes->dst.sg = dst; - aes->real_dst = dst; - - src_aligned = mtk_aes_check_aligned(src, len, &aes->src); - if (src == dst) - dst_aligned = src_aligned; - else - dst_aligned = mtk_aes_check_aligned(dst, len, &aes->dst); - - if (!src_aligned || !dst_aligned) { - if (aes->total > AES_BUF_SIZE) - return mtk_aes_complete(cryp, aes, -ENOMEM); - - if (!src_aligned) { - sg_copy_to_buffer(src, sg_nents(src), aes->buf, len); - aes->src.sg = &aes->aligned_sg; - aes->src.nents = 1; - aes->src.remainder = 0; - } - - if (!dst_aligned) { - aes->dst.sg = &aes->aligned_sg; - aes->dst.nents = 1; - aes->dst.remainder = 0; - } - - sg_init_table(&aes->aligned_sg, 1); - sg_set_buf(&aes->aligned_sg, aes->buf, aes->total); - } - - mtk_aes_gcm_info_init(cryp, aes, len); - - return mtk_aes_map(cryp, aes); -} - -/* Todo: GMAC */ -static int mtk_aes_gcm_start(struct mtk_cryp *cryp, struct mtk_aes_rec *aes) -{ - struct mtk_aes_gcm_ctx *gctx = mtk_aes_gcm_ctx_cast(aes->ctx); - struct aead_request *req = aead_request_cast(aes->areq); - struct mtk_aes_reqctx *rctx = aead_request_ctx(req); - u32 len = req->assoclen + req->cryptlen; - - mtk_aes_set_mode(aes, rctx); - - if (aes->flags & AES_FLAGS_ENCRYPT) { - u32 tag[4]; - - aes->resume = mtk_aes_transfer_complete; - /* Compute total process length. */ - aes->total = len + gctx->authsize; - /* Hardware will append authenticated tag to output buffer */ - scatterwalk_map_and_copy(tag, req->dst, len, gctx->authsize, 1); - } else { - aes->resume = mtk_aes_gcm_tag_verify; - aes->total = len; - } - - return mtk_aes_gcm_dma(cryp, aes, req->src, req->dst, len); -} - -static int mtk_aes_gcm_crypt(struct aead_request *req, u64 mode) -{ - struct mtk_aes_base_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); - struct mtk_aes_gcm_ctx *gctx = mtk_aes_gcm_ctx_cast(ctx); - struct mtk_aes_reqctx *rctx = aead_request_ctx(req); - struct mtk_cryp *cryp; - bool enc = !!(mode & AES_FLAGS_ENCRYPT); - - cryp = mtk_aes_find_dev(ctx); - if (!cryp) - return -ENODEV; - - /* Compute text length. */ - gctx->textlen = req->cryptlen - (enc ? 0 : gctx->authsize); - - /* Empty messages are not supported yet */ - if (!gctx->textlen && !req->assoclen) - return -EINVAL; - - rctx->mode = AES_FLAGS_GCM | mode; - - return mtk_aes_handle_queue(cryp, enc, &req->base); -} - -/* - * Because of the hardware limitation, we need to pre-calculate key(H) - * for the GHASH operation. The result of the encryption operation - * need to be stored in the transform state buffer. - */ -static int mtk_aes_gcm_setkey(struct crypto_aead *aead, const u8 *key, - u32 keylen) -{ - struct mtk_aes_base_ctx *ctx = crypto_aead_ctx(aead); - union { - u32 x32[SIZE_IN_WORDS(AES_BLOCK_SIZE)]; - u8 x8[AES_BLOCK_SIZE]; - } hash = {}; - struct crypto_aes_ctx aes_ctx; - int err; - int i; - - switch (keylen) { - case AES_KEYSIZE_128: - ctx->keymode = AES_TFM_128BITS; - break; - case AES_KEYSIZE_192: - ctx->keymode = AES_TFM_192BITS; - break; - case AES_KEYSIZE_256: - ctx->keymode = AES_TFM_256BITS; - break; - - default: - return -EINVAL; - } - - ctx->keylen = SIZE_IN_WORDS(keylen); - - err = aes_expandkey(&aes_ctx, key, keylen); - if (err) - return err; - - aes_encrypt(&aes_ctx, hash.x8, hash.x8); - memzero_explicit(&aes_ctx, sizeof(aes_ctx)); - - memcpy(ctx->key, key, keylen); - - /* Why do we need to do this? */ - for (i = 0; i < SIZE_IN_WORDS(AES_BLOCK_SIZE); i++) - hash.x32[i] = swab32(hash.x32[i]); - - memcpy(ctx->key + ctx->keylen, &hash, AES_BLOCK_SIZE); - - return 0; -} - -static int mtk_aes_gcm_setauthsize(struct crypto_aead *aead, - u32 authsize) -{ - struct mtk_aes_base_ctx *ctx = crypto_aead_ctx(aead); - struct mtk_aes_gcm_ctx *gctx = mtk_aes_gcm_ctx_cast(ctx); - - /* Same as crypto_gcm_authsize() from crypto/gcm.c */ - switch (authsize) { - case 8: - case 12: - case 16: - break; - default: - return -EINVAL; - } - - gctx->authsize = authsize; - return 0; -} - -static int mtk_aes_gcm_encrypt(struct aead_request *req) -{ - return mtk_aes_gcm_crypt(req, AES_FLAGS_ENCRYPT); -} - -static int mtk_aes_gcm_decrypt(struct aead_request *req) -{ - return mtk_aes_gcm_crypt(req, 0); -} - -static int mtk_aes_gcm_init(struct crypto_aead *aead) -{ - struct mtk_aes_gcm_ctx *ctx = crypto_aead_ctx(aead); - - crypto_aead_set_reqsize(aead, sizeof(struct mtk_aes_reqctx)); - ctx->base.start = mtk_aes_gcm_start; - return 0; -} - -static struct aead_alg aes_gcm_alg = { - .setkey = mtk_aes_gcm_setkey, - .setauthsize = mtk_aes_gcm_setauthsize, - .encrypt = mtk_aes_gcm_encrypt, - .decrypt = mtk_aes_gcm_decrypt, - .init = mtk_aes_gcm_init, - .ivsize = GCM_AES_IV_SIZE, - .maxauthsize = AES_BLOCK_SIZE, - - .base = { - .cra_name = "gcm(aes)", - .cra_driver_name = "gcm-aes-mtk", - .cra_priority = 400, - .cra_flags = CRYPTO_ALG_ASYNC, - .cra_blocksize = 1, - .cra_ctxsize = sizeof(struct mtk_aes_gcm_ctx), - .cra_alignmask = 0xf, - .cra_module = THIS_MODULE, - }, -}; - -static void mtk_aes_queue_task(unsigned long data) -{ - struct mtk_aes_rec *aes = (struct mtk_aes_rec *)data; - - mtk_aes_handle_queue(aes->cryp, aes->id, NULL); -} - -static void mtk_aes_done_task(unsigned long data) -{ - struct mtk_aes_rec *aes = (struct mtk_aes_rec *)data; - struct mtk_cryp *cryp = aes->cryp; - - mtk_aes_unmap(cryp, aes); - aes->resume(cryp, aes); -} - -static irqreturn_t mtk_aes_irq(int irq, void *dev_id) -{ - struct mtk_aes_rec *aes = (struct mtk_aes_rec *)dev_id; - struct mtk_cryp *cryp = aes->cryp; - u32 val = mtk_aes_read(cryp, RDR_STAT(aes->id)); - - mtk_aes_write(cryp, RDR_STAT(aes->id), val); - - if (likely(AES_FLAGS_BUSY & aes->flags)) { - mtk_aes_write(cryp, RDR_PROC_COUNT(aes->id), MTK_CNT_RST); - mtk_aes_write(cryp, RDR_THRESH(aes->id), - MTK_RDR_PROC_THRESH | MTK_RDR_PROC_MODE); - - tasklet_schedule(&aes->done_task); - } else { - dev_warn(cryp->dev, "AES interrupt when no active requests.\n"); - } - return IRQ_HANDLED; -} - -/* - * The purpose of creating encryption and decryption records is - * to process outbound/inbound data in parallel, it can improve - * performance in most use cases, such as IPSec VPN, especially - * under heavy network traffic. - */ -static int mtk_aes_record_init(struct mtk_cryp *cryp) -{ - struct mtk_aes_rec **aes = cryp->aes; - int i, err = -ENOMEM; - - for (i = 0; i < MTK_REC_NUM; i++) { - aes[i] = kzalloc(sizeof(**aes), GFP_KERNEL); - if (!aes[i]) - goto err_cleanup; - - aes[i]->buf = (void *)__get_free_pages(GFP_KERNEL, - AES_BUF_ORDER); - if (!aes[i]->buf) - goto err_cleanup; - - aes[i]->cryp = cryp; - - spin_lock_init(&aes[i]->lock); - crypto_init_queue(&aes[i]->queue, AES_QUEUE_SIZE); - - tasklet_init(&aes[i]->queue_task, mtk_aes_queue_task, - (unsigned long)aes[i]); - tasklet_init(&aes[i]->done_task, mtk_aes_done_task, - (unsigned long)aes[i]); - } - - /* Link to ring0 and ring1 respectively */ - aes[0]->id = MTK_RING0; - aes[1]->id = MTK_RING1; - - return 0; - -err_cleanup: - for (; i--; ) { - free_page((unsigned long)aes[i]->buf); - kfree(aes[i]); - } - - return err; -} - -static void mtk_aes_record_free(struct mtk_cryp *cryp) -{ - int i; - - for (i = 0; i < MTK_REC_NUM; i++) { - tasklet_kill(&cryp->aes[i]->done_task); - tasklet_kill(&cryp->aes[i]->queue_task); - - free_page((unsigned long)cryp->aes[i]->buf); - kfree(cryp->aes[i]); - } -} - -static void mtk_aes_unregister_algs(void) -{ - int i; - - crypto_unregister_aead(&aes_gcm_alg); - - for (i = 0; i < ARRAY_SIZE(aes_algs); i++) - crypto_unregister_skcipher(&aes_algs[i]); -} - -static int mtk_aes_register_algs(void) -{ - int err, i; - - for (i = 0; i < ARRAY_SIZE(aes_algs); i++) { - err = crypto_register_skcipher(&aes_algs[i]); - if (err) - goto err_aes_algs; - } - - err = crypto_register_aead(&aes_gcm_alg); - if (err) - goto err_aes_algs; - - return 0; - -err_aes_algs: - for (; i--; ) - crypto_unregister_skcipher(&aes_algs[i]); - - return err; -} - -int mtk_cipher_alg_register(struct mtk_cryp *cryp) -{ - int ret; - - INIT_LIST_HEAD(&cryp->aes_list); - - /* Initialize two cipher records */ - ret = mtk_aes_record_init(cryp); - if (ret) - goto err_record; - - ret = devm_request_irq(cryp->dev, cryp->irq[MTK_RING0], mtk_aes_irq, - 0, "mtk-aes", cryp->aes[0]); - if (ret) { - dev_err(cryp->dev, "unable to request AES irq.\n"); - goto err_res; - } - - ret = devm_request_irq(cryp->dev, cryp->irq[MTK_RING1], mtk_aes_irq, - 0, "mtk-aes", cryp->aes[1]); - if (ret) { - dev_err(cryp->dev, "unable to request AES irq.\n"); - goto err_res; - } - - /* Enable ring0 and ring1 interrupt */ - mtk_aes_write(cryp, AIC_ENABLE_SET(MTK_RING0), MTK_IRQ_RDR0); - mtk_aes_write(cryp, AIC_ENABLE_SET(MTK_RING1), MTK_IRQ_RDR1); - - spin_lock(&mtk_aes.lock); - list_add_tail(&cryp->aes_list, &mtk_aes.dev_list); - spin_unlock(&mtk_aes.lock); - - ret = mtk_aes_register_algs(); - if (ret) - goto err_algs; - - return 0; - -err_algs: - spin_lock(&mtk_aes.lock); - list_del(&cryp->aes_list); - spin_unlock(&mtk_aes.lock); -err_res: - mtk_aes_record_free(cryp); -err_record: - - dev_err(cryp->dev, "mtk-aes initialization failed.\n"); - return ret; -} - -void mtk_cipher_alg_release(struct mtk_cryp *cryp) -{ - spin_lock(&mtk_aes.lock); - list_del(&cryp->aes_list); - spin_unlock(&mtk_aes.lock); - - mtk_aes_unregister_algs(); - mtk_aes_record_free(cryp); -} diff --git a/drivers/crypto/mediatek/mtk-platform.c b/drivers/crypto/mediatek/mtk-platform.c deleted file mode 100644 index 9d878620e5c9ba790133b79cd3ae3979f076a801..0000000000000000000000000000000000000000 --- a/drivers/crypto/mediatek/mtk-platform.c +++ /dev/null @@ -1,586 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Driver for EIP97 cryptographic accelerator. - * - * Copyright (c) 2016 Ryder Lee - */ - -#include -#include -#include -#include -#include -#include -#include -#include "mtk-platform.h" - -#define MTK_BURST_SIZE_MSK GENMASK(7, 4) -#define MTK_BURST_SIZE(x) ((x) << 4) -#define MTK_DESC_SIZE(x) ((x) << 0) -#define MTK_DESC_OFFSET(x) ((x) << 16) -#define MTK_DESC_FETCH_SIZE(x) ((x) << 0) -#define MTK_DESC_FETCH_THRESH(x) ((x) << 16) -#define MTK_DESC_OVL_IRQ_EN BIT(25) -#define MTK_DESC_ATP_PRESENT BIT(30) - -#define MTK_DFSE_IDLE GENMASK(3, 0) -#define MTK_DFSE_THR_CTRL_EN BIT(30) -#define MTK_DFSE_THR_CTRL_RESET BIT(31) -#define MTK_DFSE_RING_ID(x) (((x) >> 12) & GENMASK(3, 0)) -#define MTK_DFSE_MIN_DATA(x) ((x) << 0) -#define MTK_DFSE_MAX_DATA(x) ((x) << 8) -#define MTK_DFE_MIN_CTRL(x) ((x) << 16) -#define MTK_DFE_MAX_CTRL(x) ((x) << 24) - -#define MTK_IN_BUF_MIN_THRESH(x) ((x) << 8) -#define MTK_IN_BUF_MAX_THRESH(x) ((x) << 12) -#define MTK_OUT_BUF_MIN_THRESH(x) ((x) << 0) -#define MTK_OUT_BUF_MAX_THRESH(x) ((x) << 4) -#define MTK_IN_TBUF_SIZE(x) (((x) >> 4) & GENMASK(3, 0)) -#define MTK_IN_DBUF_SIZE(x) (((x) >> 8) & GENMASK(3, 0)) -#define MTK_OUT_DBUF_SIZE(x) (((x) >> 16) & GENMASK(3, 0)) -#define MTK_CMD_FIFO_SIZE(x) (((x) >> 8) & GENMASK(3, 0)) -#define MTK_RES_FIFO_SIZE(x) (((x) >> 12) & GENMASK(3, 0)) - -#define MTK_PE_TK_LOC_AVL BIT(2) -#define MTK_PE_PROC_HELD BIT(14) -#define MTK_PE_TK_TIMEOUT_EN BIT(22) -#define MTK_PE_INPUT_DMA_ERR BIT(0) -#define MTK_PE_OUTPUT_DMA_ERR BIT(1) -#define MTK_PE_PKT_PORC_ERR BIT(2) -#define MTK_PE_PKT_TIMEOUT BIT(3) -#define MTK_PE_FATAL_ERR BIT(14) -#define MTK_PE_INPUT_DMA_ERR_EN BIT(16) -#define MTK_PE_OUTPUT_DMA_ERR_EN BIT(17) -#define MTK_PE_PKT_PORC_ERR_EN BIT(18) -#define MTK_PE_PKT_TIMEOUT_EN BIT(19) -#define MTK_PE_FATAL_ERR_EN BIT(30) -#define MTK_PE_INT_OUT_EN BIT(31) - -#define MTK_HIA_SIGNATURE ((u16)0x35ca) -#define MTK_HIA_DATA_WIDTH(x) (((x) >> 25) & GENMASK(1, 0)) -#define MTK_HIA_DMA_LENGTH(x) (((x) >> 20) & GENMASK(4, 0)) -#define MTK_CDR_STAT_CLR GENMASK(4, 0) -#define MTK_RDR_STAT_CLR GENMASK(7, 0) - -#define MTK_AIC_INT_MSK GENMASK(5, 0) -#define MTK_AIC_VER_MSK (GENMASK(15, 0) | GENMASK(27, 20)) -#define MTK_AIC_VER11 0x011036c9 -#define MTK_AIC_VER12 0x012036c9 -#define MTK_AIC_G_CLR GENMASK(30, 20) - -/** - * EIP97 is an integrated security subsystem to accelerate cryptographic - * functions and protocols to offload the host processor. - * Some important hardware modules are briefly introduced below: - * - * Host Interface Adapter(HIA) - the main interface between the host - * system and the hardware subsystem. It is responsible for attaching - * processing engine to the specific host bus interface and provides a - * standardized software view for off loading tasks to the engine. - * - * Command Descriptor Ring Manager(CDR Manager) - keeps track of how many - * CD the host has prepared in the CDR. It monitors the fill level of its - * CD-FIFO and if there's sufficient space for the next block of descriptors, - * then it fires off a DMA request to fetch a block of CDs. - * - * Data fetch engine(DFE) - It is responsible for parsing the CD and - * setting up the required control and packet data DMA transfers from - * system memory to the processing engine. - * - * Result Descriptor Ring Manager(RDR Manager) - same as CDR Manager, - * but target is result descriptors, Moreover, it also handles the RD - * updates under control of the DSE. For each packet data segment - * processed, the DSE triggers the RDR Manager to write the updated RD. - * If triggered to update, the RDR Manager sets up a DMA operation to - * copy the RD from the DSE to the correct location in the RDR. - * - * Data Store Engine(DSE) - It is responsible for parsing the prepared RD - * and setting up the required control and packet data DMA transfers from - * the processing engine to system memory. - * - * Advanced Interrupt Controllers(AICs) - receive interrupt request signals - * from various sources and combine them into one interrupt output. - * The AICs are used by: - * - One for the HIA global and processing engine interrupts. - * - The others for the descriptor ring interrupts. - */ - -/* Cryptographic engine capabilities */ -struct mtk_sys_cap { - /* host interface adapter */ - u32 hia_ver; - u32 hia_opt; - /* packet engine */ - u32 pkt_eng_opt; - /* global hardware */ - u32 hw_opt; -}; - -static void mtk_desc_ring_link(struct mtk_cryp *cryp, u32 mask) -{ - /* Assign rings to DFE/DSE thread and enable it */ - writel(MTK_DFSE_THR_CTRL_EN | mask, cryp->base + DFE_THR_CTRL); - writel(MTK_DFSE_THR_CTRL_EN | mask, cryp->base + DSE_THR_CTRL); -} - -static void mtk_dfe_dse_buf_setup(struct mtk_cryp *cryp, - struct mtk_sys_cap *cap) -{ - u32 width = MTK_HIA_DATA_WIDTH(cap->hia_opt) + 2; - u32 len = MTK_HIA_DMA_LENGTH(cap->hia_opt) - 1; - u32 ipbuf = min((u32)MTK_IN_DBUF_SIZE(cap->hw_opt) + width, len); - u32 opbuf = min((u32)MTK_OUT_DBUF_SIZE(cap->hw_opt) + width, len); - u32 itbuf = min((u32)MTK_IN_TBUF_SIZE(cap->hw_opt) + width, len); - - writel(MTK_DFSE_MIN_DATA(ipbuf - 1) | - MTK_DFSE_MAX_DATA(ipbuf) | - MTK_DFE_MIN_CTRL(itbuf - 1) | - MTK_DFE_MAX_CTRL(itbuf), - cryp->base + DFE_CFG); - - writel(MTK_DFSE_MIN_DATA(opbuf - 1) | - MTK_DFSE_MAX_DATA(opbuf), - cryp->base + DSE_CFG); - - writel(MTK_IN_BUF_MIN_THRESH(ipbuf - 1) | - MTK_IN_BUF_MAX_THRESH(ipbuf), - cryp->base + PE_IN_DBUF_THRESH); - - writel(MTK_IN_BUF_MIN_THRESH(itbuf - 1) | - MTK_IN_BUF_MAX_THRESH(itbuf), - cryp->base + PE_IN_TBUF_THRESH); - - writel(MTK_OUT_BUF_MIN_THRESH(opbuf - 1) | - MTK_OUT_BUF_MAX_THRESH(opbuf), - cryp->base + PE_OUT_DBUF_THRESH); - - writel(0, cryp->base + PE_OUT_TBUF_THRESH); - writel(0, cryp->base + PE_OUT_BUF_CTRL); -} - -static int mtk_dfe_dse_state_check(struct mtk_cryp *cryp) -{ - int ret = -EINVAL; - u32 val; - - /* Check for completion of all DMA transfers */ - val = readl(cryp->base + DFE_THR_STAT); - if (MTK_DFSE_RING_ID(val) == MTK_DFSE_IDLE) { - val = readl(cryp->base + DSE_THR_STAT); - if (MTK_DFSE_RING_ID(val) == MTK_DFSE_IDLE) - ret = 0; - } - - if (!ret) { - /* Take DFE/DSE thread out of reset */ - writel(0, cryp->base + DFE_THR_CTRL); - writel(0, cryp->base + DSE_THR_CTRL); - } else { - return -EBUSY; - } - - return 0; -} - -static int mtk_dfe_dse_reset(struct mtk_cryp *cryp) -{ - /* Reset DSE/DFE and correct system priorities for all rings. */ - writel(MTK_DFSE_THR_CTRL_RESET, cryp->base + DFE_THR_CTRL); - writel(0, cryp->base + DFE_PRIO_0); - writel(0, cryp->base + DFE_PRIO_1); - writel(0, cryp->base + DFE_PRIO_2); - writel(0, cryp->base + DFE_PRIO_3); - - writel(MTK_DFSE_THR_CTRL_RESET, cryp->base + DSE_THR_CTRL); - writel(0, cryp->base + DSE_PRIO_0); - writel(0, cryp->base + DSE_PRIO_1); - writel(0, cryp->base + DSE_PRIO_2); - writel(0, cryp->base + DSE_PRIO_3); - - return mtk_dfe_dse_state_check(cryp); -} - -static void mtk_cmd_desc_ring_setup(struct mtk_cryp *cryp, - int i, struct mtk_sys_cap *cap) -{ - /* Full descriptor that fits FIFO minus one */ - u32 count = - ((1 << MTK_CMD_FIFO_SIZE(cap->hia_opt)) / MTK_DESC_SZ) - 1; - - /* Temporarily disable external triggering */ - writel(0, cryp->base + CDR_CFG(i)); - - /* Clear CDR count */ - writel(MTK_CNT_RST, cryp->base + CDR_PREP_COUNT(i)); - writel(MTK_CNT_RST, cryp->base + CDR_PROC_COUNT(i)); - - writel(0, cryp->base + CDR_PREP_PNTR(i)); - writel(0, cryp->base + CDR_PROC_PNTR(i)); - writel(0, cryp->base + CDR_DMA_CFG(i)); - - /* Configure CDR host address space */ - writel(0, cryp->base + CDR_BASE_ADDR_HI(i)); - writel(cryp->ring[i]->cmd_dma, cryp->base + CDR_BASE_ADDR_LO(i)); - - writel(MTK_DESC_RING_SZ, cryp->base + CDR_RING_SIZE(i)); - - /* Clear and disable all CDR interrupts */ - writel(MTK_CDR_STAT_CLR, cryp->base + CDR_STAT(i)); - - /* - * Set command descriptor offset and enable additional - * token present in descriptor. - */ - writel(MTK_DESC_SIZE(MTK_DESC_SZ) | - MTK_DESC_OFFSET(MTK_DESC_OFF) | - MTK_DESC_ATP_PRESENT, - cryp->base + CDR_DESC_SIZE(i)); - - writel(MTK_DESC_FETCH_SIZE(count * MTK_DESC_OFF) | - MTK_DESC_FETCH_THRESH(count * MTK_DESC_SZ), - cryp->base + CDR_CFG(i)); -} - -static void mtk_res_desc_ring_setup(struct mtk_cryp *cryp, - int i, struct mtk_sys_cap *cap) -{ - u32 rndup = 2; - u32 count = ((1 << MTK_RES_FIFO_SIZE(cap->hia_opt)) / rndup) - 1; - - /* Temporarily disable external triggering */ - writel(0, cryp->base + RDR_CFG(i)); - - /* Clear RDR count */ - writel(MTK_CNT_RST, cryp->base + RDR_PREP_COUNT(i)); - writel(MTK_CNT_RST, cryp->base + RDR_PROC_COUNT(i)); - - writel(0, cryp->base + RDR_PREP_PNTR(i)); - writel(0, cryp->base + RDR_PROC_PNTR(i)); - writel(0, cryp->base + RDR_DMA_CFG(i)); - - /* Configure RDR host address space */ - writel(0, cryp->base + RDR_BASE_ADDR_HI(i)); - writel(cryp->ring[i]->res_dma, cryp->base + RDR_BASE_ADDR_LO(i)); - - writel(MTK_DESC_RING_SZ, cryp->base + RDR_RING_SIZE(i)); - writel(MTK_RDR_STAT_CLR, cryp->base + RDR_STAT(i)); - - /* - * RDR manager generates update interrupts on a per-completed-packet, - * and the rd_proc_thresh_irq interrupt is fired when proc_pkt_count - * for the RDR exceeds the number of packets. - */ - writel(MTK_RDR_PROC_THRESH | MTK_RDR_PROC_MODE, - cryp->base + RDR_THRESH(i)); - - /* - * Configure a threshold and time-out value for the processed - * result descriptors (or complete packets) that are written to - * the RDR. - */ - writel(MTK_DESC_SIZE(MTK_DESC_SZ) | MTK_DESC_OFFSET(MTK_DESC_OFF), - cryp->base + RDR_DESC_SIZE(i)); - - /* - * Configure HIA fetch size and fetch threshold that are used to - * fetch blocks of multiple descriptors. - */ - writel(MTK_DESC_FETCH_SIZE(count * MTK_DESC_OFF) | - MTK_DESC_FETCH_THRESH(count * rndup) | - MTK_DESC_OVL_IRQ_EN, - cryp->base + RDR_CFG(i)); -} - -static int mtk_packet_engine_setup(struct mtk_cryp *cryp) -{ - struct mtk_sys_cap cap; - int i, err; - u32 val; - - cap.hia_ver = readl(cryp->base + HIA_VERSION); - cap.hia_opt = readl(cryp->base + HIA_OPTIONS); - cap.hw_opt = readl(cryp->base + EIP97_OPTIONS); - - if (!(((u16)cap.hia_ver) == MTK_HIA_SIGNATURE)) - return -EINVAL; - - /* Configure endianness conversion method for master (DMA) interface */ - writel(0, cryp->base + EIP97_MST_CTRL); - - /* Set HIA burst size */ - val = readl(cryp->base + HIA_MST_CTRL); - val &= ~MTK_BURST_SIZE_MSK; - val |= MTK_BURST_SIZE(5); - writel(val, cryp->base + HIA_MST_CTRL); - - err = mtk_dfe_dse_reset(cryp); - if (err) { - dev_err(cryp->dev, "Failed to reset DFE and DSE.\n"); - return err; - } - - mtk_dfe_dse_buf_setup(cryp, &cap); - - /* Enable the 4 rings for the packet engines. */ - mtk_desc_ring_link(cryp, 0xf); - - for (i = 0; i < MTK_RING_MAX; i++) { - mtk_cmd_desc_ring_setup(cryp, i, &cap); - mtk_res_desc_ring_setup(cryp, i, &cap); - } - - writel(MTK_PE_TK_LOC_AVL | MTK_PE_PROC_HELD | MTK_PE_TK_TIMEOUT_EN, - cryp->base + PE_TOKEN_CTRL_STAT); - - /* Clear all pending interrupts */ - writel(MTK_AIC_G_CLR, cryp->base + AIC_G_ACK); - writel(MTK_PE_INPUT_DMA_ERR | MTK_PE_OUTPUT_DMA_ERR | - MTK_PE_PKT_PORC_ERR | MTK_PE_PKT_TIMEOUT | - MTK_PE_FATAL_ERR | MTK_PE_INPUT_DMA_ERR_EN | - MTK_PE_OUTPUT_DMA_ERR_EN | MTK_PE_PKT_PORC_ERR_EN | - MTK_PE_PKT_TIMEOUT_EN | MTK_PE_FATAL_ERR_EN | - MTK_PE_INT_OUT_EN, - cryp->base + PE_INTERRUPT_CTRL_STAT); - - return 0; -} - -static int mtk_aic_cap_check(struct mtk_cryp *cryp, int hw) -{ - u32 val; - - if (hw == MTK_RING_MAX) - val = readl(cryp->base + AIC_G_VERSION); - else - val = readl(cryp->base + AIC_VERSION(hw)); - - val &= MTK_AIC_VER_MSK; - if (val != MTK_AIC_VER11 && val != MTK_AIC_VER12) - return -ENXIO; - - if (hw == MTK_RING_MAX) - val = readl(cryp->base + AIC_G_OPTIONS); - else - val = readl(cryp->base + AIC_OPTIONS(hw)); - - val &= MTK_AIC_INT_MSK; - if (!val || val > 32) - return -ENXIO; - - return 0; -} - -static int mtk_aic_init(struct mtk_cryp *cryp, int hw) -{ - int err; - - err = mtk_aic_cap_check(cryp, hw); - if (err) - return err; - - /* Disable all interrupts and set initial configuration */ - if (hw == MTK_RING_MAX) { - writel(0, cryp->base + AIC_G_ENABLE_CTRL); - writel(0, cryp->base + AIC_G_POL_CTRL); - writel(0, cryp->base + AIC_G_TYPE_CTRL); - writel(0, cryp->base + AIC_G_ENABLE_SET); - } else { - writel(0, cryp->base + AIC_ENABLE_CTRL(hw)); - writel(0, cryp->base + AIC_POL_CTRL(hw)); - writel(0, cryp->base + AIC_TYPE_CTRL(hw)); - writel(0, cryp->base + AIC_ENABLE_SET(hw)); - } - - return 0; -} - -static int mtk_accelerator_init(struct mtk_cryp *cryp) -{ - int i, err; - - /* Initialize advanced interrupt controller(AIC) */ - for (i = 0; i < MTK_IRQ_NUM; i++) { - err = mtk_aic_init(cryp, i); - if (err) { - dev_err(cryp->dev, "Failed to initialize AIC.\n"); - return err; - } - } - - /* Initialize packet engine */ - err = mtk_packet_engine_setup(cryp); - if (err) { - dev_err(cryp->dev, "Failed to configure packet engine.\n"); - return err; - } - - return 0; -} - -static void mtk_desc_dma_free(struct mtk_cryp *cryp) -{ - int i; - - for (i = 0; i < MTK_RING_MAX; i++) { - dma_free_coherent(cryp->dev, MTK_DESC_RING_SZ, - cryp->ring[i]->res_base, - cryp->ring[i]->res_dma); - dma_free_coherent(cryp->dev, MTK_DESC_RING_SZ, - cryp->ring[i]->cmd_base, - cryp->ring[i]->cmd_dma); - kfree(cryp->ring[i]); - } -} - -static int mtk_desc_ring_alloc(struct mtk_cryp *cryp) -{ - struct mtk_ring **ring = cryp->ring; - int i; - - for (i = 0; i < MTK_RING_MAX; i++) { - ring[i] = kzalloc(sizeof(**ring), GFP_KERNEL); - if (!ring[i]) - goto err_cleanup; - - ring[i]->cmd_base = dma_alloc_coherent(cryp->dev, - MTK_DESC_RING_SZ, - &ring[i]->cmd_dma, - GFP_KERNEL); - if (!ring[i]->cmd_base) - goto err_cleanup; - - ring[i]->res_base = dma_alloc_coherent(cryp->dev, - MTK_DESC_RING_SZ, - &ring[i]->res_dma, - GFP_KERNEL); - if (!ring[i]->res_base) - goto err_cleanup; - - ring[i]->cmd_next = ring[i]->cmd_base; - ring[i]->res_next = ring[i]->res_base; - } - return 0; - -err_cleanup: - do { - dma_free_coherent(cryp->dev, MTK_DESC_RING_SZ, - ring[i]->res_base, ring[i]->res_dma); - dma_free_coherent(cryp->dev, MTK_DESC_RING_SZ, - ring[i]->cmd_base, ring[i]->cmd_dma); - kfree(ring[i]); - } while (i--); - return -ENOMEM; -} - -static int mtk_crypto_probe(struct platform_device *pdev) -{ - struct mtk_cryp *cryp; - int i, err; - - cryp = devm_kzalloc(&pdev->dev, sizeof(*cryp), GFP_KERNEL); - if (!cryp) - return -ENOMEM; - - cryp->base = devm_platform_ioremap_resource(pdev, 0); - if (IS_ERR(cryp->base)) - return PTR_ERR(cryp->base); - - for (i = 0; i < MTK_IRQ_NUM; i++) { - cryp->irq[i] = platform_get_irq(pdev, i); - if (cryp->irq[i] < 0) - return cryp->irq[i]; - } - - cryp->clk_cryp = devm_clk_get(&pdev->dev, "cryp"); - if (IS_ERR(cryp->clk_cryp)) - return -EPROBE_DEFER; - - cryp->dev = &pdev->dev; - pm_runtime_enable(cryp->dev); - pm_runtime_get_sync(cryp->dev); - - err = clk_prepare_enable(cryp->clk_cryp); - if (err) - goto err_clk_cryp; - - /* Allocate four command/result descriptor rings */ - err = mtk_desc_ring_alloc(cryp); - if (err) { - dev_err(cryp->dev, "Unable to allocate descriptor rings.\n"); - goto err_resource; - } - - /* Initialize hardware modules */ - err = mtk_accelerator_init(cryp); - if (err) { - dev_err(cryp->dev, "Failed to initialize cryptographic engine.\n"); - goto err_engine; - } - - err = mtk_cipher_alg_register(cryp); - if (err) { - dev_err(cryp->dev, "Unable to register cipher algorithm.\n"); - goto err_cipher; - } - - err = mtk_hash_alg_register(cryp); - if (err) { - dev_err(cryp->dev, "Unable to register hash algorithm.\n"); - goto err_hash; - } - - platform_set_drvdata(pdev, cryp); - return 0; - -err_hash: - mtk_cipher_alg_release(cryp); -err_cipher: - mtk_dfe_dse_reset(cryp); -err_engine: - mtk_desc_dma_free(cryp); -err_resource: - clk_disable_unprepare(cryp->clk_cryp); -err_clk_cryp: - pm_runtime_put_sync(cryp->dev); - pm_runtime_disable(cryp->dev); - - return err; -} - -static int mtk_crypto_remove(struct platform_device *pdev) -{ - struct mtk_cryp *cryp = platform_get_drvdata(pdev); - - mtk_hash_alg_release(cryp); - mtk_cipher_alg_release(cryp); - mtk_desc_dma_free(cryp); - - clk_disable_unprepare(cryp->clk_cryp); - - pm_runtime_put_sync(cryp->dev); - pm_runtime_disable(cryp->dev); - platform_set_drvdata(pdev, NULL); - - return 0; -} - -static const struct of_device_id of_crypto_id[] = { - { .compatible = "mediatek,eip97-crypto" }, - {}, -}; -MODULE_DEVICE_TABLE(of, of_crypto_id); - -static struct platform_driver mtk_crypto_driver = { - .probe = mtk_crypto_probe, - .remove = mtk_crypto_remove, - .driver = { - .name = "mtk-crypto", - .of_match_table = of_crypto_id, - }, -}; -module_platform_driver(mtk_crypto_driver); - -MODULE_LICENSE("GPL"); -MODULE_AUTHOR("Ryder Lee "); -MODULE_DESCRIPTION("Cryptographic accelerator driver for EIP97"); diff --git a/drivers/crypto/mediatek/mtk-platform.h b/drivers/crypto/mediatek/mtk-platform.h deleted file mode 100644 index 47920c51abac43ad69e651f0fd8009c6194d3a40..0000000000000000000000000000000000000000 --- a/drivers/crypto/mediatek/mtk-platform.h +++ /dev/null @@ -1,231 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Driver for EIP97 cryptographic accelerator. - * - * Copyright (c) 2016 Ryder Lee - */ - -#ifndef __MTK_PLATFORM_H_ -#define __MTK_PLATFORM_H_ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "mtk-regs.h" - -#define MTK_RDR_PROC_THRESH BIT(0) -#define MTK_RDR_PROC_MODE BIT(23) -#define MTK_CNT_RST BIT(31) -#define MTK_IRQ_RDR0 BIT(1) -#define MTK_IRQ_RDR1 BIT(3) -#define MTK_IRQ_RDR2 BIT(5) -#define MTK_IRQ_RDR3 BIT(7) - -#define SIZE_IN_WORDS(x) ((x) >> 2) - -/** - * Ring 0/1 are used by AES encrypt and decrypt. - * Ring 2/3 are used by SHA. - */ -enum { - MTK_RING0, - MTK_RING1, - MTK_RING2, - MTK_RING3, - MTK_RING_MAX -}; - -#define MTK_REC_NUM (MTK_RING_MAX / 2) -#define MTK_IRQ_NUM 5 - -/** - * struct mtk_desc - DMA descriptor - * @hdr: the descriptor control header - * @buf: DMA address of input buffer segment - * @ct: DMA address of command token that control operation flow - * @ct_hdr: the command token control header - * @tag: the user-defined field - * @tfm: DMA address of transform state - * @bound: align descriptors offset boundary - * - * Structure passed to the crypto engine to describe where source - * data needs to be fetched and how it needs to be processed. - */ -struct mtk_desc { - __le32 hdr; - __le32 buf; - __le32 ct; - __le32 ct_hdr; - __le32 tag; - __le32 tfm; - __le32 bound[2]; -}; - -#define MTK_DESC_NUM 512 -#define MTK_DESC_OFF SIZE_IN_WORDS(sizeof(struct mtk_desc)) -#define MTK_DESC_SZ (MTK_DESC_OFF - 2) -#define MTK_DESC_RING_SZ ((sizeof(struct mtk_desc) * MTK_DESC_NUM)) -#define MTK_DESC_CNT(x) ((MTK_DESC_OFF * (x)) << 2) -#define MTK_DESC_LAST cpu_to_le32(BIT(22)) -#define MTK_DESC_FIRST cpu_to_le32(BIT(23)) -#define MTK_DESC_BUF_LEN(x) cpu_to_le32(x) -#define MTK_DESC_CT_LEN(x) cpu_to_le32((x) << 24) - -/** - * struct mtk_ring - Descriptor ring - * @cmd_base: pointer to command descriptor ring base - * @cmd_next: pointer to the next command descriptor - * @cmd_dma: DMA address of command descriptor ring - * @res_base: pointer to result descriptor ring base - * @res_next: pointer to the next result descriptor - * @res_prev: pointer to the previous result descriptor - * @res_dma: DMA address of result descriptor ring - * - * A descriptor ring is a circular buffer that is used to manage - * one or more descriptors. There are two type of descriptor rings; - * the command descriptor ring and result descriptor ring. - */ -struct mtk_ring { - struct mtk_desc *cmd_base; - struct mtk_desc *cmd_next; - dma_addr_t cmd_dma; - struct mtk_desc *res_base; - struct mtk_desc *res_next; - struct mtk_desc *res_prev; - dma_addr_t res_dma; -}; - -/** - * struct mtk_aes_dma - Structure that holds sg list info - * @sg: pointer to scatter-gather list - * @nents: number of entries in the sg list - * @remainder: remainder of sg list - * @sg_len: number of entries in the sg mapped list - */ -struct mtk_aes_dma { - struct scatterlist *sg; - int nents; - u32 remainder; - u32 sg_len; -}; - -struct mtk_aes_base_ctx; -struct mtk_aes_rec; -struct mtk_cryp; - -typedef int (*mtk_aes_fn)(struct mtk_cryp *cryp, struct mtk_aes_rec *aes); - -/** - * struct mtk_aes_rec - AES operation record - * @cryp: pointer to Cryptographic device - * @queue: crypto request queue - * @areq: pointer to async request - * @done_task: the tasklet is use in AES interrupt - * @queue_task: the tasklet is used to dequeue request - * @ctx: pointer to current context - * @src: the structure that holds source sg list info - * @dst: the structure that holds destination sg list info - * @aligned_sg: the scatter list is use to alignment - * @real_dst: pointer to the destination sg list - * @resume: pointer to resume function - * @total: request buffer length - * @buf: pointer to page buffer - * @id: the current use of ring - * @flags: it's describing AES operation state - * @lock: the async queue lock - * - * Structure used to record AES execution state. - */ -struct mtk_aes_rec { - struct mtk_cryp *cryp; - struct crypto_queue queue; - struct crypto_async_request *areq; - struct tasklet_struct done_task; - struct tasklet_struct queue_task; - struct mtk_aes_base_ctx *ctx; - struct mtk_aes_dma src; - struct mtk_aes_dma dst; - - struct scatterlist aligned_sg; - struct scatterlist *real_dst; - - mtk_aes_fn resume; - - size_t total; - void *buf; - - u8 id; - unsigned long flags; - /* queue lock */ - spinlock_t lock; -}; - -/** - * struct mtk_sha_rec - SHA operation record - * @cryp: pointer to Cryptographic device - * @queue: crypto request queue - * @req: pointer to ahash request - * @done_task: the tasklet is use in SHA interrupt - * @queue_task: the tasklet is used to dequeue request - * @id: the current use of ring - * @flags: it's describing SHA operation state - * @lock: the async queue lock - * - * Structure used to record SHA execution state. - */ -struct mtk_sha_rec { - struct mtk_cryp *cryp; - struct crypto_queue queue; - struct ahash_request *req; - struct tasklet_struct done_task; - struct tasklet_struct queue_task; - - u8 id; - unsigned long flags; - /* queue lock */ - spinlock_t lock; -}; - -/** - * struct mtk_cryp - Cryptographic device - * @base: pointer to mapped register I/O base - * @dev: pointer to device - * @clk_cryp: pointer to crypto clock - * @irq: global system and rings IRQ - * @ring: pointer to descriptor rings - * @aes: pointer to operation record of AES - * @sha: pointer to operation record of SHA - * @aes_list: device list of AES - * @sha_list: device list of SHA - * @rec: it's used to select SHA record for tfm - * - * Structure storing cryptographic device information. - */ -struct mtk_cryp { - void __iomem *base; - struct device *dev; - struct clk *clk_cryp; - int irq[MTK_IRQ_NUM]; - - struct mtk_ring *ring[MTK_RING_MAX]; - struct mtk_aes_rec *aes[MTK_REC_NUM]; - struct mtk_sha_rec *sha[MTK_REC_NUM]; - - struct list_head aes_list; - struct list_head sha_list; - - bool rec; -}; - -int mtk_cipher_alg_register(struct mtk_cryp *cryp); -void mtk_cipher_alg_release(struct mtk_cryp *cryp); -int mtk_hash_alg_register(struct mtk_cryp *cryp); -void mtk_hash_alg_release(struct mtk_cryp *cryp); - -#endif /* __MTK_PLATFORM_H_ */ diff --git a/drivers/crypto/mediatek/mtk-regs.h b/drivers/crypto/mediatek/mtk-regs.h deleted file mode 100644 index d3defda7a7503797d2fa66e86beb00c354d74b5a..0000000000000000000000000000000000000000 --- a/drivers/crypto/mediatek/mtk-regs.h +++ /dev/null @@ -1,190 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Support for MediaTek cryptographic accelerator. - * - * Copyright (c) 2016 MediaTek Inc. - * Author: Ryder Lee - */ - -#ifndef __MTK_REGS_H__ -#define __MTK_REGS_H__ - -/* HIA, Command Descriptor Ring Manager */ -#define CDR_BASE_ADDR_LO(x) (0x0 + ((x) << 12)) -#define CDR_BASE_ADDR_HI(x) (0x4 + ((x) << 12)) -#define CDR_DATA_BASE_ADDR_LO(x) (0x8 + ((x) << 12)) -#define CDR_DATA_BASE_ADDR_HI(x) (0xC + ((x) << 12)) -#define CDR_ACD_BASE_ADDR_LO(x) (0x10 + ((x) << 12)) -#define CDR_ACD_BASE_ADDR_HI(x) (0x14 + ((x) << 12)) -#define CDR_RING_SIZE(x) (0x18 + ((x) << 12)) -#define CDR_DESC_SIZE(x) (0x1C + ((x) << 12)) -#define CDR_CFG(x) (0x20 + ((x) << 12)) -#define CDR_DMA_CFG(x) (0x24 + ((x) << 12)) -#define CDR_THRESH(x) (0x28 + ((x) << 12)) -#define CDR_PREP_COUNT(x) (0x2C + ((x) << 12)) -#define CDR_PROC_COUNT(x) (0x30 + ((x) << 12)) -#define CDR_PREP_PNTR(x) (0x34 + ((x) << 12)) -#define CDR_PROC_PNTR(x) (0x38 + ((x) << 12)) -#define CDR_STAT(x) (0x3C + ((x) << 12)) - -/* HIA, Result Descriptor Ring Manager */ -#define RDR_BASE_ADDR_LO(x) (0x800 + ((x) << 12)) -#define RDR_BASE_ADDR_HI(x) (0x804 + ((x) << 12)) -#define RDR_DATA_BASE_ADDR_LO(x) (0x808 + ((x) << 12)) -#define RDR_DATA_BASE_ADDR_HI(x) (0x80C + ((x) << 12)) -#define RDR_ACD_BASE_ADDR_LO(x) (0x810 + ((x) << 12)) -#define RDR_ACD_BASE_ADDR_HI(x) (0x814 + ((x) << 12)) -#define RDR_RING_SIZE(x) (0x818 + ((x) << 12)) -#define RDR_DESC_SIZE(x) (0x81C + ((x) << 12)) -#define RDR_CFG(x) (0x820 + ((x) << 12)) -#define RDR_DMA_CFG(x) (0x824 + ((x) << 12)) -#define RDR_THRESH(x) (0x828 + ((x) << 12)) -#define RDR_PREP_COUNT(x) (0x82C + ((x) << 12)) -#define RDR_PROC_COUNT(x) (0x830 + ((x) << 12)) -#define RDR_PREP_PNTR(x) (0x834 + ((x) << 12)) -#define RDR_PROC_PNTR(x) (0x838 + ((x) << 12)) -#define RDR_STAT(x) (0x83C + ((x) << 12)) - -/* HIA, Ring AIC */ -#define AIC_POL_CTRL(x) (0xE000 - ((x) << 12)) -#define AIC_TYPE_CTRL(x) (0xE004 - ((x) << 12)) -#define AIC_ENABLE_CTRL(x) (0xE008 - ((x) << 12)) -#define AIC_RAW_STAL(x) (0xE00C - ((x) << 12)) -#define AIC_ENABLE_SET(x) (0xE00C - ((x) << 12)) -#define AIC_ENABLED_STAT(x) (0xE010 - ((x) << 12)) -#define AIC_ACK(x) (0xE010 - ((x) << 12)) -#define AIC_ENABLE_CLR(x) (0xE014 - ((x) << 12)) -#define AIC_OPTIONS(x) (0xE018 - ((x) << 12)) -#define AIC_VERSION(x) (0xE01C - ((x) << 12)) - -/* HIA, Global AIC */ -#define AIC_G_POL_CTRL 0xF800 -#define AIC_G_TYPE_CTRL 0xF804 -#define AIC_G_ENABLE_CTRL 0xF808 -#define AIC_G_RAW_STAT 0xF80C -#define AIC_G_ENABLE_SET 0xF80C -#define AIC_G_ENABLED_STAT 0xF810 -#define AIC_G_ACK 0xF810 -#define AIC_G_ENABLE_CLR 0xF814 -#define AIC_G_OPTIONS 0xF818 -#define AIC_G_VERSION 0xF81C - -/* HIA, Data Fetch Engine */ -#define DFE_CFG 0xF000 -#define DFE_PRIO_0 0xF010 -#define DFE_PRIO_1 0xF014 -#define DFE_PRIO_2 0xF018 -#define DFE_PRIO_3 0xF01C - -/* HIA, Data Fetch Engine access monitoring for CDR */ -#define DFE_RING_REGION_LO(x) (0xF080 + ((x) << 3)) -#define DFE_RING_REGION_HI(x) (0xF084 + ((x) << 3)) - -/* HIA, Data Fetch Engine thread control and status for thread */ -#define DFE_THR_CTRL 0xF200 -#define DFE_THR_STAT 0xF204 -#define DFE_THR_DESC_CTRL 0xF208 -#define DFE_THR_DESC_DPTR_LO 0xF210 -#define DFE_THR_DESC_DPTR_HI 0xF214 -#define DFE_THR_DESC_ACDPTR_LO 0xF218 -#define DFE_THR_DESC_ACDPTR_HI 0xF21C - -/* HIA, Data Store Engine */ -#define DSE_CFG 0xF400 -#define DSE_PRIO_0 0xF410 -#define DSE_PRIO_1 0xF414 -#define DSE_PRIO_2 0xF418 -#define DSE_PRIO_3 0xF41C - -/* HIA, Data Store Engine access monitoring for RDR */ -#define DSE_RING_REGION_LO(x) (0xF480 + ((x) << 3)) -#define DSE_RING_REGION_HI(x) (0xF484 + ((x) << 3)) - -/* HIA, Data Store Engine thread control and status for thread */ -#define DSE_THR_CTRL 0xF600 -#define DSE_THR_STAT 0xF604 -#define DSE_THR_DESC_CTRL 0xF608 -#define DSE_THR_DESC_DPTR_LO 0xF610 -#define DSE_THR_DESC_DPTR_HI 0xF614 -#define DSE_THR_DESC_S_DPTR_LO 0xF618 -#define DSE_THR_DESC_S_DPTR_HI 0xF61C -#define DSE_THR_ERROR_STAT 0xF620 - -/* HIA Global */ -#define HIA_MST_CTRL 0xFFF4 -#define HIA_OPTIONS 0xFFF8 -#define HIA_VERSION 0xFFFC - -/* Processing Engine Input Side, Processing Engine */ -#define PE_IN_DBUF_THRESH 0x10000 -#define PE_IN_TBUF_THRESH 0x10100 - -/* Packet Engine Configuration / Status Registers */ -#define PE_TOKEN_CTRL_STAT 0x11000 -#define PE_FUNCTION_EN 0x11004 -#define PE_CONTEXT_CTRL 0x11008 -#define PE_INTERRUPT_CTRL_STAT 0x11010 -#define PE_CONTEXT_STAT 0x1100C -#define PE_OUT_TRANS_CTRL_STAT 0x11018 -#define PE_OUT_BUF_CTRL 0x1101C - -/* Packet Engine PRNG Registers */ -#define PE_PRNG_STAT 0x11040 -#define PE_PRNG_CTRL 0x11044 -#define PE_PRNG_SEED_L 0x11048 -#define PE_PRNG_SEED_H 0x1104C -#define PE_PRNG_KEY_0_L 0x11050 -#define PE_PRNG_KEY_0_H 0x11054 -#define PE_PRNG_KEY_1_L 0x11058 -#define PE_PRNG_KEY_1_H 0x1105C -#define PE_PRNG_RES_0 0x11060 -#define PE_PRNG_RES_1 0x11064 -#define PE_PRNG_RES_2 0x11068 -#define PE_PRNG_RES_3 0x1106C -#define PE_PRNG_LFSR_L 0x11070 -#define PE_PRNG_LFSR_H 0x11074 - -/* Packet Engine AIC */ -#define PE_EIP96_AIC_POL_CTRL 0x113C0 -#define PE_EIP96_AIC_TYPE_CTRL 0x113C4 -#define PE_EIP96_AIC_ENABLE_CTRL 0x113C8 -#define PE_EIP96_AIC_RAW_STAT 0x113CC -#define PE_EIP96_AIC_ENABLE_SET 0x113CC -#define PE_EIP96_AIC_ENABLED_STAT 0x113D0 -#define PE_EIP96_AIC_ACK 0x113D0 -#define PE_EIP96_AIC_ENABLE_CLR 0x113D4 -#define PE_EIP96_AIC_OPTIONS 0x113D8 -#define PE_EIP96_AIC_VERSION 0x113DC - -/* Packet Engine Options & Version Registers */ -#define PE_EIP96_OPTIONS 0x113F8 -#define PE_EIP96_VERSION 0x113FC - -/* Processing Engine Output Side */ -#define PE_OUT_DBUF_THRESH 0x11C00 -#define PE_OUT_TBUF_THRESH 0x11D00 - -/* Processing Engine Local AIC */ -#define PE_AIC_POL_CTRL 0x11F00 -#define PE_AIC_TYPE_CTRL 0x11F04 -#define PE_AIC_ENABLE_CTRL 0x11F08 -#define PE_AIC_RAW_STAT 0x11F0C -#define PE_AIC_ENABLE_SET 0x11F0C -#define PE_AIC_ENABLED_STAT 0x11F10 -#define PE_AIC_ENABLE_CLR 0x11F14 -#define PE_AIC_OPTIONS 0x11F18 -#define PE_AIC_VERSION 0x11F1C - -/* Processing Engine General Configuration and Version */ -#define PE_IN_FLIGHT 0x11FF0 -#define PE_OPTIONS 0x11FF8 -#define PE_VERSION 0x11FFC - -/* EIP-97 - Global */ -#define EIP97_CLOCK_STATE 0x1FFE4 -#define EIP97_FORCE_CLOCK_ON 0x1FFE8 -#define EIP97_FORCE_CLOCK_OFF 0x1FFEC -#define EIP97_MST_CTRL 0x1FFF4 -#define EIP97_OPTIONS 0x1FFF8 -#define EIP97_VERSION 0x1FFFC -#endif /* __MTK_REGS_H__ */ diff --git a/drivers/crypto/mediatek/mtk-sha.c b/drivers/crypto/mediatek/mtk-sha.c deleted file mode 100644 index f55aacdafbefdd8bd8410e451db24c070ea5116b..0000000000000000000000000000000000000000 --- a/drivers/crypto/mediatek/mtk-sha.c +++ /dev/null @@ -1,1353 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Cryptographic API. - * - * Driver for EIP97 SHA1/SHA2(HMAC) acceleration. - * - * Copyright (c) 2016 Ryder Lee - * - * Some ideas are from atmel-sha.c and omap-sham.c drivers. - */ - -#include -#include -#include -#include "mtk-platform.h" - -#define SHA_ALIGN_MSK (sizeof(u32) - 1) -#define SHA_QUEUE_SIZE 512 -#define SHA_BUF_SIZE ((u32)PAGE_SIZE) - -#define SHA_OP_UPDATE 1 -#define SHA_OP_FINAL 2 - -#define SHA_DATA_LEN_MSK cpu_to_le32(GENMASK(16, 0)) -#define SHA_MAX_DIGEST_BUF_SIZE 32 - -/* SHA command token */ -#define SHA_CT_SIZE 5 -#define SHA_CT_CTRL_HDR cpu_to_le32(0x02220000) -#define SHA_CMD0 cpu_to_le32(0x03020000) -#define SHA_CMD1 cpu_to_le32(0x21060000) -#define SHA_CMD2 cpu_to_le32(0xe0e63802) - -/* SHA transform information */ -#define SHA_TFM_HASH cpu_to_le32(0x2 << 0) -#define SHA_TFM_SIZE(x) cpu_to_le32((x) << 8) -#define SHA_TFM_START cpu_to_le32(0x1 << 4) -#define SHA_TFM_CONTINUE cpu_to_le32(0x1 << 5) -#define SHA_TFM_HASH_STORE cpu_to_le32(0x1 << 19) -#define SHA_TFM_SHA1 cpu_to_le32(0x2 << 23) -#define SHA_TFM_SHA256 cpu_to_le32(0x3 << 23) -#define SHA_TFM_SHA224 cpu_to_le32(0x4 << 23) -#define SHA_TFM_SHA512 cpu_to_le32(0x5 << 23) -#define SHA_TFM_SHA384 cpu_to_le32(0x6 << 23) -#define SHA_TFM_DIGEST(x) cpu_to_le32(((x) & GENMASK(3, 0)) << 24) - -/* SHA flags */ -#define SHA_FLAGS_BUSY BIT(0) -#define SHA_FLAGS_FINAL BIT(1) -#define SHA_FLAGS_FINUP BIT(2) -#define SHA_FLAGS_SG BIT(3) -#define SHA_FLAGS_ALGO_MSK GENMASK(8, 4) -#define SHA_FLAGS_SHA1 BIT(4) -#define SHA_FLAGS_SHA224 BIT(5) -#define SHA_FLAGS_SHA256 BIT(6) -#define SHA_FLAGS_SHA384 BIT(7) -#define SHA_FLAGS_SHA512 BIT(8) -#define SHA_FLAGS_HMAC BIT(9) -#define SHA_FLAGS_PAD BIT(10) - -/** - * mtk_sha_info - hardware information of AES - * @cmd: command token, hardware instruction - * @tfm: transform state of cipher algorithm. - * @state: contains keys and initial vectors. - * - */ -struct mtk_sha_info { - __le32 ctrl[2]; - __le32 cmd[3]; - __le32 tfm[2]; - __le32 digest[SHA_MAX_DIGEST_BUF_SIZE]; -}; - -struct mtk_sha_reqctx { - struct mtk_sha_info info; - unsigned long flags; - unsigned long op; - - u64 digcnt; - size_t bufcnt; - dma_addr_t dma_addr; - - __le32 ct_hdr; - u32 ct_size; - dma_addr_t ct_dma; - dma_addr_t tfm_dma; - - /* Walk state */ - struct scatterlist *sg; - u32 offset; /* Offset in current sg */ - u32 total; /* Total request */ - size_t ds; - size_t bs; - - u8 *buffer; -}; - -struct mtk_sha_hmac_ctx { - struct crypto_shash *shash; - u8 ipad[SHA512_BLOCK_SIZE] __aligned(sizeof(u32)); - u8 opad[SHA512_BLOCK_SIZE] __aligned(sizeof(u32)); -}; - -struct mtk_sha_ctx { - struct mtk_cryp *cryp; - unsigned long flags; - u8 id; - u8 buf[SHA_BUF_SIZE] __aligned(sizeof(u32)); - - struct mtk_sha_hmac_ctx base[]; -}; - -struct mtk_sha_drv { - struct list_head dev_list; - /* Device list lock */ - spinlock_t lock; -}; - -static struct mtk_sha_drv mtk_sha = { - .dev_list = LIST_HEAD_INIT(mtk_sha.dev_list), - .lock = __SPIN_LOCK_UNLOCKED(mtk_sha.lock), -}; - -static int mtk_sha_handle_queue(struct mtk_cryp *cryp, u8 id, - struct ahash_request *req); - -static inline u32 mtk_sha_read(struct mtk_cryp *cryp, u32 offset) -{ - return readl_relaxed(cryp->base + offset); -} - -static inline void mtk_sha_write(struct mtk_cryp *cryp, - u32 offset, u32 value) -{ - writel_relaxed(value, cryp->base + offset); -} - -static inline void mtk_sha_ring_shift(struct mtk_ring *ring, - struct mtk_desc **cmd_curr, - struct mtk_desc **res_curr, - int *count) -{ - *cmd_curr = ring->cmd_next++; - *res_curr = ring->res_next++; - (*count)++; - - if (ring->cmd_next == ring->cmd_base + MTK_DESC_NUM) { - ring->cmd_next = ring->cmd_base; - ring->res_next = ring->res_base; - } -} - -static struct mtk_cryp *mtk_sha_find_dev(struct mtk_sha_ctx *tctx) -{ - struct mtk_cryp *cryp = NULL; - struct mtk_cryp *tmp; - - spin_lock_bh(&mtk_sha.lock); - if (!tctx->cryp) { - list_for_each_entry(tmp, &mtk_sha.dev_list, sha_list) { - cryp = tmp; - break; - } - tctx->cryp = cryp; - } else { - cryp = tctx->cryp; - } - - /* - * Assign record id to tfm in round-robin fashion, and this - * will help tfm to bind to corresponding descriptor rings. - */ - tctx->id = cryp->rec; - cryp->rec = !cryp->rec; - - spin_unlock_bh(&mtk_sha.lock); - - return cryp; -} - -static int mtk_sha_append_sg(struct mtk_sha_reqctx *ctx) -{ - size_t count; - - while ((ctx->bufcnt < SHA_BUF_SIZE) && ctx->total) { - count = min(ctx->sg->length - ctx->offset, ctx->total); - count = min(count, SHA_BUF_SIZE - ctx->bufcnt); - - if (count <= 0) { - /* - * Check if count <= 0 because the buffer is full or - * because the sg length is 0. In the latest case, - * check if there is another sg in the list, a 0 length - * sg doesn't necessarily mean the end of the sg list. - */ - if ((ctx->sg->length == 0) && !sg_is_last(ctx->sg)) { - ctx->sg = sg_next(ctx->sg); - continue; - } else { - break; - } - } - - scatterwalk_map_and_copy(ctx->buffer + ctx->bufcnt, ctx->sg, - ctx->offset, count, 0); - - ctx->bufcnt += count; - ctx->offset += count; - ctx->total -= count; - - if (ctx->offset == ctx->sg->length) { - ctx->sg = sg_next(ctx->sg); - if (ctx->sg) - ctx->offset = 0; - else - ctx->total = 0; - } - } - - return 0; -} - -/* - * The purpose of this padding is to ensure that the padded message is a - * multiple of 512 bits (SHA1/SHA224/SHA256) or 1024 bits (SHA384/SHA512). - * The bit "1" is appended at the end of the message followed by - * "padlen-1" zero bits. Then a 64 bits block (SHA1/SHA224/SHA256) or - * 128 bits block (SHA384/SHA512) equals to the message length in bits - * is appended. - * - * For SHA1/SHA224/SHA256, padlen is calculated as followed: - * - if message length < 56 bytes then padlen = 56 - message length - * - else padlen = 64 + 56 - message length - * - * For SHA384/SHA512, padlen is calculated as followed: - * - if message length < 112 bytes then padlen = 112 - message length - * - else padlen = 128 + 112 - message length - */ -static void mtk_sha_fill_padding(struct mtk_sha_reqctx *ctx, u32 len) -{ - u32 index, padlen; - __be64 bits[2]; - u64 size = ctx->digcnt; - - size += ctx->bufcnt; - size += len; - - bits[1] = cpu_to_be64(size << 3); - bits[0] = cpu_to_be64(size >> 61); - - switch (ctx->flags & SHA_FLAGS_ALGO_MSK) { - case SHA_FLAGS_SHA384: - case SHA_FLAGS_SHA512: - index = ctx->bufcnt & 0x7f; - padlen = (index < 112) ? (112 - index) : ((128 + 112) - index); - *(ctx->buffer + ctx->bufcnt) = 0x80; - memset(ctx->buffer + ctx->bufcnt + 1, 0, padlen - 1); - memcpy(ctx->buffer + ctx->bufcnt + padlen, bits, 16); - ctx->bufcnt += padlen + 16; - ctx->flags |= SHA_FLAGS_PAD; - break; - - default: - index = ctx->bufcnt & 0x3f; - padlen = (index < 56) ? (56 - index) : ((64 + 56) - index); - *(ctx->buffer + ctx->bufcnt) = 0x80; - memset(ctx->buffer + ctx->bufcnt + 1, 0, padlen - 1); - memcpy(ctx->buffer + ctx->bufcnt + padlen, &bits[1], 8); - ctx->bufcnt += padlen + 8; - ctx->flags |= SHA_FLAGS_PAD; - break; - } -} - -/* Initialize basic transform information of SHA */ -static void mtk_sha_info_init(struct mtk_sha_reqctx *ctx) -{ - struct mtk_sha_info *info = &ctx->info; - - ctx->ct_hdr = SHA_CT_CTRL_HDR; - ctx->ct_size = SHA_CT_SIZE; - - info->tfm[0] = SHA_TFM_HASH | SHA_TFM_SIZE(SIZE_IN_WORDS(ctx->ds)); - - switch (ctx->flags & SHA_FLAGS_ALGO_MSK) { - case SHA_FLAGS_SHA1: - info->tfm[0] |= SHA_TFM_SHA1; - break; - case SHA_FLAGS_SHA224: - info->tfm[0] |= SHA_TFM_SHA224; - break; - case SHA_FLAGS_SHA256: - info->tfm[0] |= SHA_TFM_SHA256; - break; - case SHA_FLAGS_SHA384: - info->tfm[0] |= SHA_TFM_SHA384; - break; - case SHA_FLAGS_SHA512: - info->tfm[0] |= SHA_TFM_SHA512; - break; - - default: - /* Should not happen... */ - return; - } - - info->tfm[1] = SHA_TFM_HASH_STORE; - info->ctrl[0] = info->tfm[0] | SHA_TFM_CONTINUE | SHA_TFM_START; - info->ctrl[1] = info->tfm[1]; - - info->cmd[0] = SHA_CMD0; - info->cmd[1] = SHA_CMD1; - info->cmd[2] = SHA_CMD2 | SHA_TFM_DIGEST(SIZE_IN_WORDS(ctx->ds)); -} - -/* - * Update input data length field of transform information and - * map it to DMA region. - */ -static int mtk_sha_info_update(struct mtk_cryp *cryp, - struct mtk_sha_rec *sha, - size_t len1, size_t len2) -{ - struct mtk_sha_reqctx *ctx = ahash_request_ctx(sha->req); - struct mtk_sha_info *info = &ctx->info; - - ctx->ct_hdr &= ~SHA_DATA_LEN_MSK; - ctx->ct_hdr |= cpu_to_le32(len1 + len2); - info->cmd[0] &= ~SHA_DATA_LEN_MSK; - info->cmd[0] |= cpu_to_le32(len1 + len2); - - /* Setting SHA_TFM_START only for the first iteration */ - if (ctx->digcnt) - info->ctrl[0] &= ~SHA_TFM_START; - - ctx->digcnt += len1; - - ctx->ct_dma = dma_map_single(cryp->dev, info, sizeof(*info), - DMA_BIDIRECTIONAL); - if (unlikely(dma_mapping_error(cryp->dev, ctx->ct_dma))) { - dev_err(cryp->dev, "dma %zu bytes error\n", sizeof(*info)); - return -EINVAL; - } - - ctx->tfm_dma = ctx->ct_dma + sizeof(info->ctrl) + sizeof(info->cmd); - - return 0; -} - -/* - * Because of hardware limitation, we must pre-calculate the inner - * and outer digest that need to be processed firstly by engine, then - * apply the result digest to the input message. These complex hashing - * procedures limits HMAC performance, so we use fallback SW encoding. - */ -static int mtk_sha_finish_hmac(struct ahash_request *req) -{ - struct mtk_sha_ctx *tctx = crypto_tfm_ctx(req->base.tfm); - struct mtk_sha_hmac_ctx *bctx = tctx->base; - struct mtk_sha_reqctx *ctx = ahash_request_ctx(req); - - SHASH_DESC_ON_STACK(shash, bctx->shash); - - shash->tfm = bctx->shash; - - return crypto_shash_init(shash) ?: - crypto_shash_update(shash, bctx->opad, ctx->bs) ?: - crypto_shash_finup(shash, req->result, ctx->ds, req->result); -} - -/* Initialize request context */ -static int mtk_sha_init(struct ahash_request *req) -{ - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - struct mtk_sha_ctx *tctx = crypto_ahash_ctx(tfm); - struct mtk_sha_reqctx *ctx = ahash_request_ctx(req); - - ctx->flags = 0; - ctx->ds = crypto_ahash_digestsize(tfm); - - switch (ctx->ds) { - case SHA1_DIGEST_SIZE: - ctx->flags |= SHA_FLAGS_SHA1; - ctx->bs = SHA1_BLOCK_SIZE; - break; - case SHA224_DIGEST_SIZE: - ctx->flags |= SHA_FLAGS_SHA224; - ctx->bs = SHA224_BLOCK_SIZE; - break; - case SHA256_DIGEST_SIZE: - ctx->flags |= SHA_FLAGS_SHA256; - ctx->bs = SHA256_BLOCK_SIZE; - break; - case SHA384_DIGEST_SIZE: - ctx->flags |= SHA_FLAGS_SHA384; - ctx->bs = SHA384_BLOCK_SIZE; - break; - case SHA512_DIGEST_SIZE: - ctx->flags |= SHA_FLAGS_SHA512; - ctx->bs = SHA512_BLOCK_SIZE; - break; - default: - return -EINVAL; - } - - ctx->bufcnt = 0; - ctx->digcnt = 0; - ctx->buffer = tctx->buf; - - if (tctx->flags & SHA_FLAGS_HMAC) { - struct mtk_sha_hmac_ctx *bctx = tctx->base; - - memcpy(ctx->buffer, bctx->ipad, ctx->bs); - ctx->bufcnt = ctx->bs; - ctx->flags |= SHA_FLAGS_HMAC; - } - - return 0; -} - -static int mtk_sha_xmit(struct mtk_cryp *cryp, struct mtk_sha_rec *sha, - dma_addr_t addr1, size_t len1, - dma_addr_t addr2, size_t len2) -{ - struct mtk_sha_reqctx *ctx = ahash_request_ctx(sha->req); - struct mtk_ring *ring = cryp->ring[sha->id]; - struct mtk_desc *cmd, *res; - int err, count = 0; - - err = mtk_sha_info_update(cryp, sha, len1, len2); - if (err) - return err; - - /* Fill in the command/result descriptors */ - mtk_sha_ring_shift(ring, &cmd, &res, &count); - - res->hdr = MTK_DESC_FIRST | MTK_DESC_BUF_LEN(len1); - cmd->hdr = MTK_DESC_FIRST | MTK_DESC_BUF_LEN(len1) | - MTK_DESC_CT_LEN(ctx->ct_size); - cmd->buf = cpu_to_le32(addr1); - cmd->ct = cpu_to_le32(ctx->ct_dma); - cmd->ct_hdr = ctx->ct_hdr; - cmd->tfm = cpu_to_le32(ctx->tfm_dma); - - if (len2) { - mtk_sha_ring_shift(ring, &cmd, &res, &count); - - res->hdr = MTK_DESC_BUF_LEN(len2); - cmd->hdr = MTK_DESC_BUF_LEN(len2); - cmd->buf = cpu_to_le32(addr2); - } - - cmd->hdr |= MTK_DESC_LAST; - res->hdr |= MTK_DESC_LAST; - - /* - * Make sure that all changes to the DMA ring are done before we - * start engine. - */ - wmb(); - /* Start DMA transfer */ - mtk_sha_write(cryp, RDR_PREP_COUNT(sha->id), MTK_DESC_CNT(count)); - mtk_sha_write(cryp, CDR_PREP_COUNT(sha->id), MTK_DESC_CNT(count)); - - return -EINPROGRESS; -} - -static int mtk_sha_dma_map(struct mtk_cryp *cryp, - struct mtk_sha_rec *sha, - struct mtk_sha_reqctx *ctx, - size_t count) -{ - ctx->dma_addr = dma_map_single(cryp->dev, ctx->buffer, - SHA_BUF_SIZE, DMA_TO_DEVICE); - if (unlikely(dma_mapping_error(cryp->dev, ctx->dma_addr))) { - dev_err(cryp->dev, "dma map error\n"); - return -EINVAL; - } - - ctx->flags &= ~SHA_FLAGS_SG; - - return mtk_sha_xmit(cryp, sha, ctx->dma_addr, count, 0, 0); -} - -static int mtk_sha_update_slow(struct mtk_cryp *cryp, - struct mtk_sha_rec *sha) -{ - struct mtk_sha_reqctx *ctx = ahash_request_ctx(sha->req); - size_t count; - u32 final; - - mtk_sha_append_sg(ctx); - - final = (ctx->flags & SHA_FLAGS_FINUP) && !ctx->total; - - dev_dbg(cryp->dev, "slow: bufcnt: %zu\n", ctx->bufcnt); - - if (final) { - sha->flags |= SHA_FLAGS_FINAL; - mtk_sha_fill_padding(ctx, 0); - } - - if (final || (ctx->bufcnt == SHA_BUF_SIZE && ctx->total)) { - count = ctx->bufcnt; - ctx->bufcnt = 0; - - return mtk_sha_dma_map(cryp, sha, ctx, count); - } - return 0; -} - -static int mtk_sha_update_start(struct mtk_cryp *cryp, - struct mtk_sha_rec *sha) -{ - struct mtk_sha_reqctx *ctx = ahash_request_ctx(sha->req); - u32 len, final, tail; - struct scatterlist *sg; - - if (!ctx->total) - return 0; - - if (ctx->bufcnt || ctx->offset) - return mtk_sha_update_slow(cryp, sha); - - sg = ctx->sg; - - if (!IS_ALIGNED(sg->offset, sizeof(u32))) - return mtk_sha_update_slow(cryp, sha); - - if (!sg_is_last(sg) && !IS_ALIGNED(sg->length, ctx->bs)) - /* size is not ctx->bs aligned */ - return mtk_sha_update_slow(cryp, sha); - - len = min(ctx->total, sg->length); - - if (sg_is_last(sg)) { - if (!(ctx->flags & SHA_FLAGS_FINUP)) { - /* not last sg must be ctx->bs aligned */ - tail = len & (ctx->bs - 1); - len -= tail; - } - } - - ctx->total -= len; - ctx->offset = len; /* offset where to start slow */ - - final = (ctx->flags & SHA_FLAGS_FINUP) && !ctx->total; - - /* Add padding */ - if (final) { - size_t count; - - tail = len & (ctx->bs - 1); - len -= tail; - ctx->total += tail; - ctx->offset = len; /* offset where to start slow */ - - sg = ctx->sg; - mtk_sha_append_sg(ctx); - mtk_sha_fill_padding(ctx, len); - - ctx->dma_addr = dma_map_single(cryp->dev, ctx->buffer, - SHA_BUF_SIZE, DMA_TO_DEVICE); - if (unlikely(dma_mapping_error(cryp->dev, ctx->dma_addr))) { - dev_err(cryp->dev, "dma map bytes error\n"); - return -EINVAL; - } - - sha->flags |= SHA_FLAGS_FINAL; - count = ctx->bufcnt; - ctx->bufcnt = 0; - - if (len == 0) { - ctx->flags &= ~SHA_FLAGS_SG; - return mtk_sha_xmit(cryp, sha, ctx->dma_addr, - count, 0, 0); - - } else { - ctx->sg = sg; - if (!dma_map_sg(cryp->dev, ctx->sg, 1, DMA_TO_DEVICE)) { - dev_err(cryp->dev, "dma_map_sg error\n"); - return -EINVAL; - } - - ctx->flags |= SHA_FLAGS_SG; - return mtk_sha_xmit(cryp, sha, sg_dma_address(ctx->sg), - len, ctx->dma_addr, count); - } - } - - if (!dma_map_sg(cryp->dev, ctx->sg, 1, DMA_TO_DEVICE)) { - dev_err(cryp->dev, "dma_map_sg error\n"); - return -EINVAL; - } - - ctx->flags |= SHA_FLAGS_SG; - - return mtk_sha_xmit(cryp, sha, sg_dma_address(ctx->sg), - len, 0, 0); -} - -static int mtk_sha_final_req(struct mtk_cryp *cryp, - struct mtk_sha_rec *sha) -{ - struct mtk_sha_reqctx *ctx = ahash_request_ctx(sha->req); - size_t count; - - mtk_sha_fill_padding(ctx, 0); - - sha->flags |= SHA_FLAGS_FINAL; - count = ctx->bufcnt; - ctx->bufcnt = 0; - - return mtk_sha_dma_map(cryp, sha, ctx, count); -} - -/* Copy ready hash (+ finalize hmac) */ -static int mtk_sha_finish(struct ahash_request *req) -{ - struct mtk_sha_reqctx *ctx = ahash_request_ctx(req); - __le32 *digest = ctx->info.digest; - u32 *result = (u32 *)req->result; - int i; - - /* Get the hash from the digest buffer */ - for (i = 0; i < SIZE_IN_WORDS(ctx->ds); i++) - result[i] = le32_to_cpu(digest[i]); - - if (ctx->flags & SHA_FLAGS_HMAC) - return mtk_sha_finish_hmac(req); - - return 0; -} - -static void mtk_sha_finish_req(struct mtk_cryp *cryp, - struct mtk_sha_rec *sha, - int err) -{ - if (likely(!err && (SHA_FLAGS_FINAL & sha->flags))) - err = mtk_sha_finish(sha->req); - - sha->flags &= ~(SHA_FLAGS_BUSY | SHA_FLAGS_FINAL); - - sha->req->base.complete(&sha->req->base, err); - - /* Handle new request */ - tasklet_schedule(&sha->queue_task); -} - -static int mtk_sha_handle_queue(struct mtk_cryp *cryp, u8 id, - struct ahash_request *req) -{ - struct mtk_sha_rec *sha = cryp->sha[id]; - struct crypto_async_request *async_req, *backlog; - struct mtk_sha_reqctx *ctx; - unsigned long flags; - int err = 0, ret = 0; - - spin_lock_irqsave(&sha->lock, flags); - if (req) - ret = ahash_enqueue_request(&sha->queue, req); - - if (SHA_FLAGS_BUSY & sha->flags) { - spin_unlock_irqrestore(&sha->lock, flags); - return ret; - } - - backlog = crypto_get_backlog(&sha->queue); - async_req = crypto_dequeue_request(&sha->queue); - if (async_req) - sha->flags |= SHA_FLAGS_BUSY; - spin_unlock_irqrestore(&sha->lock, flags); - - if (!async_req) - return ret; - - if (backlog) - backlog->complete(backlog, -EINPROGRESS); - - req = ahash_request_cast(async_req); - ctx = ahash_request_ctx(req); - - sha->req = req; - - mtk_sha_info_init(ctx); - - if (ctx->op == SHA_OP_UPDATE) { - err = mtk_sha_update_start(cryp, sha); - if (err != -EINPROGRESS && (ctx->flags & SHA_FLAGS_FINUP)) - /* No final() after finup() */ - err = mtk_sha_final_req(cryp, sha); - } else if (ctx->op == SHA_OP_FINAL) { - err = mtk_sha_final_req(cryp, sha); - } - - if (unlikely(err != -EINPROGRESS)) - /* Task will not finish it, so do it here */ - mtk_sha_finish_req(cryp, sha, err); - - return ret; -} - -static int mtk_sha_enqueue(struct ahash_request *req, u32 op) -{ - struct mtk_sha_reqctx *ctx = ahash_request_ctx(req); - struct mtk_sha_ctx *tctx = crypto_tfm_ctx(req->base.tfm); - - ctx->op = op; - - return mtk_sha_handle_queue(tctx->cryp, tctx->id, req); -} - -static void mtk_sha_unmap(struct mtk_cryp *cryp, struct mtk_sha_rec *sha) -{ - struct mtk_sha_reqctx *ctx = ahash_request_ctx(sha->req); - - dma_unmap_single(cryp->dev, ctx->ct_dma, sizeof(ctx->info), - DMA_BIDIRECTIONAL); - - if (ctx->flags & SHA_FLAGS_SG) { - dma_unmap_sg(cryp->dev, ctx->sg, 1, DMA_TO_DEVICE); - if (ctx->sg->length == ctx->offset) { - ctx->sg = sg_next(ctx->sg); - if (ctx->sg) - ctx->offset = 0; - } - if (ctx->flags & SHA_FLAGS_PAD) { - dma_unmap_single(cryp->dev, ctx->dma_addr, - SHA_BUF_SIZE, DMA_TO_DEVICE); - } - } else - dma_unmap_single(cryp->dev, ctx->dma_addr, - SHA_BUF_SIZE, DMA_TO_DEVICE); -} - -static void mtk_sha_complete(struct mtk_cryp *cryp, - struct mtk_sha_rec *sha) -{ - int err = 0; - - err = mtk_sha_update_start(cryp, sha); - if (err != -EINPROGRESS) - mtk_sha_finish_req(cryp, sha, err); -} - -static int mtk_sha_update(struct ahash_request *req) -{ - struct mtk_sha_reqctx *ctx = ahash_request_ctx(req); - - ctx->total = req->nbytes; - ctx->sg = req->src; - ctx->offset = 0; - - if ((ctx->bufcnt + ctx->total < SHA_BUF_SIZE) && - !(ctx->flags & SHA_FLAGS_FINUP)) - return mtk_sha_append_sg(ctx); - - return mtk_sha_enqueue(req, SHA_OP_UPDATE); -} - -static int mtk_sha_final(struct ahash_request *req) -{ - struct mtk_sha_reqctx *ctx = ahash_request_ctx(req); - - ctx->flags |= SHA_FLAGS_FINUP; - - if (ctx->flags & SHA_FLAGS_PAD) - return mtk_sha_finish(req); - - return mtk_sha_enqueue(req, SHA_OP_FINAL); -} - -static int mtk_sha_finup(struct ahash_request *req) -{ - struct mtk_sha_reqctx *ctx = ahash_request_ctx(req); - int err1, err2; - - ctx->flags |= SHA_FLAGS_FINUP; - - err1 = mtk_sha_update(req); - if (err1 == -EINPROGRESS || - (err1 == -EBUSY && (ahash_request_flags(req) & - CRYPTO_TFM_REQ_MAY_BACKLOG))) - return err1; - /* - * final() has to be always called to cleanup resources - * even if update() failed - */ - err2 = mtk_sha_final(req); - - return err1 ?: err2; -} - -static int mtk_sha_digest(struct ahash_request *req) -{ - return mtk_sha_init(req) ?: mtk_sha_finup(req); -} - -static int mtk_sha_setkey(struct crypto_ahash *tfm, const u8 *key, - u32 keylen) -{ - struct mtk_sha_ctx *tctx = crypto_ahash_ctx(tfm); - struct mtk_sha_hmac_ctx *bctx = tctx->base; - size_t bs = crypto_shash_blocksize(bctx->shash); - size_t ds = crypto_shash_digestsize(bctx->shash); - int err, i; - - if (keylen > bs) { - err = crypto_shash_tfm_digest(bctx->shash, key, keylen, - bctx->ipad); - if (err) - return err; - keylen = ds; - } else { - memcpy(bctx->ipad, key, keylen); - } - - memset(bctx->ipad + keylen, 0, bs - keylen); - memcpy(bctx->opad, bctx->ipad, bs); - - for (i = 0; i < bs; i++) { - bctx->ipad[i] ^= HMAC_IPAD_VALUE; - bctx->opad[i] ^= HMAC_OPAD_VALUE; - } - - return 0; -} - -static int mtk_sha_export(struct ahash_request *req, void *out) -{ - const struct mtk_sha_reqctx *ctx = ahash_request_ctx(req); - - memcpy(out, ctx, sizeof(*ctx)); - return 0; -} - -static int mtk_sha_import(struct ahash_request *req, const void *in) -{ - struct mtk_sha_reqctx *ctx = ahash_request_ctx(req); - - memcpy(ctx, in, sizeof(*ctx)); - return 0; -} - -static int mtk_sha_cra_init_alg(struct crypto_tfm *tfm, - const char *alg_base) -{ - struct mtk_sha_ctx *tctx = crypto_tfm_ctx(tfm); - struct mtk_cryp *cryp = NULL; - - cryp = mtk_sha_find_dev(tctx); - if (!cryp) - return -ENODEV; - - crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), - sizeof(struct mtk_sha_reqctx)); - - if (alg_base) { - struct mtk_sha_hmac_ctx *bctx = tctx->base; - - tctx->flags |= SHA_FLAGS_HMAC; - bctx->shash = crypto_alloc_shash(alg_base, 0, - CRYPTO_ALG_NEED_FALLBACK); - if (IS_ERR(bctx->shash)) { - pr_err("base driver %s could not be loaded.\n", - alg_base); - - return PTR_ERR(bctx->shash); - } - } - return 0; -} - -static int mtk_sha_cra_init(struct crypto_tfm *tfm) -{ - return mtk_sha_cra_init_alg(tfm, NULL); -} - -static int mtk_sha_cra_sha1_init(struct crypto_tfm *tfm) -{ - return mtk_sha_cra_init_alg(tfm, "sha1"); -} - -static int mtk_sha_cra_sha224_init(struct crypto_tfm *tfm) -{ - return mtk_sha_cra_init_alg(tfm, "sha224"); -} - -static int mtk_sha_cra_sha256_init(struct crypto_tfm *tfm) -{ - return mtk_sha_cra_init_alg(tfm, "sha256"); -} - -static int mtk_sha_cra_sha384_init(struct crypto_tfm *tfm) -{ - return mtk_sha_cra_init_alg(tfm, "sha384"); -} - -static int mtk_sha_cra_sha512_init(struct crypto_tfm *tfm) -{ - return mtk_sha_cra_init_alg(tfm, "sha512"); -} - -static void mtk_sha_cra_exit(struct crypto_tfm *tfm) -{ - struct mtk_sha_ctx *tctx = crypto_tfm_ctx(tfm); - - if (tctx->flags & SHA_FLAGS_HMAC) { - struct mtk_sha_hmac_ctx *bctx = tctx->base; - - crypto_free_shash(bctx->shash); - } -} - -static struct ahash_alg algs_sha1_sha224_sha256[] = { -{ - .init = mtk_sha_init, - .update = mtk_sha_update, - .final = mtk_sha_final, - .finup = mtk_sha_finup, - .digest = mtk_sha_digest, - .export = mtk_sha_export, - .import = mtk_sha_import, - .halg.digestsize = SHA1_DIGEST_SIZE, - .halg.statesize = sizeof(struct mtk_sha_reqctx), - .halg.base = { - .cra_name = "sha1", - .cra_driver_name = "mtk-sha1", - .cra_priority = 400, - .cra_flags = CRYPTO_ALG_ASYNC, - .cra_blocksize = SHA1_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct mtk_sha_ctx), - .cra_alignmask = SHA_ALIGN_MSK, - .cra_module = THIS_MODULE, - .cra_init = mtk_sha_cra_init, - .cra_exit = mtk_sha_cra_exit, - } -}, -{ - .init = mtk_sha_init, - .update = mtk_sha_update, - .final = mtk_sha_final, - .finup = mtk_sha_finup, - .digest = mtk_sha_digest, - .export = mtk_sha_export, - .import = mtk_sha_import, - .halg.digestsize = SHA224_DIGEST_SIZE, - .halg.statesize = sizeof(struct mtk_sha_reqctx), - .halg.base = { - .cra_name = "sha224", - .cra_driver_name = "mtk-sha224", - .cra_priority = 400, - .cra_flags = CRYPTO_ALG_ASYNC, - .cra_blocksize = SHA224_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct mtk_sha_ctx), - .cra_alignmask = SHA_ALIGN_MSK, - .cra_module = THIS_MODULE, - .cra_init = mtk_sha_cra_init, - .cra_exit = mtk_sha_cra_exit, - } -}, -{ - .init = mtk_sha_init, - .update = mtk_sha_update, - .final = mtk_sha_final, - .finup = mtk_sha_finup, - .digest = mtk_sha_digest, - .export = mtk_sha_export, - .import = mtk_sha_import, - .halg.digestsize = SHA256_DIGEST_SIZE, - .halg.statesize = sizeof(struct mtk_sha_reqctx), - .halg.base = { - .cra_name = "sha256", - .cra_driver_name = "mtk-sha256", - .cra_priority = 400, - .cra_flags = CRYPTO_ALG_ASYNC, - .cra_blocksize = SHA256_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct mtk_sha_ctx), - .cra_alignmask = SHA_ALIGN_MSK, - .cra_module = THIS_MODULE, - .cra_init = mtk_sha_cra_init, - .cra_exit = mtk_sha_cra_exit, - } -}, -{ - .init = mtk_sha_init, - .update = mtk_sha_update, - .final = mtk_sha_final, - .finup = mtk_sha_finup, - .digest = mtk_sha_digest, - .export = mtk_sha_export, - .import = mtk_sha_import, - .setkey = mtk_sha_setkey, - .halg.digestsize = SHA1_DIGEST_SIZE, - .halg.statesize = sizeof(struct mtk_sha_reqctx), - .halg.base = { - .cra_name = "hmac(sha1)", - .cra_driver_name = "mtk-hmac-sha1", - .cra_priority = 400, - .cra_flags = CRYPTO_ALG_ASYNC | - CRYPTO_ALG_NEED_FALLBACK, - .cra_blocksize = SHA1_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct mtk_sha_ctx) + - sizeof(struct mtk_sha_hmac_ctx), - .cra_alignmask = SHA_ALIGN_MSK, - .cra_module = THIS_MODULE, - .cra_init = mtk_sha_cra_sha1_init, - .cra_exit = mtk_sha_cra_exit, - } -}, -{ - .init = mtk_sha_init, - .update = mtk_sha_update, - .final = mtk_sha_final, - .finup = mtk_sha_finup, - .digest = mtk_sha_digest, - .export = mtk_sha_export, - .import = mtk_sha_import, - .setkey = mtk_sha_setkey, - .halg.digestsize = SHA224_DIGEST_SIZE, - .halg.statesize = sizeof(struct mtk_sha_reqctx), - .halg.base = { - .cra_name = "hmac(sha224)", - .cra_driver_name = "mtk-hmac-sha224", - .cra_priority = 400, - .cra_flags = CRYPTO_ALG_ASYNC | - CRYPTO_ALG_NEED_FALLBACK, - .cra_blocksize = SHA224_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct mtk_sha_ctx) + - sizeof(struct mtk_sha_hmac_ctx), - .cra_alignmask = SHA_ALIGN_MSK, - .cra_module = THIS_MODULE, - .cra_init = mtk_sha_cra_sha224_init, - .cra_exit = mtk_sha_cra_exit, - } -}, -{ - .init = mtk_sha_init, - .update = mtk_sha_update, - .final = mtk_sha_final, - .finup = mtk_sha_finup, - .digest = mtk_sha_digest, - .export = mtk_sha_export, - .import = mtk_sha_import, - .setkey = mtk_sha_setkey, - .halg.digestsize = SHA256_DIGEST_SIZE, - .halg.statesize = sizeof(struct mtk_sha_reqctx), - .halg.base = { - .cra_name = "hmac(sha256)", - .cra_driver_name = "mtk-hmac-sha256", - .cra_priority = 400, - .cra_flags = CRYPTO_ALG_ASYNC | - CRYPTO_ALG_NEED_FALLBACK, - .cra_blocksize = SHA256_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct mtk_sha_ctx) + - sizeof(struct mtk_sha_hmac_ctx), - .cra_alignmask = SHA_ALIGN_MSK, - .cra_module = THIS_MODULE, - .cra_init = mtk_sha_cra_sha256_init, - .cra_exit = mtk_sha_cra_exit, - } -}, -}; - -static struct ahash_alg algs_sha384_sha512[] = { -{ - .init = mtk_sha_init, - .update = mtk_sha_update, - .final = mtk_sha_final, - .finup = mtk_sha_finup, - .digest = mtk_sha_digest, - .export = mtk_sha_export, - .import = mtk_sha_import, - .halg.digestsize = SHA384_DIGEST_SIZE, - .halg.statesize = sizeof(struct mtk_sha_reqctx), - .halg.base = { - .cra_name = "sha384", - .cra_driver_name = "mtk-sha384", - .cra_priority = 400, - .cra_flags = CRYPTO_ALG_ASYNC, - .cra_blocksize = SHA384_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct mtk_sha_ctx), - .cra_alignmask = SHA_ALIGN_MSK, - .cra_module = THIS_MODULE, - .cra_init = mtk_sha_cra_init, - .cra_exit = mtk_sha_cra_exit, - } -}, -{ - .init = mtk_sha_init, - .update = mtk_sha_update, - .final = mtk_sha_final, - .finup = mtk_sha_finup, - .digest = mtk_sha_digest, - .export = mtk_sha_export, - .import = mtk_sha_import, - .halg.digestsize = SHA512_DIGEST_SIZE, - .halg.statesize = sizeof(struct mtk_sha_reqctx), - .halg.base = { - .cra_name = "sha512", - .cra_driver_name = "mtk-sha512", - .cra_priority = 400, - .cra_flags = CRYPTO_ALG_ASYNC, - .cra_blocksize = SHA512_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct mtk_sha_ctx), - .cra_alignmask = SHA_ALIGN_MSK, - .cra_module = THIS_MODULE, - .cra_init = mtk_sha_cra_init, - .cra_exit = mtk_sha_cra_exit, - } -}, -{ - .init = mtk_sha_init, - .update = mtk_sha_update, - .final = mtk_sha_final, - .finup = mtk_sha_finup, - .digest = mtk_sha_digest, - .export = mtk_sha_export, - .import = mtk_sha_import, - .setkey = mtk_sha_setkey, - .halg.digestsize = SHA384_DIGEST_SIZE, - .halg.statesize = sizeof(struct mtk_sha_reqctx), - .halg.base = { - .cra_name = "hmac(sha384)", - .cra_driver_name = "mtk-hmac-sha384", - .cra_priority = 400, - .cra_flags = CRYPTO_ALG_ASYNC | - CRYPTO_ALG_NEED_FALLBACK, - .cra_blocksize = SHA384_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct mtk_sha_ctx) + - sizeof(struct mtk_sha_hmac_ctx), - .cra_alignmask = SHA_ALIGN_MSK, - .cra_module = THIS_MODULE, - .cra_init = mtk_sha_cra_sha384_init, - .cra_exit = mtk_sha_cra_exit, - } -}, -{ - .init = mtk_sha_init, - .update = mtk_sha_update, - .final = mtk_sha_final, - .finup = mtk_sha_finup, - .digest = mtk_sha_digest, - .export = mtk_sha_export, - .import = mtk_sha_import, - .setkey = mtk_sha_setkey, - .halg.digestsize = SHA512_DIGEST_SIZE, - .halg.statesize = sizeof(struct mtk_sha_reqctx), - .halg.base = { - .cra_name = "hmac(sha512)", - .cra_driver_name = "mtk-hmac-sha512", - .cra_priority = 400, - .cra_flags = CRYPTO_ALG_ASYNC | - CRYPTO_ALG_NEED_FALLBACK, - .cra_blocksize = SHA512_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct mtk_sha_ctx) + - sizeof(struct mtk_sha_hmac_ctx), - .cra_alignmask = SHA_ALIGN_MSK, - .cra_module = THIS_MODULE, - .cra_init = mtk_sha_cra_sha512_init, - .cra_exit = mtk_sha_cra_exit, - } -}, -}; - -static void mtk_sha_queue_task(unsigned long data) -{ - struct mtk_sha_rec *sha = (struct mtk_sha_rec *)data; - - mtk_sha_handle_queue(sha->cryp, sha->id - MTK_RING2, NULL); -} - -static void mtk_sha_done_task(unsigned long data) -{ - struct mtk_sha_rec *sha = (struct mtk_sha_rec *)data; - struct mtk_cryp *cryp = sha->cryp; - - mtk_sha_unmap(cryp, sha); - mtk_sha_complete(cryp, sha); -} - -static irqreturn_t mtk_sha_irq(int irq, void *dev_id) -{ - struct mtk_sha_rec *sha = (struct mtk_sha_rec *)dev_id; - struct mtk_cryp *cryp = sha->cryp; - u32 val = mtk_sha_read(cryp, RDR_STAT(sha->id)); - - mtk_sha_write(cryp, RDR_STAT(sha->id), val); - - if (likely((SHA_FLAGS_BUSY & sha->flags))) { - mtk_sha_write(cryp, RDR_PROC_COUNT(sha->id), MTK_CNT_RST); - mtk_sha_write(cryp, RDR_THRESH(sha->id), - MTK_RDR_PROC_THRESH | MTK_RDR_PROC_MODE); - - tasklet_schedule(&sha->done_task); - } else { - dev_warn(cryp->dev, "SHA interrupt when no active requests.\n"); - } - return IRQ_HANDLED; -} - -/* - * The purpose of two SHA records is used to get extra performance. - * It is similar to mtk_aes_record_init(). - */ -static int mtk_sha_record_init(struct mtk_cryp *cryp) -{ - struct mtk_sha_rec **sha = cryp->sha; - int i, err = -ENOMEM; - - for (i = 0; i < MTK_REC_NUM; i++) { - sha[i] = kzalloc(sizeof(**sha), GFP_KERNEL); - if (!sha[i]) - goto err_cleanup; - - sha[i]->cryp = cryp; - - spin_lock_init(&sha[i]->lock); - crypto_init_queue(&sha[i]->queue, SHA_QUEUE_SIZE); - - tasklet_init(&sha[i]->queue_task, mtk_sha_queue_task, - (unsigned long)sha[i]); - tasklet_init(&sha[i]->done_task, mtk_sha_done_task, - (unsigned long)sha[i]); - } - - /* Link to ring2 and ring3 respectively */ - sha[0]->id = MTK_RING2; - sha[1]->id = MTK_RING3; - - cryp->rec = 1; - - return 0; - -err_cleanup: - for (; i--; ) - kfree(sha[i]); - return err; -} - -static void mtk_sha_record_free(struct mtk_cryp *cryp) -{ - int i; - - for (i = 0; i < MTK_REC_NUM; i++) { - tasklet_kill(&cryp->sha[i]->done_task); - tasklet_kill(&cryp->sha[i]->queue_task); - - kfree(cryp->sha[i]); - } -} - -static void mtk_sha_unregister_algs(void) -{ - int i; - - for (i = 0; i < ARRAY_SIZE(algs_sha1_sha224_sha256); i++) - crypto_unregister_ahash(&algs_sha1_sha224_sha256[i]); - - for (i = 0; i < ARRAY_SIZE(algs_sha384_sha512); i++) - crypto_unregister_ahash(&algs_sha384_sha512[i]); -} - -static int mtk_sha_register_algs(void) -{ - int err, i; - - for (i = 0; i < ARRAY_SIZE(algs_sha1_sha224_sha256); i++) { - err = crypto_register_ahash(&algs_sha1_sha224_sha256[i]); - if (err) - goto err_sha_224_256_algs; - } - - for (i = 0; i < ARRAY_SIZE(algs_sha384_sha512); i++) { - err = crypto_register_ahash(&algs_sha384_sha512[i]); - if (err) - goto err_sha_384_512_algs; - } - - return 0; - -err_sha_384_512_algs: - for (; i--; ) - crypto_unregister_ahash(&algs_sha384_sha512[i]); - i = ARRAY_SIZE(algs_sha1_sha224_sha256); -err_sha_224_256_algs: - for (; i--; ) - crypto_unregister_ahash(&algs_sha1_sha224_sha256[i]); - - return err; -} - -int mtk_hash_alg_register(struct mtk_cryp *cryp) -{ - int err; - - INIT_LIST_HEAD(&cryp->sha_list); - - /* Initialize two hash records */ - err = mtk_sha_record_init(cryp); - if (err) - goto err_record; - - err = devm_request_irq(cryp->dev, cryp->irq[MTK_RING2], mtk_sha_irq, - 0, "mtk-sha", cryp->sha[0]); - if (err) { - dev_err(cryp->dev, "unable to request sha irq0.\n"); - goto err_res; - } - - err = devm_request_irq(cryp->dev, cryp->irq[MTK_RING3], mtk_sha_irq, - 0, "mtk-sha", cryp->sha[1]); - if (err) { - dev_err(cryp->dev, "unable to request sha irq1.\n"); - goto err_res; - } - - /* Enable ring2 and ring3 interrupt for hash */ - mtk_sha_write(cryp, AIC_ENABLE_SET(MTK_RING2), MTK_IRQ_RDR2); - mtk_sha_write(cryp, AIC_ENABLE_SET(MTK_RING3), MTK_IRQ_RDR3); - - spin_lock(&mtk_sha.lock); - list_add_tail(&cryp->sha_list, &mtk_sha.dev_list); - spin_unlock(&mtk_sha.lock); - - err = mtk_sha_register_algs(); - if (err) - goto err_algs; - - return 0; - -err_algs: - spin_lock(&mtk_sha.lock); - list_del(&cryp->sha_list); - spin_unlock(&mtk_sha.lock); -err_res: - mtk_sha_record_free(cryp); -err_record: - - dev_err(cryp->dev, "mtk-sha initialization failed.\n"); - return err; -} - -void mtk_hash_alg_release(struct mtk_cryp *cryp) -{ - spin_lock(&mtk_sha.lock); - list_del(&cryp->sha_list); - spin_unlock(&mtk_sha.lock); - - mtk_sha_unregister_algs(); - mtk_sha_record_free(cryp); -} diff --git a/drivers/crypto/picoxcell_crypto.c b/drivers/crypto/picoxcell_crypto.c deleted file mode 100644 index 84f9c16d984cc312c36ee1b5c436baef9581a288..0000000000000000000000000000000000000000 --- a/drivers/crypto/picoxcell_crypto.c +++ /dev/null @@ -1,1807 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Copyright (c) 2010-2011 Picochip Ltd., Jamie Iles - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "picoxcell_crypto_regs.h" - -/* - * The threshold for the number of entries in the CMD FIFO available before - * the CMD0_CNT interrupt is raised. Increasing this value will reduce the - * number of interrupts raised to the CPU. - */ -#define CMD0_IRQ_THRESHOLD 1 - -/* - * The timeout period (in jiffies) for a PDU. When the the number of PDUs in - * flight is greater than the STAT_IRQ_THRESHOLD or 0 the timer is disabled. - * When there are packets in flight but lower than the threshold, we enable - * the timer and at expiry, attempt to remove any processed packets from the - * queue and if there are still packets left, schedule the timer again. - */ -#define PACKET_TIMEOUT 1 - -/* The priority to register each algorithm with. */ -#define SPACC_CRYPTO_ALG_PRIORITY 10000 - -#define SPACC_CRYPTO_KASUMI_F8_KEY_LEN 16 -#define SPACC_CRYPTO_IPSEC_CIPHER_PG_SZ 64 -#define SPACC_CRYPTO_IPSEC_HASH_PG_SZ 64 -#define SPACC_CRYPTO_IPSEC_MAX_CTXS 32 -#define SPACC_CRYPTO_IPSEC_FIFO_SZ 32 -#define SPACC_CRYPTO_L2_CIPHER_PG_SZ 64 -#define SPACC_CRYPTO_L2_HASH_PG_SZ 64 -#define SPACC_CRYPTO_L2_MAX_CTXS 128 -#define SPACC_CRYPTO_L2_FIFO_SZ 128 - -#define MAX_DDT_LEN 16 - -/* DDT format. This must match the hardware DDT format exactly. */ -struct spacc_ddt { - dma_addr_t p; - u32 len; -}; - -/* - * Asynchronous crypto request structure. - * - * This structure defines a request that is either queued for processing or - * being processed. - */ -struct spacc_req { - struct list_head list; - struct spacc_engine *engine; - struct crypto_async_request *req; - int result; - bool is_encrypt; - unsigned ctx_id; - dma_addr_t src_addr, dst_addr; - struct spacc_ddt *src_ddt, *dst_ddt; - void (*complete)(struct spacc_req *req); - struct skcipher_request fallback_req; // keep at the end -}; - -struct spacc_aead { - unsigned long ctrl_default; - unsigned long type; - struct aead_alg alg; - struct spacc_engine *engine; - struct list_head entry; - int key_offs; - int iv_offs; -}; - -struct spacc_engine { - void __iomem *regs; - struct list_head pending; - int next_ctx; - spinlock_t hw_lock; - int in_flight; - struct list_head completed; - struct list_head in_progress; - struct tasklet_struct complete; - unsigned long fifo_sz; - void __iomem *cipher_ctx_base; - void __iomem *hash_key_base; - struct spacc_alg *algs; - unsigned num_algs; - struct list_head registered_algs; - struct spacc_aead *aeads; - unsigned num_aeads; - struct list_head registered_aeads; - size_t cipher_pg_sz; - size_t hash_pg_sz; - const char *name; - struct clk *clk; - struct device *dev; - unsigned max_ctxs; - struct timer_list packet_timeout; - unsigned stat_irq_thresh; - struct dma_pool *req_pool; -}; - -/* Algorithm type mask. */ -#define SPACC_CRYPTO_ALG_MASK 0x7 - -/* SPACC definition of a crypto algorithm. */ -struct spacc_alg { - unsigned long ctrl_default; - unsigned long type; - struct skcipher_alg alg; - struct spacc_engine *engine; - struct list_head entry; - int key_offs; - int iv_offs; -}; - -/* Generic context structure for any algorithm type. */ -struct spacc_generic_ctx { - struct spacc_engine *engine; - int flags; - int key_offs; - int iv_offs; -}; - -/* Block cipher context. */ -struct spacc_ablk_ctx { - struct spacc_generic_ctx generic; - u8 key[AES_MAX_KEY_SIZE]; - u8 key_len; - /* - * The fallback cipher. If the operation can't be done in hardware, - * fallback to a software version. - */ - struct crypto_skcipher *sw_cipher; -}; - -/* AEAD cipher context. */ -struct spacc_aead_ctx { - struct spacc_generic_ctx generic; - u8 cipher_key[AES_MAX_KEY_SIZE]; - u8 hash_ctx[SPACC_CRYPTO_IPSEC_HASH_PG_SZ]; - u8 cipher_key_len; - u8 hash_key_len; - struct crypto_aead *sw_cipher; -}; - -static int spacc_ablk_submit(struct spacc_req *req); - -static inline struct spacc_alg *to_spacc_skcipher(struct skcipher_alg *alg) -{ - return alg ? container_of(alg, struct spacc_alg, alg) : NULL; -} - -static inline struct spacc_aead *to_spacc_aead(struct aead_alg *alg) -{ - return container_of(alg, struct spacc_aead, alg); -} - -static inline int spacc_fifo_cmd_full(struct spacc_engine *engine) -{ - u32 fifo_stat = readl(engine->regs + SPA_FIFO_STAT_REG_OFFSET); - - return fifo_stat & SPA_FIFO_CMD_FULL; -} - -/* - * Given a cipher context, and a context number, get the base address of the - * context page. - * - * Returns the address of the context page where the key/context may - * be written. - */ -static inline void __iomem *spacc_ctx_page_addr(struct spacc_generic_ctx *ctx, - unsigned indx, - bool is_cipher_ctx) -{ - return is_cipher_ctx ? ctx->engine->cipher_ctx_base + - (indx * ctx->engine->cipher_pg_sz) : - ctx->engine->hash_key_base + (indx * ctx->engine->hash_pg_sz); -} - -/* The context pages can only be written with 32-bit accesses. */ -static inline void memcpy_toio32(u32 __iomem *dst, const void *src, - unsigned count) -{ - const u32 *src32 = (const u32 *) src; - - while (count--) - writel(*src32++, dst++); -} - -static void spacc_cipher_write_ctx(struct spacc_generic_ctx *ctx, - void __iomem *page_addr, const u8 *key, - size_t key_len, const u8 *iv, size_t iv_len) -{ - void __iomem *key_ptr = page_addr + ctx->key_offs; - void __iomem *iv_ptr = page_addr + ctx->iv_offs; - - memcpy_toio32(key_ptr, key, key_len / 4); - memcpy_toio32(iv_ptr, iv, iv_len / 4); -} - -/* - * Load a context into the engines context memory. - * - * Returns the index of the context page where the context was loaded. - */ -static unsigned spacc_load_ctx(struct spacc_generic_ctx *ctx, - const u8 *ciph_key, size_t ciph_len, - const u8 *iv, size_t ivlen, const u8 *hash_key, - size_t hash_len) -{ - unsigned indx = ctx->engine->next_ctx++; - void __iomem *ciph_page_addr, *hash_page_addr; - - ciph_page_addr = spacc_ctx_page_addr(ctx, indx, 1); - hash_page_addr = spacc_ctx_page_addr(ctx, indx, 0); - - ctx->engine->next_ctx &= ctx->engine->fifo_sz - 1; - spacc_cipher_write_ctx(ctx, ciph_page_addr, ciph_key, ciph_len, iv, - ivlen); - writel(ciph_len | (indx << SPA_KEY_SZ_CTX_INDEX_OFFSET) | - (1 << SPA_KEY_SZ_CIPHER_OFFSET), - ctx->engine->regs + SPA_KEY_SZ_REG_OFFSET); - - if (hash_key) { - memcpy_toio32(hash_page_addr, hash_key, hash_len / 4); - writel(hash_len | (indx << SPA_KEY_SZ_CTX_INDEX_OFFSET), - ctx->engine->regs + SPA_KEY_SZ_REG_OFFSET); - } - - return indx; -} - -static inline void ddt_set(struct spacc_ddt *ddt, dma_addr_t phys, size_t len) -{ - ddt->p = phys; - ddt->len = len; -} - -/* - * Take a crypto request and scatterlists for the data and turn them into DDTs - * for passing to the crypto engines. This also DMA maps the data so that the - * crypto engines can DMA to/from them. - */ -static struct spacc_ddt *spacc_sg_to_ddt(struct spacc_engine *engine, - struct scatterlist *payload, - unsigned nbytes, - enum dma_data_direction dir, - dma_addr_t *ddt_phys) -{ - unsigned mapped_ents; - struct scatterlist *cur; - struct spacc_ddt *ddt; - int i; - int nents; - - nents = sg_nents_for_len(payload, nbytes); - if (nents < 0) { - dev_err(engine->dev, "Invalid numbers of SG.\n"); - return NULL; - } - mapped_ents = dma_map_sg(engine->dev, payload, nents, dir); - - if (mapped_ents + 1 > MAX_DDT_LEN) - goto out; - - ddt = dma_pool_alloc(engine->req_pool, GFP_ATOMIC, ddt_phys); - if (!ddt) - goto out; - - for_each_sg(payload, cur, mapped_ents, i) - ddt_set(&ddt[i], sg_dma_address(cur), sg_dma_len(cur)); - ddt_set(&ddt[mapped_ents], 0, 0); - - return ddt; - -out: - dma_unmap_sg(engine->dev, payload, nents, dir); - return NULL; -} - -static int spacc_aead_make_ddts(struct aead_request *areq) -{ - struct crypto_aead *aead = crypto_aead_reqtfm(areq); - struct spacc_req *req = aead_request_ctx(areq); - struct spacc_engine *engine = req->engine; - struct spacc_ddt *src_ddt, *dst_ddt; - unsigned total; - int src_nents, dst_nents; - struct scatterlist *cur; - int i, dst_ents, src_ents; - - total = areq->assoclen + areq->cryptlen; - if (req->is_encrypt) - total += crypto_aead_authsize(aead); - - src_nents = sg_nents_for_len(areq->src, total); - if (src_nents < 0) { - dev_err(engine->dev, "Invalid numbers of src SG.\n"); - return src_nents; - } - if (src_nents + 1 > MAX_DDT_LEN) - return -E2BIG; - - dst_nents = 0; - if (areq->src != areq->dst) { - dst_nents = sg_nents_for_len(areq->dst, total); - if (dst_nents < 0) { - dev_err(engine->dev, "Invalid numbers of dst SG.\n"); - return dst_nents; - } - if (src_nents + 1 > MAX_DDT_LEN) - return -E2BIG; - } - - src_ddt = dma_pool_alloc(engine->req_pool, GFP_ATOMIC, &req->src_addr); - if (!src_ddt) - goto err; - - dst_ddt = dma_pool_alloc(engine->req_pool, GFP_ATOMIC, &req->dst_addr); - if (!dst_ddt) - goto err_free_src; - - req->src_ddt = src_ddt; - req->dst_ddt = dst_ddt; - - if (dst_nents) { - src_ents = dma_map_sg(engine->dev, areq->src, src_nents, - DMA_TO_DEVICE); - if (!src_ents) - goto err_free_dst; - - dst_ents = dma_map_sg(engine->dev, areq->dst, dst_nents, - DMA_FROM_DEVICE); - - if (!dst_ents) { - dma_unmap_sg(engine->dev, areq->src, src_nents, - DMA_TO_DEVICE); - goto err_free_dst; - } - } else { - src_ents = dma_map_sg(engine->dev, areq->src, src_nents, - DMA_BIDIRECTIONAL); - if (!src_ents) - goto err_free_dst; - dst_ents = src_ents; - } - - /* - * Now map in the payload for the source and destination and terminate - * with the NULL pointers. - */ - for_each_sg(areq->src, cur, src_ents, i) - ddt_set(src_ddt++, sg_dma_address(cur), sg_dma_len(cur)); - - /* For decryption we need to skip the associated data. */ - total = req->is_encrypt ? 0 : areq->assoclen; - for_each_sg(areq->dst, cur, dst_ents, i) { - unsigned len = sg_dma_len(cur); - - if (len <= total) { - total -= len; - continue; - } - - ddt_set(dst_ddt++, sg_dma_address(cur) + total, len - total); - } - - ddt_set(src_ddt, 0, 0); - ddt_set(dst_ddt, 0, 0); - - return 0; - -err_free_dst: - dma_pool_free(engine->req_pool, dst_ddt, req->dst_addr); -err_free_src: - dma_pool_free(engine->req_pool, src_ddt, req->src_addr); -err: - return -ENOMEM; -} - -static void spacc_aead_free_ddts(struct spacc_req *req) -{ - struct aead_request *areq = container_of(req->req, struct aead_request, - base); - struct crypto_aead *aead = crypto_aead_reqtfm(areq); - unsigned total = areq->assoclen + areq->cryptlen + - (req->is_encrypt ? crypto_aead_authsize(aead) : 0); - struct spacc_aead_ctx *aead_ctx = crypto_aead_ctx(aead); - struct spacc_engine *engine = aead_ctx->generic.engine; - int nents = sg_nents_for_len(areq->src, total); - - /* sg_nents_for_len should not fail since it works when mapping sg */ - if (unlikely(nents < 0)) { - dev_err(engine->dev, "Invalid numbers of src SG.\n"); - return; - } - - if (areq->src != areq->dst) { - dma_unmap_sg(engine->dev, areq->src, nents, DMA_TO_DEVICE); - nents = sg_nents_for_len(areq->dst, total); - if (unlikely(nents < 0)) { - dev_err(engine->dev, "Invalid numbers of dst SG.\n"); - return; - } - dma_unmap_sg(engine->dev, areq->dst, nents, DMA_FROM_DEVICE); - } else - dma_unmap_sg(engine->dev, areq->src, nents, DMA_BIDIRECTIONAL); - - dma_pool_free(engine->req_pool, req->src_ddt, req->src_addr); - dma_pool_free(engine->req_pool, req->dst_ddt, req->dst_addr); -} - -static void spacc_free_ddt(struct spacc_req *req, struct spacc_ddt *ddt, - dma_addr_t ddt_addr, struct scatterlist *payload, - unsigned nbytes, enum dma_data_direction dir) -{ - int nents = sg_nents_for_len(payload, nbytes); - - if (nents < 0) { - dev_err(req->engine->dev, "Invalid numbers of SG.\n"); - return; - } - - dma_unmap_sg(req->engine->dev, payload, nents, dir); - dma_pool_free(req->engine->req_pool, ddt, ddt_addr); -} - -static int spacc_aead_setkey(struct crypto_aead *tfm, const u8 *key, - unsigned int keylen) -{ - struct spacc_aead_ctx *ctx = crypto_aead_ctx(tfm); - struct crypto_authenc_keys keys; - int err; - - crypto_aead_clear_flags(ctx->sw_cipher, CRYPTO_TFM_REQ_MASK); - crypto_aead_set_flags(ctx->sw_cipher, crypto_aead_get_flags(tfm) & - CRYPTO_TFM_REQ_MASK); - err = crypto_aead_setkey(ctx->sw_cipher, key, keylen); - if (err) - return err; - - if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) - goto badkey; - - if (keys.enckeylen > AES_MAX_KEY_SIZE) - goto badkey; - - if (keys.authkeylen > sizeof(ctx->hash_ctx)) - goto badkey; - - memcpy(ctx->cipher_key, keys.enckey, keys.enckeylen); - ctx->cipher_key_len = keys.enckeylen; - - memcpy(ctx->hash_ctx, keys.authkey, keys.authkeylen); - ctx->hash_key_len = keys.authkeylen; - - memzero_explicit(&keys, sizeof(keys)); - return 0; - -badkey: - memzero_explicit(&keys, sizeof(keys)); - return -EINVAL; -} - -static int spacc_aead_setauthsize(struct crypto_aead *tfm, - unsigned int authsize) -{ - struct spacc_aead_ctx *ctx = crypto_tfm_ctx(crypto_aead_tfm(tfm)); - - return crypto_aead_setauthsize(ctx->sw_cipher, authsize); -} - -/* - * Check if an AEAD request requires a fallback operation. Some requests can't - * be completed in hardware because the hardware may not support certain key - * sizes. In these cases we need to complete the request in software. - */ -static int spacc_aead_need_fallback(struct aead_request *aead_req) -{ - struct crypto_aead *aead = crypto_aead_reqtfm(aead_req); - struct aead_alg *alg = crypto_aead_alg(aead); - struct spacc_aead *spacc_alg = to_spacc_aead(alg); - struct spacc_aead_ctx *ctx = crypto_aead_ctx(aead); - - /* - * If we have a non-supported key-length, then we need to do a - * software fallback. - */ - if ((spacc_alg->ctrl_default & SPACC_CRYPTO_ALG_MASK) == - SPA_CTRL_CIPH_ALG_AES && - ctx->cipher_key_len != AES_KEYSIZE_128 && - ctx->cipher_key_len != AES_KEYSIZE_256) - return 1; - - return 0; -} - -static int spacc_aead_do_fallback(struct aead_request *req, unsigned alg_type, - bool is_encrypt) -{ - struct crypto_tfm *old_tfm = crypto_aead_tfm(crypto_aead_reqtfm(req)); - struct spacc_aead_ctx *ctx = crypto_tfm_ctx(old_tfm); - struct aead_request *subreq = aead_request_ctx(req); - - aead_request_set_tfm(subreq, ctx->sw_cipher); - aead_request_set_callback(subreq, req->base.flags, - req->base.complete, req->base.data); - aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen, - req->iv); - aead_request_set_ad(subreq, req->assoclen); - - return is_encrypt ? crypto_aead_encrypt(subreq) : - crypto_aead_decrypt(subreq); -} - -static void spacc_aead_complete(struct spacc_req *req) -{ - spacc_aead_free_ddts(req); - req->req->complete(req->req, req->result); -} - -static int spacc_aead_submit(struct spacc_req *req) -{ - struct aead_request *aead_req = - container_of(req->req, struct aead_request, base); - struct crypto_aead *aead = crypto_aead_reqtfm(aead_req); - unsigned int authsize = crypto_aead_authsize(aead); - struct spacc_aead_ctx *ctx = crypto_aead_ctx(aead); - struct aead_alg *alg = crypto_aead_alg(aead); - struct spacc_aead *spacc_alg = to_spacc_aead(alg); - struct spacc_engine *engine = ctx->generic.engine; - u32 ctrl, proc_len, assoc_len; - - req->result = -EINPROGRESS; - req->ctx_id = spacc_load_ctx(&ctx->generic, ctx->cipher_key, - ctx->cipher_key_len, aead_req->iv, crypto_aead_ivsize(aead), - ctx->hash_ctx, ctx->hash_key_len); - - /* Set the source and destination DDT pointers. */ - writel(req->src_addr, engine->regs + SPA_SRC_PTR_REG_OFFSET); - writel(req->dst_addr, engine->regs + SPA_DST_PTR_REG_OFFSET); - writel(0, engine->regs + SPA_OFFSET_REG_OFFSET); - - assoc_len = aead_req->assoclen; - proc_len = aead_req->cryptlen + assoc_len; - - /* - * If we are decrypting, we need to take the length of the ICV out of - * the processing length. - */ - if (!req->is_encrypt) - proc_len -= authsize; - - writel(proc_len, engine->regs + SPA_PROC_LEN_REG_OFFSET); - writel(assoc_len, engine->regs + SPA_AAD_LEN_REG_OFFSET); - writel(authsize, engine->regs + SPA_ICV_LEN_REG_OFFSET); - writel(0, engine->regs + SPA_ICV_OFFSET_REG_OFFSET); - writel(0, engine->regs + SPA_AUX_INFO_REG_OFFSET); - - ctrl = spacc_alg->ctrl_default | (req->ctx_id << SPA_CTRL_CTX_IDX) | - (1 << SPA_CTRL_ICV_APPEND); - if (req->is_encrypt) - ctrl |= (1 << SPA_CTRL_ENCRYPT_IDX) | (1 << SPA_CTRL_AAD_COPY); - else - ctrl |= (1 << SPA_CTRL_KEY_EXP); - - mod_timer(&engine->packet_timeout, jiffies + PACKET_TIMEOUT); - - writel(ctrl, engine->regs + SPA_CTRL_REG_OFFSET); - - return -EINPROGRESS; -} - -static int spacc_req_submit(struct spacc_req *req); - -static void spacc_push(struct spacc_engine *engine) -{ - struct spacc_req *req; - - while (!list_empty(&engine->pending) && - engine->in_flight + 1 <= engine->fifo_sz) { - - ++engine->in_flight; - req = list_first_entry(&engine->pending, struct spacc_req, - list); - list_move_tail(&req->list, &engine->in_progress); - - req->result = spacc_req_submit(req); - } -} - -/* - * Setup an AEAD request for processing. This will configure the engine, load - * the context and then start the packet processing. - */ -static int spacc_aead_setup(struct aead_request *req, - unsigned alg_type, bool is_encrypt) -{ - struct crypto_aead *aead = crypto_aead_reqtfm(req); - struct aead_alg *alg = crypto_aead_alg(aead); - struct spacc_engine *engine = to_spacc_aead(alg)->engine; - struct spacc_req *dev_req = aead_request_ctx(req); - int err; - unsigned long flags; - - dev_req->req = &req->base; - dev_req->is_encrypt = is_encrypt; - dev_req->result = -EBUSY; - dev_req->engine = engine; - dev_req->complete = spacc_aead_complete; - - if (unlikely(spacc_aead_need_fallback(req) || - ((err = spacc_aead_make_ddts(req)) == -E2BIG))) - return spacc_aead_do_fallback(req, alg_type, is_encrypt); - - if (err) - goto out; - - err = -EINPROGRESS; - spin_lock_irqsave(&engine->hw_lock, flags); - if (unlikely(spacc_fifo_cmd_full(engine)) || - engine->in_flight + 1 > engine->fifo_sz) { - if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { - err = -EBUSY; - spin_unlock_irqrestore(&engine->hw_lock, flags); - goto out_free_ddts; - } - list_add_tail(&dev_req->list, &engine->pending); - } else { - list_add_tail(&dev_req->list, &engine->pending); - spacc_push(engine); - } - spin_unlock_irqrestore(&engine->hw_lock, flags); - - goto out; - -out_free_ddts: - spacc_aead_free_ddts(dev_req); -out: - return err; -} - -static int spacc_aead_encrypt(struct aead_request *req) -{ - struct crypto_aead *aead = crypto_aead_reqtfm(req); - struct spacc_aead *alg = to_spacc_aead(crypto_aead_alg(aead)); - - return spacc_aead_setup(req, alg->type, 1); -} - -static int spacc_aead_decrypt(struct aead_request *req) -{ - struct crypto_aead *aead = crypto_aead_reqtfm(req); - struct spacc_aead *alg = to_spacc_aead(crypto_aead_alg(aead)); - - return spacc_aead_setup(req, alg->type, 0); -} - -/* - * Initialise a new AEAD context. This is responsible for allocating the - * fallback cipher and initialising the context. - */ -static int spacc_aead_cra_init(struct crypto_aead *tfm) -{ - struct spacc_aead_ctx *ctx = crypto_aead_ctx(tfm); - struct aead_alg *alg = crypto_aead_alg(tfm); - struct spacc_aead *spacc_alg = to_spacc_aead(alg); - struct spacc_engine *engine = spacc_alg->engine; - - ctx->generic.flags = spacc_alg->type; - ctx->generic.engine = engine; - ctx->sw_cipher = crypto_alloc_aead(alg->base.cra_name, 0, - CRYPTO_ALG_NEED_FALLBACK); - if (IS_ERR(ctx->sw_cipher)) - return PTR_ERR(ctx->sw_cipher); - ctx->generic.key_offs = spacc_alg->key_offs; - ctx->generic.iv_offs = spacc_alg->iv_offs; - - crypto_aead_set_reqsize( - tfm, - max(sizeof(struct spacc_req), - sizeof(struct aead_request) + - crypto_aead_reqsize(ctx->sw_cipher))); - - return 0; -} - -/* - * Destructor for an AEAD context. This is called when the transform is freed - * and must free the fallback cipher. - */ -static void spacc_aead_cra_exit(struct crypto_aead *tfm) -{ - struct spacc_aead_ctx *ctx = crypto_aead_ctx(tfm); - - crypto_free_aead(ctx->sw_cipher); -} - -/* - * Set the DES key for a block cipher transform. This also performs weak key - * checking if the transform has requested it. - */ -static int spacc_des_setkey(struct crypto_skcipher *cipher, const u8 *key, - unsigned int len) -{ - struct spacc_ablk_ctx *ctx = crypto_skcipher_ctx(cipher); - int err; - - err = verify_skcipher_des_key(cipher, key); - if (err) - return err; - - memcpy(ctx->key, key, len); - ctx->key_len = len; - - return 0; -} - -/* - * Set the 3DES key for a block cipher transform. This also performs weak key - * checking if the transform has requested it. - */ -static int spacc_des3_setkey(struct crypto_skcipher *cipher, const u8 *key, - unsigned int len) -{ - struct spacc_ablk_ctx *ctx = crypto_skcipher_ctx(cipher); - int err; - - err = verify_skcipher_des3_key(cipher, key); - if (err) - return err; - - memcpy(ctx->key, key, len); - ctx->key_len = len; - - return 0; -} - -/* - * Set the key for an AES block cipher. Some key lengths are not supported in - * hardware so this must also check whether a fallback is needed. - */ -static int spacc_aes_setkey(struct crypto_skcipher *cipher, const u8 *key, - unsigned int len) -{ - struct crypto_tfm *tfm = crypto_skcipher_tfm(cipher); - struct spacc_ablk_ctx *ctx = crypto_tfm_ctx(tfm); - int err = 0; - - if (len > AES_MAX_KEY_SIZE) - return -EINVAL; - - /* - * IPSec engine only supports 128 and 256 bit AES keys. If we get a - * request for any other size (192 bits) then we need to do a software - * fallback. - */ - if (len != AES_KEYSIZE_128 && len != AES_KEYSIZE_256) { - if (!ctx->sw_cipher) - return -EINVAL; - - /* - * Set the fallback transform to use the same request flags as - * the hardware transform. - */ - crypto_skcipher_clear_flags(ctx->sw_cipher, - CRYPTO_TFM_REQ_MASK); - crypto_skcipher_set_flags(ctx->sw_cipher, - cipher->base.crt_flags & - CRYPTO_TFM_REQ_MASK); - - err = crypto_skcipher_setkey(ctx->sw_cipher, key, len); - if (err) - goto sw_setkey_failed; - } - - memcpy(ctx->key, key, len); - ctx->key_len = len; - -sw_setkey_failed: - return err; -} - -static int spacc_kasumi_f8_setkey(struct crypto_skcipher *cipher, - const u8 *key, unsigned int len) -{ - struct crypto_tfm *tfm = crypto_skcipher_tfm(cipher); - struct spacc_ablk_ctx *ctx = crypto_tfm_ctx(tfm); - int err = 0; - - if (len > AES_MAX_KEY_SIZE) { - err = -EINVAL; - goto out; - } - - memcpy(ctx->key, key, len); - ctx->key_len = len; - -out: - return err; -} - -static int spacc_ablk_need_fallback(struct spacc_req *req) -{ - struct skcipher_request *ablk_req = skcipher_request_cast(req->req); - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(ablk_req); - struct spacc_alg *spacc_alg = to_spacc_skcipher(crypto_skcipher_alg(tfm)); - struct spacc_ablk_ctx *ctx; - - ctx = crypto_skcipher_ctx(tfm); - - return (spacc_alg->ctrl_default & SPACC_CRYPTO_ALG_MASK) == - SPA_CTRL_CIPH_ALG_AES && - ctx->key_len != AES_KEYSIZE_128 && - ctx->key_len != AES_KEYSIZE_256; -} - -static void spacc_ablk_complete(struct spacc_req *req) -{ - struct skcipher_request *ablk_req = skcipher_request_cast(req->req); - - if (ablk_req->src != ablk_req->dst) { - spacc_free_ddt(req, req->src_ddt, req->src_addr, ablk_req->src, - ablk_req->cryptlen, DMA_TO_DEVICE); - spacc_free_ddt(req, req->dst_ddt, req->dst_addr, ablk_req->dst, - ablk_req->cryptlen, DMA_FROM_DEVICE); - } else - spacc_free_ddt(req, req->dst_ddt, req->dst_addr, ablk_req->dst, - ablk_req->cryptlen, DMA_BIDIRECTIONAL); - - req->req->complete(req->req, req->result); -} - -static int spacc_ablk_submit(struct spacc_req *req) -{ - struct skcipher_request *ablk_req = skcipher_request_cast(req->req); - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(ablk_req); - struct skcipher_alg *alg = crypto_skcipher_alg(tfm); - struct spacc_alg *spacc_alg = to_spacc_skcipher(alg); - struct spacc_ablk_ctx *ctx = crypto_skcipher_ctx(tfm); - struct spacc_engine *engine = ctx->generic.engine; - u32 ctrl; - - req->ctx_id = spacc_load_ctx(&ctx->generic, ctx->key, - ctx->key_len, ablk_req->iv, alg->ivsize, - NULL, 0); - - writel(req->src_addr, engine->regs + SPA_SRC_PTR_REG_OFFSET); - writel(req->dst_addr, engine->regs + SPA_DST_PTR_REG_OFFSET); - writel(0, engine->regs + SPA_OFFSET_REG_OFFSET); - - writel(ablk_req->cryptlen, engine->regs + SPA_PROC_LEN_REG_OFFSET); - writel(0, engine->regs + SPA_ICV_OFFSET_REG_OFFSET); - writel(0, engine->regs + SPA_AUX_INFO_REG_OFFSET); - writel(0, engine->regs + SPA_AAD_LEN_REG_OFFSET); - - ctrl = spacc_alg->ctrl_default | (req->ctx_id << SPA_CTRL_CTX_IDX) | - (req->is_encrypt ? (1 << SPA_CTRL_ENCRYPT_IDX) : - (1 << SPA_CTRL_KEY_EXP)); - - mod_timer(&engine->packet_timeout, jiffies + PACKET_TIMEOUT); - - writel(ctrl, engine->regs + SPA_CTRL_REG_OFFSET); - - return -EINPROGRESS; -} - -static int spacc_ablk_do_fallback(struct skcipher_request *req, - unsigned alg_type, bool is_encrypt) -{ - struct crypto_tfm *old_tfm = - crypto_skcipher_tfm(crypto_skcipher_reqtfm(req)); - struct spacc_ablk_ctx *ctx = crypto_tfm_ctx(old_tfm); - struct spacc_req *dev_req = skcipher_request_ctx(req); - int err; - - /* - * Change the request to use the software fallback transform, and once - * the ciphering has completed, put the old transform back into the - * request. - */ - skcipher_request_set_tfm(&dev_req->fallback_req, ctx->sw_cipher); - skcipher_request_set_callback(&dev_req->fallback_req, req->base.flags, - req->base.complete, req->base.data); - skcipher_request_set_crypt(&dev_req->fallback_req, req->src, req->dst, - req->cryptlen, req->iv); - err = is_encrypt ? crypto_skcipher_encrypt(&dev_req->fallback_req) : - crypto_skcipher_decrypt(&dev_req->fallback_req); - - return err; -} - -static int spacc_ablk_setup(struct skcipher_request *req, unsigned alg_type, - bool is_encrypt) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct skcipher_alg *alg = crypto_skcipher_alg(tfm); - struct spacc_engine *engine = to_spacc_skcipher(alg)->engine; - struct spacc_req *dev_req = skcipher_request_ctx(req); - unsigned long flags; - int err = -ENOMEM; - - dev_req->req = &req->base; - dev_req->is_encrypt = is_encrypt; - dev_req->engine = engine; - dev_req->complete = spacc_ablk_complete; - dev_req->result = -EINPROGRESS; - - if (unlikely(spacc_ablk_need_fallback(dev_req))) - return spacc_ablk_do_fallback(req, alg_type, is_encrypt); - - /* - * Create the DDT's for the engine. If we share the same source and - * destination then we can optimize by reusing the DDT's. - */ - if (req->src != req->dst) { - dev_req->src_ddt = spacc_sg_to_ddt(engine, req->src, - req->cryptlen, DMA_TO_DEVICE, &dev_req->src_addr); - if (!dev_req->src_ddt) - goto out; - - dev_req->dst_ddt = spacc_sg_to_ddt(engine, req->dst, - req->cryptlen, DMA_FROM_DEVICE, &dev_req->dst_addr); - if (!dev_req->dst_ddt) - goto out_free_src; - } else { - dev_req->dst_ddt = spacc_sg_to_ddt(engine, req->dst, - req->cryptlen, DMA_BIDIRECTIONAL, &dev_req->dst_addr); - if (!dev_req->dst_ddt) - goto out; - - dev_req->src_ddt = NULL; - dev_req->src_addr = dev_req->dst_addr; - } - - err = -EINPROGRESS; - spin_lock_irqsave(&engine->hw_lock, flags); - /* - * Check if the engine will accept the operation now. If it won't then - * we either stick it on the end of a pending list if we can backlog, - * or bailout with an error if not. - */ - if (unlikely(spacc_fifo_cmd_full(engine)) || - engine->in_flight + 1 > engine->fifo_sz) { - if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { - err = -EBUSY; - spin_unlock_irqrestore(&engine->hw_lock, flags); - goto out_free_ddts; - } - list_add_tail(&dev_req->list, &engine->pending); - } else { - list_add_tail(&dev_req->list, &engine->pending); - spacc_push(engine); - } - spin_unlock_irqrestore(&engine->hw_lock, flags); - - goto out; - -out_free_ddts: - spacc_free_ddt(dev_req, dev_req->dst_ddt, dev_req->dst_addr, req->dst, - req->cryptlen, req->src == req->dst ? - DMA_BIDIRECTIONAL : DMA_FROM_DEVICE); -out_free_src: - if (req->src != req->dst) - spacc_free_ddt(dev_req, dev_req->src_ddt, dev_req->src_addr, - req->src, req->cryptlen, DMA_TO_DEVICE); -out: - return err; -} - -static int spacc_ablk_init_tfm(struct crypto_skcipher *tfm) -{ - struct spacc_ablk_ctx *ctx = crypto_skcipher_ctx(tfm); - struct skcipher_alg *alg = crypto_skcipher_alg(tfm); - struct spacc_alg *spacc_alg = to_spacc_skcipher(alg); - struct spacc_engine *engine = spacc_alg->engine; - - ctx->generic.flags = spacc_alg->type; - ctx->generic.engine = engine; - if (alg->base.cra_flags & CRYPTO_ALG_NEED_FALLBACK) { - ctx->sw_cipher = crypto_alloc_skcipher(alg->base.cra_name, 0, - CRYPTO_ALG_NEED_FALLBACK); - if (IS_ERR(ctx->sw_cipher)) { - dev_warn(engine->dev, "failed to allocate fallback for %s\n", - alg->base.cra_name); - return PTR_ERR(ctx->sw_cipher); - } - crypto_skcipher_set_reqsize(tfm, sizeof(struct spacc_req) + - crypto_skcipher_reqsize(ctx->sw_cipher)); - } else { - /* take the size without the fallback skcipher_request at the end */ - crypto_skcipher_set_reqsize(tfm, offsetof(struct spacc_req, - fallback_req)); - } - - ctx->generic.key_offs = spacc_alg->key_offs; - ctx->generic.iv_offs = spacc_alg->iv_offs; - - return 0; -} - -static void spacc_ablk_exit_tfm(struct crypto_skcipher *tfm) -{ - struct spacc_ablk_ctx *ctx = crypto_skcipher_ctx(tfm); - - crypto_free_skcipher(ctx->sw_cipher); -} - -static int spacc_ablk_encrypt(struct skcipher_request *req) -{ - struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req); - struct skcipher_alg *alg = crypto_skcipher_alg(cipher); - struct spacc_alg *spacc_alg = to_spacc_skcipher(alg); - - return spacc_ablk_setup(req, spacc_alg->type, 1); -} - -static int spacc_ablk_decrypt(struct skcipher_request *req) -{ - struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req); - struct skcipher_alg *alg = crypto_skcipher_alg(cipher); - struct spacc_alg *spacc_alg = to_spacc_skcipher(alg); - - return spacc_ablk_setup(req, spacc_alg->type, 0); -} - -static inline int spacc_fifo_stat_empty(struct spacc_engine *engine) -{ - return readl(engine->regs + SPA_FIFO_STAT_REG_OFFSET) & - SPA_FIFO_STAT_EMPTY; -} - -static void spacc_process_done(struct spacc_engine *engine) -{ - struct spacc_req *req; - unsigned long flags; - - spin_lock_irqsave(&engine->hw_lock, flags); - - while (!spacc_fifo_stat_empty(engine)) { - req = list_first_entry(&engine->in_progress, struct spacc_req, - list); - list_move_tail(&req->list, &engine->completed); - --engine->in_flight; - - /* POP the status register. */ - writel(~0, engine->regs + SPA_STAT_POP_REG_OFFSET); - req->result = (readl(engine->regs + SPA_STATUS_REG_OFFSET) & - SPA_STATUS_RES_CODE_MASK) >> SPA_STATUS_RES_CODE_OFFSET; - - /* - * Convert the SPAcc error status into the standard POSIX error - * codes. - */ - if (unlikely(req->result)) { - switch (req->result) { - case SPA_STATUS_ICV_FAIL: - req->result = -EBADMSG; - break; - - case SPA_STATUS_MEMORY_ERROR: - dev_warn(engine->dev, - "memory error triggered\n"); - req->result = -EFAULT; - break; - - case SPA_STATUS_BLOCK_ERROR: - dev_warn(engine->dev, - "block error triggered\n"); - req->result = -EIO; - break; - } - } - } - - tasklet_schedule(&engine->complete); - - spin_unlock_irqrestore(&engine->hw_lock, flags); -} - -static irqreturn_t spacc_spacc_irq(int irq, void *dev) -{ - struct spacc_engine *engine = (struct spacc_engine *)dev; - u32 spacc_irq_stat = readl(engine->regs + SPA_IRQ_STAT_REG_OFFSET); - - writel(spacc_irq_stat, engine->regs + SPA_IRQ_STAT_REG_OFFSET); - spacc_process_done(engine); - - return IRQ_HANDLED; -} - -static void spacc_packet_timeout(struct timer_list *t) -{ - struct spacc_engine *engine = from_timer(engine, t, packet_timeout); - - spacc_process_done(engine); -} - -static int spacc_req_submit(struct spacc_req *req) -{ - struct crypto_alg *alg = req->req->tfm->__crt_alg; - - if (CRYPTO_ALG_TYPE_AEAD == (CRYPTO_ALG_TYPE_MASK & alg->cra_flags)) - return spacc_aead_submit(req); - else - return spacc_ablk_submit(req); -} - -static void spacc_spacc_complete(unsigned long data) -{ - struct spacc_engine *engine = (struct spacc_engine *)data; - struct spacc_req *req, *tmp; - unsigned long flags; - LIST_HEAD(completed); - - spin_lock_irqsave(&engine->hw_lock, flags); - - list_splice_init(&engine->completed, &completed); - spacc_push(engine); - if (engine->in_flight) - mod_timer(&engine->packet_timeout, jiffies + PACKET_TIMEOUT); - - spin_unlock_irqrestore(&engine->hw_lock, flags); - - list_for_each_entry_safe(req, tmp, &completed, list) { - list_del(&req->list); - req->complete(req); - } -} - -#ifdef CONFIG_PM -static int spacc_suspend(struct device *dev) -{ - struct spacc_engine *engine = dev_get_drvdata(dev); - - /* - * We only support standby mode. All we have to do is gate the clock to - * the spacc. The hardware will preserve state until we turn it back - * on again. - */ - clk_disable(engine->clk); - - return 0; -} - -static int spacc_resume(struct device *dev) -{ - struct spacc_engine *engine = dev_get_drvdata(dev); - - return clk_enable(engine->clk); -} - -static const struct dev_pm_ops spacc_pm_ops = { - .suspend = spacc_suspend, - .resume = spacc_resume, -}; -#endif /* CONFIG_PM */ - -static inline struct spacc_engine *spacc_dev_to_engine(struct device *dev) -{ - return dev ? dev_get_drvdata(dev) : NULL; -} - -static ssize_t spacc_stat_irq_thresh_show(struct device *dev, - struct device_attribute *attr, - char *buf) -{ - struct spacc_engine *engine = spacc_dev_to_engine(dev); - - return snprintf(buf, PAGE_SIZE, "%u\n", engine->stat_irq_thresh); -} - -static ssize_t spacc_stat_irq_thresh_store(struct device *dev, - struct device_attribute *attr, - const char *buf, size_t len) -{ - struct spacc_engine *engine = spacc_dev_to_engine(dev); - unsigned long thresh; - - if (kstrtoul(buf, 0, &thresh)) - return -EINVAL; - - thresh = clamp(thresh, 1UL, engine->fifo_sz - 1); - - engine->stat_irq_thresh = thresh; - writel(engine->stat_irq_thresh << SPA_IRQ_CTRL_STAT_CNT_OFFSET, - engine->regs + SPA_IRQ_CTRL_REG_OFFSET); - - return len; -} -static DEVICE_ATTR(stat_irq_thresh, 0644, spacc_stat_irq_thresh_show, - spacc_stat_irq_thresh_store); - -static struct spacc_alg ipsec_engine_algs[] = { - { - .ctrl_default = SPA_CTRL_CIPH_ALG_AES | SPA_CTRL_CIPH_MODE_CBC, - .key_offs = 0, - .iv_offs = AES_MAX_KEY_SIZE, - .alg = { - .base.cra_name = "cbc(aes)", - .base.cra_driver_name = "cbc-aes-picoxcell", - .base.cra_priority = SPACC_CRYPTO_ALG_PRIORITY, - .base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | - CRYPTO_ALG_ASYNC | - CRYPTO_ALG_ALLOCATES_MEMORY | - CRYPTO_ALG_NEED_FALLBACK, - .base.cra_blocksize = AES_BLOCK_SIZE, - .base.cra_ctxsize = sizeof(struct spacc_ablk_ctx), - .base.cra_module = THIS_MODULE, - - .setkey = spacc_aes_setkey, - .encrypt = spacc_ablk_encrypt, - .decrypt = spacc_ablk_decrypt, - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .ivsize = AES_BLOCK_SIZE, - .init = spacc_ablk_init_tfm, - .exit = spacc_ablk_exit_tfm, - }, - }, - { - .key_offs = 0, - .iv_offs = AES_MAX_KEY_SIZE, - .ctrl_default = SPA_CTRL_CIPH_ALG_AES | SPA_CTRL_CIPH_MODE_ECB, - .alg = { - .base.cra_name = "ecb(aes)", - .base.cra_driver_name = "ecb-aes-picoxcell", - .base.cra_priority = SPACC_CRYPTO_ALG_PRIORITY, - .base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | - CRYPTO_ALG_ASYNC | - CRYPTO_ALG_ALLOCATES_MEMORY | - CRYPTO_ALG_NEED_FALLBACK, - .base.cra_blocksize = AES_BLOCK_SIZE, - .base.cra_ctxsize = sizeof(struct spacc_ablk_ctx), - .base.cra_module = THIS_MODULE, - - .setkey = spacc_aes_setkey, - .encrypt = spacc_ablk_encrypt, - .decrypt = spacc_ablk_decrypt, - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .init = spacc_ablk_init_tfm, - .exit = spacc_ablk_exit_tfm, - }, - }, - { - .key_offs = DES_BLOCK_SIZE, - .iv_offs = 0, - .ctrl_default = SPA_CTRL_CIPH_ALG_DES | SPA_CTRL_CIPH_MODE_CBC, - .alg = { - .base.cra_name = "cbc(des)", - .base.cra_driver_name = "cbc-des-picoxcell", - .base.cra_priority = SPACC_CRYPTO_ALG_PRIORITY, - .base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | - CRYPTO_ALG_ASYNC | - CRYPTO_ALG_ALLOCATES_MEMORY, - .base.cra_blocksize = DES_BLOCK_SIZE, - .base.cra_ctxsize = sizeof(struct spacc_ablk_ctx), - .base.cra_module = THIS_MODULE, - - .setkey = spacc_des_setkey, - .encrypt = spacc_ablk_encrypt, - .decrypt = spacc_ablk_decrypt, - .min_keysize = DES_KEY_SIZE, - .max_keysize = DES_KEY_SIZE, - .ivsize = DES_BLOCK_SIZE, - .init = spacc_ablk_init_tfm, - .exit = spacc_ablk_exit_tfm, - }, - }, - { - .key_offs = DES_BLOCK_SIZE, - .iv_offs = 0, - .ctrl_default = SPA_CTRL_CIPH_ALG_DES | SPA_CTRL_CIPH_MODE_ECB, - .alg = { - .base.cra_name = "ecb(des)", - .base.cra_driver_name = "ecb-des-picoxcell", - .base.cra_priority = SPACC_CRYPTO_ALG_PRIORITY, - .base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | - CRYPTO_ALG_ASYNC | - CRYPTO_ALG_ALLOCATES_MEMORY, - .base.cra_blocksize = DES_BLOCK_SIZE, - .base.cra_ctxsize = sizeof(struct spacc_ablk_ctx), - .base.cra_module = THIS_MODULE, - - .setkey = spacc_des_setkey, - .encrypt = spacc_ablk_encrypt, - .decrypt = spacc_ablk_decrypt, - .min_keysize = DES_KEY_SIZE, - .max_keysize = DES_KEY_SIZE, - .init = spacc_ablk_init_tfm, - .exit = spacc_ablk_exit_tfm, - }, - }, - { - .key_offs = DES_BLOCK_SIZE, - .iv_offs = 0, - .ctrl_default = SPA_CTRL_CIPH_ALG_DES | SPA_CTRL_CIPH_MODE_CBC, - .alg = { - .base.cra_name = "cbc(des3_ede)", - .base.cra_driver_name = "cbc-des3-ede-picoxcell", - .base.cra_priority = SPACC_CRYPTO_ALG_PRIORITY, - .base.cra_flags = CRYPTO_ALG_ASYNC | - CRYPTO_ALG_ALLOCATES_MEMORY | - CRYPTO_ALG_KERN_DRIVER_ONLY, - .base.cra_blocksize = DES3_EDE_BLOCK_SIZE, - .base.cra_ctxsize = sizeof(struct spacc_ablk_ctx), - .base.cra_module = THIS_MODULE, - - .setkey = spacc_des3_setkey, - .encrypt = spacc_ablk_encrypt, - .decrypt = spacc_ablk_decrypt, - .min_keysize = DES3_EDE_KEY_SIZE, - .max_keysize = DES3_EDE_KEY_SIZE, - .ivsize = DES3_EDE_BLOCK_SIZE, - .init = spacc_ablk_init_tfm, - .exit = spacc_ablk_exit_tfm, - }, - }, - { - .key_offs = DES_BLOCK_SIZE, - .iv_offs = 0, - .ctrl_default = SPA_CTRL_CIPH_ALG_DES | SPA_CTRL_CIPH_MODE_ECB, - .alg = { - .base.cra_name = "ecb(des3_ede)", - .base.cra_driver_name = "ecb-des3-ede-picoxcell", - .base.cra_priority = SPACC_CRYPTO_ALG_PRIORITY, - .base.cra_flags = CRYPTO_ALG_ASYNC | - CRYPTO_ALG_ALLOCATES_MEMORY | - CRYPTO_ALG_KERN_DRIVER_ONLY, - .base.cra_blocksize = DES3_EDE_BLOCK_SIZE, - .base.cra_ctxsize = sizeof(struct spacc_ablk_ctx), - .base.cra_module = THIS_MODULE, - - .setkey = spacc_des3_setkey, - .encrypt = spacc_ablk_encrypt, - .decrypt = spacc_ablk_decrypt, - .min_keysize = DES3_EDE_KEY_SIZE, - .max_keysize = DES3_EDE_KEY_SIZE, - .init = spacc_ablk_init_tfm, - .exit = spacc_ablk_exit_tfm, - }, - }, -}; - -static struct spacc_aead ipsec_engine_aeads[] = { - { - .ctrl_default = SPA_CTRL_CIPH_ALG_AES | - SPA_CTRL_CIPH_MODE_CBC | - SPA_CTRL_HASH_ALG_SHA | - SPA_CTRL_HASH_MODE_HMAC, - .key_offs = 0, - .iv_offs = AES_MAX_KEY_SIZE, - .alg = { - .base = { - .cra_name = "authenc(hmac(sha1),cbc(aes))", - .cra_driver_name = "authenc-hmac-sha1-" - "cbc-aes-picoxcell", - .cra_priority = SPACC_CRYPTO_ALG_PRIORITY, - .cra_flags = CRYPTO_ALG_ASYNC | - CRYPTO_ALG_ALLOCATES_MEMORY | - CRYPTO_ALG_NEED_FALLBACK | - CRYPTO_ALG_KERN_DRIVER_ONLY, - .cra_blocksize = AES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct spacc_aead_ctx), - .cra_module = THIS_MODULE, - }, - .setkey = spacc_aead_setkey, - .setauthsize = spacc_aead_setauthsize, - .encrypt = spacc_aead_encrypt, - .decrypt = spacc_aead_decrypt, - .ivsize = AES_BLOCK_SIZE, - .maxauthsize = SHA1_DIGEST_SIZE, - .init = spacc_aead_cra_init, - .exit = spacc_aead_cra_exit, - }, - }, - { - .ctrl_default = SPA_CTRL_CIPH_ALG_AES | - SPA_CTRL_CIPH_MODE_CBC | - SPA_CTRL_HASH_ALG_SHA256 | - SPA_CTRL_HASH_MODE_HMAC, - .key_offs = 0, - .iv_offs = AES_MAX_KEY_SIZE, - .alg = { - .base = { - .cra_name = "authenc(hmac(sha256),cbc(aes))", - .cra_driver_name = "authenc-hmac-sha256-" - "cbc-aes-picoxcell", - .cra_priority = SPACC_CRYPTO_ALG_PRIORITY, - .cra_flags = CRYPTO_ALG_ASYNC | - CRYPTO_ALG_ALLOCATES_MEMORY | - CRYPTO_ALG_NEED_FALLBACK | - CRYPTO_ALG_KERN_DRIVER_ONLY, - .cra_blocksize = AES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct spacc_aead_ctx), - .cra_module = THIS_MODULE, - }, - .setkey = spacc_aead_setkey, - .setauthsize = spacc_aead_setauthsize, - .encrypt = spacc_aead_encrypt, - .decrypt = spacc_aead_decrypt, - .ivsize = AES_BLOCK_SIZE, - .maxauthsize = SHA256_DIGEST_SIZE, - .init = spacc_aead_cra_init, - .exit = spacc_aead_cra_exit, - }, - }, - { - .key_offs = 0, - .iv_offs = AES_MAX_KEY_SIZE, - .ctrl_default = SPA_CTRL_CIPH_ALG_AES | - SPA_CTRL_CIPH_MODE_CBC | - SPA_CTRL_HASH_ALG_MD5 | - SPA_CTRL_HASH_MODE_HMAC, - .alg = { - .base = { - .cra_name = "authenc(hmac(md5),cbc(aes))", - .cra_driver_name = "authenc-hmac-md5-" - "cbc-aes-picoxcell", - .cra_priority = SPACC_CRYPTO_ALG_PRIORITY, - .cra_flags = CRYPTO_ALG_ASYNC | - CRYPTO_ALG_ALLOCATES_MEMORY | - CRYPTO_ALG_NEED_FALLBACK | - CRYPTO_ALG_KERN_DRIVER_ONLY, - .cra_blocksize = AES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct spacc_aead_ctx), - .cra_module = THIS_MODULE, - }, - .setkey = spacc_aead_setkey, - .setauthsize = spacc_aead_setauthsize, - .encrypt = spacc_aead_encrypt, - .decrypt = spacc_aead_decrypt, - .ivsize = AES_BLOCK_SIZE, - .maxauthsize = MD5_DIGEST_SIZE, - .init = spacc_aead_cra_init, - .exit = spacc_aead_cra_exit, - }, - }, - { - .key_offs = DES_BLOCK_SIZE, - .iv_offs = 0, - .ctrl_default = SPA_CTRL_CIPH_ALG_DES | - SPA_CTRL_CIPH_MODE_CBC | - SPA_CTRL_HASH_ALG_SHA | - SPA_CTRL_HASH_MODE_HMAC, - .alg = { - .base = { - .cra_name = "authenc(hmac(sha1),cbc(des3_ede))", - .cra_driver_name = "authenc-hmac-sha1-" - "cbc-3des-picoxcell", - .cra_priority = SPACC_CRYPTO_ALG_PRIORITY, - .cra_flags = CRYPTO_ALG_ASYNC | - CRYPTO_ALG_ALLOCATES_MEMORY | - CRYPTO_ALG_NEED_FALLBACK | - CRYPTO_ALG_KERN_DRIVER_ONLY, - .cra_blocksize = DES3_EDE_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct spacc_aead_ctx), - .cra_module = THIS_MODULE, - }, - .setkey = spacc_aead_setkey, - .setauthsize = spacc_aead_setauthsize, - .encrypt = spacc_aead_encrypt, - .decrypt = spacc_aead_decrypt, - .ivsize = DES3_EDE_BLOCK_SIZE, - .maxauthsize = SHA1_DIGEST_SIZE, - .init = spacc_aead_cra_init, - .exit = spacc_aead_cra_exit, - }, - }, - { - .key_offs = DES_BLOCK_SIZE, - .iv_offs = 0, - .ctrl_default = SPA_CTRL_CIPH_ALG_AES | - SPA_CTRL_CIPH_MODE_CBC | - SPA_CTRL_HASH_ALG_SHA256 | - SPA_CTRL_HASH_MODE_HMAC, - .alg = { - .base = { - .cra_name = "authenc(hmac(sha256)," - "cbc(des3_ede))", - .cra_driver_name = "authenc-hmac-sha256-" - "cbc-3des-picoxcell", - .cra_priority = SPACC_CRYPTO_ALG_PRIORITY, - .cra_flags = CRYPTO_ALG_ASYNC | - CRYPTO_ALG_ALLOCATES_MEMORY | - CRYPTO_ALG_NEED_FALLBACK | - CRYPTO_ALG_KERN_DRIVER_ONLY, - .cra_blocksize = DES3_EDE_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct spacc_aead_ctx), - .cra_module = THIS_MODULE, - }, - .setkey = spacc_aead_setkey, - .setauthsize = spacc_aead_setauthsize, - .encrypt = spacc_aead_encrypt, - .decrypt = spacc_aead_decrypt, - .ivsize = DES3_EDE_BLOCK_SIZE, - .maxauthsize = SHA256_DIGEST_SIZE, - .init = spacc_aead_cra_init, - .exit = spacc_aead_cra_exit, - }, - }, - { - .key_offs = DES_BLOCK_SIZE, - .iv_offs = 0, - .ctrl_default = SPA_CTRL_CIPH_ALG_DES | - SPA_CTRL_CIPH_MODE_CBC | - SPA_CTRL_HASH_ALG_MD5 | - SPA_CTRL_HASH_MODE_HMAC, - .alg = { - .base = { - .cra_name = "authenc(hmac(md5),cbc(des3_ede))", - .cra_driver_name = "authenc-hmac-md5-" - "cbc-3des-picoxcell", - .cra_priority = SPACC_CRYPTO_ALG_PRIORITY, - .cra_flags = CRYPTO_ALG_ASYNC | - CRYPTO_ALG_ALLOCATES_MEMORY | - CRYPTO_ALG_NEED_FALLBACK | - CRYPTO_ALG_KERN_DRIVER_ONLY, - .cra_blocksize = DES3_EDE_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct spacc_aead_ctx), - .cra_module = THIS_MODULE, - }, - .setkey = spacc_aead_setkey, - .setauthsize = spacc_aead_setauthsize, - .encrypt = spacc_aead_encrypt, - .decrypt = spacc_aead_decrypt, - .ivsize = DES3_EDE_BLOCK_SIZE, - .maxauthsize = MD5_DIGEST_SIZE, - .init = spacc_aead_cra_init, - .exit = spacc_aead_cra_exit, - }, - }, -}; - -static struct spacc_alg l2_engine_algs[] = { - { - .key_offs = 0, - .iv_offs = SPACC_CRYPTO_KASUMI_F8_KEY_LEN, - .ctrl_default = SPA_CTRL_CIPH_ALG_KASUMI | - SPA_CTRL_CIPH_MODE_F8, - .alg = { - .base.cra_name = "f8(kasumi)", - .base.cra_driver_name = "f8-kasumi-picoxcell", - .base.cra_priority = SPACC_CRYPTO_ALG_PRIORITY, - .base.cra_flags = CRYPTO_ALG_ASYNC | - CRYPTO_ALG_ALLOCATES_MEMORY | - CRYPTO_ALG_KERN_DRIVER_ONLY, - .base.cra_blocksize = 8, - .base.cra_ctxsize = sizeof(struct spacc_ablk_ctx), - .base.cra_module = THIS_MODULE, - - .setkey = spacc_kasumi_f8_setkey, - .encrypt = spacc_ablk_encrypt, - .decrypt = spacc_ablk_decrypt, - .min_keysize = 16, - .max_keysize = 16, - .ivsize = 8, - .init = spacc_ablk_init_tfm, - .exit = spacc_ablk_exit_tfm, - }, - }, -}; - -#ifdef CONFIG_OF -static const struct of_device_id spacc_of_id_table[] = { - { .compatible = "picochip,spacc-ipsec" }, - { .compatible = "picochip,spacc-l2" }, - {} -}; -MODULE_DEVICE_TABLE(of, spacc_of_id_table); -#endif /* CONFIG_OF */ - -static void spacc_tasklet_kill(void *data) -{ - tasklet_kill(data); -} - -static int spacc_probe(struct platform_device *pdev) -{ - int i, err, ret; - struct resource *irq; - struct device_node *np = pdev->dev.of_node; - struct spacc_engine *engine = devm_kzalloc(&pdev->dev, sizeof(*engine), - GFP_KERNEL); - if (!engine) - return -ENOMEM; - - if (of_device_is_compatible(np, "picochip,spacc-ipsec")) { - engine->max_ctxs = SPACC_CRYPTO_IPSEC_MAX_CTXS; - engine->cipher_pg_sz = SPACC_CRYPTO_IPSEC_CIPHER_PG_SZ; - engine->hash_pg_sz = SPACC_CRYPTO_IPSEC_HASH_PG_SZ; - engine->fifo_sz = SPACC_CRYPTO_IPSEC_FIFO_SZ; - engine->algs = ipsec_engine_algs; - engine->num_algs = ARRAY_SIZE(ipsec_engine_algs); - engine->aeads = ipsec_engine_aeads; - engine->num_aeads = ARRAY_SIZE(ipsec_engine_aeads); - } else if (of_device_is_compatible(np, "picochip,spacc-l2")) { - engine->max_ctxs = SPACC_CRYPTO_L2_MAX_CTXS; - engine->cipher_pg_sz = SPACC_CRYPTO_L2_CIPHER_PG_SZ; - engine->hash_pg_sz = SPACC_CRYPTO_L2_HASH_PG_SZ; - engine->fifo_sz = SPACC_CRYPTO_L2_FIFO_SZ; - engine->algs = l2_engine_algs; - engine->num_algs = ARRAY_SIZE(l2_engine_algs); - } else { - return -EINVAL; - } - - engine->name = dev_name(&pdev->dev); - - engine->regs = devm_platform_ioremap_resource(pdev, 0); - if (IS_ERR(engine->regs)) - return PTR_ERR(engine->regs); - - irq = platform_get_resource(pdev, IORESOURCE_IRQ, 0); - if (!irq) { - dev_err(&pdev->dev, "no memory/irq resource for engine\n"); - return -ENXIO; - } - - tasklet_init(&engine->complete, spacc_spacc_complete, - (unsigned long)engine); - - ret = devm_add_action(&pdev->dev, spacc_tasklet_kill, - &engine->complete); - if (ret) - return ret; - - if (devm_request_irq(&pdev->dev, irq->start, spacc_spacc_irq, 0, - engine->name, engine)) { - dev_err(engine->dev, "failed to request IRQ\n"); - return -EBUSY; - } - - engine->dev = &pdev->dev; - engine->cipher_ctx_base = engine->regs + SPA_CIPH_KEY_BASE_REG_OFFSET; - engine->hash_key_base = engine->regs + SPA_HASH_KEY_BASE_REG_OFFSET; - - engine->req_pool = dmam_pool_create(engine->name, engine->dev, - MAX_DDT_LEN * sizeof(struct spacc_ddt), 8, SZ_64K); - if (!engine->req_pool) - return -ENOMEM; - - spin_lock_init(&engine->hw_lock); - - engine->clk = clk_get(&pdev->dev, "ref"); - if (IS_ERR(engine->clk)) { - dev_info(&pdev->dev, "clk unavailable\n"); - return PTR_ERR(engine->clk); - } - - if (clk_prepare_enable(engine->clk)) { - dev_info(&pdev->dev, "unable to prepare/enable clk\n"); - ret = -EIO; - goto err_clk_put; - } - - /* - * Use an IRQ threshold of 50% as a default. This seems to be a - * reasonable trade off of latency against throughput but can be - * changed at runtime. - */ - engine->stat_irq_thresh = (engine->fifo_sz / 2); - - ret = device_create_file(&pdev->dev, &dev_attr_stat_irq_thresh); - if (ret) - goto err_clk_disable; - - /* - * Configure the interrupts. We only use the STAT_CNT interrupt as we - * only submit a new packet for processing when we complete another in - * the queue. This minimizes time spent in the interrupt handler. - */ - writel(engine->stat_irq_thresh << SPA_IRQ_CTRL_STAT_CNT_OFFSET, - engine->regs + SPA_IRQ_CTRL_REG_OFFSET); - writel(SPA_IRQ_EN_STAT_EN | SPA_IRQ_EN_GLBL_EN, - engine->regs + SPA_IRQ_EN_REG_OFFSET); - - timer_setup(&engine->packet_timeout, spacc_packet_timeout, 0); - - INIT_LIST_HEAD(&engine->pending); - INIT_LIST_HEAD(&engine->completed); - INIT_LIST_HEAD(&engine->in_progress); - engine->in_flight = 0; - - platform_set_drvdata(pdev, engine); - - ret = -EINVAL; - INIT_LIST_HEAD(&engine->registered_algs); - for (i = 0; i < engine->num_algs; ++i) { - engine->algs[i].engine = engine; - err = crypto_register_skcipher(&engine->algs[i].alg); - if (!err) { - list_add_tail(&engine->algs[i].entry, - &engine->registered_algs); - ret = 0; - } - if (err) - dev_err(engine->dev, "failed to register alg \"%s\"\n", - engine->algs[i].alg.base.cra_name); - else - dev_dbg(engine->dev, "registered alg \"%s\"\n", - engine->algs[i].alg.base.cra_name); - } - - INIT_LIST_HEAD(&engine->registered_aeads); - for (i = 0; i < engine->num_aeads; ++i) { - engine->aeads[i].engine = engine; - err = crypto_register_aead(&engine->aeads[i].alg); - if (!err) { - list_add_tail(&engine->aeads[i].entry, - &engine->registered_aeads); - ret = 0; - } - if (err) - dev_err(engine->dev, "failed to register alg \"%s\"\n", - engine->aeads[i].alg.base.cra_name); - else - dev_dbg(engine->dev, "registered alg \"%s\"\n", - engine->aeads[i].alg.base.cra_name); - } - - if (!ret) - return 0; - - del_timer_sync(&engine->packet_timeout); - device_remove_file(&pdev->dev, &dev_attr_stat_irq_thresh); -err_clk_disable: - clk_disable_unprepare(engine->clk); -err_clk_put: - clk_put(engine->clk); - - return ret; -} - -static int spacc_remove(struct platform_device *pdev) -{ - struct spacc_aead *aead, *an; - struct spacc_alg *alg, *next; - struct spacc_engine *engine = platform_get_drvdata(pdev); - - del_timer_sync(&engine->packet_timeout); - device_remove_file(&pdev->dev, &dev_attr_stat_irq_thresh); - - list_for_each_entry_safe(aead, an, &engine->registered_aeads, entry) { - list_del(&aead->entry); - crypto_unregister_aead(&aead->alg); - } - - list_for_each_entry_safe(alg, next, &engine->registered_algs, entry) { - list_del(&alg->entry); - crypto_unregister_skcipher(&alg->alg); - } - - clk_disable_unprepare(engine->clk); - clk_put(engine->clk); - - return 0; -} - -static struct platform_driver spacc_driver = { - .probe = spacc_probe, - .remove = spacc_remove, - .driver = { - .name = "picochip,spacc", -#ifdef CONFIG_PM - .pm = &spacc_pm_ops, -#endif /* CONFIG_PM */ - .of_match_table = of_match_ptr(spacc_of_id_table), - }, -}; - -module_platform_driver(spacc_driver); - -MODULE_LICENSE("GPL"); -MODULE_AUTHOR("Jamie Iles"); diff --git a/drivers/crypto/picoxcell_crypto_regs.h b/drivers/crypto/picoxcell_crypto_regs.h deleted file mode 100644 index b870a50238baad727d2ee596d3972b356e87f2ba..0000000000000000000000000000000000000000 --- a/drivers/crypto/picoxcell_crypto_regs.h +++ /dev/null @@ -1,115 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * Copyright (c) 2010 Picochip Ltd., Jamie Iles - */ -#ifndef __PICOXCELL_CRYPTO_REGS_H__ -#define __PICOXCELL_CRYPTO_REGS_H__ - -#define SPA_STATUS_OK 0 -#define SPA_STATUS_ICV_FAIL 1 -#define SPA_STATUS_MEMORY_ERROR 2 -#define SPA_STATUS_BLOCK_ERROR 3 - -#define SPA_IRQ_CTRL_STAT_CNT_OFFSET 16 -#define SPA_IRQ_STAT_STAT_MASK (1 << 4) -#define SPA_FIFO_STAT_STAT_OFFSET 16 -#define SPA_FIFO_STAT_STAT_CNT_MASK (0x3F << SPA_FIFO_STAT_STAT_OFFSET) -#define SPA_STATUS_RES_CODE_OFFSET 24 -#define SPA_STATUS_RES_CODE_MASK (0x3 << SPA_STATUS_RES_CODE_OFFSET) -#define SPA_KEY_SZ_CTX_INDEX_OFFSET 8 -#define SPA_KEY_SZ_CIPHER_OFFSET 31 - -#define SPA_IRQ_EN_REG_OFFSET 0x00000000 -#define SPA_IRQ_STAT_REG_OFFSET 0x00000004 -#define SPA_IRQ_CTRL_REG_OFFSET 0x00000008 -#define SPA_FIFO_STAT_REG_OFFSET 0x0000000C -#define SPA_SDMA_BRST_SZ_REG_OFFSET 0x00000010 -#define SPA_SRC_PTR_REG_OFFSET 0x00000020 -#define SPA_DST_PTR_REG_OFFSET 0x00000024 -#define SPA_OFFSET_REG_OFFSET 0x00000028 -#define SPA_AAD_LEN_REG_OFFSET 0x0000002C -#define SPA_PROC_LEN_REG_OFFSET 0x00000030 -#define SPA_ICV_LEN_REG_OFFSET 0x00000034 -#define SPA_ICV_OFFSET_REG_OFFSET 0x00000038 -#define SPA_SW_CTRL_REG_OFFSET 0x0000003C -#define SPA_CTRL_REG_OFFSET 0x00000040 -#define SPA_AUX_INFO_REG_OFFSET 0x0000004C -#define SPA_STAT_POP_REG_OFFSET 0x00000050 -#define SPA_STATUS_REG_OFFSET 0x00000054 -#define SPA_KEY_SZ_REG_OFFSET 0x00000100 -#define SPA_CIPH_KEY_BASE_REG_OFFSET 0x00004000 -#define SPA_HASH_KEY_BASE_REG_OFFSET 0x00008000 -#define SPA_RC4_CTX_BASE_REG_OFFSET 0x00020000 - -#define SPA_IRQ_EN_REG_RESET 0x00000000 -#define SPA_IRQ_CTRL_REG_RESET 0x00000000 -#define SPA_FIFO_STAT_REG_RESET 0x00000000 -#define SPA_SDMA_BRST_SZ_REG_RESET 0x00000000 -#define SPA_SRC_PTR_REG_RESET 0x00000000 -#define SPA_DST_PTR_REG_RESET 0x00000000 -#define SPA_OFFSET_REG_RESET 0x00000000 -#define SPA_AAD_LEN_REG_RESET 0x00000000 -#define SPA_PROC_LEN_REG_RESET 0x00000000 -#define SPA_ICV_LEN_REG_RESET 0x00000000 -#define SPA_ICV_OFFSET_REG_RESET 0x00000000 -#define SPA_SW_CTRL_REG_RESET 0x00000000 -#define SPA_CTRL_REG_RESET 0x00000000 -#define SPA_AUX_INFO_REG_RESET 0x00000000 -#define SPA_STAT_POP_REG_RESET 0x00000000 -#define SPA_STATUS_REG_RESET 0x00000000 -#define SPA_KEY_SZ_REG_RESET 0x00000000 - -#define SPA_CTRL_HASH_ALG_IDX 4 -#define SPA_CTRL_CIPH_MODE_IDX 8 -#define SPA_CTRL_HASH_MODE_IDX 12 -#define SPA_CTRL_CTX_IDX 16 -#define SPA_CTRL_ENCRYPT_IDX 24 -#define SPA_CTRL_AAD_COPY 25 -#define SPA_CTRL_ICV_PT 26 -#define SPA_CTRL_ICV_ENC 27 -#define SPA_CTRL_ICV_APPEND 28 -#define SPA_CTRL_KEY_EXP 29 - -#define SPA_KEY_SZ_CXT_IDX 8 -#define SPA_KEY_SZ_CIPHER_IDX 31 - -#define SPA_IRQ_EN_CMD0_EN (1 << 0) -#define SPA_IRQ_EN_STAT_EN (1 << 4) -#define SPA_IRQ_EN_GLBL_EN (1 << 31) - -#define SPA_CTRL_CIPH_ALG_NULL 0x00 -#define SPA_CTRL_CIPH_ALG_DES 0x01 -#define SPA_CTRL_CIPH_ALG_AES 0x02 -#define SPA_CTRL_CIPH_ALG_RC4 0x03 -#define SPA_CTRL_CIPH_ALG_MULTI2 0x04 -#define SPA_CTRL_CIPH_ALG_KASUMI 0x05 - -#define SPA_CTRL_HASH_ALG_NULL (0x00 << SPA_CTRL_HASH_ALG_IDX) -#define SPA_CTRL_HASH_ALG_MD5 (0x01 << SPA_CTRL_HASH_ALG_IDX) -#define SPA_CTRL_HASH_ALG_SHA (0x02 << SPA_CTRL_HASH_ALG_IDX) -#define SPA_CTRL_HASH_ALG_SHA224 (0x03 << SPA_CTRL_HASH_ALG_IDX) -#define SPA_CTRL_HASH_ALG_SHA256 (0x04 << SPA_CTRL_HASH_ALG_IDX) -#define SPA_CTRL_HASH_ALG_SHA384 (0x05 << SPA_CTRL_HASH_ALG_IDX) -#define SPA_CTRL_HASH_ALG_SHA512 (0x06 << SPA_CTRL_HASH_ALG_IDX) -#define SPA_CTRL_HASH_ALG_AESMAC (0x07 << SPA_CTRL_HASH_ALG_IDX) -#define SPA_CTRL_HASH_ALG_AESCMAC (0x08 << SPA_CTRL_HASH_ALG_IDX) -#define SPA_CTRL_HASH_ALG_KASF9 (0x09 << SPA_CTRL_HASH_ALG_IDX) - -#define SPA_CTRL_CIPH_MODE_NULL (0x00 << SPA_CTRL_CIPH_MODE_IDX) -#define SPA_CTRL_CIPH_MODE_ECB (0x00 << SPA_CTRL_CIPH_MODE_IDX) -#define SPA_CTRL_CIPH_MODE_CBC (0x01 << SPA_CTRL_CIPH_MODE_IDX) -#define SPA_CTRL_CIPH_MODE_CTR (0x02 << SPA_CTRL_CIPH_MODE_IDX) -#define SPA_CTRL_CIPH_MODE_CCM (0x03 << SPA_CTRL_CIPH_MODE_IDX) -#define SPA_CTRL_CIPH_MODE_GCM (0x05 << SPA_CTRL_CIPH_MODE_IDX) -#define SPA_CTRL_CIPH_MODE_OFB (0x07 << SPA_CTRL_CIPH_MODE_IDX) -#define SPA_CTRL_CIPH_MODE_CFB (0x08 << SPA_CTRL_CIPH_MODE_IDX) -#define SPA_CTRL_CIPH_MODE_F8 (0x09 << SPA_CTRL_CIPH_MODE_IDX) - -#define SPA_CTRL_HASH_MODE_RAW (0x00 << SPA_CTRL_HASH_MODE_IDX) -#define SPA_CTRL_HASH_MODE_SSLMAC (0x01 << SPA_CTRL_HASH_MODE_IDX) -#define SPA_CTRL_HASH_MODE_HMAC (0x02 << SPA_CTRL_HASH_MODE_IDX) - -#define SPA_FIFO_STAT_EMPTY (1 << 31) -#define SPA_FIFO_CMD_FULL (1 << 7) - -#endif /* __PICOXCELL_CRYPTO_REGS_H__ */ diff --git a/drivers/crypto/qat/Kconfig b/drivers/crypto/qat/Kconfig index 846a3d90b41a352bf4ae0788e545c00d4a396c0e..77783feb62b25f12e4a5f8a9bc72c2ff9339fc11 100644 --- a/drivers/crypto/qat/Kconfig +++ b/drivers/crypto/qat/Kconfig @@ -11,7 +11,7 @@ config CRYPTO_DEV_QAT select CRYPTO_SHA1 select CRYPTO_SHA256 select CRYPTO_SHA512 - select CRYPTO_AES + select CRYPTO_LIB_AES select FW_LOADER config CRYPTO_DEV_QAT_DH895xCC diff --git a/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c b/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c index 344bfae45bff8e5f525b3ef15f803658793afe56..6a9be01fdf33c2bb176d41e2e723ec338589ea50 100644 --- a/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c +++ b/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c @@ -19,7 +19,7 @@ static struct adf_fw_config adf_4xxx_fw_config[] = { }; /* Worker thread to service arbiter mappings */ -static u32 thrd_to_arb_map[] = { +static const u32 thrd_to_arb_map[ADF_4XXX_MAX_ACCELENGINES] = { 0x5555555, 0x5555555, 0x5555555, 0x5555555, 0xAAAAAAA, 0xAAAAAAA, 0xAAAAAAA, 0xAAAAAAA, 0x0 @@ -119,17 +119,9 @@ static enum dev_sku_info get_sku(struct adf_hw_device_data *self) return DEV_SKU_1; } -static void adf_get_arbiter_mapping(struct adf_accel_dev *accel_dev, - u32 const **arb_map_config) +static const u32 *adf_get_arbiter_mapping(void) { - struct adf_hw_device_data *hw_device = accel_dev->hw_device; - unsigned long ae_mask = hw_device->ae_mask; - int i; - - for_each_clear_bit(i, &ae_mask, ADF_4XXX_MAX_ACCELENGINES) - thrd_to_arb_map[i] = 0; - - *arb_map_config = thrd_to_arb_map; + return thrd_to_arb_map; } static void get_arb_info(struct arb_info *arb_info) diff --git a/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c b/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c index eb45f1b1ae3ebe630e42e506d155af184c499544..f5990d042c9a15dce7bcafdccfef2d6ac85bcf42 100644 --- a/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c +++ b/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c @@ -7,8 +7,8 @@ #include "adf_c3xxx_hw_data.h" #include "icp_qat_hw.h" -/* Worker thread to service arbiter mappings based on dev SKUs */ -static const u32 thrd_to_arb_map_6_me_sku[] = { +/* Worker thread to service arbiter mappings */ +static const u32 thrd_to_arb_map[ADF_C3XXX_MAX_ACCELENGINES] = { 0x12222AAA, 0x11222AAA, 0x12222AAA, 0x11222AAA, 0x12222AAA, 0x11222AAA }; @@ -101,18 +101,9 @@ static enum dev_sku_info get_sku(struct adf_hw_device_data *self) return DEV_SKU_UNKNOWN; } -static void adf_get_arbiter_mapping(struct adf_accel_dev *accel_dev, - u32 const **arb_map_config) +static const u32 *adf_get_arbiter_mapping(void) { - switch (accel_dev->accel_pci_dev.sku) { - case DEV_SKU_4: - *arb_map_config = thrd_to_arb_map_6_me_sku; - break; - default: - dev_err(&GET_DEV(accel_dev), - "The configuration doesn't match any SKU"); - *arb_map_config = NULL; - } + return thrd_to_arb_map; } static u32 get_pf2vf_offset(u32 i) diff --git a/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c b/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c index babdffbcb846f8a92f163d0b9b84e26065644c86..cadcf12884c8b4f517f1fd6fcd3f0f08bcb46e51 100644 --- a/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c +++ b/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c @@ -7,13 +7,8 @@ #include "adf_c62x_hw_data.h" #include "icp_qat_hw.h" -/* Worker thread to service arbiter mappings based on dev SKUs */ -static const u32 thrd_to_arb_map_8_me_sku[] = { - 0x12222AAA, 0x11222AAA, 0x12222AAA, 0x11222AAA, 0x12222AAA, - 0x11222AAA, 0x12222AAA, 0x11222AAA, 0, 0 -}; - -static const u32 thrd_to_arb_map_10_me_sku[] = { +/* Worker thread to service arbiter mappings */ +static const u32 thrd_to_arb_map[ADF_C62X_MAX_ACCELENGINES] = { 0x12222AAA, 0x11222AAA, 0x12222AAA, 0x11222AAA, 0x12222AAA, 0x11222AAA, 0x12222AAA, 0x11222AAA, 0x12222AAA, 0x11222AAA }; @@ -108,21 +103,9 @@ static enum dev_sku_info get_sku(struct adf_hw_device_data *self) return DEV_SKU_UNKNOWN; } -static void adf_get_arbiter_mapping(struct adf_accel_dev *accel_dev, - u32 const **arb_map_config) -{ - switch (accel_dev->accel_pci_dev.sku) { - case DEV_SKU_2: - *arb_map_config = thrd_to_arb_map_8_me_sku; - break; - case DEV_SKU_4: - *arb_map_config = thrd_to_arb_map_10_me_sku; - break; - default: - dev_err(&GET_DEV(accel_dev), - "The configuration doesn't match any SKU"); - *arb_map_config = NULL; - } +static const u32 *adf_get_arbiter_mapping(void) +{ + return thrd_to_arb_map; } static u32 get_pf2vf_offset(u32 i) diff --git a/drivers/crypto/qat/qat_common/adf_accel_devices.h b/drivers/crypto/qat/qat_common/adf_accel_devices.h index c46a5805b294072c5370fd8051500aa63d35dd3b..5527344546e5e6b3011e9819153b14e6499637ae 100644 --- a/drivers/crypto/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/qat/qat_common/adf_accel_devices.h @@ -168,8 +168,7 @@ struct adf_hw_device_data { int (*send_admin_init)(struct adf_accel_dev *accel_dev); int (*init_arb)(struct adf_accel_dev *accel_dev); void (*exit_arb)(struct adf_accel_dev *accel_dev); - void (*get_arb_mapping)(struct adf_accel_dev *accel_dev, - const u32 **cfg); + const u32 *(*get_arb_mapping)(void); void (*disable_iov)(struct adf_accel_dev *accel_dev); void (*configure_iov_threads)(struct adf_accel_dev *accel_dev, bool enable); diff --git a/drivers/crypto/qat/qat_common/adf_ctl_drv.c b/drivers/crypto/qat/qat_common/adf_ctl_drv.c index eb9b3be9d8ebeda6a5df2968301966ef9c25340c..96b437bfe3de2b494b3918a9fb0b6da13173eede 100644 --- a/drivers/crypto/qat/qat_common/adf_ctl_drv.c +++ b/drivers/crypto/qat/qat_common/adf_ctl_drv.c @@ -464,3 +464,4 @@ MODULE_AUTHOR("Intel"); MODULE_DESCRIPTION("Intel(R) QuickAssist Technology"); MODULE_ALIAS_CRYPTO("intel_qat"); MODULE_VERSION(ADF_DRV_VERSION); +MODULE_IMPORT_NS(CRYPTO_INTERNAL); diff --git a/drivers/crypto/qat/qat_common/adf_hw_arbiter.c b/drivers/crypto/qat/qat_common/adf_hw_arbiter.c index 9f5240d9488bf628e0f0fa42bc82b8aee11b1f38..64e4596a24f400fdf8c6de52514ff6bf2bef701b 100644 --- a/drivers/crypto/qat/qat_common/adf_hw_arbiter.c +++ b/drivers/crypto/qat/qat_common/adf_hw_arbiter.c @@ -19,6 +19,7 @@ int adf_init_arb(struct adf_accel_dev *accel_dev) { struct adf_hw_device_data *hw_data = accel_dev->hw_device; void __iomem *csr = accel_dev->transport->banks[0].csr_addr; + unsigned long ae_mask = hw_data->ae_mask; u32 arb_off, wt_off, arb_cfg; const u32 *thd_2_arb_cfg; struct arb_info info; @@ -35,12 +36,9 @@ int adf_init_arb(struct adf_accel_dev *accel_dev) WRITE_CSR_ARB_SARCONFIG(csr, arb_off, arb, arb_cfg); /* Map worker threads to service arbiters */ - hw_data->get_arb_mapping(accel_dev, &thd_2_arb_cfg); + thd_2_arb_cfg = hw_data->get_arb_mapping(); - if (!thd_2_arb_cfg) - return -EFAULT; - - for (i = 0; i < hw_data->num_engines; i++) + for_each_set_bit(i, &ae_mask, hw_data->num_engines) WRITE_CSR_ARB_WT2SAM(csr, arb_off, wt_off, i, thd_2_arb_cfg[i]); return 0; diff --git a/drivers/crypto/qat/qat_common/adf_transport.c b/drivers/crypto/qat/qat_common/adf_transport.c index 5a7030acdc334563234724347e2dad27decf99ca..888c1e0472952591ea8105f6d1377db787730486 100644 --- a/drivers/crypto/qat/qat_common/adf_transport.c +++ b/drivers/crypto/qat/qat_common/adf_transport.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) /* Copyright(c) 2014 - 2020 Intel Corporation */ #include +#include #include "adf_accel_devices.h" #include "adf_transport_internal.h" #include "adf_transport_access_macros.h" @@ -246,6 +247,7 @@ int adf_create_ring(struct adf_accel_dev *accel_dev, const char *section, return -EFAULT; } + ring_num = array_index_nospec(ring_num, num_rings_per_bank); bank = &transport_data->banks[bank_num]; if (adf_reserve_ring(bank, ring_num)) { dev_err(&GET_DEV(accel_dev), "Ring %d, %s already exists.\n", diff --git a/drivers/crypto/qat/qat_common/adf_transport_debug.c b/drivers/crypto/qat/qat_common/adf_transport_debug.c index 1205186ad51ecfeb3503a72b9afc3d416f242c87..e69e5907f5950e60e2b74d9ba03ee6acc4a11ae3 100644 --- a/drivers/crypto/qat/qat_common/adf_transport_debug.c +++ b/drivers/crypto/qat/qat_common/adf_transport_debug.c @@ -62,8 +62,8 @@ static int adf_ring_show(struct seq_file *sfile, void *v) seq_printf(sfile, "head %x, tail %x, empty: %d\n", head, tail, (empty & 1 << ring->ring_number) >> ring->ring_number); - seq_printf(sfile, "ring size %d, msg size %d\n", - ADF_SIZE_TO_RING_SIZE_IN_BYTES(ring->ring_size), + seq_printf(sfile, "ring size %lld, msg size %d\n", + (long long)ADF_SIZE_TO_RING_SIZE_IN_BYTES(ring->ring_size), ADF_MSG_SIZE_TO_BYTES(ring->msg_size)); seq_puts(sfile, "----------- Ring data ------------\n"); return 0; diff --git a/drivers/crypto/qat/qat_common/qat_algs.c b/drivers/crypto/qat/qat_common/qat_algs.c index 31c7a206a6296cc11b87f4f2baddeddd2f94a366..ff78c73c47e382c87fd84543c449967618e5ee5b 100644 --- a/drivers/crypto/qat/qat_common/qat_algs.c +++ b/drivers/crypto/qat/qat_common/qat_algs.c @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include diff --git a/drivers/crypto/qat/qat_common/qat_asym_algs.c b/drivers/crypto/qat/qat_common/qat_asym_algs.c index 2c863d25327a3841473f74c06e9723b9ca5fc809..b0b78445418bb48135587cba5884a09d8cb3c208 100644 --- a/drivers/crypto/qat/qat_common/qat_asym_algs.c +++ b/drivers/crypto/qat/qat_common/qat_asym_algs.c @@ -321,13 +321,13 @@ static int qat_dh_compute_value(struct kpp_request *req) qat_req->out.dh.out_tab[1] = 0; /* Mapping in.in.b or in.in_g2.xa is the same */ qat_req->phy_in = dma_map_single(dev, &qat_req->in.dh.in.b, - sizeof(struct qat_dh_input_params), + sizeof(qat_req->in.dh.in.b), DMA_TO_DEVICE); if (unlikely(dma_mapping_error(dev, qat_req->phy_in))) goto unmap_dst; qat_req->phy_out = dma_map_single(dev, &qat_req->out.dh.r, - sizeof(struct qat_dh_output_params), + sizeof(qat_req->out.dh.r), DMA_TO_DEVICE); if (unlikely(dma_mapping_error(dev, qat_req->phy_out))) goto unmap_in_params; @@ -716,13 +716,13 @@ static int qat_rsa_enc(struct akcipher_request *req) qat_req->in.rsa.in_tab[3] = 0; qat_req->out.rsa.out_tab[1] = 0; qat_req->phy_in = dma_map_single(dev, &qat_req->in.rsa.enc.m, - sizeof(struct qat_rsa_input_params), + sizeof(qat_req->in.rsa.enc.m), DMA_TO_DEVICE); if (unlikely(dma_mapping_error(dev, qat_req->phy_in))) goto unmap_dst; qat_req->phy_out = dma_map_single(dev, &qat_req->out.rsa.enc.c, - sizeof(struct qat_rsa_output_params), + sizeof(qat_req->out.rsa.enc.c), DMA_TO_DEVICE); if (unlikely(dma_mapping_error(dev, qat_req->phy_out))) goto unmap_in_params; @@ -864,13 +864,13 @@ static int qat_rsa_dec(struct akcipher_request *req) qat_req->in.rsa.in_tab[3] = 0; qat_req->out.rsa.out_tab[1] = 0; qat_req->phy_in = dma_map_single(dev, &qat_req->in.rsa.dec.c, - sizeof(struct qat_rsa_input_params), + sizeof(qat_req->in.rsa.dec.c), DMA_TO_DEVICE); if (unlikely(dma_mapping_error(dev, qat_req->phy_in))) goto unmap_dst; qat_req->phy_out = dma_map_single(dev, &qat_req->out.rsa.dec.m, - sizeof(struct qat_rsa_output_params), + sizeof(qat_req->out.rsa.dec.m), DMA_TO_DEVICE); if (unlikely(dma_mapping_error(dev, qat_req->phy_out))) goto unmap_in_params; diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c index 1e83d9397b116b4fd132d213f12e0cac4468e79e..7dd7cd6c3ef84bd5887e9f8687acc8fe787b151d 100644 --- a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c +++ b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c @@ -7,14 +7,8 @@ #include "adf_dh895xcc_hw_data.h" #include "icp_qat_hw.h" -/* Worker thread to service arbiter mappings based on dev SKUs */ -static const u32 thrd_to_arb_map_sku4[] = { - 0x12222AAA, 0x11666666, 0x12222AAA, 0x11666666, - 0x12222AAA, 0x11222222, 0x12222AAA, 0x11222222, - 0x00000000, 0x00000000, 0x00000000, 0x00000000 -}; - -static const u32 thrd_to_arb_map_sku6[] = { +/* Worker thread to service arbiter mappings */ +static const u32 thrd_to_arb_map[ADF_DH895XCC_MAX_ACCELENGINES] = { 0x12222AAA, 0x11666666, 0x12222AAA, 0x11666666, 0x12222AAA, 0x11222222, 0x12222AAA, 0x11222222, 0x12222AAA, 0x11222222, 0x12222AAA, 0x11222222 @@ -127,23 +121,9 @@ static enum dev_sku_info get_sku(struct adf_hw_device_data *self) return DEV_SKU_UNKNOWN; } -static void adf_get_arbiter_mapping(struct adf_accel_dev *accel_dev, - u32 const **arb_map_config) +static const u32 *adf_get_arbiter_mapping(void) { - switch (accel_dev->accel_pci_dev.sku) { - case DEV_SKU_1: - *arb_map_config = thrd_to_arb_map_sku4; - break; - - case DEV_SKU_2: - case DEV_SKU_4: - *arb_map_config = thrd_to_arb_map_sku6; - break; - default: - dev_err(&GET_DEV(accel_dev), - "The configuration doesn't match any SKU"); - *arb_map_config = NULL; - } + return thrd_to_arb_map; } static u32 get_pf2vf_offset(u32 i) diff --git a/drivers/crypto/sahara.c b/drivers/crypto/sahara.c index 8b5be29cb4dcb5fd9ec4e33a4f5d2d609f3410c2..457084b344c170733e2c1331dc00a0ecb8e297ce 100644 --- a/drivers/crypto/sahara.c +++ b/drivers/crypto/sahara.c @@ -1350,12 +1350,6 @@ static void sahara_unregister_algs(struct sahara_dev *dev) crypto_unregister_ahash(&sha_v4_algs[i]); } -static const struct platform_device_id sahara_platform_ids[] = { - { .name = "sahara-imx27" }, - { /* sentinel */ } -}; -MODULE_DEVICE_TABLE(platform, sahara_platform_ids); - static const struct of_device_id sahara_dt_ids[] = { { .compatible = "fsl,imx53-sahara" }, { .compatible = "fsl,imx27-sahara" }, @@ -1540,7 +1534,6 @@ static struct platform_driver sahara_driver = { .name = SAHARA_NAME, .of_match_table = sahara_dt_ids, }, - .id_table = sahara_platform_ids, }; module_platform_driver(sahara_driver); diff --git a/drivers/crypto/stm32/stm32-cryp.c b/drivers/crypto/stm32/stm32-cryp.c index 2670c30332fad9f3638d831607c9790f2966562a..2a4793176c713a98370bca7742614872b7aa238f 100644 --- a/drivers/crypto/stm32/stm32-cryp.c +++ b/drivers/crypto/stm32/stm32-cryp.c @@ -1229,7 +1229,7 @@ static void stm32_cryp_check_ctr_counter(struct stm32_cryp *cryp) cr = stm32_cryp_read(cryp, CRYP_CR); stm32_cryp_write(cryp, CRYP_CR, cr & ~CR_CRYPEN); - stm32_cryp_hw_write_iv(cryp, (u32 *)cryp->last_ctr); + stm32_cryp_hw_write_iv(cryp, (__be32 *)cryp->last_ctr); stm32_cryp_write(cryp, CRYP_CR, cr); } diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index 4fd85f31630ac96de562b7fced035526dafb587e..25c9f825b8b5424d098fa450b6145f6e10de850d 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -1093,11 +1093,12 @@ static void ipsec_esp_decrypt_hwauth_done(struct device *dev, */ static int sg_to_link_tbl_offset(struct scatterlist *sg, int sg_count, unsigned int offset, int datalen, int elen, - struct talitos_ptr *link_tbl_ptr) + struct talitos_ptr *link_tbl_ptr, int align) { int n_sg = elen ? sg_count + 1 : sg_count; int count = 0; int cryptlen = datalen + elen; + int padding = ALIGN(cryptlen, align) - cryptlen; while (cryptlen && sg && n_sg--) { unsigned int len = sg_dma_len(sg); @@ -1121,7 +1122,7 @@ static int sg_to_link_tbl_offset(struct scatterlist *sg, int sg_count, offset += datalen; } to_talitos_ptr(link_tbl_ptr + count, - sg_dma_address(sg) + offset, len, 0); + sg_dma_address(sg) + offset, sg_next(sg) ? len : len + padding, 0); to_talitos_ptr_ext_set(link_tbl_ptr + count, 0, 0); count++; cryptlen -= len; @@ -1144,10 +1145,11 @@ static int talitos_sg_map_ext(struct device *dev, struct scatterlist *src, unsigned int len, struct talitos_edesc *edesc, struct talitos_ptr *ptr, int sg_count, unsigned int offset, int tbl_off, int elen, - bool force) + bool force, int align) { struct talitos_private *priv = dev_get_drvdata(dev); bool is_sec1 = has_ftr_sec1(priv); + int aligned_len = ALIGN(len, align); if (!src) { to_talitos_ptr(ptr, 0, 0, is_sec1); @@ -1155,22 +1157,22 @@ static int talitos_sg_map_ext(struct device *dev, struct scatterlist *src, } to_talitos_ptr_ext_set(ptr, elen, is_sec1); if (sg_count == 1 && !force) { - to_talitos_ptr(ptr, sg_dma_address(src) + offset, len, is_sec1); + to_talitos_ptr(ptr, sg_dma_address(src) + offset, aligned_len, is_sec1); return sg_count; } if (is_sec1) { - to_talitos_ptr(ptr, edesc->dma_link_tbl + offset, len, is_sec1); + to_talitos_ptr(ptr, edesc->dma_link_tbl + offset, aligned_len, is_sec1); return sg_count; } sg_count = sg_to_link_tbl_offset(src, sg_count, offset, len, elen, - &edesc->link_tbl[tbl_off]); + &edesc->link_tbl[tbl_off], align); if (sg_count == 1 && !force) { /* Only one segment now, so no link tbl needed*/ copy_talitos_ptr(ptr, &edesc->link_tbl[tbl_off], is_sec1); return sg_count; } to_talitos_ptr(ptr, edesc->dma_link_tbl + - tbl_off * sizeof(struct talitos_ptr), len, is_sec1); + tbl_off * sizeof(struct talitos_ptr), aligned_len, is_sec1); to_talitos_ptr_ext_or(ptr, DESC_PTR_LNKTBL_JUMP, is_sec1); return sg_count; @@ -1182,7 +1184,7 @@ static int talitos_sg_map(struct device *dev, struct scatterlist *src, unsigned int offset, int tbl_off) { return talitos_sg_map_ext(dev, src, len, edesc, ptr, sg_count, offset, - tbl_off, 0, false); + tbl_off, 0, false, 1); } /* @@ -1251,7 +1253,7 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, ret = talitos_sg_map_ext(dev, areq->src, cryptlen, edesc, &desc->ptr[4], sg_count, areq->assoclen, tbl_off, elen, - false); + false, 1); if (ret > 1) { tbl_off += ret; @@ -1271,7 +1273,7 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, elen = 0; ret = talitos_sg_map_ext(dev, areq->dst, cryptlen, edesc, &desc->ptr[5], sg_count, areq->assoclen, tbl_off, elen, - is_ipsec_esp && !encrypt); + is_ipsec_esp && !encrypt, 1); tbl_off += ret; if (!encrypt && is_ipsec_esp) { @@ -1577,6 +1579,8 @@ static int common_nonsnoop(struct talitos_edesc *edesc, bool sync_needed = false; struct talitos_private *priv = dev_get_drvdata(dev); bool is_sec1 = has_ftr_sec1(priv); + bool is_ctr = (desc->hdr & DESC_HDR_SEL0_MASK) == DESC_HDR_SEL0_AESU && + (desc->hdr & DESC_HDR_MODE0_AESU_MASK) == DESC_HDR_MODE0_AESU_CTR; /* first DWORD empty */ @@ -1597,8 +1601,8 @@ static int common_nonsnoop(struct talitos_edesc *edesc, /* * cipher in */ - sg_count = talitos_sg_map(dev, areq->src, cryptlen, edesc, - &desc->ptr[3], sg_count, 0, 0); + sg_count = talitos_sg_map_ext(dev, areq->src, cryptlen, edesc, &desc->ptr[3], + sg_count, 0, 0, 0, false, is_ctr ? 16 : 1); if (sg_count > 1) sync_needed = true; @@ -2761,6 +2765,22 @@ static struct talitos_alg_template driver_algs[] = { DESC_HDR_SEL0_AESU | DESC_HDR_MODE0_AESU_CTR, }, + { .type = CRYPTO_ALG_TYPE_SKCIPHER, + .alg.skcipher = { + .base.cra_name = "ctr(aes)", + .base.cra_driver_name = "ctr-aes-talitos", + .base.cra_blocksize = 1, + .base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = skcipher_aes_setkey, + }, + .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | + DESC_HDR_SEL0_AESU | + DESC_HDR_MODE0_AESU_CTR, + }, { .type = CRYPTO_ALG_TYPE_SKCIPHER, .alg.skcipher = { .base.cra_name = "ecb(des)", @@ -3178,6 +3198,12 @@ static struct talitos_crypto_alg *talitos_alg_alloc(struct device *dev, t_alg->algt.alg.skcipher.setkey ?: skcipher_setkey; t_alg->algt.alg.skcipher.encrypt = skcipher_encrypt; t_alg->algt.alg.skcipher.decrypt = skcipher_decrypt; + if (!strcmp(alg->cra_name, "ctr(aes)") && !has_ftr_sec1(priv) && + DESC_TYPE(t_alg->algt.desc_hdr_template) != + DESC_TYPE(DESC_HDR_TYPE_AESU_CTR_NONSNOOP)) { + devm_kfree(dev, t_alg); + return ERR_PTR(-ENOTSUPP); + } break; case CRYPTO_ALG_TYPE_AEAD: alg = &t_alg->algt.alg.aead.base; diff --git a/drivers/crypto/talitos.h b/drivers/crypto/talitos.h index 1469b956948abf459e82bc71cb139845e5b91cb0..32825119e8805f24174e92458074c7c32a84153f 100644 --- a/drivers/crypto/talitos.h +++ b/drivers/crypto/talitos.h @@ -344,6 +344,7 @@ static inline bool has_ftr_sec1(struct talitos_private *priv) /* primary execution unit mode (MODE0) and derivatives */ #define DESC_HDR_MODE0_ENCRYPT cpu_to_be32(0x00100000) +#define DESC_HDR_MODE0_AESU_MASK cpu_to_be32(0x00600000) #define DESC_HDR_MODE0_AESU_CBC cpu_to_be32(0x00200000) #define DESC_HDR_MODE0_AESU_CTR cpu_to_be32(0x00600000) #define DESC_HDR_MODE0_DEU_CBC cpu_to_be32(0x00400000) diff --git a/drivers/crypto/vmx/aes.c b/drivers/crypto/vmx/aes.c index 2bc5d4e1adf4b459b758dc84a823136edd97b02e..d05c02baebcf0ce541a85f74cc02c688b33bfe35 100644 --- a/drivers/crypto/vmx/aes.c +++ b/drivers/crypto/vmx/aes.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include "aesp8-ppc.h" diff --git a/drivers/crypto/vmx/aesp8-ppc.h b/drivers/crypto/vmx/aesp8-ppc.h index 01774a4d26a2555e6a8098e8737c386d83413cff..5764d443838850ef7ac04eaabb69aa33528c4d65 100644 --- a/drivers/crypto/vmx/aesp8-ppc.h +++ b/drivers/crypto/vmx/aesp8-ppc.h @@ -7,6 +7,12 @@ struct aes_key { int rounds; }; +extern struct shash_alg p8_ghash_alg; +extern struct crypto_alg p8_aes_alg; +extern struct skcipher_alg p8_aes_cbc_alg; +extern struct skcipher_alg p8_aes_ctr_alg; +extern struct skcipher_alg p8_aes_xts_alg; + int aes_p8_set_encrypt_key(const u8 *userKey, const int bits, struct aes_key *key); int aes_p8_set_decrypt_key(const u8 *userKey, const int bits, diff --git a/drivers/crypto/vmx/vmx.c b/drivers/crypto/vmx/vmx.c index 3e0335fb406c703193ecca00e0e3f3e32b9ad9df..a40d08e75fc0bae31c5e379f03170d9f9dae134d 100644 --- a/drivers/crypto/vmx/vmx.c +++ b/drivers/crypto/vmx/vmx.c @@ -17,11 +17,7 @@ #include #include -extern struct shash_alg p8_ghash_alg; -extern struct crypto_alg p8_aes_alg; -extern struct skcipher_alg p8_aes_cbc_alg; -extern struct skcipher_alg p8_aes_ctr_alg; -extern struct skcipher_alg p8_aes_xts_alg; +#include "aesp8-ppc.h" static int __init p8_init(void) { @@ -78,3 +74,4 @@ MODULE_DESCRIPTION("IBM VMX cryptographic acceleration instructions " "support on Power 8"); MODULE_LICENSE("GPL"); MODULE_VERSION("1.0.0"); +MODULE_IMPORT_NS(CRYPTO_INTERNAL); diff --git a/drivers/cxl/Kconfig b/drivers/cxl/Kconfig new file mode 100644 index 0000000000000000000000000000000000000000..97dc4d751651197dcea752bac0a865d8b63e269c --- /dev/null +++ b/drivers/cxl/Kconfig @@ -0,0 +1,53 @@ +# SPDX-License-Identifier: GPL-2.0-only +menuconfig CXL_BUS + tristate "CXL (Compute Express Link) Devices Support" + depends on PCI + help + CXL is a bus that is electrically compatible with PCI Express, but + layers three protocols on that signalling (CXL.io, CXL.cache, and + CXL.mem). The CXL.cache protocol allows devices to hold cachelines + locally, the CXL.mem protocol allows devices to be fully coherent + memory targets, the CXL.io protocol is equivalent to PCI Express. + Say 'y' to enable support for the configuration and management of + devices supporting these protocols. + +if CXL_BUS + +config CXL_MEM + tristate "CXL.mem: Memory Devices" + help + The CXL.mem protocol allows a device to act as a provider of + "System RAM" and/or "Persistent Memory" that is fully coherent + as if the memory was attached to the typical CPU memory + controller. + + Say 'y/m' to enable a driver (named "cxl_mem.ko" when built as + a module) that will attach to CXL.mem devices for + configuration, provisioning, and health monitoring. This + driver is required for dynamic provisioning of CXL.mem + attached memory which is a prerequisite for persistent memory + support. Typically volatile memory is mapped by platform + firmware and included in the platform memory map, but in some + cases the OS is responsible for mapping that memory. See + Chapter 2.3 Type 3 CXL Device in the CXL 2.0 specification. + + If unsure say 'm'. + +config CXL_MEM_RAW_COMMANDS + bool "RAW Command Interface for Memory Devices" + depends on CXL_MEM + help + Enable CXL RAW command interface. + + The CXL driver ioctl interface may assign a kernel ioctl command + number for each specification defined opcode. At any given point in + time the number of opcodes that the specification defines and a device + may implement may exceed the kernel's set of associated ioctl function + numbers. The mismatch is either by omission, specification is too new, + or by design. When prototyping new hardware, or developing / debugging + the driver it is useful to be able to submit any possible command to + the hardware, even commands that may crash the kernel due to their + potential impact to memory currently in use by the kernel. + + If developing CXL hardware or the driver say Y, otherwise say N. +endif diff --git a/drivers/cxl/Makefile b/drivers/cxl/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..a314a1891f4d37911da3c65b1de936c4978ca6ed --- /dev/null +++ b/drivers/cxl/Makefile @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_CXL_BUS) += cxl_bus.o +obj-$(CONFIG_CXL_MEM) += cxl_mem.o + +ccflags-y += -DDEFAULT_SYMBOL_NAMESPACE=CXL +cxl_bus-y := bus.o +cxl_mem-y := mem.o diff --git a/drivers/cxl/bus.c b/drivers/cxl/bus.c new file mode 100644 index 0000000000000000000000000000000000000000..58f74796d525b21050c3f2f56eda8a5410495d17 --- /dev/null +++ b/drivers/cxl/bus.c @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2020 Intel Corporation. All rights reserved. */ +#include +#include + +/** + * DOC: cxl bus + * + * The CXL bus provides namespace for control devices and a rendezvous + * point for cross-device interleave coordination. + */ +struct bus_type cxl_bus_type = { + .name = "cxl", +}; +EXPORT_SYMBOL_GPL(cxl_bus_type); + +static __init int cxl_bus_init(void) +{ + return bus_register(&cxl_bus_type); +} + +static void cxl_bus_exit(void) +{ + bus_unregister(&cxl_bus_type); +} + +module_init(cxl_bus_init); +module_exit(cxl_bus_exit); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h new file mode 100644 index 0000000000000000000000000000000000000000..6f14838c2d25706f9948b54d360cfc87c5b474a2 --- /dev/null +++ b/drivers/cxl/cxl.h @@ -0,0 +1,95 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2020 Intel Corporation. */ + +#ifndef __CXL_H__ +#define __CXL_H__ + +#include +#include +#include + +/* CXL 2.0 8.2.8.1 Device Capabilities Array Register */ +#define CXLDEV_CAP_ARRAY_OFFSET 0x0 +#define CXLDEV_CAP_ARRAY_CAP_ID 0 +#define CXLDEV_CAP_ARRAY_ID_MASK GENMASK_ULL(15, 0) +#define CXLDEV_CAP_ARRAY_COUNT_MASK GENMASK_ULL(47, 32) +/* CXL 2.0 8.2.8.2 CXL Device Capability Header Register */ +#define CXLDEV_CAP_HDR_CAP_ID_MASK GENMASK(15, 0) +/* CXL 2.0 8.2.8.2.1 CXL Device Capabilities */ +#define CXLDEV_CAP_CAP_ID_DEVICE_STATUS 0x1 +#define CXLDEV_CAP_CAP_ID_PRIMARY_MAILBOX 0x2 +#define CXLDEV_CAP_CAP_ID_SECONDARY_MAILBOX 0x3 +#define CXLDEV_CAP_CAP_ID_MEMDEV 0x4000 + +/* CXL 2.0 8.2.8.4 Mailbox Registers */ +#define CXLDEV_MBOX_CAPS_OFFSET 0x00 +#define CXLDEV_MBOX_CAP_PAYLOAD_SIZE_MASK GENMASK(4, 0) +#define CXLDEV_MBOX_CTRL_OFFSET 0x04 +#define CXLDEV_MBOX_CTRL_DOORBELL BIT(0) +#define CXLDEV_MBOX_CMD_OFFSET 0x08 +#define CXLDEV_MBOX_CMD_COMMAND_OPCODE_MASK GENMASK_ULL(15, 0) +#define CXLDEV_MBOX_CMD_PAYLOAD_LENGTH_MASK GENMASK_ULL(36, 16) +#define CXLDEV_MBOX_STATUS_OFFSET 0x10 +#define CXLDEV_MBOX_STATUS_RET_CODE_MASK GENMASK_ULL(47, 32) +#define CXLDEV_MBOX_BG_CMD_STATUS_OFFSET 0x18 +#define CXLDEV_MBOX_PAYLOAD_OFFSET 0x20 + +/* CXL 2.0 8.2.8.5.1.1 Memory Device Status Register */ +#define CXLMDEV_STATUS_OFFSET 0x0 +#define CXLMDEV_DEV_FATAL BIT(0) +#define CXLMDEV_FW_HALT BIT(1) +#define CXLMDEV_STATUS_MEDIA_STATUS_MASK GENMASK(3, 2) +#define CXLMDEV_MS_NOT_READY 0 +#define CXLMDEV_MS_READY 1 +#define CXLMDEV_MS_ERROR 2 +#define CXLMDEV_MS_DISABLED 3 +#define CXLMDEV_READY(status) \ + (FIELD_GET(CXLMDEV_STATUS_MEDIA_STATUS_MASK, status) == \ + CXLMDEV_MS_READY) +#define CXLMDEV_MBOX_IF_READY BIT(4) +#define CXLMDEV_RESET_NEEDED_MASK GENMASK(7, 5) +#define CXLMDEV_RESET_NEEDED_NOT 0 +#define CXLMDEV_RESET_NEEDED_COLD 1 +#define CXLMDEV_RESET_NEEDED_WARM 2 +#define CXLMDEV_RESET_NEEDED_HOT 3 +#define CXLMDEV_RESET_NEEDED_CXL 4 +#define CXLMDEV_RESET_NEEDED(status) \ + (FIELD_GET(CXLMDEV_RESET_NEEDED_MASK, status) != \ + CXLMDEV_RESET_NEEDED_NOT) + +struct cxl_memdev; +/** + * struct cxl_mem - A CXL memory device + * @pdev: The PCI device associated with this CXL device. + * @regs: IO mappings to the device's MMIO + * @status_regs: CXL 2.0 8.2.8.3 Device Status Registers + * @mbox_regs: CXL 2.0 8.2.8.4 Mailbox Registers + * @memdev_regs: CXL 2.0 8.2.8.5 Memory Device Registers + * @payload_size: Size of space for payload + * (CXL 2.0 8.2.8.4.3 Mailbox Capabilities Register) + * @mbox_mutex: Mutex to synchronize mailbox access. + * @firmware_version: Firmware version for the memory device. + * @enabled_commands: Hardware commands found enabled in CEL. + * @pmem_range: Persistent memory capacity information. + * @ram_range: Volatile memory capacity information. + */ +struct cxl_mem { + struct pci_dev *pdev; + void __iomem *regs; + struct cxl_memdev *cxlmd; + + void __iomem *status_regs; + void __iomem *mbox_regs; + void __iomem *memdev_regs; + + size_t payload_size; + struct mutex mbox_mutex; /* Protects device mailbox and firmware */ + char firmware_version[0x10]; + unsigned long *enabled_cmds; + + struct range pmem_range; + struct range ram_range; +}; + +extern struct bus_type cxl_bus_type; +#endif /* __CXL_H__ */ diff --git a/drivers/cxl/mem.c b/drivers/cxl/mem.c new file mode 100644 index 0000000000000000000000000000000000000000..244cb7d89678b24a9cd21de12e2142530fd553ec --- /dev/null +++ b/drivers/cxl/mem.c @@ -0,0 +1,1552 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2020 Intel Corporation. All rights reserved. */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "pci.h" +#include "cxl.h" + +/** + * DOC: cxl mem + * + * This implements a CXL memory device ("type-3") as it is defined by the + * Compute Express Link specification. + * + * The driver has several responsibilities, mainly: + * - Create the memX device and register on the CXL bus. + * - Enumerate device's register interface and map them. + * - Probe the device attributes to establish sysfs interface. + * - Provide an IOCTL interface to userspace to communicate with the device for + * things like firmware update. + * - Support management of interleave sets. + * - Handle and manage error conditions. + */ + +/* + * An entire PCI topology full of devices should be enough for any + * config + */ +#define CXL_MEM_MAX_DEVS 65536 + +#define cxl_doorbell_busy(cxlm) \ + (readl((cxlm)->mbox_regs + CXLDEV_MBOX_CTRL_OFFSET) & \ + CXLDEV_MBOX_CTRL_DOORBELL) + +/* CXL 2.0 - 8.2.8.4 */ +#define CXL_MAILBOX_TIMEOUT_MS (2 * HZ) + +enum opcode { + CXL_MBOX_OP_INVALID = 0x0000, + CXL_MBOX_OP_RAW = CXL_MBOX_OP_INVALID, + CXL_MBOX_OP_GET_FW_INFO = 0x0200, + CXL_MBOX_OP_ACTIVATE_FW = 0x0202, + CXL_MBOX_OP_GET_SUPPORTED_LOGS = 0x0400, + CXL_MBOX_OP_GET_LOG = 0x0401, + CXL_MBOX_OP_IDENTIFY = 0x4000, + CXL_MBOX_OP_GET_PARTITION_INFO = 0x4100, + CXL_MBOX_OP_SET_PARTITION_INFO = 0x4101, + CXL_MBOX_OP_GET_LSA = 0x4102, + CXL_MBOX_OP_SET_LSA = 0x4103, + CXL_MBOX_OP_GET_HEALTH_INFO = 0x4200, + CXL_MBOX_OP_SET_SHUTDOWN_STATE = 0x4204, + CXL_MBOX_OP_SCAN_MEDIA = 0x4304, + CXL_MBOX_OP_GET_SCAN_MEDIA = 0x4305, + CXL_MBOX_OP_MAX = 0x10000 +}; + +/** + * struct mbox_cmd - A command to be submitted to hardware. + * @opcode: (input) The command set and command submitted to hardware. + * @payload_in: (input) Pointer to the input payload. + * @payload_out: (output) Pointer to the output payload. Must be allocated by + * the caller. + * @size_in: (input) Number of bytes to load from @payload_in. + * @size_out: (input) Max number of bytes loaded into @payload_out. + * (output) Number of bytes generated by the device. For fixed size + * outputs commands this is always expected to be deterministic. For + * variable sized output commands, it tells the exact number of bytes + * written. + * @return_code: (output) Error code returned from hardware. + * + * This is the primary mechanism used to send commands to the hardware. + * All the fields except @payload_* correspond exactly to the fields described in + * Command Register section of the CXL 2.0 8.2.8.4.5. @payload_in and + * @payload_out are written to, and read from the Command Payload Registers + * defined in CXL 2.0 8.2.8.4.8. + */ +struct mbox_cmd { + u16 opcode; + void *payload_in; + void *payload_out; + size_t size_in; + size_t size_out; + u16 return_code; +#define CXL_MBOX_SUCCESS 0 +}; + +/** + * struct cxl_memdev - CXL bus object representing a Type-3 Memory Device + * @dev: driver core device object + * @cdev: char dev core object for ioctl operations + * @cxlm: pointer to the parent device driver data + * @ops_active: active user of @cxlm in ops handlers + * @ops_dead: completion when all @cxlm ops users have exited + * @id: id number of this memdev instance. + */ +struct cxl_memdev { + struct device dev; + struct cdev cdev; + struct cxl_mem *cxlm; + struct percpu_ref ops_active; + struct completion ops_dead; + int id; +}; + +static int cxl_mem_major; +static DEFINE_IDA(cxl_memdev_ida); +static struct dentry *cxl_debugfs; +static bool cxl_raw_allow_all; + +enum { + CEL_UUID, + VENDOR_DEBUG_UUID, +}; + +/* See CXL 2.0 Table 170. Get Log Input Payload */ +static const uuid_t log_uuid[] = { + [CEL_UUID] = UUID_INIT(0xda9c0b5, 0xbf41, 0x4b78, 0x8f, 0x79, 0x96, + 0xb1, 0x62, 0x3b, 0x3f, 0x17), + [VENDOR_DEBUG_UUID] = UUID_INIT(0xe1819d9, 0x11a9, 0x400c, 0x81, 0x1f, + 0xd6, 0x07, 0x19, 0x40, 0x3d, 0x86), +}; + +/** + * struct cxl_mem_command - Driver representation of a memory device command + * @info: Command information as it exists for the UAPI + * @opcode: The actual bits used for the mailbox protocol + * @flags: Set of flags effecting driver behavior. + * + * * %CXL_CMD_FLAG_FORCE_ENABLE: In cases of error, commands with this flag + * will be enabled by the driver regardless of what hardware may have + * advertised. + * + * The cxl_mem_command is the driver's internal representation of commands that + * are supported by the driver. Some of these commands may not be supported by + * the hardware. The driver will use @info to validate the fields passed in by + * the user then submit the @opcode to the hardware. + * + * See struct cxl_command_info. + */ +struct cxl_mem_command { + struct cxl_command_info info; + enum opcode opcode; + u32 flags; +#define CXL_CMD_FLAG_NONE 0 +#define CXL_CMD_FLAG_FORCE_ENABLE BIT(0) +}; + +#define CXL_CMD(_id, sin, sout, _flags) \ + [CXL_MEM_COMMAND_ID_##_id] = { \ + .info = { \ + .id = CXL_MEM_COMMAND_ID_##_id, \ + .size_in = sin, \ + .size_out = sout, \ + }, \ + .opcode = CXL_MBOX_OP_##_id, \ + .flags = _flags, \ + } + +/* + * This table defines the supported mailbox commands for the driver. This table + * is made up of a UAPI structure. Non-negative values as parameters in the + * table will be validated against the user's input. For example, if size_in is + * 0, and the user passed in 1, it is an error. + */ +static struct cxl_mem_command mem_commands[] = { + CXL_CMD(IDENTIFY, 0, 0x43, CXL_CMD_FLAG_FORCE_ENABLE), +#ifdef CONFIG_CXL_MEM_RAW_COMMANDS + CXL_CMD(RAW, ~0, ~0, 0), +#endif + CXL_CMD(GET_SUPPORTED_LOGS, 0, ~0, CXL_CMD_FLAG_FORCE_ENABLE), + CXL_CMD(GET_FW_INFO, 0, 0x50, 0), + CXL_CMD(GET_PARTITION_INFO, 0, 0x20, 0), + CXL_CMD(GET_LSA, 0x8, ~0, 0), + CXL_CMD(GET_HEALTH_INFO, 0, 0x12, 0), + CXL_CMD(GET_LOG, 0x18, ~0, CXL_CMD_FLAG_FORCE_ENABLE), +}; + +/* + * Commands that RAW doesn't permit. The rationale for each: + * + * CXL_MBOX_OP_ACTIVATE_FW: Firmware activation requires adjustment / + * coordination of transaction timeout values at the root bridge level. + * + * CXL_MBOX_OP_SET_PARTITION_INFO: The device memory map may change live + * and needs to be coordinated with HDM updates. + * + * CXL_MBOX_OP_SET_LSA: The label storage area may be cached by the + * driver and any writes from userspace invalidates those contents. + * + * CXL_MBOX_OP_SET_SHUTDOWN_STATE: Set shutdown state assumes no writes + * to the device after it is marked clean, userspace can not make that + * assertion. + * + * CXL_MBOX_OP_[GET_]SCAN_MEDIA: The kernel provides a native error list that + * is kept up to date with patrol notifications and error management. + */ +static u16 cxl_disabled_raw_commands[] = { + CXL_MBOX_OP_ACTIVATE_FW, + CXL_MBOX_OP_SET_PARTITION_INFO, + CXL_MBOX_OP_SET_LSA, + CXL_MBOX_OP_SET_SHUTDOWN_STATE, + CXL_MBOX_OP_SCAN_MEDIA, + CXL_MBOX_OP_GET_SCAN_MEDIA, +}; + +/* + * Command sets that RAW doesn't permit. All opcodes in this set are + * disabled because they pass plain text security payloads over the + * user/kernel boundary. This functionality is intended to be wrapped + * behind the keys ABI which allows for encrypted payloads in the UAPI + */ +static u8 security_command_sets[] = { + 0x44, /* Sanitize */ + 0x45, /* Persistent Memory Data-at-rest Security */ + 0x46, /* Security Passthrough */ +}; + +#define cxl_for_each_cmd(cmd) \ + for ((cmd) = &mem_commands[0]; \ + ((cmd) - mem_commands) < ARRAY_SIZE(mem_commands); (cmd)++) + +#define cxl_cmd_count ARRAY_SIZE(mem_commands) + +static int cxl_mem_wait_for_doorbell(struct cxl_mem *cxlm) +{ + const unsigned long start = jiffies; + unsigned long end = start; + + while (cxl_doorbell_busy(cxlm)) { + end = jiffies; + + if (time_after(end, start + CXL_MAILBOX_TIMEOUT_MS)) { + /* Check again in case preempted before timeout test */ + if (!cxl_doorbell_busy(cxlm)) + break; + return -ETIMEDOUT; + } + cpu_relax(); + } + + dev_dbg(&cxlm->pdev->dev, "Doorbell wait took %dms", + jiffies_to_msecs(end) - jiffies_to_msecs(start)); + return 0; +} + +static bool cxl_is_security_command(u16 opcode) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(security_command_sets); i++) + if (security_command_sets[i] == (opcode >> 8)) + return true; + return false; +} + +static void cxl_mem_mbox_timeout(struct cxl_mem *cxlm, + struct mbox_cmd *mbox_cmd) +{ + struct device *dev = &cxlm->pdev->dev; + + dev_dbg(dev, "Mailbox command (opcode: %#x size: %zub) timed out\n", + mbox_cmd->opcode, mbox_cmd->size_in); +} + +/** + * __cxl_mem_mbox_send_cmd() - Execute a mailbox command + * @cxlm: The CXL memory device to communicate with. + * @mbox_cmd: Command to send to the memory device. + * + * Context: Any context. Expects mbox_mutex to be held. + * Return: -ETIMEDOUT if timeout occurred waiting for completion. 0 on success. + * Caller should check the return code in @mbox_cmd to make sure it + * succeeded. + * + * This is a generic form of the CXL mailbox send command thus only using the + * registers defined by the mailbox capability ID - CXL 2.0 8.2.8.4. Memory + * devices, and perhaps other types of CXL devices may have further information + * available upon error conditions. Driver facilities wishing to send mailbox + * commands should use the wrapper command. + * + * The CXL spec allows for up to two mailboxes. The intention is for the primary + * mailbox to be OS controlled and the secondary mailbox to be used by system + * firmware. This allows the OS and firmware to communicate with the device and + * not need to coordinate with each other. The driver only uses the primary + * mailbox. + */ +static int __cxl_mem_mbox_send_cmd(struct cxl_mem *cxlm, + struct mbox_cmd *mbox_cmd) +{ + void __iomem *payload = cxlm->mbox_regs + CXLDEV_MBOX_PAYLOAD_OFFSET; + u64 cmd_reg, status_reg; + size_t out_len; + int rc; + + lockdep_assert_held(&cxlm->mbox_mutex); + + /* + * Here are the steps from 8.2.8.4 of the CXL 2.0 spec. + * 1. Caller reads MB Control Register to verify doorbell is clear + * 2. Caller writes Command Register + * 3. Caller writes Command Payload Registers if input payload is non-empty + * 4. Caller writes MB Control Register to set doorbell + * 5. Caller either polls for doorbell to be clear or waits for interrupt if configured + * 6. Caller reads MB Status Register to fetch Return code + * 7. If command successful, Caller reads Command Register to get Payload Length + * 8. If output payload is non-empty, host reads Command Payload Registers + * + * Hardware is free to do whatever it wants before the doorbell is rung, + * and isn't allowed to change anything after it clears the doorbell. As + * such, steps 2 and 3 can happen in any order, and steps 6, 7, 8 can + * also happen in any order (though some orders might not make sense). + */ + + /* #1 */ + if (cxl_doorbell_busy(cxlm)) { + dev_err_ratelimited(&cxlm->pdev->dev, + "Mailbox re-busy after acquiring\n"); + return -EBUSY; + } + + cmd_reg = FIELD_PREP(CXLDEV_MBOX_CMD_COMMAND_OPCODE_MASK, + mbox_cmd->opcode); + if (mbox_cmd->size_in) { + if (WARN_ON(!mbox_cmd->payload_in)) + return -EINVAL; + + cmd_reg |= FIELD_PREP(CXLDEV_MBOX_CMD_PAYLOAD_LENGTH_MASK, + mbox_cmd->size_in); + memcpy_toio(payload, mbox_cmd->payload_in, mbox_cmd->size_in); + } + + /* #2, #3 */ + writeq(cmd_reg, cxlm->mbox_regs + CXLDEV_MBOX_CMD_OFFSET); + + /* #4 */ + dev_dbg(&cxlm->pdev->dev, "Sending command\n"); + writel(CXLDEV_MBOX_CTRL_DOORBELL, + cxlm->mbox_regs + CXLDEV_MBOX_CTRL_OFFSET); + + /* #5 */ + rc = cxl_mem_wait_for_doorbell(cxlm); + if (rc == -ETIMEDOUT) { + cxl_mem_mbox_timeout(cxlm, mbox_cmd); + return rc; + } + + /* #6 */ + status_reg = readq(cxlm->mbox_regs + CXLDEV_MBOX_STATUS_OFFSET); + mbox_cmd->return_code = + FIELD_GET(CXLDEV_MBOX_STATUS_RET_CODE_MASK, status_reg); + + if (mbox_cmd->return_code != 0) { + dev_dbg(&cxlm->pdev->dev, "Mailbox operation had an error\n"); + return 0; + } + + /* #7 */ + cmd_reg = readq(cxlm->mbox_regs + CXLDEV_MBOX_CMD_OFFSET); + out_len = FIELD_GET(CXLDEV_MBOX_CMD_PAYLOAD_LENGTH_MASK, cmd_reg); + + /* #8 */ + if (out_len && mbox_cmd->payload_out) { + /* + * Sanitize the copy. If hardware misbehaves, out_len per the + * spec can actually be greater than the max allowed size (21 + * bits available but spec defined 1M max). The caller also may + * have requested less data than the hardware supplied even + * within spec. + */ + size_t n = min3(mbox_cmd->size_out, cxlm->payload_size, out_len); + + memcpy_fromio(mbox_cmd->payload_out, payload, n); + mbox_cmd->size_out = n; + } else { + mbox_cmd->size_out = 0; + } + + return 0; +} + +/** + * cxl_mem_mbox_get() - Acquire exclusive access to the mailbox. + * @cxlm: The memory device to gain access to. + * + * Context: Any context. Takes the mbox_mutex. + * Return: 0 if exclusive access was acquired. + */ +static int cxl_mem_mbox_get(struct cxl_mem *cxlm) +{ + struct device *dev = &cxlm->pdev->dev; + u64 md_status; + int rc; + + mutex_lock_io(&cxlm->mbox_mutex); + + /* + * XXX: There is some amount of ambiguity in the 2.0 version of the spec + * around the mailbox interface ready (8.2.8.5.1.1). The purpose of the + * bit is to allow firmware running on the device to notify the driver + * that it's ready to receive commands. It is unclear if the bit needs + * to be read for each transaction mailbox, ie. the firmware can switch + * it on and off as needed. Second, there is no defined timeout for + * mailbox ready, like there is for the doorbell interface. + * + * Assumptions: + * 1. The firmware might toggle the Mailbox Interface Ready bit, check + * it for every command. + * + * 2. If the doorbell is clear, the firmware should have first set the + * Mailbox Interface Ready bit. Therefore, waiting for the doorbell + * to be ready is sufficient. + */ + rc = cxl_mem_wait_for_doorbell(cxlm); + if (rc) { + dev_warn(dev, "Mailbox interface not ready\n"); + goto out; + } + + md_status = readq(cxlm->memdev_regs + CXLMDEV_STATUS_OFFSET); + if (!(md_status & CXLMDEV_MBOX_IF_READY && CXLMDEV_READY(md_status))) { + dev_err(dev, "mbox: reported doorbell ready, but not mbox ready\n"); + rc = -EBUSY; + goto out; + } + + /* + * Hardware shouldn't allow a ready status but also have failure bits + * set. Spit out an error, this should be a bug report + */ + rc = -EFAULT; + if (md_status & CXLMDEV_DEV_FATAL) { + dev_err(dev, "mbox: reported ready, but fatal\n"); + goto out; + } + if (md_status & CXLMDEV_FW_HALT) { + dev_err(dev, "mbox: reported ready, but halted\n"); + goto out; + } + if (CXLMDEV_RESET_NEEDED(md_status)) { + dev_err(dev, "mbox: reported ready, but reset needed\n"); + goto out; + } + + /* with lock held */ + return 0; + +out: + mutex_unlock(&cxlm->mbox_mutex); + return rc; +} + +/** + * cxl_mem_mbox_put() - Release exclusive access to the mailbox. + * @cxlm: The CXL memory device to communicate with. + * + * Context: Any context. Expects mbox_mutex to be held. + */ +static void cxl_mem_mbox_put(struct cxl_mem *cxlm) +{ + mutex_unlock(&cxlm->mbox_mutex); +} + +/** + * handle_mailbox_cmd_from_user() - Dispatch a mailbox command for userspace. + * @cxlm: The CXL memory device to communicate with. + * @cmd: The validated command. + * @in_payload: Pointer to userspace's input payload. + * @out_payload: Pointer to userspace's output payload. + * @size_out: (Input) Max payload size to copy out. + * (Output) Payload size hardware generated. + * @retval: Hardware generated return code from the operation. + * + * Return: + * * %0 - Mailbox transaction succeeded. This implies the mailbox + * protocol completed successfully not that the operation itself + * was successful. + * * %-ENOMEM - Couldn't allocate a bounce buffer. + * * %-EFAULT - Something happened with copy_to/from_user. + * * %-EINTR - Mailbox acquisition interrupted. + * * %-EXXX - Transaction level failures. + * + * Creates the appropriate mailbox command and dispatches it on behalf of a + * userspace request. The input and output payloads are copied between + * userspace. + * + * See cxl_send_cmd(). + */ +static int handle_mailbox_cmd_from_user(struct cxl_mem *cxlm, + const struct cxl_mem_command *cmd, + u64 in_payload, u64 out_payload, + s32 *size_out, u32 *retval) +{ + struct device *dev = &cxlm->pdev->dev; + struct mbox_cmd mbox_cmd = { + .opcode = cmd->opcode, + .size_in = cmd->info.size_in, + .size_out = cmd->info.size_out, + }; + int rc; + + if (cmd->info.size_out) { + mbox_cmd.payload_out = kvzalloc(cmd->info.size_out, GFP_KERNEL); + if (!mbox_cmd.payload_out) + return -ENOMEM; + } + + if (cmd->info.size_in) { + mbox_cmd.payload_in = vmemdup_user(u64_to_user_ptr(in_payload), + cmd->info.size_in); + if (IS_ERR(mbox_cmd.payload_in)) { + kvfree(mbox_cmd.payload_out); + return PTR_ERR(mbox_cmd.payload_in); + } + } + + rc = cxl_mem_mbox_get(cxlm); + if (rc) + goto out; + + dev_dbg(dev, + "Submitting %s command for user\n" + "\topcode: %x\n" + "\tsize: %ub\n", + cxl_command_names[cmd->info.id].name, mbox_cmd.opcode, + cmd->info.size_in); + + dev_WARN_ONCE(dev, cmd->info.id == CXL_MEM_COMMAND_ID_RAW, + "raw command path used\n"); + + rc = __cxl_mem_mbox_send_cmd(cxlm, &mbox_cmd); + cxl_mem_mbox_put(cxlm); + if (rc) + goto out; + + /* + * @size_out contains the max size that's allowed to be written back out + * to userspace. While the payload may have written more output than + * this it will have to be ignored. + */ + if (mbox_cmd.size_out) { + dev_WARN_ONCE(dev, mbox_cmd.size_out > *size_out, + "Invalid return size\n"); + if (copy_to_user(u64_to_user_ptr(out_payload), + mbox_cmd.payload_out, mbox_cmd.size_out)) { + rc = -EFAULT; + goto out; + } + } + + *size_out = mbox_cmd.size_out; + *retval = mbox_cmd.return_code; + +out: + kvfree(mbox_cmd.payload_in); + kvfree(mbox_cmd.payload_out); + return rc; +} + +static bool cxl_mem_raw_command_allowed(u16 opcode) +{ + int i; + + if (!IS_ENABLED(CONFIG_CXL_MEM_RAW_COMMANDS)) + return false; + + if (security_locked_down(LOCKDOWN_NONE)) + return false; + + if (cxl_raw_allow_all) + return true; + + if (cxl_is_security_command(opcode)) + return false; + + for (i = 0; i < ARRAY_SIZE(cxl_disabled_raw_commands); i++) + if (cxl_disabled_raw_commands[i] == opcode) + return false; + + return true; +} + +/** + * cxl_validate_cmd_from_user() - Check fields for CXL_MEM_SEND_COMMAND. + * @cxlm: &struct cxl_mem device whose mailbox will be used. + * @send_cmd: &struct cxl_send_command copied in from userspace. + * @out_cmd: Sanitized and populated &struct cxl_mem_command. + * + * Return: + * * %0 - @out_cmd is ready to send. + * * %-ENOTTY - Invalid command specified. + * * %-EINVAL - Reserved fields or invalid values were used. + * * %-ENOMEM - Input or output buffer wasn't sized properly. + * * %-EPERM - Attempted to use a protected command. + * + * The result of this command is a fully validated command in @out_cmd that is + * safe to send to the hardware. + * + * See handle_mailbox_cmd_from_user() + */ +static int cxl_validate_cmd_from_user(struct cxl_mem *cxlm, + const struct cxl_send_command *send_cmd, + struct cxl_mem_command *out_cmd) +{ + const struct cxl_command_info *info; + struct cxl_mem_command *c; + + if (send_cmd->id == 0 || send_cmd->id >= CXL_MEM_COMMAND_ID_MAX) + return -ENOTTY; + + /* + * The user can never specify an input payload larger than what hardware + * supports, but output can be arbitrarily large (simply write out as + * much data as the hardware provides). + */ + if (send_cmd->in.size > cxlm->payload_size) + return -EINVAL; + + /* + * Checks are bypassed for raw commands but a WARN/taint will occur + * later in the callchain + */ + if (send_cmd->id == CXL_MEM_COMMAND_ID_RAW) { + const struct cxl_mem_command temp = { + .info = { + .id = CXL_MEM_COMMAND_ID_RAW, + .flags = 0, + .size_in = send_cmd->in.size, + .size_out = send_cmd->out.size, + }, + .opcode = send_cmd->raw.opcode + }; + + if (send_cmd->raw.rsvd) + return -EINVAL; + + /* + * Unlike supported commands, the output size of RAW commands + * gets passed along without further checking, so it must be + * validated here. + */ + if (send_cmd->out.size > cxlm->payload_size) + return -EINVAL; + + if (!cxl_mem_raw_command_allowed(send_cmd->raw.opcode)) + return -EPERM; + + memcpy(out_cmd, &temp, sizeof(temp)); + + return 0; + } + + if (send_cmd->flags & ~CXL_MEM_COMMAND_FLAG_MASK) + return -EINVAL; + + if (send_cmd->rsvd) + return -EINVAL; + + if (send_cmd->in.rsvd || send_cmd->out.rsvd) + return -EINVAL; + + /* Convert user's command into the internal representation */ + c = &mem_commands[send_cmd->id]; + info = &c->info; + + /* Check that the command is enabled for hardware */ + if (!test_bit(info->id, cxlm->enabled_cmds)) + return -ENOTTY; + + /* Check the input buffer is the expected size */ + if (info->size_in >= 0 && info->size_in != send_cmd->in.size) + return -ENOMEM; + + /* Check the output buffer is at least large enough */ + if (info->size_out >= 0 && send_cmd->out.size < info->size_out) + return -ENOMEM; + + memcpy(out_cmd, c, sizeof(*c)); + out_cmd->info.size_in = send_cmd->in.size; + /* + * XXX: out_cmd->info.size_out will be controlled by the driver, and the + * specified number of bytes @send_cmd->out.size will be copied back out + * to userspace. + */ + + return 0; +} + +static int cxl_query_cmd(struct cxl_memdev *cxlmd, + struct cxl_mem_query_commands __user *q) +{ + struct device *dev = &cxlmd->dev; + struct cxl_mem_command *cmd; + u32 n_commands; + int j = 0; + + dev_dbg(dev, "Query IOCTL\n"); + + if (get_user(n_commands, &q->n_commands)) + return -EFAULT; + + /* returns the total number if 0 elements are requested. */ + if (n_commands == 0) + return put_user(cxl_cmd_count, &q->n_commands); + + /* + * otherwise, return max(n_commands, total commands) cxl_command_info + * structures. + */ + cxl_for_each_cmd(cmd) { + const struct cxl_command_info *info = &cmd->info; + + if (copy_to_user(&q->commands[j++], info, sizeof(*info))) + return -EFAULT; + + if (j == n_commands) + break; + } + + return 0; +} + +static int cxl_send_cmd(struct cxl_memdev *cxlmd, + struct cxl_send_command __user *s) +{ + struct cxl_mem *cxlm = cxlmd->cxlm; + struct device *dev = &cxlmd->dev; + struct cxl_send_command send; + struct cxl_mem_command c; + int rc; + + dev_dbg(dev, "Send IOCTL\n"); + + if (copy_from_user(&send, s, sizeof(send))) + return -EFAULT; + + rc = cxl_validate_cmd_from_user(cxlmd->cxlm, &send, &c); + if (rc) + return rc; + + /* Prepare to handle a full payload for variable sized output */ + if (c.info.size_out < 0) + c.info.size_out = cxlm->payload_size; + + rc = handle_mailbox_cmd_from_user(cxlm, &c, send.in.payload, + send.out.payload, &send.out.size, + &send.retval); + if (rc) + return rc; + + if (copy_to_user(s, &send, sizeof(send))) + return -EFAULT; + + return 0; +} + +static long __cxl_memdev_ioctl(struct cxl_memdev *cxlmd, unsigned int cmd, + unsigned long arg) +{ + switch (cmd) { + case CXL_MEM_QUERY_COMMANDS: + return cxl_query_cmd(cxlmd, (void __user *)arg); + case CXL_MEM_SEND_COMMAND: + return cxl_send_cmd(cxlmd, (void __user *)arg); + default: + return -ENOTTY; + } +} + +static long cxl_memdev_ioctl(struct file *file, unsigned int cmd, + unsigned long arg) +{ + struct cxl_memdev *cxlmd; + struct inode *inode; + int rc = -ENOTTY; + + inode = file_inode(file); + cxlmd = container_of(inode->i_cdev, typeof(*cxlmd), cdev); + + if (!percpu_ref_tryget_live(&cxlmd->ops_active)) + return -ENXIO; + + rc = __cxl_memdev_ioctl(cxlmd, cmd, arg); + + percpu_ref_put(&cxlmd->ops_active); + + return rc; +} + +static const struct file_operations cxl_memdev_fops = { + .owner = THIS_MODULE, + .unlocked_ioctl = cxl_memdev_ioctl, + .compat_ioctl = compat_ptr_ioctl, + .llseek = noop_llseek, +}; + +static inline struct cxl_mem_command *cxl_mem_find_command(u16 opcode) +{ + struct cxl_mem_command *c; + + cxl_for_each_cmd(c) + if (c->opcode == opcode) + return c; + + return NULL; +} + +/** + * cxl_mem_mbox_send_cmd() - Send a mailbox command to a memory device. + * @cxlm: The CXL memory device to communicate with. + * @opcode: Opcode for the mailbox command. + * @in: The input payload for the mailbox command. + * @in_size: The length of the input payload + * @out: Caller allocated buffer for the output. + * @out_size: Expected size of output. + * + * Context: Any context. Will acquire and release mbox_mutex. + * Return: + * * %>=0 - Number of bytes returned in @out. + * * %-E2BIG - Payload is too large for hardware. + * * %-EBUSY - Couldn't acquire exclusive mailbox access. + * * %-EFAULT - Hardware error occurred. + * * %-ENXIO - Command completed, but device reported an error. + * * %-EIO - Unexpected output size. + * + * Mailbox commands may execute successfully yet the device itself reported an + * error. While this distinction can be useful for commands from userspace, the + * kernel will only be able to use results when both are successful. + * + * See __cxl_mem_mbox_send_cmd() + */ +static int cxl_mem_mbox_send_cmd(struct cxl_mem *cxlm, u16 opcode, + void *in, size_t in_size, + void *out, size_t out_size) +{ + const struct cxl_mem_command *cmd = cxl_mem_find_command(opcode); + struct mbox_cmd mbox_cmd = { + .opcode = opcode, + .payload_in = in, + .size_in = in_size, + .size_out = out_size, + .payload_out = out, + }; + int rc; + + if (out_size > cxlm->payload_size) + return -E2BIG; + + rc = cxl_mem_mbox_get(cxlm); + if (rc) + return rc; + + rc = __cxl_mem_mbox_send_cmd(cxlm, &mbox_cmd); + cxl_mem_mbox_put(cxlm); + if (rc) + return rc; + + /* TODO: Map return code to proper kernel style errno */ + if (mbox_cmd.return_code != CXL_MBOX_SUCCESS) + return -ENXIO; + + /* + * Variable sized commands can't be validated and so it's up to the + * caller to do that if they wish. + */ + if (cmd->info.size_out >= 0 && mbox_cmd.size_out != out_size) + return -EIO; + + return 0; +} + +/** + * cxl_mem_setup_regs() - Setup necessary MMIO. + * @cxlm: The CXL memory device to communicate with. + * + * Return: 0 if all necessary registers mapped. + * + * A memory device is required by spec to implement a certain set of MMIO + * regions. The purpose of this function is to enumerate and map those + * registers. + */ +static int cxl_mem_setup_regs(struct cxl_mem *cxlm) +{ + struct device *dev = &cxlm->pdev->dev; + int cap, cap_count; + u64 cap_array; + + cap_array = readq(cxlm->regs + CXLDEV_CAP_ARRAY_OFFSET); + if (FIELD_GET(CXLDEV_CAP_ARRAY_ID_MASK, cap_array) != + CXLDEV_CAP_ARRAY_CAP_ID) + return -ENODEV; + + cap_count = FIELD_GET(CXLDEV_CAP_ARRAY_COUNT_MASK, cap_array); + + for (cap = 1; cap <= cap_count; cap++) { + void __iomem *register_block; + u32 offset; + u16 cap_id; + + cap_id = FIELD_GET(CXLDEV_CAP_HDR_CAP_ID_MASK, + readl(cxlm->regs + cap * 0x10)); + offset = readl(cxlm->regs + cap * 0x10 + 0x4); + register_block = cxlm->regs + offset; + + switch (cap_id) { + case CXLDEV_CAP_CAP_ID_DEVICE_STATUS: + dev_dbg(dev, "found Status capability (0x%x)\n", offset); + cxlm->status_regs = register_block; + break; + case CXLDEV_CAP_CAP_ID_PRIMARY_MAILBOX: + dev_dbg(dev, "found Mailbox capability (0x%x)\n", offset); + cxlm->mbox_regs = register_block; + break; + case CXLDEV_CAP_CAP_ID_SECONDARY_MAILBOX: + dev_dbg(dev, "found Secondary Mailbox capability (0x%x)\n", offset); + break; + case CXLDEV_CAP_CAP_ID_MEMDEV: + dev_dbg(dev, "found Memory Device capability (0x%x)\n", offset); + cxlm->memdev_regs = register_block; + break; + default: + dev_dbg(dev, "Unknown cap ID: %d (0x%x)\n", cap_id, offset); + break; + } + } + + if (!cxlm->status_regs || !cxlm->mbox_regs || !cxlm->memdev_regs) { + dev_err(dev, "registers not found: %s%s%s\n", + !cxlm->status_regs ? "status " : "", + !cxlm->mbox_regs ? "mbox " : "", + !cxlm->memdev_regs ? "memdev" : ""); + return -ENXIO; + } + + return 0; +} + +static int cxl_mem_setup_mailbox(struct cxl_mem *cxlm) +{ + const int cap = readl(cxlm->mbox_regs + CXLDEV_MBOX_CAPS_OFFSET); + + cxlm->payload_size = + 1 << FIELD_GET(CXLDEV_MBOX_CAP_PAYLOAD_SIZE_MASK, cap); + + /* + * CXL 2.0 8.2.8.4.3 Mailbox Capabilities Register + * + * If the size is too small, mandatory commands will not work and so + * there's no point in going forward. If the size is too large, there's + * no harm is soft limiting it. + */ + cxlm->payload_size = min_t(size_t, cxlm->payload_size, SZ_1M); + if (cxlm->payload_size < 256) { + dev_err(&cxlm->pdev->dev, "Mailbox is too small (%zub)", + cxlm->payload_size); + return -ENXIO; + } + + dev_dbg(&cxlm->pdev->dev, "Mailbox payload sized %zu", + cxlm->payload_size); + + return 0; +} + +static struct cxl_mem *cxl_mem_create(struct pci_dev *pdev, u32 reg_lo, + u32 reg_hi) +{ + struct device *dev = &pdev->dev; + struct cxl_mem *cxlm; + void __iomem *regs; + u64 offset; + u8 bar; + int rc; + + cxlm = devm_kzalloc(&pdev->dev, sizeof(*cxlm), GFP_KERNEL); + if (!cxlm) { + dev_err(dev, "No memory available\n"); + return NULL; + } + + offset = ((u64)reg_hi << 32) | FIELD_GET(CXL_REGLOC_ADDR_MASK, reg_lo); + bar = FIELD_GET(CXL_REGLOC_BIR_MASK, reg_lo); + + /* Basic sanity check that BAR is big enough */ + if (pci_resource_len(pdev, bar) < offset) { + dev_err(dev, "BAR%d: %pr: too small (offset: %#llx)\n", bar, + &pdev->resource[bar], (unsigned long long)offset); + return NULL; + } + + rc = pcim_iomap_regions(pdev, BIT(bar), pci_name(pdev)); + if (rc) { + dev_err(dev, "failed to map registers\n"); + return NULL; + } + regs = pcim_iomap_table(pdev)[bar]; + + mutex_init(&cxlm->mbox_mutex); + cxlm->pdev = pdev; + cxlm->regs = regs + offset; + cxlm->enabled_cmds = + devm_kmalloc_array(dev, BITS_TO_LONGS(cxl_cmd_count), + sizeof(unsigned long), + GFP_KERNEL | __GFP_ZERO); + if (!cxlm->enabled_cmds) { + dev_err(dev, "No memory available for bitmap\n"); + return NULL; + } + + dev_dbg(dev, "Mapped CXL Memory Device resource\n"); + return cxlm; +} + +static int cxl_mem_dvsec(struct pci_dev *pdev, int dvsec) +{ + int pos; + + pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_DVSEC); + if (!pos) + return 0; + + while (pos) { + u16 vendor, id; + + pci_read_config_word(pdev, pos + PCI_DVSEC_HEADER1, &vendor); + pci_read_config_word(pdev, pos + PCI_DVSEC_HEADER2, &id); + if (vendor == PCI_DVSEC_VENDOR_ID_CXL && dvsec == id) + return pos; + + pos = pci_find_next_ext_capability(pdev, pos, + PCI_EXT_CAP_ID_DVSEC); + } + + return 0; +} + +static struct cxl_memdev *to_cxl_memdev(struct device *dev) +{ + return container_of(dev, struct cxl_memdev, dev); +} + +static void cxl_memdev_release(struct device *dev) +{ + struct cxl_memdev *cxlmd = to_cxl_memdev(dev); + + percpu_ref_exit(&cxlmd->ops_active); + ida_free(&cxl_memdev_ida, cxlmd->id); + kfree(cxlmd); +} + +static char *cxl_memdev_devnode(struct device *dev, umode_t *mode, kuid_t *uid, + kgid_t *gid) +{ + return kasprintf(GFP_KERNEL, "cxl/%s", dev_name(dev)); +} + +static ssize_t firmware_version_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct cxl_memdev *cxlmd = to_cxl_memdev(dev); + struct cxl_mem *cxlm = cxlmd->cxlm; + + return sprintf(buf, "%.16s\n", cxlm->firmware_version); +} +static DEVICE_ATTR_RO(firmware_version); + +static ssize_t payload_max_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct cxl_memdev *cxlmd = to_cxl_memdev(dev); + struct cxl_mem *cxlm = cxlmd->cxlm; + + return sprintf(buf, "%zu\n", cxlm->payload_size); +} +static DEVICE_ATTR_RO(payload_max); + +static ssize_t ram_size_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct cxl_memdev *cxlmd = to_cxl_memdev(dev); + struct cxl_mem *cxlm = cxlmd->cxlm; + unsigned long long len = range_len(&cxlm->ram_range); + + return sprintf(buf, "%#llx\n", len); +} + +static struct device_attribute dev_attr_ram_size = + __ATTR(size, 0444, ram_size_show, NULL); + +static ssize_t pmem_size_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct cxl_memdev *cxlmd = to_cxl_memdev(dev); + struct cxl_mem *cxlm = cxlmd->cxlm; + unsigned long long len = range_len(&cxlm->pmem_range); + + return sprintf(buf, "%#llx\n", len); +} + +static struct device_attribute dev_attr_pmem_size = + __ATTR(size, 0444, pmem_size_show, NULL); + +static struct attribute *cxl_memdev_attributes[] = { + &dev_attr_firmware_version.attr, + &dev_attr_payload_max.attr, + NULL, +}; + +static struct attribute *cxl_memdev_pmem_attributes[] = { + &dev_attr_pmem_size.attr, + NULL, +}; + +static struct attribute *cxl_memdev_ram_attributes[] = { + &dev_attr_ram_size.attr, + NULL, +}; + +static struct attribute_group cxl_memdev_attribute_group = { + .attrs = cxl_memdev_attributes, +}; + +static struct attribute_group cxl_memdev_ram_attribute_group = { + .name = "ram", + .attrs = cxl_memdev_ram_attributes, +}; + +static struct attribute_group cxl_memdev_pmem_attribute_group = { + .name = "pmem", + .attrs = cxl_memdev_pmem_attributes, +}; + +static const struct attribute_group *cxl_memdev_attribute_groups[] = { + &cxl_memdev_attribute_group, + &cxl_memdev_ram_attribute_group, + &cxl_memdev_pmem_attribute_group, + NULL, +}; + +static const struct device_type cxl_memdev_type = { + .name = "cxl_memdev", + .release = cxl_memdev_release, + .devnode = cxl_memdev_devnode, + .groups = cxl_memdev_attribute_groups, +}; + +static void cxlmdev_unregister(void *_cxlmd) +{ + struct cxl_memdev *cxlmd = _cxlmd; + struct device *dev = &cxlmd->dev; + + percpu_ref_kill(&cxlmd->ops_active); + cdev_device_del(&cxlmd->cdev, dev); + wait_for_completion(&cxlmd->ops_dead); + cxlmd->cxlm = NULL; + put_device(dev); +} + +static void cxlmdev_ops_active_release(struct percpu_ref *ref) +{ + struct cxl_memdev *cxlmd = + container_of(ref, typeof(*cxlmd), ops_active); + + complete(&cxlmd->ops_dead); +} + +static int cxl_mem_add_memdev(struct cxl_mem *cxlm) +{ + struct pci_dev *pdev = cxlm->pdev; + struct cxl_memdev *cxlmd; + struct device *dev; + struct cdev *cdev; + int rc; + + cxlmd = kzalloc(sizeof(*cxlmd), GFP_KERNEL); + if (!cxlmd) + return -ENOMEM; + init_completion(&cxlmd->ops_dead); + + /* + * @cxlm is deallocated when the driver unbinds so operations + * that are using it need to hold a live reference. + */ + cxlmd->cxlm = cxlm; + rc = percpu_ref_init(&cxlmd->ops_active, cxlmdev_ops_active_release, 0, + GFP_KERNEL); + if (rc) + goto err_ref; + + rc = ida_alloc_range(&cxl_memdev_ida, 0, CXL_MEM_MAX_DEVS, GFP_KERNEL); + if (rc < 0) + goto err_id; + cxlmd->id = rc; + + dev = &cxlmd->dev; + device_initialize(dev); + dev->parent = &pdev->dev; + dev->bus = &cxl_bus_type; + dev->devt = MKDEV(cxl_mem_major, cxlmd->id); + dev->type = &cxl_memdev_type; + dev_set_name(dev, "mem%d", cxlmd->id); + + cdev = &cxlmd->cdev; + cdev_init(cdev, &cxl_memdev_fops); + + rc = cdev_device_add(cdev, dev); + if (rc) + goto err_add; + + return devm_add_action_or_reset(dev->parent, cxlmdev_unregister, cxlmd); + +err_add: + ida_free(&cxl_memdev_ida, cxlmd->id); +err_id: + /* + * Theoretically userspace could have already entered the fops, + * so flush ops_active. + */ + percpu_ref_kill(&cxlmd->ops_active); + wait_for_completion(&cxlmd->ops_dead); + percpu_ref_exit(&cxlmd->ops_active); +err_ref: + kfree(cxlmd); + + return rc; +} + +static int cxl_xfer_log(struct cxl_mem *cxlm, uuid_t *uuid, u32 size, u8 *out) +{ + u32 remaining = size; + u32 offset = 0; + + while (remaining) { + u32 xfer_size = min_t(u32, remaining, cxlm->payload_size); + struct cxl_mbox_get_log { + uuid_t uuid; + __le32 offset; + __le32 length; + } __packed log = { + .uuid = *uuid, + .offset = cpu_to_le32(offset), + .length = cpu_to_le32(xfer_size) + }; + int rc; + + rc = cxl_mem_mbox_send_cmd(cxlm, CXL_MBOX_OP_GET_LOG, &log, + sizeof(log), out, xfer_size); + if (rc < 0) + return rc; + + out += xfer_size; + remaining -= xfer_size; + offset += xfer_size; + } + + return 0; +} + +/** + * cxl_walk_cel() - Walk through the Command Effects Log. + * @cxlm: Device. + * @size: Length of the Command Effects Log. + * @cel: CEL + * + * Iterate over each entry in the CEL and determine if the driver supports the + * command. If so, the command is enabled for the device and can be used later. + */ +static void cxl_walk_cel(struct cxl_mem *cxlm, size_t size, u8 *cel) +{ + struct cel_entry { + __le16 opcode; + __le16 effect; + } __packed * cel_entry; + const int cel_entries = size / sizeof(*cel_entry); + int i; + + cel_entry = (struct cel_entry *)cel; + + for (i = 0; i < cel_entries; i++) { + u16 opcode = le16_to_cpu(cel_entry[i].opcode); + struct cxl_mem_command *cmd = cxl_mem_find_command(opcode); + + if (!cmd) { + dev_dbg(&cxlm->pdev->dev, + "Opcode 0x%04x unsupported by driver", opcode); + continue; + } + + set_bit(cmd->info.id, cxlm->enabled_cmds); + } +} + +struct cxl_mbox_get_supported_logs { + __le16 entries; + u8 rsvd[6]; + struct gsl_entry { + uuid_t uuid; + __le32 size; + } __packed entry[]; +} __packed; + +static struct cxl_mbox_get_supported_logs *cxl_get_gsl(struct cxl_mem *cxlm) +{ + struct cxl_mbox_get_supported_logs *ret; + int rc; + + ret = kvmalloc(cxlm->payload_size, GFP_KERNEL); + if (!ret) + return ERR_PTR(-ENOMEM); + + rc = cxl_mem_mbox_send_cmd(cxlm, CXL_MBOX_OP_GET_SUPPORTED_LOGS, NULL, + 0, ret, cxlm->payload_size); + if (rc < 0) { + kvfree(ret); + return ERR_PTR(rc); + } + + return ret; +} + +/** + * cxl_mem_enumerate_cmds() - Enumerate commands for a device. + * @cxlm: The device. + * + * Returns 0 if enumerate completed successfully. + * + * CXL devices have optional support for certain commands. This function will + * determine the set of supported commands for the hardware and update the + * enabled_cmds bitmap in the @cxlm. + */ +static int cxl_mem_enumerate_cmds(struct cxl_mem *cxlm) +{ + struct cxl_mbox_get_supported_logs *gsl; + struct device *dev = &cxlm->pdev->dev; + struct cxl_mem_command *cmd; + int i, rc; + + gsl = cxl_get_gsl(cxlm); + if (IS_ERR(gsl)) + return PTR_ERR(gsl); + + rc = -ENOENT; + for (i = 0; i < le16_to_cpu(gsl->entries); i++) { + u32 size = le32_to_cpu(gsl->entry[i].size); + uuid_t uuid = gsl->entry[i].uuid; + u8 *log; + + dev_dbg(dev, "Found LOG type %pU of size %d", &uuid, size); + + if (!uuid_equal(&uuid, &log_uuid[CEL_UUID])) + continue; + + log = kvmalloc(size, GFP_KERNEL); + if (!log) { + rc = -ENOMEM; + goto out; + } + + rc = cxl_xfer_log(cxlm, &uuid, size, log); + if (rc) { + kvfree(log); + goto out; + } + + cxl_walk_cel(cxlm, size, log); + kvfree(log); + + /* In case CEL was bogus, enable some default commands. */ + cxl_for_each_cmd(cmd) + if (cmd->flags & CXL_CMD_FLAG_FORCE_ENABLE) + set_bit(cmd->info.id, cxlm->enabled_cmds); + + /* Found the required CEL */ + rc = 0; + } + +out: + kvfree(gsl); + return rc; +} + +/** + * cxl_mem_identify() - Send the IDENTIFY command to the device. + * @cxlm: The device to identify. + * + * Return: 0 if identify was executed successfully. + * + * This will dispatch the identify command to the device and on success populate + * structures to be exported to sysfs. + */ +static int cxl_mem_identify(struct cxl_mem *cxlm) +{ + struct cxl_mbox_identify { + char fw_revision[0x10]; + __le64 total_capacity; + __le64 volatile_capacity; + __le64 persistent_capacity; + __le64 partition_align; + __le16 info_event_log_size; + __le16 warning_event_log_size; + __le16 failure_event_log_size; + __le16 fatal_event_log_size; + __le32 lsa_size; + u8 poison_list_max_mer[3]; + __le16 inject_poison_limit; + u8 poison_caps; + u8 qos_telemetry_caps; + } __packed id; + int rc; + + rc = cxl_mem_mbox_send_cmd(cxlm, CXL_MBOX_OP_IDENTIFY, NULL, 0, &id, + sizeof(id)); + if (rc < 0) + return rc; + + /* + * TODO: enumerate DPA map, as 'ram' and 'pmem' do not alias. + * For now, only the capacity is exported in sysfs + */ + cxlm->ram_range.start = 0; + cxlm->ram_range.end = le64_to_cpu(id.volatile_capacity) - 1; + + cxlm->pmem_range.start = 0; + cxlm->pmem_range.end = le64_to_cpu(id.persistent_capacity) - 1; + + memcpy(cxlm->firmware_version, id.fw_revision, sizeof(id.fw_revision)); + + return 0; +} + +static int cxl_mem_probe(struct pci_dev *pdev, const struct pci_device_id *id) +{ + struct device *dev = &pdev->dev; + struct cxl_mem *cxlm = NULL; + u32 regloc_size, regblocks; + int rc, regloc, i; + + rc = pcim_enable_device(pdev); + if (rc) + return rc; + + regloc = cxl_mem_dvsec(pdev, PCI_DVSEC_ID_CXL_REGLOC_OFFSET); + if (!regloc) { + dev_err(dev, "register location dvsec not found\n"); + return -ENXIO; + } + + /* Get the size of the Register Locator DVSEC */ + pci_read_config_dword(pdev, regloc + PCI_DVSEC_HEADER1, ®loc_size); + regloc_size = FIELD_GET(PCI_DVSEC_HEADER1_LENGTH_MASK, regloc_size); + + regloc += PCI_DVSEC_ID_CXL_REGLOC_BLOCK1_OFFSET; + regblocks = (regloc_size - PCI_DVSEC_ID_CXL_REGLOC_BLOCK1_OFFSET) / 8; + + for (i = 0; i < regblocks; i++, regloc += 8) { + u32 reg_lo, reg_hi; + u8 reg_type; + + /* "register low and high" contain other bits */ + pci_read_config_dword(pdev, regloc, ®_lo); + pci_read_config_dword(pdev, regloc + 4, ®_hi); + + reg_type = FIELD_GET(CXL_REGLOC_RBI_MASK, reg_lo); + + if (reg_type == CXL_REGLOC_RBI_MEMDEV) { + cxlm = cxl_mem_create(pdev, reg_lo, reg_hi); + break; + } + } + + if (!cxlm) + return -ENODEV; + + rc = cxl_mem_setup_regs(cxlm); + if (rc) + return rc; + + rc = cxl_mem_setup_mailbox(cxlm); + if (rc) + return rc; + + rc = cxl_mem_enumerate_cmds(cxlm); + if (rc) + return rc; + + rc = cxl_mem_identify(cxlm); + if (rc) + return rc; + + return cxl_mem_add_memdev(cxlm); +} + +static const struct pci_device_id cxl_mem_pci_tbl[] = { + /* PCI class code for CXL.mem Type-3 Devices */ + { PCI_DEVICE_CLASS((PCI_CLASS_MEMORY_CXL << 8 | CXL_MEMORY_PROGIF), ~0)}, + { /* terminate list */ }, +}; +MODULE_DEVICE_TABLE(pci, cxl_mem_pci_tbl); + +static struct pci_driver cxl_mem_driver = { + .name = KBUILD_MODNAME, + .id_table = cxl_mem_pci_tbl, + .probe = cxl_mem_probe, + .driver = { + .probe_type = PROBE_PREFER_ASYNCHRONOUS, + }, +}; + +static __init int cxl_mem_init(void) +{ + struct dentry *mbox_debugfs; + dev_t devt; + int rc; + + rc = alloc_chrdev_region(&devt, 0, CXL_MEM_MAX_DEVS, "cxl"); + if (rc) + return rc; + + cxl_mem_major = MAJOR(devt); + + rc = pci_register_driver(&cxl_mem_driver); + if (rc) { + unregister_chrdev_region(MKDEV(cxl_mem_major, 0), + CXL_MEM_MAX_DEVS); + return rc; + } + + cxl_debugfs = debugfs_create_dir("cxl", NULL); + mbox_debugfs = debugfs_create_dir("mbox", cxl_debugfs); + debugfs_create_bool("raw_allow_all", 0600, mbox_debugfs, + &cxl_raw_allow_all); + + return 0; +} + +static __exit void cxl_mem_exit(void) +{ + debugfs_remove_recursive(cxl_debugfs); + pci_unregister_driver(&cxl_mem_driver); + unregister_chrdev_region(MKDEV(cxl_mem_major, 0), CXL_MEM_MAX_DEVS); +} + +MODULE_LICENSE("GPL v2"); +module_init(cxl_mem_init); +module_exit(cxl_mem_exit); +MODULE_IMPORT_NS(CXL); diff --git a/drivers/cxl/pci.h b/drivers/cxl/pci.h new file mode 100644 index 0000000000000000000000000000000000000000..af3ec078cf6cbe7cf43173178abe84f826e1b095 --- /dev/null +++ b/drivers/cxl/pci.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2020 Intel Corporation. All rights reserved. */ +#ifndef __CXL_PCI_H__ +#define __CXL_PCI_H__ + +#define CXL_MEMORY_PROGIF 0x10 + +/* + * See section 8.1 Configuration Space Registers in the CXL 2.0 + * Specification + */ +#define PCI_DVSEC_HEADER1_LENGTH_MASK GENMASK(31, 20) +#define PCI_DVSEC_VENDOR_ID_CXL 0x1E98 +#define PCI_DVSEC_ID_CXL 0x0 + +#define PCI_DVSEC_ID_CXL_REGLOC_OFFSET 0x8 +#define PCI_DVSEC_ID_CXL_REGLOC_BLOCK1_OFFSET 0xC + +/* BAR Indicator Register (BIR) */ +#define CXL_REGLOC_BIR_MASK GENMASK(2, 0) + +/* Register Block Identifier (RBI) */ +#define CXL_REGLOC_RBI_MASK GENMASK(15, 8) +#define CXL_REGLOC_RBI_EMPTY 0 +#define CXL_REGLOC_RBI_COMPONENT 1 +#define CXL_REGLOC_RBI_VIRT 2 +#define CXL_REGLOC_RBI_MEMDEV 3 + +#define CXL_REGLOC_ADDR_MASK GENMASK(31, 16) + +#endif /* __CXL_PCI_H__ */ diff --git a/drivers/dax/bus.c b/drivers/dax/bus.c index 737b207c9e30d73f1378830462104f682d624da4..452e85ae87a836611056720c5dcebb9938bdf612 100644 --- a/drivers/dax/bus.c +++ b/drivers/dax/bus.c @@ -179,7 +179,10 @@ static int dax_bus_remove(struct device *dev) struct dax_device_driver *dax_drv = to_dax_drv(dev->driver); struct dev_dax *dev_dax = to_dev_dax(dev); - return dax_drv->remove(dev_dax); + if (dax_drv->remove) + dax_drv->remove(dev_dax); + + return 0; } static struct bus_type dax_bus_type = { @@ -1038,7 +1041,7 @@ static ssize_t range_parse(const char *opt, size_t len, struct range *range) { unsigned long long addr = 0; char *start, *end, *str; - ssize_t rc = EINVAL; + ssize_t rc = -EINVAL; str = kstrdup(opt, GFP_KERNEL); if (!str) @@ -1392,6 +1395,13 @@ int __dax_driver_register(struct dax_device_driver *dax_drv, struct device_driver *drv = &dax_drv->drv; int rc = 0; + /* + * dax_bus_probe() calls dax_drv->probe() unconditionally. + * So better be safe than sorry and ensure it is provided. + */ + if (!dax_drv->probe) + return -EINVAL; + INIT_LIST_HEAD(&dax_drv->ids); drv->owner = module; drv->name = mod_name; @@ -1409,7 +1419,15 @@ int __dax_driver_register(struct dax_device_driver *dax_drv, mutex_unlock(&dax_bus_lock); if (rc) return rc; - return driver_register(drv); + + rc = driver_register(drv); + if (rc && dax_drv->match_always) { + mutex_lock(&dax_bus_lock); + match_always_count -= dax_drv->match_always; + mutex_unlock(&dax_bus_lock); + } + + return rc; } EXPORT_SYMBOL_GPL(__dax_driver_register); diff --git a/drivers/dax/bus.h b/drivers/dax/bus.h index 72b92f95509f7ece65ab7e7b22dfb66915e019c0..1e946ad7780a82b40f6ccdc472429c41005ee960 100644 --- a/drivers/dax/bus.h +++ b/drivers/dax/bus.h @@ -39,7 +39,7 @@ struct dax_device_driver { struct list_head ids; int match_always; int (*probe)(struct dev_dax *dev); - int (*remove)(struct dev_dax *dev); + void (*remove)(struct dev_dax *dev); }; int __dax_driver_register(struct dax_device_driver *dax_drv, diff --git a/drivers/dax/device.c b/drivers/dax/device.c index 5da2980bb16b99cb699de486bf72a1a522081022..db92573c94e864109680d613e52b194f7774a874 100644 --- a/drivers/dax/device.c +++ b/drivers/dax/device.c @@ -452,15 +452,9 @@ int dev_dax_probe(struct dev_dax *dev_dax) } EXPORT_SYMBOL_GPL(dev_dax_probe); -static int dev_dax_remove(struct dev_dax *dev_dax) -{ - /* all probe actions are unwound by devm */ - return 0; -} - static struct dax_device_driver device_dax_driver = { .probe = dev_dax_probe, - .remove = dev_dax_remove, + /* all probe actions are unwound by devm, so .remove isn't necessary */ .match_always = 1, }; diff --git a/drivers/dax/kmem.c b/drivers/dax/kmem.c index 403ec42472d1d9ff9480e2db0b3a12ee03fc4004..ac231cc3635959ad7a92a9e59e91a0c19300ed4b 100644 --- a/drivers/dax/kmem.c +++ b/drivers/dax/kmem.c @@ -136,7 +136,7 @@ static int dev_dax_kmem_probe(struct dev_dax *dev_dax) } #ifdef CONFIG_MEMORY_HOTREMOVE -static int dev_dax_kmem_remove(struct dev_dax *dev_dax) +static void dev_dax_kmem_remove(struct dev_dax *dev_dax) { int i, success = 0; struct device *dev = &dev_dax->dev; @@ -176,11 +176,9 @@ static int dev_dax_kmem_remove(struct dev_dax *dev_dax) kfree(data); dev_set_drvdata(dev, NULL); } - - return 0; } #else -static int dev_dax_kmem_remove(struct dev_dax *dev_dax) +static void dev_dax_kmem_remove(struct dev_dax *dev_dax) { /* * Without hotremove purposely leak the request_mem_region() for the @@ -190,7 +188,6 @@ static int dev_dax_kmem_remove(struct dev_dax *dev_dax) * request_mem_region(). */ any_hotremove_failed = true; - return 0; } #endif /* CONFIG_MEMORY_HOTREMOVE */ diff --git a/drivers/dax/pmem/compat.c b/drivers/dax/pmem/compat.c index 863c114fd88ce1cb810ca59ec6eec9e1445160c0..d81dc35fd65d6efcacf226df89d7f2f9fdd9efef 100644 --- a/drivers/dax/pmem/compat.c +++ b/drivers/dax/pmem/compat.c @@ -41,10 +41,9 @@ static int dax_pmem_compat_release(struct device *dev, void *data) return 0; } -static int dax_pmem_compat_remove(struct device *dev) +static void dax_pmem_compat_remove(struct device *dev) { device_for_each_child(dev, NULL, dax_pmem_compat_release); - return 0; } static struct nd_device_driver dax_pmem_compat_driver = { diff --git a/drivers/dax/super.c b/drivers/dax/super.c index cadbd0a1a1ef0220296faa93736f53b1b359783c..5fa6ae9dbc8b02b137496231b5f04175f0e5376c 100644 --- a/drivers/dax/super.c +++ b/drivers/dax/super.c @@ -480,7 +480,7 @@ static void dax_free_inode(struct inode *inode) kfree(dax_dev->host); dax_dev->host = NULL; if (inode->i_rdev) - ida_simple_remove(&dax_minor_ida, MINOR(inode->i_rdev)); + ida_simple_remove(&dax_minor_ida, iminor(inode)); kmem_cache_free(dax_cache, dax_dev); } diff --git a/drivers/dma-buf/Kconfig b/drivers/dma-buf/Kconfig index 4f8224a6ac9562c0ad10b781c9c55b851c5b7453..4e16c71c24b71d27e5df9cc3368d5209af2607c5 100644 --- a/drivers/dma-buf/Kconfig +++ b/drivers/dma-buf/Kconfig @@ -50,6 +50,14 @@ config DMABUF_MOVE_NOTIFY This is marked experimental because we don't yet have a consistent execution context and memory management between drivers. +config DMABUF_DEBUG + bool "DMA-BUF debug checks" + default y if DMA_API_DEBUG + help + This option enables additional checks for DMA-BUF importers and + exporters. Specifically it validates that importers do not peek at the + underlying struct page when they import a buffer. + config DMABUF_SELFTESTS tristate "Selftests for the dma-buf interfaces" default n diff --git a/drivers/dma-buf/dma-buf.c b/drivers/dma-buf/dma-buf.c index 9ad6397aaa97e10b580a23b54836162e765ae7ff..f264b70c383eb4ec5db083bc455296929668dc4d 100644 --- a/drivers/dma-buf/dma-buf.c +++ b/drivers/dma-buf/dma-buf.c @@ -493,7 +493,7 @@ static struct file *dma_buf_getfile(struct dma_buf *dmabuf, int flags) * * 4. Once a driver is done with a shared buffer it needs to call * dma_buf_detach() (after cleaning up any mappings) and then release the - * reference acquired with dma_buf_get by calling dma_buf_put(). + * reference acquired with dma_buf_get() by calling dma_buf_put(). * * For the detailed semantics exporters are expected to implement see * &dma_buf_ops. @@ -509,9 +509,10 @@ static struct file *dma_buf_getfile(struct dma_buf *dmabuf, int flags) * by the exporter. see &struct dma_buf_export_info * for further details. * - * Returns, on success, a newly created dma_buf object, which wraps the - * supplied private data and operations for dma_buf_ops. On either missing - * ops, or error in allocating struct dma_buf, will return negative error. + * Returns, on success, a newly created struct dma_buf object, which wraps the + * supplied private data and operations for struct dma_buf_ops. On either + * missing ops, or error in allocating struct dma_buf, will return negative + * error. * * For most cases the easiest way to create @exp_info is through the * %DEFINE_DMA_BUF_EXPORT_INFO macro. @@ -597,7 +598,7 @@ struct dma_buf *dma_buf_export(const struct dma_buf_export_info *exp_info) EXPORT_SYMBOL_GPL(dma_buf_export); /** - * dma_buf_fd - returns a file descriptor for the given dma_buf + * dma_buf_fd - returns a file descriptor for the given struct dma_buf * @dmabuf: [in] pointer to dma_buf for which fd is required. * @flags: [in] flags to give to fd * @@ -621,10 +622,10 @@ int dma_buf_fd(struct dma_buf *dmabuf, int flags) EXPORT_SYMBOL_GPL(dma_buf_fd); /** - * dma_buf_get - returns the dma_buf structure related to an fd - * @fd: [in] fd associated with the dma_buf to be returned + * dma_buf_get - returns the struct dma_buf related to an fd + * @fd: [in] fd associated with the struct dma_buf to be returned * - * On success, returns the dma_buf structure associated with an fd; uses + * On success, returns the struct dma_buf associated with an fd; uses * file's refcounting done by fget to increase refcount. returns ERR_PTR * otherwise. */ @@ -665,9 +666,36 @@ void dma_buf_put(struct dma_buf *dmabuf) } EXPORT_SYMBOL_GPL(dma_buf_put); +static void mangle_sg_table(struct sg_table *sg_table) +{ +#ifdef CONFIG_DMABUF_DEBUG + int i; + struct scatterlist *sg; + + /* To catch abuse of the underlying struct page by importers mix + * up the bits, but take care to preserve the low SG_ bits to + * not corrupt the sgt. The mixing is undone in __unmap_dma_buf + * before passing the sgt back to the exporter. */ + for_each_sgtable_sg(sg_table, sg, i) + sg->page_link ^= ~0xffUL; +#endif + +} +static struct sg_table * __map_dma_buf(struct dma_buf_attachment *attach, + enum dma_data_direction direction) +{ + struct sg_table *sg_table; + + sg_table = attach->dmabuf->ops->map_dma_buf(attach, direction); + + if (!IS_ERR_OR_NULL(sg_table)) + mangle_sg_table(sg_table); + + return sg_table; +} + /** - * dma_buf_dynamic_attach - Add the device to dma_buf's attachments list; optionally, - * calls attach() of dma_buf_ops to allow device-specific attach functionality + * dma_buf_dynamic_attach - Add the device to dma_buf's attachments list * @dmabuf: [in] buffer to attach device to. * @dev: [in] device to be attached. * @importer_ops: [in] importer operations for the attachment @@ -676,6 +704,9 @@ EXPORT_SYMBOL_GPL(dma_buf_put); * Returns struct dma_buf_attachment pointer for this attachment. Attachments * must be cleaned up by calling dma_buf_detach(). * + * Optionally this calls &dma_buf_ops.attach to allow device-specific attach + * functionality. + * * Returns: * * A pointer to newly created &dma_buf_attachment on success, or a negative @@ -734,7 +765,7 @@ dma_buf_dynamic_attach(struct dma_buf *dmabuf, struct device *dev, goto err_unlock; } - sgt = dmabuf->ops->map_dma_buf(attach, DMA_BIDIRECTIONAL); + sgt = __map_dma_buf(attach, DMA_BIDIRECTIONAL); if (!sgt) sgt = ERR_PTR(-ENOMEM); if (IS_ERR(sgt)) { @@ -781,13 +812,24 @@ struct dma_buf_attachment *dma_buf_attach(struct dma_buf *dmabuf, } EXPORT_SYMBOL_GPL(dma_buf_attach); +static void __unmap_dma_buf(struct dma_buf_attachment *attach, + struct sg_table *sg_table, + enum dma_data_direction direction) +{ + /* uses XOR, hence this unmangles */ + mangle_sg_table(sg_table); + + attach->dmabuf->ops->unmap_dma_buf(attach, sg_table, direction); +} + /** - * dma_buf_detach - Remove the given attachment from dmabuf's attachments list; - * optionally calls detach() of dma_buf_ops for device-specific detach + * dma_buf_detach - Remove the given attachment from dmabuf's attachments list * @dmabuf: [in] buffer to detach from. * @attach: [in] attachment to be detached; is free'd after this call. * * Clean up a device attachment obtained by calling dma_buf_attach(). + * + * Optionally this calls &dma_buf_ops.detach for device-specific detach. */ void dma_buf_detach(struct dma_buf *dmabuf, struct dma_buf_attachment *attach) { @@ -798,7 +840,7 @@ void dma_buf_detach(struct dma_buf *dmabuf, struct dma_buf_attachment *attach) if (dma_buf_is_dynamic(attach->dmabuf)) dma_resv_lock(attach->dmabuf->resv, NULL); - dmabuf->ops->unmap_dma_buf(attach, attach->sgt, attach->dir); + __unmap_dma_buf(attach, attach->sgt, attach->dir); if (dma_buf_is_dynamic(attach->dmabuf)) { dma_buf_unpin(attach); @@ -818,9 +860,15 @@ EXPORT_SYMBOL_GPL(dma_buf_detach); /** * dma_buf_pin - Lock down the DMA-buf - * * @attach: [in] attachment which should be pinned * + * Only dynamic importers (who set up @attach with dma_buf_dynamic_attach()) may + * call this, and only for limited use cases like scanout and not for temporary + * pin operations. It is not permitted to allow userspace to pin arbitrary + * amounts of buffers through this interface. + * + * Buffers must be unpinned by calling dma_buf_unpin(). + * * Returns: * 0 on success, negative error code on failure. */ @@ -829,6 +877,8 @@ int dma_buf_pin(struct dma_buf_attachment *attach) struct dma_buf *dmabuf = attach->dmabuf; int ret = 0; + WARN_ON(!dma_buf_attachment_is_dynamic(attach)); + dma_resv_assert_held(dmabuf->resv); if (dmabuf->ops->pin) @@ -839,14 +889,19 @@ int dma_buf_pin(struct dma_buf_attachment *attach) EXPORT_SYMBOL_GPL(dma_buf_pin); /** - * dma_buf_unpin - Remove lock from DMA-buf - * + * dma_buf_unpin - Unpin a DMA-buf * @attach: [in] attachment which should be unpinned + * + * This unpins a buffer pinned by dma_buf_pin() and allows the exporter to move + * any mapping of @attach again and inform the importer through + * &dma_buf_attach_ops.move_notify. */ void dma_buf_unpin(struct dma_buf_attachment *attach) { struct dma_buf *dmabuf = attach->dmabuf; + WARN_ON(!dma_buf_attachment_is_dynamic(attach)); + dma_resv_assert_held(dmabuf->resv); if (dmabuf->ops->unpin) @@ -907,7 +962,7 @@ struct sg_table *dma_buf_map_attachment(struct dma_buf_attachment *attach, } } - sg_table = attach->dmabuf->ops->map_dma_buf(attach, direction); + sg_table = __map_dma_buf(attach, direction); if (!sg_table) sg_table = ERR_PTR(-ENOMEM); @@ -970,7 +1025,7 @@ void dma_buf_unmap_attachment(struct dma_buf_attachment *attach, if (dma_buf_is_dynamic(attach->dmabuf)) dma_resv_assert_held(attach->dmabuf->resv); - attach->dmabuf->ops->unmap_dma_buf(attach, sg_table, direction); + __unmap_dma_buf(attach, sg_table, direction); if (dma_buf_is_dynamic(attach->dmabuf) && !IS_ENABLED(CONFIG_DMABUF_MOVE_NOTIFY)) @@ -1014,15 +1069,15 @@ EXPORT_SYMBOL_GPL(dma_buf_move_notify); * vmalloc space might be limited and result in vmap calls failing. * * Interfaces:: + * * void \*dma_buf_vmap(struct dma_buf \*dmabuf) * void dma_buf_vunmap(struct dma_buf \*dmabuf, void \*vaddr) * * The vmap call can fail if there is no vmap support in the exporter, or if - * it runs out of vmalloc space. Fallback to kmap should be implemented. Note - * that the dma-buf layer keeps a reference count for all vmap access and - * calls down into the exporter's vmap function only when no vmapping exists, - * and only unmaps it once. Protection against concurrent vmap/vunmap calls is - * provided by taking the dma_buf->lock mutex. + * it runs out of vmalloc space. Note that the dma-buf layer keeps a reference + * count for all vmap access and calls down into the exporter's vmap function + * only when no vmapping exists, and only unmaps it once. Protection against + * concurrent vmap/vunmap calls is provided by taking the &dma_buf.lock mutex. * * - For full compatibility on the importer side with existing userspace * interfaces, which might already support mmap'ing buffers. This is needed in @@ -1074,11 +1129,12 @@ EXPORT_SYMBOL_GPL(dma_buf_move_notify); * shootdowns would increase the complexity quite a bit. * * Interface:: + * * int dma_buf_mmap(struct dma_buf \*, struct vm_area_struct \*, * unsigned long); * * If the importing subsystem simply provides a special-purpose mmap call to - * set up a mapping in userspace, calling do_mmap with dma_buf->file will + * set up a mapping in userspace, calling do_mmap with &dma_buf.file will * equally achieve that for a dma-buf object. */ @@ -1111,6 +1167,11 @@ static int __dma_buf_begin_cpu_access(struct dma_buf *dmabuf, * dma_buf_end_cpu_access(). Only when cpu access is braketed by both calls is * it guaranteed to be coherent with other DMA access. * + * This function will also wait for any DMA transactions tracked through + * implicit synchronization in &dma_buf.resv. For DMA transactions with explicit + * synchronization this function will only ensure cache coherency, callers must + * ensure synchronization with such DMA transactions on their own. + * * Can return negative error values, returns 0 on success. */ int dma_buf_begin_cpu_access(struct dma_buf *dmabuf, @@ -1121,6 +1182,8 @@ int dma_buf_begin_cpu_access(struct dma_buf *dmabuf, if (WARN_ON(!dmabuf)) return -EINVAL; + might_lock(&dmabuf->resv->lock.base); + if (dmabuf->ops->begin_cpu_access) ret = dmabuf->ops->begin_cpu_access(dmabuf, direction); @@ -1154,6 +1217,8 @@ int dma_buf_end_cpu_access(struct dma_buf *dmabuf, WARN_ON(!dmabuf); + might_lock(&dmabuf->resv->lock.base); + if (dmabuf->ops->end_cpu_access) ret = dmabuf->ops->end_cpu_access(dmabuf, direction); @@ -1212,7 +1277,10 @@ EXPORT_SYMBOL_GPL(dma_buf_mmap); * This call may fail due to lack of virtual mapping address space. * These calls are optional in drivers. The intended use for them * is for mapping objects linear in kernel space for high use objects. - * Please attempt to use kmap/kunmap before thinking about these interfaces. + * + * To ensure coherency users must call dma_buf_begin_cpu_access() and + * dma_buf_end_cpu_access() around any cpu access performed through this + * mapping. * * Returns 0 on success, or a negative errno code otherwise. */ diff --git a/drivers/dma-buf/dma-fence.c b/drivers/dma-buf/dma-fence.c index 7475e09b0680835e0606b5acbb3a16133d71e658..d64fc03929be3182dc5da2f7d3a0e7767b62be23 100644 --- a/drivers/dma-buf/dma-fence.c +++ b/drivers/dma-buf/dma-fence.c @@ -312,22 +312,25 @@ void __dma_fence_might_wait(void) /** - * dma_fence_signal_locked - signal completion of a fence + * dma_fence_signal_timestamp_locked - signal completion of a fence * @fence: the fence to signal + * @timestamp: fence signal timestamp in kernel's CLOCK_MONOTONIC time domain * * Signal completion for software callbacks on a fence, this will unblock * dma_fence_wait() calls and run all the callbacks added with * dma_fence_add_callback(). Can be called multiple times, but since a fence * can only go from the unsignaled to the signaled state and not back, it will - * only be effective the first time. + * only be effective the first time. Set the timestamp provided as the fence + * signal timestamp. * - * Unlike dma_fence_signal(), this function must be called with &dma_fence.lock - * held. + * Unlike dma_fence_signal_timestamp(), this function must be called with + * &dma_fence.lock held. * * Returns 0 on success and a negative error value when @fence has been * signalled already. */ -int dma_fence_signal_locked(struct dma_fence *fence) +int dma_fence_signal_timestamp_locked(struct dma_fence *fence, + ktime_t timestamp) { struct dma_fence_cb *cur, *tmp; struct list_head cb_list; @@ -341,7 +344,7 @@ int dma_fence_signal_locked(struct dma_fence *fence) /* Stash the cb_list before replacing it with the timestamp */ list_replace(&fence->cb_list, &cb_list); - fence->timestamp = ktime_get(); + fence->timestamp = timestamp; set_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, &fence->flags); trace_dma_fence_signaled(fence); @@ -352,6 +355,59 @@ int dma_fence_signal_locked(struct dma_fence *fence) return 0; } +EXPORT_SYMBOL(dma_fence_signal_timestamp_locked); + +/** + * dma_fence_signal_timestamp - signal completion of a fence + * @fence: the fence to signal + * @timestamp: fence signal timestamp in kernel's CLOCK_MONOTONIC time domain + * + * Signal completion for software callbacks on a fence, this will unblock + * dma_fence_wait() calls and run all the callbacks added with + * dma_fence_add_callback(). Can be called multiple times, but since a fence + * can only go from the unsignaled to the signaled state and not back, it will + * only be effective the first time. Set the timestamp provided as the fence + * signal timestamp. + * + * Returns 0 on success and a negative error value when @fence has been + * signalled already. + */ +int dma_fence_signal_timestamp(struct dma_fence *fence, ktime_t timestamp) +{ + unsigned long flags; + int ret; + + if (!fence) + return -EINVAL; + + spin_lock_irqsave(fence->lock, flags); + ret = dma_fence_signal_timestamp_locked(fence, timestamp); + spin_unlock_irqrestore(fence->lock, flags); + + return ret; +} +EXPORT_SYMBOL(dma_fence_signal_timestamp); + +/** + * dma_fence_signal_locked - signal completion of a fence + * @fence: the fence to signal + * + * Signal completion for software callbacks on a fence, this will unblock + * dma_fence_wait() calls and run all the callbacks added with + * dma_fence_add_callback(). Can be called multiple times, but since a fence + * can only go from the unsignaled to the signaled state and not back, it will + * only be effective the first time. + * + * Unlike dma_fence_signal(), this function must be called with &dma_fence.lock + * held. + * + * Returns 0 on success and a negative error value when @fence has been + * signalled already. + */ +int dma_fence_signal_locked(struct dma_fence *fence) +{ + return dma_fence_signal_timestamp_locked(fence, ktime_get()); +} EXPORT_SYMBOL(dma_fence_signal_locked); /** @@ -379,7 +435,7 @@ int dma_fence_signal(struct dma_fence *fence) tmp = dma_fence_begin_signalling(); spin_lock_irqsave(fence->lock, flags); - ret = dma_fence_signal_locked(fence); + ret = dma_fence_signal_timestamp_locked(fence, ktime_get()); spin_unlock_irqrestore(fence->lock, flags); dma_fence_end_signalling(tmp); diff --git a/drivers/dma-buf/dma-heap.c b/drivers/dma-buf/dma-heap.c index afd22c9dbdcfa16864f07e2fdab2e1ed8f5992fb..6b5db954569fc4b9c6dcdcc2bc78c157b2beb482 100644 --- a/drivers/dma-buf/dma-heap.c +++ b/drivers/dma-buf/dma-heap.c @@ -52,6 +52,9 @@ static int dma_heap_buffer_alloc(struct dma_heap *heap, size_t len, unsigned int fd_flags, unsigned int heap_flags) { + struct dma_buf *dmabuf; + int fd; + /* * Allocations from all heaps have to begin * and end on page boundaries. @@ -60,7 +63,16 @@ static int dma_heap_buffer_alloc(struct dma_heap *heap, size_t len, if (!len) return -EINVAL; - return heap->ops->allocate(heap, len, fd_flags, heap_flags); + dmabuf = heap->ops->allocate(heap, len, fd_flags, heap_flags); + if (IS_ERR(dmabuf)) + return PTR_ERR(dmabuf); + + fd = dma_buf_fd(dmabuf, fd_flags); + if (fd < 0) { + dma_buf_put(dmabuf); + /* just return, as put will call release and that will free */ + } + return fd; } static int dma_heap_open(struct inode *inode, struct file *file) diff --git a/drivers/dma-buf/heaps/cma_heap.c b/drivers/dma-buf/heaps/cma_heap.c index 364fc2f3e499549cd3eb89673b1a3dffd8af7977..5d64eccd21d638203ed25e72d143ffbfa8e94327 100644 --- a/drivers/dma-buf/heaps/cma_heap.c +++ b/drivers/dma-buf/heaps/cma_heap.c @@ -271,10 +271,10 @@ static const struct dma_buf_ops cma_heap_buf_ops = { .release = cma_heap_dma_buf_release, }; -static int cma_heap_allocate(struct dma_heap *heap, - unsigned long len, - unsigned long fd_flags, - unsigned long heap_flags) +static struct dma_buf *cma_heap_allocate(struct dma_heap *heap, + unsigned long len, + unsigned long fd_flags, + unsigned long heap_flags) { struct cma_heap *cma_heap = dma_heap_get_drvdata(heap); struct cma_heap_buffer *buffer; @@ -289,7 +289,7 @@ static int cma_heap_allocate(struct dma_heap *heap, buffer = kzalloc(sizeof(*buffer), GFP_KERNEL); if (!buffer) - return -ENOMEM; + return ERR_PTR(-ENOMEM); INIT_LIST_HEAD(&buffer->attachments); mutex_init(&buffer->lock); @@ -348,15 +348,7 @@ static int cma_heap_allocate(struct dma_heap *heap, ret = PTR_ERR(dmabuf); goto free_pages; } - - ret = dma_buf_fd(dmabuf, fd_flags); - if (ret < 0) { - dma_buf_put(dmabuf); - /* just return, as put will call release and that will free */ - return ret; - } - - return ret; + return dmabuf; free_pages: kfree(buffer->pages); @@ -365,7 +357,7 @@ static int cma_heap_allocate(struct dma_heap *heap, free_buffer: kfree(buffer); - return ret; + return ERR_PTR(ret); } static const struct dma_heap_ops cma_heap_ops = { diff --git a/drivers/dma-buf/heaps/system_heap.c b/drivers/dma-buf/heaps/system_heap.c index 17e0e9a68baff4695f17c079b5f8ae29ec1e7ad3..29e49ac1725135ee6254ed9e0726e42bc885e7b3 100644 --- a/drivers/dma-buf/heaps/system_heap.c +++ b/drivers/dma-buf/heaps/system_heap.c @@ -331,10 +331,10 @@ static struct page *alloc_largest_available(unsigned long size, return NULL; } -static int system_heap_allocate(struct dma_heap *heap, - unsigned long len, - unsigned long fd_flags, - unsigned long heap_flags) +static struct dma_buf *system_heap_allocate(struct dma_heap *heap, + unsigned long len, + unsigned long fd_flags, + unsigned long heap_flags) { struct system_heap_buffer *buffer; DEFINE_DMA_BUF_EXPORT_INFO(exp_info); @@ -349,7 +349,7 @@ static int system_heap_allocate(struct dma_heap *heap, buffer = kzalloc(sizeof(*buffer), GFP_KERNEL); if (!buffer) - return -ENOMEM; + return ERR_PTR(-ENOMEM); INIT_LIST_HEAD(&buffer->attachments); mutex_init(&buffer->lock); @@ -363,8 +363,10 @@ static int system_heap_allocate(struct dma_heap *heap, * Avoid trying to allocate memory if the process * has been killed by SIGKILL */ - if (fatal_signal_pending(current)) + if (fatal_signal_pending(current)) { + ret = -EINTR; goto free_buffer; + } page = alloc_largest_available(size_remaining, max_order); if (!page) @@ -397,14 +399,7 @@ static int system_heap_allocate(struct dma_heap *heap, ret = PTR_ERR(dmabuf); goto free_pages; } - - ret = dma_buf_fd(dmabuf, fd_flags); - if (ret < 0) { - dma_buf_put(dmabuf); - /* just return, as put will call release and that will free */ - return ret; - } - return ret; + return dmabuf; free_pages: for_each_sgtable_sg(table, sg, i) { @@ -418,7 +413,7 @@ static int system_heap_allocate(struct dma_heap *heap, __free_pages(page, compound_order(page)); kfree(buffer); - return ret; + return ERR_PTR(ret); } static const struct dma_heap_ops system_heap_ops = { diff --git a/drivers/dma-buf/st-dma-fence.c b/drivers/dma-buf/st-dma-fence.c index e593064341c8c2a973546eeab98acc8f16ef0f71..c8a12d7ad71ab6baef2298b60619b0d25edc2d03 100644 --- a/drivers/dma-buf/st-dma-fence.c +++ b/drivers/dma-buf/st-dma-fence.c @@ -471,8 +471,11 @@ static int thread_signal_callback(void *arg) dma_fence_signal(f1); smp_store_mb(cb.seen, false); - if (!f2 || dma_fence_add_callback(f2, &cb.cb, simple_callback)) - miss++, cb.seen = true; + if (!f2 || + dma_fence_add_callback(f2, &cb.cb, simple_callback)) { + miss++; + cb.seen = true; + } if (!t->before) dma_fence_signal(f1); diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig index d242c76326217a6907dfa0814d459ae66f2ee360..0c2827fd8c195bcc3525ef8892ac4e9809c7a734 100644 --- a/drivers/dma/Kconfig +++ b/drivers/dma/Kconfig @@ -124,13 +124,6 @@ config BCM_SBA_RAID has the capability to offload memcpy, xor and pq computation for raid5/6. -config COH901318 - bool "ST-Ericsson COH901318 DMA support" - select DMA_ENGINE - depends on ARCH_U300 || COMPILE_TEST - help - Enable support for ST-Ericsson COH 901 318 DMA. - config DMA_BCM2835 tristate "BCM2835 DMA engine support" depends on ARCH_BCM2835 @@ -179,6 +172,7 @@ config DMA_SUN6I config DW_AXI_DMAC tristate "Synopsys DesignWare AXI DMA support" depends on OF || COMPILE_TEST + depends on HAS_IOMEM select DMA_ENGINE select DMA_VIRTUAL_CHANNELS help @@ -378,14 +372,14 @@ config MILBEAUT_XDMAC XDMAC device. config MMP_PDMA - bool "MMP PDMA support" + tristate "MMP PDMA support" depends on ARCH_MMP || ARCH_PXA || COMPILE_TEST select DMA_ENGINE help Support the MMP PDMA engine for PXA and MMP platform. config MMP_TDMA - bool "MMP Two-Channel DMA support" + tristate "MMP Two-Channel DMA support" depends on ARCH_MMP || COMPILE_TEST select DMA_ENGINE select GENERIC_ALLOCATOR @@ -519,13 +513,6 @@ config PLX_DMA These are exposed via extra functions on the switch's upstream port. Each function exposes one DMA channel. -config SIRF_DMA - tristate "CSR SiRFprimaII/SiRFmarco DMA support" - depends on ARCH_SIRF - select DMA_ENGINE - help - Enable support for the CSR SiRFprimaII DMA engine. - config STE_DMA40 bool "ST-Ericsson DMA40 support" depends on ARCH_U8500 @@ -710,15 +697,6 @@ config XILINX_ZYNQMP_DPDMA driver provides the dmaengine required by the DisplayPort subsystem display driver. -config ZX_DMA - tristate "ZTE ZX DMA support" - depends on ARCH_ZX || COMPILE_TEST - select DMA_ENGINE - select DMA_VIRTUAL_CHANNELS - help - Support the DMA engine for ZTE ZX family platform devices. - - # driver files source "drivers/dma/bestcomm/Kconfig" @@ -740,6 +718,8 @@ source "drivers/dma/ti/Kconfig" source "drivers/dma/fsl-dpaa2-qdma/Kconfig" +source "drivers/dma/lgm/Kconfig" + # clients comment "DMA Clients" depends on DMA_ENGINE diff --git a/drivers/dma/Makefile b/drivers/dma/Makefile index 948a8da05f8b6829c498d9961b9c3e8dfc665586..aa69094e35470ca029d52b5633936df06d4411b8 100644 --- a/drivers/dma/Makefile +++ b/drivers/dma/Makefile @@ -20,7 +20,6 @@ obj-$(CONFIG_AT_HDMAC) += at_hdmac.o obj-$(CONFIG_AT_XDMAC) += at_xdmac.o obj-$(CONFIG_AXI_DMAC) += dma-axi-dmac.o obj-$(CONFIG_BCM_SBA_RAID) += bcm-sba-raid.o -obj-$(CONFIG_COH901318) += coh901318.o coh901318_lli.o obj-$(CONFIG_DMA_BCM2835) += bcm2835-dma.o obj-$(CONFIG_DMA_JZ4780) += dma-jz4780.o obj-$(CONFIG_DMA_SA11X0) += sa11x0-dma.o @@ -65,7 +64,6 @@ obj-$(CONFIG_PPC_BESTCOMM) += bestcomm/ obj-$(CONFIG_PXA_DMA) += pxa_dma.o obj-$(CONFIG_RENESAS_DMA) += sh/ obj-$(CONFIG_SF_PDMA) += sf-pdma/ -obj-$(CONFIG_SIRF_DMA) += sirf-dma.o obj-$(CONFIG_STE_DMA40) += ste_dma40.o ste_dma40_ll.o obj-$(CONFIG_STM32_DMA) += stm32-dma.o obj-$(CONFIG_STM32_DMAMUX) += stm32-dmamux.o @@ -79,9 +77,9 @@ obj-$(CONFIG_TIMB_DMA) += timb_dma.o obj-$(CONFIG_UNIPHIER_MDMAC) += uniphier-mdmac.o obj-$(CONFIG_UNIPHIER_XDMAC) += uniphier-xdmac.o obj-$(CONFIG_XGENE_DMA) += xgene-dma.o -obj-$(CONFIG_ZX_DMA) += zx_dma.o obj-$(CONFIG_ST_FDMA) += st_fdma.o obj-$(CONFIG_FSL_DPAA2_QDMA) += fsl-dpaa2-qdma/ +obj-$(CONFIG_INTEL_LDMA) += lgm/ obj-y += mediatek/ obj-y += qcom/ diff --git a/drivers/dma/at_hdmac.c b/drivers/dma/at_hdmac.c index 7eaee5b705b1bf47bb7c78301550cd5751720ee2..30ae36124b1db9d031affc1c4a7b6b4cc2564a7c 100644 --- a/drivers/dma/at_hdmac.c +++ b/drivers/dma/at_hdmac.c @@ -54,6 +54,25 @@ module_param(init_nr_desc_per_channel, uint, 0644); MODULE_PARM_DESC(init_nr_desc_per_channel, "initial descriptors per channel (default: 64)"); +/** + * struct at_dma_platform_data - Controller configuration parameters + * @nr_channels: Number of channels supported by hardware (max 8) + * @cap_mask: dma_capability flags supported by the platform + */ +struct at_dma_platform_data { + unsigned int nr_channels; + dma_cap_mask_t cap_mask; +}; + +/** + * struct at_dma_slave - Controller-specific information about a slave + * @dma_dev: required DMA master device + * @cfg: Platform-specific initializer for the CFG register + */ +struct at_dma_slave { + struct device *dma_dev; + u32 cfg; +}; /* prototypes */ static dma_cookie_t atc_tx_submit(struct dma_async_tx_descriptor *tx); diff --git a/drivers/dma/at_hdmac_regs.h b/drivers/dma/at_hdmac_regs.h index 80fc2fe8c77ea9be26dc84a18beaaad2198e429c..4d1ebc040031ce8e5f5f21a998af37bd3b6f2313 100644 --- a/drivers/dma/at_hdmac_regs.h +++ b/drivers/dma/at_hdmac_regs.h @@ -7,8 +7,6 @@ #ifndef AT_HDMAC_REGS_H #define AT_HDMAC_REGS_H -#include - #define AT_DMA_MAX_NR_CHANNELS 8 @@ -148,7 +146,31 @@ #define ATC_AUTO (0x1 << 31) /* Auto multiple buffer tx enable */ /* Bitfields in CFG */ -/* are in at_hdmac.h */ +#define ATC_PER_MSB(h) ((0x30U & (h)) >> 4) /* Extract most significant bits of a handshaking identifier */ + +#define ATC_SRC_PER(h) (0xFU & (h)) /* Channel src rq associated with periph handshaking ifc h */ +#define ATC_DST_PER(h) ((0xFU & (h)) << 4) /* Channel dst rq associated with periph handshaking ifc h */ +#define ATC_SRC_REP (0x1 << 8) /* Source Replay Mod */ +#define ATC_SRC_H2SEL (0x1 << 9) /* Source Handshaking Mod */ +#define ATC_SRC_H2SEL_SW (0x0 << 9) +#define ATC_SRC_H2SEL_HW (0x1 << 9) +#define ATC_SRC_PER_MSB(h) (ATC_PER_MSB(h) << 10) /* Channel src rq (most significant bits) */ +#define ATC_DST_REP (0x1 << 12) /* Destination Replay Mod */ +#define ATC_DST_H2SEL (0x1 << 13) /* Destination Handshaking Mod */ +#define ATC_DST_H2SEL_SW (0x0 << 13) +#define ATC_DST_H2SEL_HW (0x1 << 13) +#define ATC_DST_PER_MSB(h) (ATC_PER_MSB(h) << 14) /* Channel dst rq (most significant bits) */ +#define ATC_SOD (0x1 << 16) /* Stop On Done */ +#define ATC_LOCK_IF (0x1 << 20) /* Interface Lock */ +#define ATC_LOCK_B (0x1 << 21) /* AHB Bus Lock */ +#define ATC_LOCK_IF_L (0x1 << 22) /* Master Interface Arbiter Lock */ +#define ATC_LOCK_IF_L_CHUNK (0x0 << 22) +#define ATC_LOCK_IF_L_BUFFER (0x1 << 22) +#define ATC_AHB_PROT_MASK (0x7 << 24) /* AHB Protection */ +#define ATC_FIFOCFG_MASK (0x3 << 28) /* FIFO Request Configuration */ +#define ATC_FIFOCFG_LARGESTBURST (0x0 << 28) +#define ATC_FIFOCFG_HALFFIFO (0x1 << 28) +#define ATC_FIFOCFG_ENOUGHSPACE (0x2 << 28) /* Bitfields in SPIP */ #define ATC_SPIP_HOLE(x) (0xFFFFU & (x)) diff --git a/drivers/dma/coh901318.c b/drivers/dma/coh901318.c deleted file mode 100644 index 95b9b2f5358ef1073964afc7c6a5d1afe5f51ccb..0000000000000000000000000000000000000000 --- a/drivers/dma/coh901318.c +++ /dev/null @@ -1,2808 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * driver/dma/coh901318.c - * - * Copyright (C) 2007-2009 ST-Ericsson - * DMA driver for COH 901 318 - * Author: Per Friden - */ - -#include -#include -#include /* printk() */ -#include /* everything... */ -#include -#include /* kmalloc() */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "coh901318.h" -#include "dmaengine.h" - -#define COH901318_MOD32_MASK (0x1F) -#define COH901318_WORD_MASK (0xFFFFFFFF) -/* INT_STATUS - Interrupt Status Registers 32bit (R/-) */ -#define COH901318_INT_STATUS1 (0x0000) -#define COH901318_INT_STATUS2 (0x0004) -/* TC_INT_STATUS - Terminal Count Interrupt Status Registers 32bit (R/-) */ -#define COH901318_TC_INT_STATUS1 (0x0008) -#define COH901318_TC_INT_STATUS2 (0x000C) -/* TC_INT_CLEAR - Terminal Count Interrupt Clear Registers 32bit (-/W) */ -#define COH901318_TC_INT_CLEAR1 (0x0010) -#define COH901318_TC_INT_CLEAR2 (0x0014) -/* RAW_TC_INT_STATUS - Raw Term Count Interrupt Status Registers 32bit (R/-) */ -#define COH901318_RAW_TC_INT_STATUS1 (0x0018) -#define COH901318_RAW_TC_INT_STATUS2 (0x001C) -/* BE_INT_STATUS - Bus Error Interrupt Status Registers 32bit (R/-) */ -#define COH901318_BE_INT_STATUS1 (0x0020) -#define COH901318_BE_INT_STATUS2 (0x0024) -/* BE_INT_CLEAR - Bus Error Interrupt Clear Registers 32bit (-/W) */ -#define COH901318_BE_INT_CLEAR1 (0x0028) -#define COH901318_BE_INT_CLEAR2 (0x002C) -/* RAW_BE_INT_STATUS - Raw Term Count Interrupt Status Registers 32bit (R/-) */ -#define COH901318_RAW_BE_INT_STATUS1 (0x0030) -#define COH901318_RAW_BE_INT_STATUS2 (0x0034) - -/* - * CX_CFG - Channel Configuration Registers 32bit (R/W) - */ -#define COH901318_CX_CFG (0x0100) -#define COH901318_CX_CFG_SPACING (0x04) -/* Channel enable activates tha dma job */ -#define COH901318_CX_CFG_CH_ENABLE (0x00000001) -#define COH901318_CX_CFG_CH_DISABLE (0x00000000) -/* Request Mode */ -#define COH901318_CX_CFG_RM_MASK (0x00000006) -#define COH901318_CX_CFG_RM_MEMORY_TO_MEMORY (0x0 << 1) -#define COH901318_CX_CFG_RM_PRIMARY_TO_MEMORY (0x1 << 1) -#define COH901318_CX_CFG_RM_MEMORY_TO_PRIMARY (0x1 << 1) -#define COH901318_CX_CFG_RM_PRIMARY_TO_SECONDARY (0x3 << 1) -#define COH901318_CX_CFG_RM_SECONDARY_TO_PRIMARY (0x3 << 1) -/* Linked channel request field. RM must == 11 */ -#define COH901318_CX_CFG_LCRF_SHIFT 3 -#define COH901318_CX_CFG_LCRF_MASK (0x000001F8) -#define COH901318_CX_CFG_LCR_DISABLE (0x00000000) -/* Terminal Counter Interrupt Request Mask */ -#define COH901318_CX_CFG_TC_IRQ_ENABLE (0x00000200) -#define COH901318_CX_CFG_TC_IRQ_DISABLE (0x00000000) -/* Bus Error interrupt Mask */ -#define COH901318_CX_CFG_BE_IRQ_ENABLE (0x00000400) -#define COH901318_CX_CFG_BE_IRQ_DISABLE (0x00000000) - -/* - * CX_STAT - Channel Status Registers 32bit (R/-) - */ -#define COH901318_CX_STAT (0x0200) -#define COH901318_CX_STAT_SPACING (0x04) -#define COH901318_CX_STAT_RBE_IRQ_IND (0x00000008) -#define COH901318_CX_STAT_RTC_IRQ_IND (0x00000004) -#define COH901318_CX_STAT_ACTIVE (0x00000002) -#define COH901318_CX_STAT_ENABLED (0x00000001) - -/* - * CX_CTRL - Channel Control Registers 32bit (R/W) - */ -#define COH901318_CX_CTRL (0x0400) -#define COH901318_CX_CTRL_SPACING (0x10) -/* Transfer Count Enable */ -#define COH901318_CX_CTRL_TC_ENABLE (0x00001000) -#define COH901318_CX_CTRL_TC_DISABLE (0x00000000) -/* Transfer Count Value 0 - 4095 */ -#define COH901318_CX_CTRL_TC_VALUE_MASK (0x00000FFF) -/* Burst count */ -#define COH901318_CX_CTRL_BURST_COUNT_MASK (0x0000E000) -#define COH901318_CX_CTRL_BURST_COUNT_64_BYTES (0x7 << 13) -#define COH901318_CX_CTRL_BURST_COUNT_48_BYTES (0x6 << 13) -#define COH901318_CX_CTRL_BURST_COUNT_32_BYTES (0x5 << 13) -#define COH901318_CX_CTRL_BURST_COUNT_16_BYTES (0x4 << 13) -#define COH901318_CX_CTRL_BURST_COUNT_8_BYTES (0x3 << 13) -#define COH901318_CX_CTRL_BURST_COUNT_4_BYTES (0x2 << 13) -#define COH901318_CX_CTRL_BURST_COUNT_2_BYTES (0x1 << 13) -#define COH901318_CX_CTRL_BURST_COUNT_1_BYTE (0x0 << 13) -/* Source bus size */ -#define COH901318_CX_CTRL_SRC_BUS_SIZE_MASK (0x00030000) -#define COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS (0x2 << 16) -#define COH901318_CX_CTRL_SRC_BUS_SIZE_16_BITS (0x1 << 16) -#define COH901318_CX_CTRL_SRC_BUS_SIZE_8_BITS (0x0 << 16) -/* Source address increment */ -#define COH901318_CX_CTRL_SRC_ADDR_INC_ENABLE (0x00040000) -#define COH901318_CX_CTRL_SRC_ADDR_INC_DISABLE (0x00000000) -/* Destination Bus Size */ -#define COH901318_CX_CTRL_DST_BUS_SIZE_MASK (0x00180000) -#define COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS (0x2 << 19) -#define COH901318_CX_CTRL_DST_BUS_SIZE_16_BITS (0x1 << 19) -#define COH901318_CX_CTRL_DST_BUS_SIZE_8_BITS (0x0 << 19) -/* Destination address increment */ -#define COH901318_CX_CTRL_DST_ADDR_INC_ENABLE (0x00200000) -#define COH901318_CX_CTRL_DST_ADDR_INC_DISABLE (0x00000000) -/* Master Mode (Master2 is only connected to MSL) */ -#define COH901318_CX_CTRL_MASTER_MODE_MASK (0x00C00000) -#define COH901318_CX_CTRL_MASTER_MODE_M2R_M1W (0x3 << 22) -#define COH901318_CX_CTRL_MASTER_MODE_M1R_M2W (0x2 << 22) -#define COH901318_CX_CTRL_MASTER_MODE_M2RW (0x1 << 22) -#define COH901318_CX_CTRL_MASTER_MODE_M1RW (0x0 << 22) -/* Terminal Count flag to PER enable */ -#define COH901318_CX_CTRL_TCP_ENABLE (0x01000000) -#define COH901318_CX_CTRL_TCP_DISABLE (0x00000000) -/* Terminal Count flags to CPU enable */ -#define COH901318_CX_CTRL_TC_IRQ_ENABLE (0x02000000) -#define COH901318_CX_CTRL_TC_IRQ_DISABLE (0x00000000) -/* Hand shake to peripheral */ -#define COH901318_CX_CTRL_HSP_ENABLE (0x04000000) -#define COH901318_CX_CTRL_HSP_DISABLE (0x00000000) -#define COH901318_CX_CTRL_HSS_ENABLE (0x08000000) -#define COH901318_CX_CTRL_HSS_DISABLE (0x00000000) -/* DMA mode */ -#define COH901318_CX_CTRL_DDMA_MASK (0x30000000) -#define COH901318_CX_CTRL_DDMA_LEGACY (0x0 << 28) -#define COH901318_CX_CTRL_DDMA_DEMAND_DMA1 (0x1 << 28) -#define COH901318_CX_CTRL_DDMA_DEMAND_DMA2 (0x2 << 28) -/* Primary Request Data Destination */ -#define COH901318_CX_CTRL_PRDD_MASK (0x40000000) -#define COH901318_CX_CTRL_PRDD_DEST (0x1 << 30) -#define COH901318_CX_CTRL_PRDD_SOURCE (0x0 << 30) - -/* - * CX_SRC_ADDR - Channel Source Address Registers 32bit (R/W) - */ -#define COH901318_CX_SRC_ADDR (0x0404) -#define COH901318_CX_SRC_ADDR_SPACING (0x10) - -/* - * CX_DST_ADDR - Channel Destination Address Registers 32bit R/W - */ -#define COH901318_CX_DST_ADDR (0x0408) -#define COH901318_CX_DST_ADDR_SPACING (0x10) - -/* - * CX_LNK_ADDR - Channel Link Address Registers 32bit (R/W) - */ -#define COH901318_CX_LNK_ADDR (0x040C) -#define COH901318_CX_LNK_ADDR_SPACING (0x10) -#define COH901318_CX_LNK_LINK_IMMEDIATE (0x00000001) - -/** - * struct coh901318_params - parameters for DMAC configuration - * @config: DMA config register - * @ctrl_lli_last: DMA control register for the last lli in the list - * @ctrl_lli: DMA control register for an lli - * @ctrl_lli_chained: DMA control register for a chained lli - */ -struct coh901318_params { - u32 config; - u32 ctrl_lli_last; - u32 ctrl_lli; - u32 ctrl_lli_chained; -}; - -/** - * struct coh_dma_channel - dma channel base - * @name: ascii name of dma channel - * @number: channel id number - * @desc_nbr_max: number of preallocated descriptors - * @priority_high: prio of channel, 0 low otherwise high. - * @param: configuration parameters - */ -struct coh_dma_channel { - const char name[32]; - const int number; - const int desc_nbr_max; - const int priority_high; - const struct coh901318_params param; -}; - -/** - * struct powersave - DMA power save structure - * @lock: lock protecting data in this struct - * @started_channels: bit mask indicating active dma channels - */ -struct powersave { - spinlock_t lock; - u64 started_channels; -}; - -/* points out all dma slave channels. - * Syntax is [A1, B1, A2, B2, .... ,-1,-1] - * Select all channels from A to B, end of list is marked with -1,-1 - */ -static int dma_slave_channels[] = { - U300_DMA_MSL_TX_0, U300_DMA_SPI_RX, - U300_DMA_UART1_TX, U300_DMA_UART1_RX, -1, -1}; - -/* points out all dma memcpy channels. */ -static int dma_memcpy_channels[] = { - U300_DMA_GENERAL_PURPOSE_0, U300_DMA_GENERAL_PURPOSE_8, -1, -1}; - -#define flags_memcpy_config (COH901318_CX_CFG_CH_DISABLE | \ - COH901318_CX_CFG_RM_MEMORY_TO_MEMORY | \ - COH901318_CX_CFG_LCR_DISABLE | \ - COH901318_CX_CFG_TC_IRQ_ENABLE | \ - COH901318_CX_CFG_BE_IRQ_ENABLE) -#define flags_memcpy_lli_chained (COH901318_CX_CTRL_TC_ENABLE | \ - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | \ - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | \ - COH901318_CX_CTRL_SRC_ADDR_INC_ENABLE | \ - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | \ - COH901318_CX_CTRL_DST_ADDR_INC_ENABLE | \ - COH901318_CX_CTRL_MASTER_MODE_M1RW | \ - COH901318_CX_CTRL_TCP_DISABLE | \ - COH901318_CX_CTRL_TC_IRQ_DISABLE | \ - COH901318_CX_CTRL_HSP_DISABLE | \ - COH901318_CX_CTRL_HSS_DISABLE | \ - COH901318_CX_CTRL_DDMA_LEGACY | \ - COH901318_CX_CTRL_PRDD_SOURCE) -#define flags_memcpy_lli (COH901318_CX_CTRL_TC_ENABLE | \ - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | \ - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | \ - COH901318_CX_CTRL_SRC_ADDR_INC_ENABLE | \ - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | \ - COH901318_CX_CTRL_DST_ADDR_INC_ENABLE | \ - COH901318_CX_CTRL_MASTER_MODE_M1RW | \ - COH901318_CX_CTRL_TCP_DISABLE | \ - COH901318_CX_CTRL_TC_IRQ_DISABLE | \ - COH901318_CX_CTRL_HSP_DISABLE | \ - COH901318_CX_CTRL_HSS_DISABLE | \ - COH901318_CX_CTRL_DDMA_LEGACY | \ - COH901318_CX_CTRL_PRDD_SOURCE) -#define flags_memcpy_lli_last (COH901318_CX_CTRL_TC_ENABLE | \ - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | \ - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | \ - COH901318_CX_CTRL_SRC_ADDR_INC_ENABLE | \ - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | \ - COH901318_CX_CTRL_DST_ADDR_INC_ENABLE | \ - COH901318_CX_CTRL_MASTER_MODE_M1RW | \ - COH901318_CX_CTRL_TCP_DISABLE | \ - COH901318_CX_CTRL_TC_IRQ_ENABLE | \ - COH901318_CX_CTRL_HSP_DISABLE | \ - COH901318_CX_CTRL_HSS_DISABLE | \ - COH901318_CX_CTRL_DDMA_LEGACY | \ - COH901318_CX_CTRL_PRDD_SOURCE) - -static const struct coh_dma_channel chan_config[U300_DMA_CHANNELS] = { - { - .number = U300_DMA_MSL_TX_0, - .name = "MSL TX 0", - .priority_high = 0, - }, - { - .number = U300_DMA_MSL_TX_1, - .name = "MSL TX 1", - .priority_high = 0, - .param.config = COH901318_CX_CFG_CH_DISABLE | - COH901318_CX_CFG_LCR_DISABLE | - COH901318_CX_CFG_TC_IRQ_ENABLE | - COH901318_CX_CFG_BE_IRQ_ENABLE, - .param.ctrl_lli_chained = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_ENABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_DISABLE | - COH901318_CX_CTRL_MASTER_MODE_M1R_M2W | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY | - COH901318_CX_CTRL_PRDD_SOURCE, - .param.ctrl_lli = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_ENABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_DISABLE | - COH901318_CX_CTRL_MASTER_MODE_M1R_M2W | - COH901318_CX_CTRL_TCP_ENABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY | - COH901318_CX_CTRL_PRDD_SOURCE, - .param.ctrl_lli_last = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_ENABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_DISABLE | - COH901318_CX_CTRL_MASTER_MODE_M1R_M2W | - COH901318_CX_CTRL_TCP_ENABLE | - COH901318_CX_CTRL_TC_IRQ_ENABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY | - COH901318_CX_CTRL_PRDD_SOURCE, - }, - { - .number = U300_DMA_MSL_TX_2, - .name = "MSL TX 2", - .priority_high = 0, - .param.config = COH901318_CX_CFG_CH_DISABLE | - COH901318_CX_CFG_LCR_DISABLE | - COH901318_CX_CFG_TC_IRQ_ENABLE | - COH901318_CX_CFG_BE_IRQ_ENABLE, - .param.ctrl_lli_chained = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_ENABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_DISABLE | - COH901318_CX_CTRL_MASTER_MODE_M1R_M2W | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY | - COH901318_CX_CTRL_PRDD_SOURCE, - .param.ctrl_lli = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_ENABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_DISABLE | - COH901318_CX_CTRL_MASTER_MODE_M1R_M2W | - COH901318_CX_CTRL_TCP_ENABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY | - COH901318_CX_CTRL_PRDD_SOURCE, - .param.ctrl_lli_last = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_ENABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_DISABLE | - COH901318_CX_CTRL_MASTER_MODE_M1R_M2W | - COH901318_CX_CTRL_TCP_ENABLE | - COH901318_CX_CTRL_TC_IRQ_ENABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY | - COH901318_CX_CTRL_PRDD_SOURCE, - .desc_nbr_max = 10, - }, - { - .number = U300_DMA_MSL_TX_3, - .name = "MSL TX 3", - .priority_high = 0, - .param.config = COH901318_CX_CFG_CH_DISABLE | - COH901318_CX_CFG_LCR_DISABLE | - COH901318_CX_CFG_TC_IRQ_ENABLE | - COH901318_CX_CFG_BE_IRQ_ENABLE, - .param.ctrl_lli_chained = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_ENABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_DISABLE | - COH901318_CX_CTRL_MASTER_MODE_M1R_M2W | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY | - COH901318_CX_CTRL_PRDD_SOURCE, - .param.ctrl_lli = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_ENABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_DISABLE | - COH901318_CX_CTRL_MASTER_MODE_M1R_M2W | - COH901318_CX_CTRL_TCP_ENABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY | - COH901318_CX_CTRL_PRDD_SOURCE, - .param.ctrl_lli_last = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_ENABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_DISABLE | - COH901318_CX_CTRL_MASTER_MODE_M1R_M2W | - COH901318_CX_CTRL_TCP_ENABLE | - COH901318_CX_CTRL_TC_IRQ_ENABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY | - COH901318_CX_CTRL_PRDD_SOURCE, - }, - { - .number = U300_DMA_MSL_TX_4, - .name = "MSL TX 4", - .priority_high = 0, - .param.config = COH901318_CX_CFG_CH_DISABLE | - COH901318_CX_CFG_LCR_DISABLE | - COH901318_CX_CFG_TC_IRQ_ENABLE | - COH901318_CX_CFG_BE_IRQ_ENABLE, - .param.ctrl_lli_chained = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_ENABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_DISABLE | - COH901318_CX_CTRL_MASTER_MODE_M1R_M2W | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY | - COH901318_CX_CTRL_PRDD_SOURCE, - .param.ctrl_lli = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_ENABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_DISABLE | - COH901318_CX_CTRL_MASTER_MODE_M1R_M2W | - COH901318_CX_CTRL_TCP_ENABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY | - COH901318_CX_CTRL_PRDD_SOURCE, - .param.ctrl_lli_last = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_ENABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_DISABLE | - COH901318_CX_CTRL_MASTER_MODE_M1R_M2W | - COH901318_CX_CTRL_TCP_ENABLE | - COH901318_CX_CTRL_TC_IRQ_ENABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY | - COH901318_CX_CTRL_PRDD_SOURCE, - }, - { - .number = U300_DMA_MSL_TX_5, - .name = "MSL TX 5", - .priority_high = 0, - }, - { - .number = U300_DMA_MSL_TX_6, - .name = "MSL TX 6", - .priority_high = 0, - }, - { - .number = U300_DMA_MSL_RX_0, - .name = "MSL RX 0", - .priority_high = 0, - }, - { - .number = U300_DMA_MSL_RX_1, - .name = "MSL RX 1", - .priority_high = 0, - .param.config = COH901318_CX_CFG_CH_DISABLE | - COH901318_CX_CFG_LCR_DISABLE | - COH901318_CX_CFG_TC_IRQ_ENABLE | - COH901318_CX_CFG_BE_IRQ_ENABLE, - .param.ctrl_lli_chained = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_DISABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M2R_M1W | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_DEMAND_DMA1 | - COH901318_CX_CTRL_PRDD_DEST, - .param.ctrl_lli = 0, - .param.ctrl_lli_last = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_DISABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M2R_M1W | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_ENABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_DEMAND_DMA1 | - COH901318_CX_CTRL_PRDD_DEST, - }, - { - .number = U300_DMA_MSL_RX_2, - .name = "MSL RX 2", - .priority_high = 0, - .param.config = COH901318_CX_CFG_CH_DISABLE | - COH901318_CX_CFG_LCR_DISABLE | - COH901318_CX_CFG_TC_IRQ_ENABLE | - COH901318_CX_CFG_BE_IRQ_ENABLE, - .param.ctrl_lli_chained = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_DISABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M2R_M1W | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_DEMAND_DMA1 | - COH901318_CX_CTRL_PRDD_DEST, - .param.ctrl_lli = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_DISABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M2R_M1W | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_ENABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_DEMAND_DMA1 | - COH901318_CX_CTRL_PRDD_DEST, - .param.ctrl_lli_last = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_DISABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M2R_M1W | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_ENABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_DEMAND_DMA1 | - COH901318_CX_CTRL_PRDD_DEST, - }, - { - .number = U300_DMA_MSL_RX_3, - .name = "MSL RX 3", - .priority_high = 0, - .param.config = COH901318_CX_CFG_CH_DISABLE | - COH901318_CX_CFG_LCR_DISABLE | - COH901318_CX_CFG_TC_IRQ_ENABLE | - COH901318_CX_CFG_BE_IRQ_ENABLE, - .param.ctrl_lli_chained = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_DISABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M2R_M1W | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_DEMAND_DMA1 | - COH901318_CX_CTRL_PRDD_DEST, - .param.ctrl_lli = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_DISABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M2R_M1W | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_ENABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_DEMAND_DMA1 | - COH901318_CX_CTRL_PRDD_DEST, - .param.ctrl_lli_last = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_DISABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M2R_M1W | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_ENABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_DEMAND_DMA1 | - COH901318_CX_CTRL_PRDD_DEST, - }, - { - .number = U300_DMA_MSL_RX_4, - .name = "MSL RX 4", - .priority_high = 0, - .param.config = COH901318_CX_CFG_CH_DISABLE | - COH901318_CX_CFG_LCR_DISABLE | - COH901318_CX_CFG_TC_IRQ_ENABLE | - COH901318_CX_CFG_BE_IRQ_ENABLE, - .param.ctrl_lli_chained = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_DISABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M2R_M1W | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_DEMAND_DMA1 | - COH901318_CX_CTRL_PRDD_DEST, - .param.ctrl_lli = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_DISABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M2R_M1W | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_ENABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_DEMAND_DMA1 | - COH901318_CX_CTRL_PRDD_DEST, - .param.ctrl_lli_last = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_DISABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M2R_M1W | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_ENABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_DEMAND_DMA1 | - COH901318_CX_CTRL_PRDD_DEST, - }, - { - .number = U300_DMA_MSL_RX_5, - .name = "MSL RX 5", - .priority_high = 0, - .param.config = COH901318_CX_CFG_CH_DISABLE | - COH901318_CX_CFG_LCR_DISABLE | - COH901318_CX_CFG_TC_IRQ_ENABLE | - COH901318_CX_CFG_BE_IRQ_ENABLE, - .param.ctrl_lli_chained = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_DISABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M2R_M1W | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_DEMAND_DMA1 | - COH901318_CX_CTRL_PRDD_DEST, - .param.ctrl_lli = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_DISABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M2R_M1W | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_ENABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_DEMAND_DMA1 | - COH901318_CX_CTRL_PRDD_DEST, - .param.ctrl_lli_last = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_32_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_DISABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M2R_M1W | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_ENABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_DEMAND_DMA1 | - COH901318_CX_CTRL_PRDD_DEST, - }, - { - .number = U300_DMA_MSL_RX_6, - .name = "MSL RX 6", - .priority_high = 0, - }, - /* - * Don't set up device address, burst count or size of src - * or dst bus for this peripheral - handled by PrimeCell - * DMA extension. - */ - { - .number = U300_DMA_MMCSD_RX_TX, - .name = "MMCSD RX TX", - .priority_high = 0, - .param.config = COH901318_CX_CFG_CH_DISABLE | - COH901318_CX_CFG_LCR_DISABLE | - COH901318_CX_CFG_TC_IRQ_ENABLE | - COH901318_CX_CFG_BE_IRQ_ENABLE, - .param.ctrl_lli_chained = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M1RW | - COH901318_CX_CTRL_TCP_ENABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY, - .param.ctrl_lli = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M1RW | - COH901318_CX_CTRL_TCP_ENABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY, - .param.ctrl_lli_last = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M1RW | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_ENABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY, - - }, - { - .number = U300_DMA_MSPRO_TX, - .name = "MSPRO TX", - .priority_high = 0, - }, - { - .number = U300_DMA_MSPRO_RX, - .name = "MSPRO RX", - .priority_high = 0, - }, - /* - * Don't set up device address, burst count or size of src - * or dst bus for this peripheral - handled by PrimeCell - * DMA extension. - */ - { - .number = U300_DMA_UART0_TX, - .name = "UART0 TX", - .priority_high = 0, - .param.config = COH901318_CX_CFG_CH_DISABLE | - COH901318_CX_CFG_LCR_DISABLE | - COH901318_CX_CFG_TC_IRQ_ENABLE | - COH901318_CX_CFG_BE_IRQ_ENABLE, - .param.ctrl_lli_chained = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M1RW | - COH901318_CX_CTRL_TCP_ENABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY, - .param.ctrl_lli = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M1RW | - COH901318_CX_CTRL_TCP_ENABLE | - COH901318_CX_CTRL_TC_IRQ_ENABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY, - .param.ctrl_lli_last = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M1RW | - COH901318_CX_CTRL_TCP_ENABLE | - COH901318_CX_CTRL_TC_IRQ_ENABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY, - }, - { - .number = U300_DMA_UART0_RX, - .name = "UART0 RX", - .priority_high = 0, - .param.config = COH901318_CX_CFG_CH_DISABLE | - COH901318_CX_CFG_LCR_DISABLE | - COH901318_CX_CFG_TC_IRQ_ENABLE | - COH901318_CX_CFG_BE_IRQ_ENABLE, - .param.ctrl_lli_chained = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M1RW | - COH901318_CX_CTRL_TCP_ENABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY, - .param.ctrl_lli = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M1RW | - COH901318_CX_CTRL_TCP_ENABLE | - COH901318_CX_CTRL_TC_IRQ_ENABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY, - .param.ctrl_lli_last = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M1RW | - COH901318_CX_CTRL_TCP_ENABLE | - COH901318_CX_CTRL_TC_IRQ_ENABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY, - }, - { - .number = U300_DMA_APEX_TX, - .name = "APEX TX", - .priority_high = 0, - }, - { - .number = U300_DMA_APEX_RX, - .name = "APEX RX", - .priority_high = 0, - }, - { - .number = U300_DMA_PCM_I2S0_TX, - .name = "PCM I2S0 TX", - .priority_high = 1, - .param.config = COH901318_CX_CFG_CH_DISABLE | - COH901318_CX_CFG_LCR_DISABLE | - COH901318_CX_CFG_TC_IRQ_ENABLE | - COH901318_CX_CFG_BE_IRQ_ENABLE, - .param.ctrl_lli_chained = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_16_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_ENABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_DISABLE | - COH901318_CX_CTRL_MASTER_MODE_M1RW | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY | - COH901318_CX_CTRL_PRDD_SOURCE, - .param.ctrl_lli = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_16_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_ENABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_DISABLE | - COH901318_CX_CTRL_MASTER_MODE_M1RW | - COH901318_CX_CTRL_TCP_ENABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY | - COH901318_CX_CTRL_PRDD_SOURCE, - .param.ctrl_lli_last = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_16_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_ENABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_DISABLE | - COH901318_CX_CTRL_MASTER_MODE_M1RW | - COH901318_CX_CTRL_TCP_ENABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY | - COH901318_CX_CTRL_PRDD_SOURCE, - }, - { - .number = U300_DMA_PCM_I2S0_RX, - .name = "PCM I2S0 RX", - .priority_high = 1, - .param.config = COH901318_CX_CFG_CH_DISABLE | - COH901318_CX_CFG_LCR_DISABLE | - COH901318_CX_CFG_TC_IRQ_ENABLE | - COH901318_CX_CFG_BE_IRQ_ENABLE, - .param.ctrl_lli_chained = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_16_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_DISABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M1RW | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY | - COH901318_CX_CTRL_PRDD_DEST, - .param.ctrl_lli = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_16_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_DISABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M1RW | - COH901318_CX_CTRL_TCP_ENABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY | - COH901318_CX_CTRL_PRDD_DEST, - .param.ctrl_lli_last = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_16_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_DISABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M1RW | - COH901318_CX_CTRL_TCP_ENABLE | - COH901318_CX_CTRL_TC_IRQ_ENABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY | - COH901318_CX_CTRL_PRDD_DEST, - }, - { - .number = U300_DMA_PCM_I2S1_TX, - .name = "PCM I2S1 TX", - .priority_high = 1, - .param.config = COH901318_CX_CFG_CH_DISABLE | - COH901318_CX_CFG_LCR_DISABLE | - COH901318_CX_CFG_TC_IRQ_ENABLE | - COH901318_CX_CFG_BE_IRQ_ENABLE, - .param.ctrl_lli_chained = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_16_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_ENABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_DISABLE | - COH901318_CX_CTRL_MASTER_MODE_M1RW | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY | - COH901318_CX_CTRL_PRDD_SOURCE, - .param.ctrl_lli = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_16_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_ENABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_DISABLE | - COH901318_CX_CTRL_MASTER_MODE_M1RW | - COH901318_CX_CTRL_TCP_ENABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY | - COH901318_CX_CTRL_PRDD_SOURCE, - .param.ctrl_lli_last = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_16_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_ENABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_DISABLE | - COH901318_CX_CTRL_MASTER_MODE_M1RW | - COH901318_CX_CTRL_TCP_ENABLE | - COH901318_CX_CTRL_TC_IRQ_ENABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY | - COH901318_CX_CTRL_PRDD_SOURCE, - }, - { - .number = U300_DMA_PCM_I2S1_RX, - .name = "PCM I2S1 RX", - .priority_high = 1, - .param.config = COH901318_CX_CFG_CH_DISABLE | - COH901318_CX_CFG_LCR_DISABLE | - COH901318_CX_CFG_TC_IRQ_ENABLE | - COH901318_CX_CFG_BE_IRQ_ENABLE, - .param.ctrl_lli_chained = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_16_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_DISABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M1RW | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY | - COH901318_CX_CTRL_PRDD_DEST, - .param.ctrl_lli = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_16_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_DISABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M1RW | - COH901318_CX_CTRL_TCP_ENABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY | - COH901318_CX_CTRL_PRDD_DEST, - .param.ctrl_lli_last = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_BURST_COUNT_16_BYTES | - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_SRC_ADDR_INC_DISABLE | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_ADDR_INC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M1RW | - COH901318_CX_CTRL_TCP_ENABLE | - COH901318_CX_CTRL_TC_IRQ_ENABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY | - COH901318_CX_CTRL_PRDD_DEST, - }, - { - .number = U300_DMA_XGAM_CDI, - .name = "XGAM CDI", - .priority_high = 0, - }, - { - .number = U300_DMA_XGAM_PDI, - .name = "XGAM PDI", - .priority_high = 0, - }, - /* - * Don't set up device address, burst count or size of src - * or dst bus for this peripheral - handled by PrimeCell - * DMA extension. - */ - { - .number = U300_DMA_SPI_TX, - .name = "SPI TX", - .priority_high = 0, - .param.config = COH901318_CX_CFG_CH_DISABLE | - COH901318_CX_CFG_LCR_DISABLE | - COH901318_CX_CFG_TC_IRQ_ENABLE | - COH901318_CX_CFG_BE_IRQ_ENABLE, - .param.ctrl_lli_chained = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M1RW | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY, - .param.ctrl_lli = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M1RW | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_ENABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY, - .param.ctrl_lli_last = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M1RW | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_ENABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY, - }, - { - .number = U300_DMA_SPI_RX, - .name = "SPI RX", - .priority_high = 0, - .param.config = COH901318_CX_CFG_CH_DISABLE | - COH901318_CX_CFG_LCR_DISABLE | - COH901318_CX_CFG_TC_IRQ_ENABLE | - COH901318_CX_CFG_BE_IRQ_ENABLE, - .param.ctrl_lli_chained = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M1RW | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_DISABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY, - .param.ctrl_lli = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M1RW | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_ENABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY, - .param.ctrl_lli_last = 0 | - COH901318_CX_CTRL_TC_ENABLE | - COH901318_CX_CTRL_MASTER_MODE_M1RW | - COH901318_CX_CTRL_TCP_DISABLE | - COH901318_CX_CTRL_TC_IRQ_ENABLE | - COH901318_CX_CTRL_HSP_ENABLE | - COH901318_CX_CTRL_HSS_DISABLE | - COH901318_CX_CTRL_DDMA_LEGACY, - - }, - { - .number = U300_DMA_GENERAL_PURPOSE_0, - .name = "GENERAL 00", - .priority_high = 0, - - .param.config = flags_memcpy_config, - .param.ctrl_lli_chained = flags_memcpy_lli_chained, - .param.ctrl_lli = flags_memcpy_lli, - .param.ctrl_lli_last = flags_memcpy_lli_last, - }, - { - .number = U300_DMA_GENERAL_PURPOSE_1, - .name = "GENERAL 01", - .priority_high = 0, - - .param.config = flags_memcpy_config, - .param.ctrl_lli_chained = flags_memcpy_lli_chained, - .param.ctrl_lli = flags_memcpy_lli, - .param.ctrl_lli_last = flags_memcpy_lli_last, - }, - { - .number = U300_DMA_GENERAL_PURPOSE_2, - .name = "GENERAL 02", - .priority_high = 0, - - .param.config = flags_memcpy_config, - .param.ctrl_lli_chained = flags_memcpy_lli_chained, - .param.ctrl_lli = flags_memcpy_lli, - .param.ctrl_lli_last = flags_memcpy_lli_last, - }, - { - .number = U300_DMA_GENERAL_PURPOSE_3, - .name = "GENERAL 03", - .priority_high = 0, - - .param.config = flags_memcpy_config, - .param.ctrl_lli_chained = flags_memcpy_lli_chained, - .param.ctrl_lli = flags_memcpy_lli, - .param.ctrl_lli_last = flags_memcpy_lli_last, - }, - { - .number = U300_DMA_GENERAL_PURPOSE_4, - .name = "GENERAL 04", - .priority_high = 0, - - .param.config = flags_memcpy_config, - .param.ctrl_lli_chained = flags_memcpy_lli_chained, - .param.ctrl_lli = flags_memcpy_lli, - .param.ctrl_lli_last = flags_memcpy_lli_last, - }, - { - .number = U300_DMA_GENERAL_PURPOSE_5, - .name = "GENERAL 05", - .priority_high = 0, - - .param.config = flags_memcpy_config, - .param.ctrl_lli_chained = flags_memcpy_lli_chained, - .param.ctrl_lli = flags_memcpy_lli, - .param.ctrl_lli_last = flags_memcpy_lli_last, - }, - { - .number = U300_DMA_GENERAL_PURPOSE_6, - .name = "GENERAL 06", - .priority_high = 0, - - .param.config = flags_memcpy_config, - .param.ctrl_lli_chained = flags_memcpy_lli_chained, - .param.ctrl_lli = flags_memcpy_lli, - .param.ctrl_lli_last = flags_memcpy_lli_last, - }, - { - .number = U300_DMA_GENERAL_PURPOSE_7, - .name = "GENERAL 07", - .priority_high = 0, - - .param.config = flags_memcpy_config, - .param.ctrl_lli_chained = flags_memcpy_lli_chained, - .param.ctrl_lli = flags_memcpy_lli, - .param.ctrl_lli_last = flags_memcpy_lli_last, - }, - { - .number = U300_DMA_GENERAL_PURPOSE_8, - .name = "GENERAL 08", - .priority_high = 0, - - .param.config = flags_memcpy_config, - .param.ctrl_lli_chained = flags_memcpy_lli_chained, - .param.ctrl_lli = flags_memcpy_lli, - .param.ctrl_lli_last = flags_memcpy_lli_last, - }, - { - .number = U300_DMA_UART1_TX, - .name = "UART1 TX", - .priority_high = 0, - }, - { - .number = U300_DMA_UART1_RX, - .name = "UART1 RX", - .priority_high = 0, - } -}; - -#define COHC_2_DEV(cohc) (&cohc->chan.dev->device) - -#ifdef VERBOSE_DEBUG -#define COH_DBG(x) ({ if (1) x; 0; }) -#else -#define COH_DBG(x) ({ if (0) x; 0; }) -#endif - -struct coh901318_desc { - struct dma_async_tx_descriptor desc; - struct list_head node; - struct scatterlist *sg; - unsigned int sg_len; - struct coh901318_lli *lli; - enum dma_transfer_direction dir; - unsigned long flags; - u32 head_config; - u32 head_ctrl; -}; - -struct coh901318_base { - struct device *dev; - void __iomem *virtbase; - unsigned int irq; - struct coh901318_pool pool; - struct powersave pm; - struct dma_device dma_slave; - struct dma_device dma_memcpy; - struct coh901318_chan *chans; -}; - -struct coh901318_chan { - spinlock_t lock; - int allocated; - int id; - int stopped; - - struct work_struct free_work; - struct dma_chan chan; - - struct tasklet_struct tasklet; - - struct list_head active; - struct list_head queue; - struct list_head free; - - unsigned long nbr_active_done; - unsigned long busy; - - struct dma_slave_config config; - u32 addr; - u32 ctrl; - - struct coh901318_base *base; -}; - -static void coh901318_list_print(struct coh901318_chan *cohc, - struct coh901318_lli *lli) -{ - struct coh901318_lli *l = lli; - int i = 0; - - while (l) { - dev_vdbg(COHC_2_DEV(cohc), "i %d, lli %p, ctrl 0x%x, src %pad" - ", dst %pad, link %pad virt_link_addr 0x%p\n", - i, l, l->control, &l->src_addr, &l->dst_addr, - &l->link_addr, l->virt_link_addr); - i++; - l = l->virt_link_addr; - } -} - -#ifdef CONFIG_DEBUG_FS - -#define COH901318_DEBUGFS_ASSIGN(x, y) (x = y) - -static struct coh901318_base *debugfs_dma_base; -static struct dentry *dma_dentry; - -static ssize_t coh901318_debugfs_read(struct file *file, char __user *buf, - size_t count, loff_t *f_pos) -{ - u64 started_channels = debugfs_dma_base->pm.started_channels; - int pool_count = debugfs_dma_base->pool.debugfs_pool_counter; - char *dev_buf; - char *tmp; - int ret; - int i; - - dev_buf = kmalloc(4*1024, GFP_KERNEL); - if (dev_buf == NULL) - return -ENOMEM; - tmp = dev_buf; - - tmp += sprintf(tmp, "DMA -- enabled dma channels\n"); - - for (i = 0; i < U300_DMA_CHANNELS; i++) { - if (started_channels & (1ULL << i)) - tmp += sprintf(tmp, "channel %d\n", i); - } - - tmp += sprintf(tmp, "Pool alloc nbr %d\n", pool_count); - - ret = simple_read_from_buffer(buf, count, f_pos, dev_buf, - tmp - dev_buf); - kfree(dev_buf); - return ret; -} - -static const struct file_operations coh901318_debugfs_status_operations = { - .open = simple_open, - .read = coh901318_debugfs_read, - .llseek = default_llseek, -}; - - -static int __init init_coh901318_debugfs(void) -{ - - dma_dentry = debugfs_create_dir("dma", NULL); - - debugfs_create_file("status", S_IFREG | S_IRUGO, dma_dentry, NULL, - &coh901318_debugfs_status_operations); - return 0; -} - -static void __exit exit_coh901318_debugfs(void) -{ - debugfs_remove_recursive(dma_dentry); -} - -module_init(init_coh901318_debugfs); -module_exit(exit_coh901318_debugfs); -#else - -#define COH901318_DEBUGFS_ASSIGN(x, y) - -#endif /* CONFIG_DEBUG_FS */ - -static inline struct coh901318_chan *to_coh901318_chan(struct dma_chan *chan) -{ - return container_of(chan, struct coh901318_chan, chan); -} - -static int coh901318_dma_set_runtimeconfig(struct dma_chan *chan, - struct dma_slave_config *config, - enum dma_transfer_direction direction); - -static inline const struct coh901318_params * -cohc_chan_param(struct coh901318_chan *cohc) -{ - return &chan_config[cohc->id].param; -} - -static inline const struct coh_dma_channel * -cohc_chan_conf(struct coh901318_chan *cohc) -{ - return &chan_config[cohc->id]; -} - -static void enable_powersave(struct coh901318_chan *cohc) -{ - unsigned long flags; - struct powersave *pm = &cohc->base->pm; - - spin_lock_irqsave(&pm->lock, flags); - - pm->started_channels &= ~(1ULL << cohc->id); - - spin_unlock_irqrestore(&pm->lock, flags); -} -static void disable_powersave(struct coh901318_chan *cohc) -{ - unsigned long flags; - struct powersave *pm = &cohc->base->pm; - - spin_lock_irqsave(&pm->lock, flags); - - pm->started_channels |= (1ULL << cohc->id); - - spin_unlock_irqrestore(&pm->lock, flags); -} - -static inline int coh901318_set_ctrl(struct coh901318_chan *cohc, u32 control) -{ - int channel = cohc->id; - void __iomem *virtbase = cohc->base->virtbase; - - writel(control, - virtbase + COH901318_CX_CTRL + - COH901318_CX_CTRL_SPACING * channel); - return 0; -} - -static inline int coh901318_set_conf(struct coh901318_chan *cohc, u32 conf) -{ - int channel = cohc->id; - void __iomem *virtbase = cohc->base->virtbase; - - writel(conf, - virtbase + COH901318_CX_CFG + - COH901318_CX_CFG_SPACING*channel); - return 0; -} - - -static int coh901318_start(struct coh901318_chan *cohc) -{ - u32 val; - int channel = cohc->id; - void __iomem *virtbase = cohc->base->virtbase; - - disable_powersave(cohc); - - val = readl(virtbase + COH901318_CX_CFG + - COH901318_CX_CFG_SPACING * channel); - - /* Enable channel */ - val |= COH901318_CX_CFG_CH_ENABLE; - writel(val, virtbase + COH901318_CX_CFG + - COH901318_CX_CFG_SPACING * channel); - - return 0; -} - -static int coh901318_prep_linked_list(struct coh901318_chan *cohc, - struct coh901318_lli *lli) -{ - int channel = cohc->id; - void __iomem *virtbase = cohc->base->virtbase; - - BUG_ON(readl(virtbase + COH901318_CX_STAT + - COH901318_CX_STAT_SPACING*channel) & - COH901318_CX_STAT_ACTIVE); - - writel(lli->src_addr, - virtbase + COH901318_CX_SRC_ADDR + - COH901318_CX_SRC_ADDR_SPACING * channel); - - writel(lli->dst_addr, virtbase + - COH901318_CX_DST_ADDR + - COH901318_CX_DST_ADDR_SPACING * channel); - - writel(lli->link_addr, virtbase + COH901318_CX_LNK_ADDR + - COH901318_CX_LNK_ADDR_SPACING * channel); - - writel(lli->control, virtbase + COH901318_CX_CTRL + - COH901318_CX_CTRL_SPACING * channel); - - return 0; -} - -static struct coh901318_desc * -coh901318_desc_get(struct coh901318_chan *cohc) -{ - struct coh901318_desc *desc; - - if (list_empty(&cohc->free)) { - /* alloc new desc because we're out of used ones - * TODO: alloc a pile of descs instead of just one, - * avoid many small allocations. - */ - desc = kzalloc(sizeof(struct coh901318_desc), GFP_NOWAIT); - if (desc == NULL) - goto out; - INIT_LIST_HEAD(&desc->node); - dma_async_tx_descriptor_init(&desc->desc, &cohc->chan); - } else { - /* Reuse an old desc. */ - desc = list_first_entry(&cohc->free, - struct coh901318_desc, - node); - list_del(&desc->node); - /* Initialize it a bit so it's not insane */ - desc->sg = NULL; - desc->sg_len = 0; - desc->desc.callback = NULL; - desc->desc.callback_param = NULL; - } - - out: - return desc; -} - -static void -coh901318_desc_free(struct coh901318_chan *cohc, struct coh901318_desc *cohd) -{ - list_add_tail(&cohd->node, &cohc->free); -} - -/* call with irq lock held */ -static void -coh901318_desc_submit(struct coh901318_chan *cohc, struct coh901318_desc *desc) -{ - list_add_tail(&desc->node, &cohc->active); -} - -static struct coh901318_desc * -coh901318_first_active_get(struct coh901318_chan *cohc) -{ - return list_first_entry_or_null(&cohc->active, struct coh901318_desc, - node); -} - -static void -coh901318_desc_remove(struct coh901318_desc *cohd) -{ - list_del(&cohd->node); -} - -static void -coh901318_desc_queue(struct coh901318_chan *cohc, struct coh901318_desc *desc) -{ - list_add_tail(&desc->node, &cohc->queue); -} - -static struct coh901318_desc * -coh901318_first_queued(struct coh901318_chan *cohc) -{ - return list_first_entry_or_null(&cohc->queue, struct coh901318_desc, - node); -} - -static inline u32 coh901318_get_bytes_in_lli(struct coh901318_lli *in_lli) -{ - struct coh901318_lli *lli = in_lli; - u32 bytes = 0; - - while (lli) { - bytes += lli->control & COH901318_CX_CTRL_TC_VALUE_MASK; - lli = lli->virt_link_addr; - } - return bytes; -} - -/* - * Get the number of bytes left to transfer on this channel, - * it is unwise to call this before stopping the channel for - * absolute measures, but for a rough guess you can still call - * it. - */ -static u32 coh901318_get_bytes_left(struct dma_chan *chan) -{ - struct coh901318_chan *cohc = to_coh901318_chan(chan); - struct coh901318_desc *cohd; - struct list_head *pos; - unsigned long flags; - u32 left = 0; - int i = 0; - - spin_lock_irqsave(&cohc->lock, flags); - - /* - * If there are many queued jobs, we iterate and add the - * size of them all. We take a special look on the first - * job though, since it is probably active. - */ - list_for_each(pos, &cohc->active) { - /* - * The first job in the list will be working on the - * hardware. The job can be stopped but still active, - * so that the transfer counter is somewhere inside - * the buffer. - */ - cohd = list_entry(pos, struct coh901318_desc, node); - - if (i == 0) { - struct coh901318_lli *lli; - dma_addr_t ladd; - - /* Read current transfer count value */ - left = readl(cohc->base->virtbase + - COH901318_CX_CTRL + - COH901318_CX_CTRL_SPACING * cohc->id) & - COH901318_CX_CTRL_TC_VALUE_MASK; - - /* See if the transfer is linked... */ - ladd = readl(cohc->base->virtbase + - COH901318_CX_LNK_ADDR + - COH901318_CX_LNK_ADDR_SPACING * - cohc->id) & - ~COH901318_CX_LNK_LINK_IMMEDIATE; - /* Single transaction */ - if (!ladd) - continue; - - /* - * Linked transaction, follow the lli, find the - * currently processing lli, and proceed to the next - */ - lli = cohd->lli; - while (lli && lli->link_addr != ladd) - lli = lli->virt_link_addr; - - if (lli) - lli = lli->virt_link_addr; - - /* - * Follow remaining lli links around to count the total - * number of bytes left - */ - left += coh901318_get_bytes_in_lli(lli); - } else { - left += coh901318_get_bytes_in_lli(cohd->lli); - } - i++; - } - - /* Also count bytes in the queued jobs */ - list_for_each(pos, &cohc->queue) { - cohd = list_entry(pos, struct coh901318_desc, node); - left += coh901318_get_bytes_in_lli(cohd->lli); - } - - spin_unlock_irqrestore(&cohc->lock, flags); - - return left; -} - -/* - * Pauses a transfer without losing data. Enables power save. - * Use this function in conjunction with coh901318_resume. - */ -static int coh901318_pause(struct dma_chan *chan) -{ - u32 val; - unsigned long flags; - struct coh901318_chan *cohc = to_coh901318_chan(chan); - int channel = cohc->id; - void __iomem *virtbase = cohc->base->virtbase; - - spin_lock_irqsave(&cohc->lock, flags); - - /* Disable channel in HW */ - val = readl(virtbase + COH901318_CX_CFG + - COH901318_CX_CFG_SPACING * channel); - - /* Stopping infinite transfer */ - if ((val & COH901318_CX_CTRL_TC_ENABLE) == 0 && - (val & COH901318_CX_CFG_CH_ENABLE)) - cohc->stopped = 1; - - - val &= ~COH901318_CX_CFG_CH_ENABLE; - /* Enable twice, HW bug work around */ - writel(val, virtbase + COH901318_CX_CFG + - COH901318_CX_CFG_SPACING * channel); - writel(val, virtbase + COH901318_CX_CFG + - COH901318_CX_CFG_SPACING * channel); - - /* Spin-wait for it to actually go inactive */ - while (readl(virtbase + COH901318_CX_STAT+COH901318_CX_STAT_SPACING * - channel) & COH901318_CX_STAT_ACTIVE) - cpu_relax(); - - /* Check if we stopped an active job */ - if ((readl(virtbase + COH901318_CX_CTRL+COH901318_CX_CTRL_SPACING * - channel) & COH901318_CX_CTRL_TC_VALUE_MASK) > 0) - cohc->stopped = 1; - - enable_powersave(cohc); - - spin_unlock_irqrestore(&cohc->lock, flags); - return 0; -} - -/* Resumes a transfer that has been stopped via 300_dma_stop(..). - Power save is handled. -*/ -static int coh901318_resume(struct dma_chan *chan) -{ - u32 val; - unsigned long flags; - struct coh901318_chan *cohc = to_coh901318_chan(chan); - int channel = cohc->id; - - spin_lock_irqsave(&cohc->lock, flags); - - disable_powersave(cohc); - - if (cohc->stopped) { - /* Enable channel in HW */ - val = readl(cohc->base->virtbase + COH901318_CX_CFG + - COH901318_CX_CFG_SPACING * channel); - - val |= COH901318_CX_CFG_CH_ENABLE; - - writel(val, cohc->base->virtbase + COH901318_CX_CFG + - COH901318_CX_CFG_SPACING*channel); - - cohc->stopped = 0; - } - - spin_unlock_irqrestore(&cohc->lock, flags); - return 0; -} - -bool coh901318_filter_id(struct dma_chan *chan, void *chan_id) -{ - unsigned long ch_nr = (unsigned long) chan_id; - - if (ch_nr == to_coh901318_chan(chan)->id) - return true; - - return false; -} -EXPORT_SYMBOL(coh901318_filter_id); - -struct coh901318_filter_args { - struct coh901318_base *base; - unsigned int ch_nr; -}; - -static bool coh901318_filter_base_and_id(struct dma_chan *chan, void *data) -{ - struct coh901318_filter_args *args = data; - - if (&args->base->dma_slave == chan->device && - args->ch_nr == to_coh901318_chan(chan)->id) - return true; - - return false; -} - -static struct dma_chan *coh901318_xlate(struct of_phandle_args *dma_spec, - struct of_dma *ofdma) -{ - struct coh901318_filter_args args = { - .base = ofdma->of_dma_data, - .ch_nr = dma_spec->args[0], - }; - dma_cap_mask_t cap; - dma_cap_zero(cap); - dma_cap_set(DMA_SLAVE, cap); - - return dma_request_channel(cap, coh901318_filter_base_and_id, &args); -} -/* - * DMA channel allocation - */ -static int coh901318_config(struct coh901318_chan *cohc, - struct coh901318_params *param) -{ - const struct coh901318_params *p; - int channel = cohc->id; - void __iomem *virtbase = cohc->base->virtbase; - - if (param) - p = param; - else - p = cohc_chan_param(cohc); - - /* Clear any pending BE or TC interrupt */ - if (channel < 32) { - writel(1 << channel, virtbase + COH901318_BE_INT_CLEAR1); - writel(1 << channel, virtbase + COH901318_TC_INT_CLEAR1); - } else { - writel(1 << (channel - 32), virtbase + - COH901318_BE_INT_CLEAR2); - writel(1 << (channel - 32), virtbase + - COH901318_TC_INT_CLEAR2); - } - - coh901318_set_conf(cohc, p->config); - coh901318_set_ctrl(cohc, p->ctrl_lli_last); - - return 0; -} - -/* must lock when calling this function - * start queued jobs, if any - * TODO: start all queued jobs in one go - * - * Returns descriptor if queued job is started otherwise NULL. - * If the queue is empty NULL is returned. - */ -static struct coh901318_desc *coh901318_queue_start(struct coh901318_chan *cohc) -{ - struct coh901318_desc *cohd; - - /* - * start queued jobs, if any - * TODO: transmit all queued jobs in one go - */ - cohd = coh901318_first_queued(cohc); - - if (cohd != NULL) { - /* Remove from queue */ - coh901318_desc_remove(cohd); - /* initiate DMA job */ - cohc->busy = 1; - - coh901318_desc_submit(cohc, cohd); - - /* Program the transaction head */ - coh901318_set_conf(cohc, cohd->head_config); - coh901318_set_ctrl(cohc, cohd->head_ctrl); - coh901318_prep_linked_list(cohc, cohd->lli); - - /* start dma job on this channel */ - coh901318_start(cohc); - - } - - return cohd; -} - -/* - * This tasklet is called from the interrupt handler to - * handle each descriptor (DMA job) that is sent to a channel. - */ -static void dma_tasklet(struct tasklet_struct *t) -{ - struct coh901318_chan *cohc = from_tasklet(cohc, t, tasklet); - struct coh901318_desc *cohd_fin; - unsigned long flags; - struct dmaengine_desc_callback cb; - - dev_vdbg(COHC_2_DEV(cohc), "[%s] chan_id %d" - " nbr_active_done %ld\n", __func__, - cohc->id, cohc->nbr_active_done); - - spin_lock_irqsave(&cohc->lock, flags); - - /* get first active descriptor entry from list */ - cohd_fin = coh901318_first_active_get(cohc); - - if (cohd_fin == NULL) - goto err; - - /* locate callback to client */ - dmaengine_desc_get_callback(&cohd_fin->desc, &cb); - - /* sign this job as completed on the channel */ - dma_cookie_complete(&cohd_fin->desc); - - /* release the lli allocation and remove the descriptor */ - coh901318_lli_free(&cohc->base->pool, &cohd_fin->lli); - - /* return desc to free-list */ - coh901318_desc_remove(cohd_fin); - coh901318_desc_free(cohc, cohd_fin); - - spin_unlock_irqrestore(&cohc->lock, flags); - - /* Call the callback when we're done */ - dmaengine_desc_callback_invoke(&cb, NULL); - - spin_lock_irqsave(&cohc->lock, flags); - - /* - * If another interrupt fired while the tasklet was scheduling, - * we don't get called twice, so we have this number of active - * counter that keep track of the number of IRQs expected to - * be handled for this channel. If there happen to be more than - * one IRQ to be ack:ed, we simply schedule this tasklet again. - */ - cohc->nbr_active_done--; - if (cohc->nbr_active_done) { - dev_dbg(COHC_2_DEV(cohc), "scheduling tasklet again, new IRQs " - "came in while we were scheduling this tasklet\n"); - if (cohc_chan_conf(cohc)->priority_high) - tasklet_hi_schedule(&cohc->tasklet); - else - tasklet_schedule(&cohc->tasklet); - } - - spin_unlock_irqrestore(&cohc->lock, flags); - - return; - - err: - spin_unlock_irqrestore(&cohc->lock, flags); - dev_err(COHC_2_DEV(cohc), "[%s] No active dma desc\n", __func__); -} - - -/* called from interrupt context */ -static void dma_tc_handle(struct coh901318_chan *cohc) -{ - /* - * If the channel is not allocated, then we shouldn't have - * any TC interrupts on it. - */ - if (!cohc->allocated) { - dev_err(COHC_2_DEV(cohc), "spurious interrupt from " - "unallocated channel\n"); - return; - } - - /* - * When we reach this point, at least one queue item - * should have been moved over from cohc->queue to - * cohc->active and run to completion, that is why we're - * getting a terminal count interrupt is it not? - * If you get this BUG() the most probable cause is that - * the individual nodes in the lli chain have IRQ enabled, - * so check your platform config for lli chain ctrl. - */ - BUG_ON(list_empty(&cohc->active)); - - cohc->nbr_active_done++; - - /* - * This attempt to take a job from cohc->queue, put it - * into cohc->active and start it. - */ - if (coh901318_queue_start(cohc) == NULL) - cohc->busy = 0; - - /* - * This tasklet will remove items from cohc->active - * and thus terminates them. - */ - if (cohc_chan_conf(cohc)->priority_high) - tasklet_hi_schedule(&cohc->tasklet); - else - tasklet_schedule(&cohc->tasklet); -} - - -static irqreturn_t dma_irq_handler(int irq, void *dev_id) -{ - u32 status1; - u32 status2; - int i; - int ch; - struct coh901318_base *base = dev_id; - struct coh901318_chan *cohc; - void __iomem *virtbase = base->virtbase; - - status1 = readl(virtbase + COH901318_INT_STATUS1); - status2 = readl(virtbase + COH901318_INT_STATUS2); - - if (unlikely(status1 == 0 && status2 == 0)) { - dev_warn(base->dev, "spurious DMA IRQ from no channel!\n"); - return IRQ_HANDLED; - } - - /* TODO: consider handle IRQ in tasklet here to - * minimize interrupt latency */ - - /* Check the first 32 DMA channels for IRQ */ - while (status1) { - /* Find first bit set, return as a number. */ - i = ffs(status1) - 1; - ch = i; - - cohc = &base->chans[ch]; - spin_lock(&cohc->lock); - - /* Mask off this bit */ - status1 &= ~(1 << i); - /* Check the individual channel bits */ - if (test_bit(i, virtbase + COH901318_BE_INT_STATUS1)) { - dev_crit(COHC_2_DEV(cohc), - "DMA bus error on channel %d!\n", ch); - BUG_ON(1); - /* Clear BE interrupt */ - __set_bit(i, virtbase + COH901318_BE_INT_CLEAR1); - } else { - /* Caused by TC, really? */ - if (unlikely(!test_bit(i, virtbase + - COH901318_TC_INT_STATUS1))) { - dev_warn(COHC_2_DEV(cohc), - "ignoring interrupt not caused by terminal count on channel %d\n", ch); - /* Clear TC interrupt */ - BUG_ON(1); - __set_bit(i, virtbase + COH901318_TC_INT_CLEAR1); - } else { - /* Enable powersave if transfer has finished */ - if (!(readl(virtbase + COH901318_CX_STAT + - COH901318_CX_STAT_SPACING*ch) & - COH901318_CX_STAT_ENABLED)) { - enable_powersave(cohc); - } - - /* Must clear TC interrupt before calling - * dma_tc_handle - * in case tc_handle initiate a new dma job - */ - __set_bit(i, virtbase + COH901318_TC_INT_CLEAR1); - - dma_tc_handle(cohc); - } - } - spin_unlock(&cohc->lock); - } - - /* Check the remaining 32 DMA channels for IRQ */ - while (status2) { - /* Find first bit set, return as a number. */ - i = ffs(status2) - 1; - ch = i + 32; - cohc = &base->chans[ch]; - spin_lock(&cohc->lock); - - /* Mask off this bit */ - status2 &= ~(1 << i); - /* Check the individual channel bits */ - if (test_bit(i, virtbase + COH901318_BE_INT_STATUS2)) { - dev_crit(COHC_2_DEV(cohc), - "DMA bus error on channel %d!\n", ch); - /* Clear BE interrupt */ - BUG_ON(1); - __set_bit(i, virtbase + COH901318_BE_INT_CLEAR2); - } else { - /* Caused by TC, really? */ - if (unlikely(!test_bit(i, virtbase + - COH901318_TC_INT_STATUS2))) { - dev_warn(COHC_2_DEV(cohc), - "ignoring interrupt not caused by terminal count on channel %d\n", ch); - /* Clear TC interrupt */ - __set_bit(i, virtbase + COH901318_TC_INT_CLEAR2); - BUG_ON(1); - } else { - /* Enable powersave if transfer has finished */ - if (!(readl(virtbase + COH901318_CX_STAT + - COH901318_CX_STAT_SPACING*ch) & - COH901318_CX_STAT_ENABLED)) { - enable_powersave(cohc); - } - /* Must clear TC interrupt before calling - * dma_tc_handle - * in case tc_handle initiate a new dma job - */ - __set_bit(i, virtbase + COH901318_TC_INT_CLEAR2); - - dma_tc_handle(cohc); - } - } - spin_unlock(&cohc->lock); - } - - return IRQ_HANDLED; -} - -static int coh901318_terminate_all(struct dma_chan *chan) -{ - unsigned long flags; - struct coh901318_chan *cohc = to_coh901318_chan(chan); - struct coh901318_desc *cohd; - void __iomem *virtbase = cohc->base->virtbase; - - /* The remainder of this function terminates the transfer */ - coh901318_pause(chan); - spin_lock_irqsave(&cohc->lock, flags); - - /* Clear any pending BE or TC interrupt */ - if (cohc->id < 32) { - writel(1 << cohc->id, virtbase + COH901318_BE_INT_CLEAR1); - writel(1 << cohc->id, virtbase + COH901318_TC_INT_CLEAR1); - } else { - writel(1 << (cohc->id - 32), virtbase + - COH901318_BE_INT_CLEAR2); - writel(1 << (cohc->id - 32), virtbase + - COH901318_TC_INT_CLEAR2); - } - - enable_powersave(cohc); - - while ((cohd = coh901318_first_active_get(cohc))) { - /* release the lli allocation*/ - coh901318_lli_free(&cohc->base->pool, &cohd->lli); - - /* return desc to free-list */ - coh901318_desc_remove(cohd); - coh901318_desc_free(cohc, cohd); - } - - while ((cohd = coh901318_first_queued(cohc))) { - /* release the lli allocation*/ - coh901318_lli_free(&cohc->base->pool, &cohd->lli); - - /* return desc to free-list */ - coh901318_desc_remove(cohd); - coh901318_desc_free(cohc, cohd); - } - - - cohc->nbr_active_done = 0; - cohc->busy = 0; - - spin_unlock_irqrestore(&cohc->lock, flags); - - return 0; -} - -static int coh901318_alloc_chan_resources(struct dma_chan *chan) -{ - struct coh901318_chan *cohc = to_coh901318_chan(chan); - unsigned long flags; - - dev_vdbg(COHC_2_DEV(cohc), "[%s] DMA channel %d\n", - __func__, cohc->id); - - if (chan->client_count > 1) - return -EBUSY; - - spin_lock_irqsave(&cohc->lock, flags); - - coh901318_config(cohc, NULL); - - cohc->allocated = 1; - dma_cookie_init(chan); - - spin_unlock_irqrestore(&cohc->lock, flags); - - return 1; -} - -static void -coh901318_free_chan_resources(struct dma_chan *chan) -{ - struct coh901318_chan *cohc = to_coh901318_chan(chan); - int channel = cohc->id; - unsigned long flags; - - spin_lock_irqsave(&cohc->lock, flags); - - /* Disable HW */ - writel(0x00000000U, cohc->base->virtbase + COH901318_CX_CFG + - COH901318_CX_CFG_SPACING*channel); - writel(0x00000000U, cohc->base->virtbase + COH901318_CX_CTRL + - COH901318_CX_CTRL_SPACING*channel); - - cohc->allocated = 0; - - spin_unlock_irqrestore(&cohc->lock, flags); - - coh901318_terminate_all(chan); -} - - -static dma_cookie_t -coh901318_tx_submit(struct dma_async_tx_descriptor *tx) -{ - struct coh901318_desc *cohd = container_of(tx, struct coh901318_desc, - desc); - struct coh901318_chan *cohc = to_coh901318_chan(tx->chan); - unsigned long flags; - dma_cookie_t cookie; - - spin_lock_irqsave(&cohc->lock, flags); - cookie = dma_cookie_assign(tx); - - coh901318_desc_queue(cohc, cohd); - - spin_unlock_irqrestore(&cohc->lock, flags); - - return cookie; -} - -static struct dma_async_tx_descriptor * -coh901318_prep_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, - size_t size, unsigned long flags) -{ - struct coh901318_lli *lli; - struct coh901318_desc *cohd; - unsigned long flg; - struct coh901318_chan *cohc = to_coh901318_chan(chan); - int lli_len; - u32 ctrl_last = cohc_chan_param(cohc)->ctrl_lli_last; - int ret; - - spin_lock_irqsave(&cohc->lock, flg); - - dev_vdbg(COHC_2_DEV(cohc), - "[%s] channel %d src %pad dest %pad size %zu\n", - __func__, cohc->id, &src, &dest, size); - - if (flags & DMA_PREP_INTERRUPT) - /* Trigger interrupt after last lli */ - ctrl_last |= COH901318_CX_CTRL_TC_IRQ_ENABLE; - - lli_len = size >> MAX_DMA_PACKET_SIZE_SHIFT; - if ((lli_len << MAX_DMA_PACKET_SIZE_SHIFT) < size) - lli_len++; - - lli = coh901318_lli_alloc(&cohc->base->pool, lli_len); - - if (lli == NULL) - goto err; - - ret = coh901318_lli_fill_memcpy( - &cohc->base->pool, lli, src, size, dest, - cohc_chan_param(cohc)->ctrl_lli_chained, - ctrl_last); - if (ret) - goto err; - - COH_DBG(coh901318_list_print(cohc, lli)); - - /* Pick a descriptor to handle this transfer */ - cohd = coh901318_desc_get(cohc); - cohd->lli = lli; - cohd->flags = flags; - cohd->desc.tx_submit = coh901318_tx_submit; - - spin_unlock_irqrestore(&cohc->lock, flg); - - return &cohd->desc; - err: - spin_unlock_irqrestore(&cohc->lock, flg); - return NULL; -} - -static struct dma_async_tx_descriptor * -coh901318_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, - unsigned int sg_len, enum dma_transfer_direction direction, - unsigned long flags, void *context) -{ - struct coh901318_chan *cohc = to_coh901318_chan(chan); - struct coh901318_lli *lli; - struct coh901318_desc *cohd; - const struct coh901318_params *params; - struct scatterlist *sg; - int len = 0; - int size; - int i; - u32 ctrl_chained = cohc_chan_param(cohc)->ctrl_lli_chained; - u32 ctrl = cohc_chan_param(cohc)->ctrl_lli; - u32 ctrl_last = cohc_chan_param(cohc)->ctrl_lli_last; - u32 config; - unsigned long flg; - int ret; - - if (!sgl) - goto out; - if (sg_dma_len(sgl) == 0) - goto out; - - spin_lock_irqsave(&cohc->lock, flg); - - dev_vdbg(COHC_2_DEV(cohc), "[%s] sg_len %d dir %d\n", - __func__, sg_len, direction); - - if (flags & DMA_PREP_INTERRUPT) - /* Trigger interrupt after last lli */ - ctrl_last |= COH901318_CX_CTRL_TC_IRQ_ENABLE; - - params = cohc_chan_param(cohc); - config = params->config; - /* - * Add runtime-specific control on top, make - * sure the bits you set per peripheral channel are - * cleared in the default config from the platform. - */ - ctrl_chained |= cohc->ctrl; - ctrl_last |= cohc->ctrl; - ctrl |= cohc->ctrl; - - if (direction == DMA_MEM_TO_DEV) { - u32 tx_flags = COH901318_CX_CTRL_PRDD_SOURCE | - COH901318_CX_CTRL_SRC_ADDR_INC_ENABLE; - - config |= COH901318_CX_CFG_RM_MEMORY_TO_PRIMARY; - ctrl_chained |= tx_flags; - ctrl_last |= tx_flags; - ctrl |= tx_flags; - } else if (direction == DMA_DEV_TO_MEM) { - u32 rx_flags = COH901318_CX_CTRL_PRDD_DEST | - COH901318_CX_CTRL_DST_ADDR_INC_ENABLE; - - config |= COH901318_CX_CFG_RM_PRIMARY_TO_MEMORY; - ctrl_chained |= rx_flags; - ctrl_last |= rx_flags; - ctrl |= rx_flags; - } else - goto err_direction; - - /* The dma only supports transmitting packages up to - * MAX_DMA_PACKET_SIZE. Calculate to total number of - * dma elemts required to send the entire sg list - */ - for_each_sg(sgl, sg, sg_len, i) { - unsigned int factor; - size = sg_dma_len(sg); - - if (size <= MAX_DMA_PACKET_SIZE) { - len++; - continue; - } - - factor = size >> MAX_DMA_PACKET_SIZE_SHIFT; - if ((factor << MAX_DMA_PACKET_SIZE_SHIFT) < size) - factor++; - - len += factor; - } - - pr_debug("Allocate %d lli:s for this transfer\n", len); - lli = coh901318_lli_alloc(&cohc->base->pool, len); - - if (lli == NULL) - goto err_dma_alloc; - - coh901318_dma_set_runtimeconfig(chan, &cohc->config, direction); - - /* initiate allocated lli list */ - ret = coh901318_lli_fill_sg(&cohc->base->pool, lli, sgl, sg_len, - cohc->addr, - ctrl_chained, - ctrl, - ctrl_last, - direction, COH901318_CX_CTRL_TC_IRQ_ENABLE); - if (ret) - goto err_lli_fill; - - - COH_DBG(coh901318_list_print(cohc, lli)); - - /* Pick a descriptor to handle this transfer */ - cohd = coh901318_desc_get(cohc); - cohd->head_config = config; - /* - * Set the default head ctrl for the channel to the one from the - * lli, things may have changed due to odd buffer alignment - * etc. - */ - cohd->head_ctrl = lli->control; - cohd->dir = direction; - cohd->flags = flags; - cohd->desc.tx_submit = coh901318_tx_submit; - cohd->lli = lli; - - spin_unlock_irqrestore(&cohc->lock, flg); - - return &cohd->desc; - err_lli_fill: - err_dma_alloc: - err_direction: - spin_unlock_irqrestore(&cohc->lock, flg); - out: - return NULL; -} - -static enum dma_status -coh901318_tx_status(struct dma_chan *chan, dma_cookie_t cookie, - struct dma_tx_state *txstate) -{ - struct coh901318_chan *cohc = to_coh901318_chan(chan); - enum dma_status ret; - - ret = dma_cookie_status(chan, cookie, txstate); - if (ret == DMA_COMPLETE || !txstate) - return ret; - - dma_set_residue(txstate, coh901318_get_bytes_left(chan)); - - if (ret == DMA_IN_PROGRESS && cohc->stopped) - ret = DMA_PAUSED; - - return ret; -} - -static void -coh901318_issue_pending(struct dma_chan *chan) -{ - struct coh901318_chan *cohc = to_coh901318_chan(chan); - unsigned long flags; - - spin_lock_irqsave(&cohc->lock, flags); - - /* - * Busy means that pending jobs are already being processed, - * and then there is no point in starting the queue: the - * terminal count interrupt on the channel will take the next - * job on the queue and execute it anyway. - */ - if (!cohc->busy) - coh901318_queue_start(cohc); - - spin_unlock_irqrestore(&cohc->lock, flags); -} - -/* - * Here we wrap in the runtime dma control interface - */ -struct burst_table { - int burst_8bit; - int burst_16bit; - int burst_32bit; - u32 reg; -}; - -static const struct burst_table burst_sizes[] = { - { - .burst_8bit = 64, - .burst_16bit = 32, - .burst_32bit = 16, - .reg = COH901318_CX_CTRL_BURST_COUNT_64_BYTES, - }, - { - .burst_8bit = 48, - .burst_16bit = 24, - .burst_32bit = 12, - .reg = COH901318_CX_CTRL_BURST_COUNT_48_BYTES, - }, - { - .burst_8bit = 32, - .burst_16bit = 16, - .burst_32bit = 8, - .reg = COH901318_CX_CTRL_BURST_COUNT_32_BYTES, - }, - { - .burst_8bit = 16, - .burst_16bit = 8, - .burst_32bit = 4, - .reg = COH901318_CX_CTRL_BURST_COUNT_16_BYTES, - }, - { - .burst_8bit = 8, - .burst_16bit = 4, - .burst_32bit = 2, - .reg = COH901318_CX_CTRL_BURST_COUNT_8_BYTES, - }, - { - .burst_8bit = 4, - .burst_16bit = 2, - .burst_32bit = 1, - .reg = COH901318_CX_CTRL_BURST_COUNT_4_BYTES, - }, - { - .burst_8bit = 2, - .burst_16bit = 1, - .burst_32bit = 0, - .reg = COH901318_CX_CTRL_BURST_COUNT_2_BYTES, - }, - { - .burst_8bit = 1, - .burst_16bit = 0, - .burst_32bit = 0, - .reg = COH901318_CX_CTRL_BURST_COUNT_1_BYTE, - }, -}; - -static int coh901318_dma_set_runtimeconfig(struct dma_chan *chan, - struct dma_slave_config *config, - enum dma_transfer_direction direction) -{ - struct coh901318_chan *cohc = to_coh901318_chan(chan); - dma_addr_t addr; - enum dma_slave_buswidth addr_width; - u32 maxburst; - u32 ctrl = 0; - int i = 0; - - /* We only support mem to per or per to mem transfers */ - if (direction == DMA_DEV_TO_MEM) { - addr = config->src_addr; - addr_width = config->src_addr_width; - maxburst = config->src_maxburst; - } else if (direction == DMA_MEM_TO_DEV) { - addr = config->dst_addr; - addr_width = config->dst_addr_width; - maxburst = config->dst_maxburst; - } else { - dev_err(COHC_2_DEV(cohc), "illegal channel mode\n"); - return -EINVAL; - } - - dev_dbg(COHC_2_DEV(cohc), "configure channel for %d byte transfers\n", - addr_width); - switch (addr_width) { - case DMA_SLAVE_BUSWIDTH_1_BYTE: - ctrl |= - COH901318_CX_CTRL_SRC_BUS_SIZE_8_BITS | - COH901318_CX_CTRL_DST_BUS_SIZE_8_BITS; - - while (i < ARRAY_SIZE(burst_sizes)) { - if (burst_sizes[i].burst_8bit <= maxburst) - break; - i++; - } - - break; - case DMA_SLAVE_BUSWIDTH_2_BYTES: - ctrl |= - COH901318_CX_CTRL_SRC_BUS_SIZE_16_BITS | - COH901318_CX_CTRL_DST_BUS_SIZE_16_BITS; - - while (i < ARRAY_SIZE(burst_sizes)) { - if (burst_sizes[i].burst_16bit <= maxburst) - break; - i++; - } - - break; - case DMA_SLAVE_BUSWIDTH_4_BYTES: - /* Direction doesn't matter here, it's 32/32 bits */ - ctrl |= - COH901318_CX_CTRL_SRC_BUS_SIZE_32_BITS | - COH901318_CX_CTRL_DST_BUS_SIZE_32_BITS; - - while (i < ARRAY_SIZE(burst_sizes)) { - if (burst_sizes[i].burst_32bit <= maxburst) - break; - i++; - } - - break; - default: - dev_err(COHC_2_DEV(cohc), - "bad runtimeconfig: alien address width\n"); - return -EINVAL; - } - - ctrl |= burst_sizes[i].reg; - dev_dbg(COHC_2_DEV(cohc), - "selected burst size %d bytes for address width %d bytes, maxburst %d\n", - burst_sizes[i].burst_8bit, addr_width, maxburst); - - cohc->addr = addr; - cohc->ctrl = ctrl; - - return 0; -} - -static int coh901318_dma_slave_config(struct dma_chan *chan, - struct dma_slave_config *config) -{ - struct coh901318_chan *cohc = to_coh901318_chan(chan); - - memcpy(&cohc->config, config, sizeof(*config)); - - return 0; -} - -static void coh901318_base_init(struct dma_device *dma, const int *pick_chans, - struct coh901318_base *base) -{ - int chans_i; - int i = 0; - struct coh901318_chan *cohc; - - INIT_LIST_HEAD(&dma->channels); - - for (chans_i = 0; pick_chans[chans_i] != -1; chans_i += 2) { - for (i = pick_chans[chans_i]; i <= pick_chans[chans_i+1]; i++) { - cohc = &base->chans[i]; - - cohc->base = base; - cohc->chan.device = dma; - cohc->id = i; - - /* TODO: do we really need this lock if only one - * client is connected to each channel? - */ - - spin_lock_init(&cohc->lock); - - cohc->nbr_active_done = 0; - cohc->busy = 0; - INIT_LIST_HEAD(&cohc->free); - INIT_LIST_HEAD(&cohc->active); - INIT_LIST_HEAD(&cohc->queue); - - tasklet_setup(&cohc->tasklet, dma_tasklet); - - list_add_tail(&cohc->chan.device_node, - &dma->channels); - } - } -} - -static int __init coh901318_probe(struct platform_device *pdev) -{ - int err = 0; - struct coh901318_base *base; - int irq; - struct resource *io; - - io = platform_get_resource(pdev, IORESOURCE_MEM, 0); - if (!io) - return -ENODEV; - - /* Map DMA controller registers to virtual memory */ - if (devm_request_mem_region(&pdev->dev, - io->start, - resource_size(io), - pdev->dev.driver->name) == NULL) - return -ENOMEM; - - base = devm_kzalloc(&pdev->dev, - ALIGN(sizeof(struct coh901318_base), 4) + - U300_DMA_CHANNELS * - sizeof(struct coh901318_chan), - GFP_KERNEL); - if (!base) - return -ENOMEM; - - base->chans = ((void *)base) + ALIGN(sizeof(struct coh901318_base), 4); - - base->virtbase = devm_ioremap(&pdev->dev, io->start, resource_size(io)); - if (!base->virtbase) - return -ENOMEM; - - base->dev = &pdev->dev; - spin_lock_init(&base->pm.lock); - base->pm.started_channels = 0; - - COH901318_DEBUGFS_ASSIGN(debugfs_dma_base, base); - - irq = platform_get_irq(pdev, 0); - if (irq < 0) - return irq; - - err = devm_request_irq(&pdev->dev, irq, dma_irq_handler, 0, - "coh901318", base); - if (err) - return err; - - base->irq = irq; - - err = coh901318_pool_create(&base->pool, &pdev->dev, - sizeof(struct coh901318_lli), - 32); - if (err) - return err; - - /* init channels for device transfers */ - coh901318_base_init(&base->dma_slave, dma_slave_channels, - base); - - dma_cap_zero(base->dma_slave.cap_mask); - dma_cap_set(DMA_SLAVE, base->dma_slave.cap_mask); - - base->dma_slave.device_alloc_chan_resources = coh901318_alloc_chan_resources; - base->dma_slave.device_free_chan_resources = coh901318_free_chan_resources; - base->dma_slave.device_prep_slave_sg = coh901318_prep_slave_sg; - base->dma_slave.device_tx_status = coh901318_tx_status; - base->dma_slave.device_issue_pending = coh901318_issue_pending; - base->dma_slave.device_config = coh901318_dma_slave_config; - base->dma_slave.device_pause = coh901318_pause; - base->dma_slave.device_resume = coh901318_resume; - base->dma_slave.device_terminate_all = coh901318_terminate_all; - base->dma_slave.dev = &pdev->dev; - - err = dma_async_device_register(&base->dma_slave); - - if (err) - goto err_register_slave; - - /* init channels for memcpy */ - coh901318_base_init(&base->dma_memcpy, dma_memcpy_channels, - base); - - dma_cap_zero(base->dma_memcpy.cap_mask); - dma_cap_set(DMA_MEMCPY, base->dma_memcpy.cap_mask); - - base->dma_memcpy.device_alloc_chan_resources = coh901318_alloc_chan_resources; - base->dma_memcpy.device_free_chan_resources = coh901318_free_chan_resources; - base->dma_memcpy.device_prep_dma_memcpy = coh901318_prep_memcpy; - base->dma_memcpy.device_tx_status = coh901318_tx_status; - base->dma_memcpy.device_issue_pending = coh901318_issue_pending; - base->dma_memcpy.device_config = coh901318_dma_slave_config; - base->dma_memcpy.device_pause = coh901318_pause; - base->dma_memcpy.device_resume = coh901318_resume; - base->dma_memcpy.device_terminate_all = coh901318_terminate_all; - base->dma_memcpy.dev = &pdev->dev; - /* - * This controller can only access address at even 32bit boundaries, - * i.e. 2^2 - */ - base->dma_memcpy.copy_align = DMAENGINE_ALIGN_4_BYTES; - err = dma_async_device_register(&base->dma_memcpy); - - if (err) - goto err_register_memcpy; - - err = of_dma_controller_register(pdev->dev.of_node, coh901318_xlate, - base); - if (err) - goto err_register_of_dma; - - platform_set_drvdata(pdev, base); - dev_info(&pdev->dev, "Initialized COH901318 DMA on virtual base 0x%p\n", - base->virtbase); - - return err; - - err_register_of_dma: - dma_async_device_unregister(&base->dma_memcpy); - err_register_memcpy: - dma_async_device_unregister(&base->dma_slave); - err_register_slave: - coh901318_pool_destroy(&base->pool); - return err; -} -static void coh901318_base_remove(struct coh901318_base *base, const int *pick_chans) -{ - int chans_i; - int i = 0; - struct coh901318_chan *cohc; - - for (chans_i = 0; pick_chans[chans_i] != -1; chans_i += 2) { - for (i = pick_chans[chans_i]; i <= pick_chans[chans_i+1]; i++) { - cohc = &base->chans[i]; - - tasklet_kill(&cohc->tasklet); - } - } - -} - -static int coh901318_remove(struct platform_device *pdev) -{ - struct coh901318_base *base = platform_get_drvdata(pdev); - - devm_free_irq(&pdev->dev, base->irq, base); - - coh901318_base_remove(base, dma_slave_channels); - coh901318_base_remove(base, dma_memcpy_channels); - - of_dma_controller_free(pdev->dev.of_node); - dma_async_device_unregister(&base->dma_memcpy); - dma_async_device_unregister(&base->dma_slave); - coh901318_pool_destroy(&base->pool); - return 0; -} - -static const struct of_device_id coh901318_dt_match[] = { - { .compatible = "stericsson,coh901318" }, - {}, -}; - -static struct platform_driver coh901318_driver = { - .remove = coh901318_remove, - .driver = { - .name = "coh901318", - .of_match_table = coh901318_dt_match, - }, -}; - -static int __init coh901318_init(void) -{ - return platform_driver_probe(&coh901318_driver, coh901318_probe); -} -subsys_initcall(coh901318_init); - -static void __exit coh901318_exit(void) -{ - platform_driver_unregister(&coh901318_driver); -} -module_exit(coh901318_exit); - -MODULE_LICENSE("GPL"); -MODULE_AUTHOR("Per Friden"); diff --git a/drivers/dma/coh901318.h b/drivers/dma/coh901318.h deleted file mode 100644 index bbf5336005587d8986c363497ab82d22c2fdfa8f..0000000000000000000000000000000000000000 --- a/drivers/dma/coh901318.h +++ /dev/null @@ -1,141 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (C) 2007-2013 ST-Ericsson - * DMA driver for COH 901 318 - * Author: Per Friden - */ - -#ifndef COH901318_H -#define COH901318_H - -#define MAX_DMA_PACKET_SIZE_SHIFT 11 -#define MAX_DMA_PACKET_SIZE (1 << MAX_DMA_PACKET_SIZE_SHIFT) - -struct device; - -struct coh901318_pool { - spinlock_t lock; - struct dma_pool *dmapool; - struct device *dev; - -#ifdef CONFIG_DEBUG_FS - int debugfs_pool_counter; -#endif -}; - -/** - * struct coh901318_lli - linked list item for DMAC - * @control: control settings for DMAC - * @src_addr: transfer source address - * @dst_addr: transfer destination address - * @link_addr: physical address to next lli - * @virt_link_addr: virtual address of next lli (only used by pool_free) - * @phy_this: physical address of current lli (only used by pool_free) - */ -struct coh901318_lli { - u32 control; - dma_addr_t src_addr; - dma_addr_t dst_addr; - dma_addr_t link_addr; - - void *virt_link_addr; - dma_addr_t phy_this; -}; - -/** - * coh901318_pool_create() - Creates an dma pool for lli:s - * @pool: pool handle - * @dev: dma device - * @lli_nbr: number of lli:s in the pool - * @algin: address alignemtn of lli:s - * returns 0 on success otherwise none zero - */ -int coh901318_pool_create(struct coh901318_pool *pool, - struct device *dev, - size_t lli_nbr, size_t align); - -/** - * coh901318_pool_destroy() - Destroys the dma pool - * @pool: pool handle - * returns 0 on success otherwise none zero - */ -int coh901318_pool_destroy(struct coh901318_pool *pool); - -/** - * coh901318_lli_alloc() - Allocates a linked list - * - * @pool: pool handle - * @len: length to list - * return: none NULL if success otherwise NULL - */ -struct coh901318_lli * -coh901318_lli_alloc(struct coh901318_pool *pool, - unsigned int len); - -/** - * coh901318_lli_free() - Returns the linked list items to the pool - * @pool: pool handle - * @lli: reference to lli pointer to be freed - */ -void coh901318_lli_free(struct coh901318_pool *pool, - struct coh901318_lli **lli); - -/** - * coh901318_lli_fill_memcpy() - Prepares the lli:s for dma memcpy - * @pool: pool handle - * @lli: allocated lli - * @src: src address - * @size: transfer size - * @dst: destination address - * @ctrl_chained: ctrl for chained lli - * @ctrl_last: ctrl for the last lli - * returns number of CPU interrupts for the lli, negative on error. - */ -int -coh901318_lli_fill_memcpy(struct coh901318_pool *pool, - struct coh901318_lli *lli, - dma_addr_t src, unsigned int size, - dma_addr_t dst, u32 ctrl_chained, u32 ctrl_last); - -/** - * coh901318_lli_fill_single() - Prepares the lli:s for dma single transfer - * @pool: pool handle - * @lli: allocated lli - * @buf: transfer buffer - * @size: transfer size - * @dev_addr: address of periphal - * @ctrl_chained: ctrl for chained lli - * @ctrl_last: ctrl for the last lli - * @dir: direction of transfer (to or from device) - * returns number of CPU interrupts for the lli, negative on error. - */ -int -coh901318_lli_fill_single(struct coh901318_pool *pool, - struct coh901318_lli *lli, - dma_addr_t buf, unsigned int size, - dma_addr_t dev_addr, u32 ctrl_chained, u32 ctrl_last, - enum dma_transfer_direction dir); - -/** - * coh901318_lli_fill_single() - Prepares the lli:s for dma scatter list transfer - * @pool: pool handle - * @lli: allocated lli - * @sg: scatter gather list - * @nents: number of entries in sg - * @dev_addr: address of periphal - * @ctrl_chained: ctrl for chained lli - * @ctrl: ctrl of middle lli - * @ctrl_last: ctrl for the last lli - * @dir: direction of transfer (to or from device) - * @ctrl_irq_mask: ctrl mask for CPU interrupt - * returns number of CPU interrupts for the lli, negative on error. - */ -int -coh901318_lli_fill_sg(struct coh901318_pool *pool, - struct coh901318_lli *lli, - struct scatterlist *sg, unsigned int nents, - dma_addr_t dev_addr, u32 ctrl_chained, - u32 ctrl, u32 ctrl_last, - enum dma_transfer_direction dir, u32 ctrl_irq_mask); - -#endif /* COH901318_H */ diff --git a/drivers/dma/coh901318_lli.c b/drivers/dma/coh901318_lli.c deleted file mode 100644 index 6b6c2fd0865a589871eae966d21f3ccd209b5ae9..0000000000000000000000000000000000000000 --- a/drivers/dma/coh901318_lli.c +++ /dev/null @@ -1,313 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * driver/dma/coh901318_lli.c - * - * Copyright (C) 2007-2009 ST-Ericsson - * Support functions for handling lli for dma - * Author: Per Friden - */ - -#include -#include -#include -#include -#include - -#include "coh901318.h" - -#if (defined(CONFIG_DEBUG_FS) && defined(CONFIG_U300_DEBUG)) -#define DEBUGFS_POOL_COUNTER_RESET(pool) (pool->debugfs_pool_counter = 0) -#define DEBUGFS_POOL_COUNTER_ADD(pool, add) (pool->debugfs_pool_counter += add) -#else -#define DEBUGFS_POOL_COUNTER_RESET(pool) -#define DEBUGFS_POOL_COUNTER_ADD(pool, add) -#endif - -static struct coh901318_lli * -coh901318_lli_next(struct coh901318_lli *data) -{ - if (data == NULL || data->link_addr == 0) - return NULL; - - return (struct coh901318_lli *) data->virt_link_addr; -} - -int coh901318_pool_create(struct coh901318_pool *pool, - struct device *dev, - size_t size, size_t align) -{ - spin_lock_init(&pool->lock); - pool->dev = dev; - pool->dmapool = dma_pool_create("lli_pool", dev, size, align, 0); - - DEBUGFS_POOL_COUNTER_RESET(pool); - return 0; -} - -int coh901318_pool_destroy(struct coh901318_pool *pool) -{ - - dma_pool_destroy(pool->dmapool); - return 0; -} - -struct coh901318_lli * -coh901318_lli_alloc(struct coh901318_pool *pool, unsigned int len) -{ - int i; - struct coh901318_lli *head; - struct coh901318_lli *lli; - struct coh901318_lli *lli_prev; - dma_addr_t phy; - - if (len == 0) - return NULL; - - spin_lock(&pool->lock); - - head = dma_pool_alloc(pool->dmapool, GFP_NOWAIT, &phy); - - if (head == NULL) - goto err; - - DEBUGFS_POOL_COUNTER_ADD(pool, 1); - - lli = head; - lli->phy_this = phy; - lli->link_addr = 0x00000000; - lli->virt_link_addr = NULL; - - for (i = 1; i < len; i++) { - lli_prev = lli; - - lli = dma_pool_alloc(pool->dmapool, GFP_NOWAIT, &phy); - - if (lli == NULL) - goto err_clean_up; - - DEBUGFS_POOL_COUNTER_ADD(pool, 1); - lli->phy_this = phy; - lli->link_addr = 0x00000000; - lli->virt_link_addr = NULL; - - lli_prev->link_addr = phy; - lli_prev->virt_link_addr = lli; - } - - spin_unlock(&pool->lock); - - return head; - - err: - spin_unlock(&pool->lock); - return NULL; - - err_clean_up: - lli_prev->link_addr = 0x00000000U; - spin_unlock(&pool->lock); - coh901318_lli_free(pool, &head); - return NULL; -} - -void coh901318_lli_free(struct coh901318_pool *pool, - struct coh901318_lli **lli) -{ - struct coh901318_lli *l; - struct coh901318_lli *next; - - if (lli == NULL) - return; - - l = *lli; - - if (l == NULL) - return; - - spin_lock(&pool->lock); - - while (l->link_addr) { - next = l->virt_link_addr; - dma_pool_free(pool->dmapool, l, l->phy_this); - DEBUGFS_POOL_COUNTER_ADD(pool, -1); - l = next; - } - dma_pool_free(pool->dmapool, l, l->phy_this); - DEBUGFS_POOL_COUNTER_ADD(pool, -1); - - spin_unlock(&pool->lock); - *lli = NULL; -} - -int -coh901318_lli_fill_memcpy(struct coh901318_pool *pool, - struct coh901318_lli *lli, - dma_addr_t source, unsigned int size, - dma_addr_t destination, u32 ctrl_chained, - u32 ctrl_eom) -{ - int s = size; - dma_addr_t src = source; - dma_addr_t dst = destination; - - lli->src_addr = src; - lli->dst_addr = dst; - - while (lli->link_addr) { - lli->control = ctrl_chained | MAX_DMA_PACKET_SIZE; - lli->src_addr = src; - lli->dst_addr = dst; - - s -= MAX_DMA_PACKET_SIZE; - lli = coh901318_lli_next(lli); - - src += MAX_DMA_PACKET_SIZE; - dst += MAX_DMA_PACKET_SIZE; - } - - lli->control = ctrl_eom | s; - lli->src_addr = src; - lli->dst_addr = dst; - - return 0; -} - -int -coh901318_lli_fill_single(struct coh901318_pool *pool, - struct coh901318_lli *lli, - dma_addr_t buf, unsigned int size, - dma_addr_t dev_addr, u32 ctrl_chained, u32 ctrl_eom, - enum dma_transfer_direction dir) -{ - int s = size; - dma_addr_t src; - dma_addr_t dst; - - - if (dir == DMA_MEM_TO_DEV) { - src = buf; - dst = dev_addr; - - } else if (dir == DMA_DEV_TO_MEM) { - - src = dev_addr; - dst = buf; - } else { - return -EINVAL; - } - - while (lli->link_addr) { - size_t block_size = MAX_DMA_PACKET_SIZE; - lli->control = ctrl_chained | MAX_DMA_PACKET_SIZE; - - /* If we are on the next-to-final block and there will - * be less than half a DMA packet left for the last - * block, then we want to make this block a little - * smaller to balance the sizes. This is meant to - * avoid too small transfers if the buffer size is - * (MAX_DMA_PACKET_SIZE*N + 1) */ - if (s < (MAX_DMA_PACKET_SIZE + MAX_DMA_PACKET_SIZE/2)) - block_size = MAX_DMA_PACKET_SIZE/2; - - s -= block_size; - lli->src_addr = src; - lli->dst_addr = dst; - - lli = coh901318_lli_next(lli); - - if (dir == DMA_MEM_TO_DEV) - src += block_size; - else if (dir == DMA_DEV_TO_MEM) - dst += block_size; - } - - lli->control = ctrl_eom | s; - lli->src_addr = src; - lli->dst_addr = dst; - - return 0; -} - -int -coh901318_lli_fill_sg(struct coh901318_pool *pool, - struct coh901318_lli *lli, - struct scatterlist *sgl, unsigned int nents, - dma_addr_t dev_addr, u32 ctrl_chained, u32 ctrl, - u32 ctrl_last, - enum dma_transfer_direction dir, u32 ctrl_irq_mask) -{ - int i; - struct scatterlist *sg; - u32 ctrl_sg; - dma_addr_t src = 0; - dma_addr_t dst = 0; - u32 bytes_to_transfer; - u32 elem_size; - - if (lli == NULL) - goto err; - - spin_lock(&pool->lock); - - if (dir == DMA_MEM_TO_DEV) - dst = dev_addr; - else if (dir == DMA_DEV_TO_MEM) - src = dev_addr; - else - goto err; - - for_each_sg(sgl, sg, nents, i) { - if (sg_is_chain(sg)) { - /* sg continues to the next sg-element don't - * send ctrl_finish until the last - * sg-element in the chain - */ - ctrl_sg = ctrl_chained; - } else if (i == nents - 1) - ctrl_sg = ctrl_last; - else - ctrl_sg = ctrl ? ctrl : ctrl_last; - - - if (dir == DMA_MEM_TO_DEV) - /* increment source address */ - src = sg_dma_address(sg); - else - /* increment destination address */ - dst = sg_dma_address(sg); - - bytes_to_transfer = sg_dma_len(sg); - - while (bytes_to_transfer) { - u32 val; - - if (bytes_to_transfer > MAX_DMA_PACKET_SIZE) { - elem_size = MAX_DMA_PACKET_SIZE; - val = ctrl_chained; - } else { - elem_size = bytes_to_transfer; - val = ctrl_sg; - } - - lli->control = val | elem_size; - lli->src_addr = src; - lli->dst_addr = dst; - - if (dir == DMA_DEV_TO_MEM) - dst += elem_size; - else - src += elem_size; - - BUG_ON(lli->link_addr & 3); - - bytes_to_transfer -= elem_size; - lli = coh901318_lli_next(lli); - } - - } - spin_unlock(&pool->lock); - - return 0; - err: - spin_unlock(&pool->lock); - return -EINVAL; -} diff --git a/drivers/dma/dma-jz4780.c b/drivers/dma/dma-jz4780.c index 612d353648cf70c888daaf69ff0bca1db3321500..ebee94dbd63030933e3c0c79eed002f67cbfb473 100644 --- a/drivers/dma/dma-jz4780.c +++ b/drivers/dma/dma-jz4780.c @@ -1004,6 +1004,18 @@ static const struct jz4780_dma_soc_data jz4725b_dma_soc_data = { JZ_SOC_DATA_BREAK_LINKS, }; +static const struct jz4780_dma_soc_data jz4760_dma_soc_data = { + .nb_channels = 5, + .transfer_ord_max = 6, + .flags = JZ_SOC_DATA_PER_CHAN_PM | JZ_SOC_DATA_NO_DCKES_DCKEC, +}; + +static const struct jz4780_dma_soc_data jz4760b_dma_soc_data = { + .nb_channels = 5, + .transfer_ord_max = 6, + .flags = JZ_SOC_DATA_PER_CHAN_PM, +}; + static const struct jz4780_dma_soc_data jz4770_dma_soc_data = { .nb_channels = 6, .transfer_ord_max = 6, @@ -1031,6 +1043,8 @@ static const struct jz4780_dma_soc_data x1830_dma_soc_data = { static const struct of_device_id jz4780_dma_dt_match[] = { { .compatible = "ingenic,jz4740-dma", .data = &jz4740_dma_soc_data }, { .compatible = "ingenic,jz4725b-dma", .data = &jz4725b_dma_soc_data }, + { .compatible = "ingenic,jz4760-dma", .data = &jz4760_dma_soc_data }, + { .compatible = "ingenic,jz4760b-dma", .data = &jz4760b_dma_soc_data }, { .compatible = "ingenic,jz4770-dma", .data = &jz4770_dma_soc_data }, { .compatible = "ingenic,jz4780-dma", .data = &jz4780_dma_soc_data }, { .compatible = "ingenic,x1000-dma", .data = &x1000_dma_soc_data }, diff --git a/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c b/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c index e164f3295f5d7acaca24245a719e3383918d8e7c..d9e4ac3edb4ea11e4e8c4ce11c710cf25cf60723 100644 --- a/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c +++ b/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c @@ -12,15 +12,20 @@ #include #include #include +#include #include #include #include +#include +#include #include #include #include +#include #include #include #include +#include #include #include "dw-axi-dmac.h" @@ -195,43 +200,56 @@ static inline const char *axi_chan_name(struct axi_dma_chan *chan) return dma_chan_name(&chan->vc.chan); } -static struct axi_dma_desc *axi_desc_get(struct axi_dma_chan *chan) +static struct axi_dma_desc *axi_desc_alloc(u32 num) { - struct dw_axi_dma *dw = chan->chip->dw; struct axi_dma_desc *desc; + + desc = kzalloc(sizeof(*desc), GFP_NOWAIT); + if (!desc) + return NULL; + + desc->hw_desc = kcalloc(num, sizeof(*desc->hw_desc), GFP_NOWAIT); + if (!desc->hw_desc) { + kfree(desc); + return NULL; + } + + return desc; +} + +static struct axi_dma_lli *axi_desc_get(struct axi_dma_chan *chan, + dma_addr_t *addr) +{ + struct axi_dma_lli *lli; dma_addr_t phys; - desc = dma_pool_zalloc(dw->desc_pool, GFP_NOWAIT, &phys); - if (unlikely(!desc)) { + lli = dma_pool_zalloc(chan->desc_pool, GFP_NOWAIT, &phys); + if (unlikely(!lli)) { dev_err(chan2dev(chan), "%s: not enough descriptors available\n", axi_chan_name(chan)); return NULL; } atomic_inc(&chan->descs_allocated); - INIT_LIST_HEAD(&desc->xfer_list); - desc->vd.tx.phys = phys; - desc->chan = chan; + *addr = phys; - return desc; + return lli; } static void axi_desc_put(struct axi_dma_desc *desc) { struct axi_dma_chan *chan = desc->chan; - struct dw_axi_dma *dw = chan->chip->dw; - struct axi_dma_desc *child, *_next; - unsigned int descs_put = 0; + int count = atomic_read(&chan->descs_allocated); + struct axi_dma_hw_desc *hw_desc; + int descs_put; - list_for_each_entry_safe(child, _next, &desc->xfer_list, xfer_list) { - list_del(&child->xfer_list); - dma_pool_free(dw->desc_pool, child, child->vd.tx.phys); - descs_put++; + for (descs_put = 0; descs_put < count; descs_put++) { + hw_desc = &desc->hw_desc[descs_put]; + dma_pool_free(chan->desc_pool, hw_desc->lli, hw_desc->llp); } - dma_pool_free(dw->desc_pool, desc, desc->vd.tx.phys); - descs_put++; - + kfree(desc->hw_desc); + kfree(desc); atomic_sub(descs_put, &chan->descs_allocated); dev_vdbg(chan2dev(chan), "%s: %d descs put, %d still allocated\n", axi_chan_name(chan), descs_put, @@ -248,19 +266,41 @@ dma_chan_tx_status(struct dma_chan *dchan, dma_cookie_t cookie, struct dma_tx_state *txstate) { struct axi_dma_chan *chan = dchan_to_axi_dma_chan(dchan); - enum dma_status ret; + struct virt_dma_desc *vdesc; + enum dma_status status; + u32 completed_length; + unsigned long flags; + u32 completed_blocks; + size_t bytes = 0; + u32 length; + u32 len; - ret = dma_cookie_status(dchan, cookie, txstate); + status = dma_cookie_status(dchan, cookie, txstate); + if (status == DMA_COMPLETE || !txstate) + return status; - if (chan->is_paused && ret == DMA_IN_PROGRESS) - ret = DMA_PAUSED; + spin_lock_irqsave(&chan->vc.lock, flags); - return ret; + vdesc = vchan_find_desc(&chan->vc, cookie); + if (vdesc) { + length = vd_to_axi_desc(vdesc)->length; + completed_blocks = vd_to_axi_desc(vdesc)->completed_blocks; + len = vd_to_axi_desc(vdesc)->hw_desc[0].len; + completed_length = completed_blocks * len; + bytes = length - completed_length; + } else { + bytes = vd_to_axi_desc(vdesc)->length; + } + + spin_unlock_irqrestore(&chan->vc.lock, flags); + dma_set_residue(txstate, bytes); + + return status; } -static void write_desc_llp(struct axi_dma_desc *desc, dma_addr_t adr) +static void write_desc_llp(struct axi_dma_hw_desc *desc, dma_addr_t adr) { - desc->lli.llp = cpu_to_le64(adr); + desc->lli->llp = cpu_to_le64(adr); } static void write_chan_llp(struct axi_dma_chan *chan, dma_addr_t adr) @@ -268,6 +308,29 @@ static void write_chan_llp(struct axi_dma_chan *chan, dma_addr_t adr) axi_chan_iowrite64(chan, CH_LLP, adr); } +static void dw_axi_dma_set_byte_halfword(struct axi_dma_chan *chan, bool set) +{ + u32 offset = DMAC_APB_BYTE_WR_CH_EN; + u32 reg_width, val; + + if (!chan->chip->apb_regs) { + dev_dbg(chan->chip->dev, "apb_regs not initialized\n"); + return; + } + + reg_width = __ffs(chan->config.dst_addr_width); + if (reg_width == DWAXIDMAC_TRANS_WIDTH_16) + offset = DMAC_APB_HALFWORD_WR_CH_EN; + + val = ioread32(chan->chip->apb_regs + offset); + + if (set) + val |= BIT(chan->id); + else + val &= ~BIT(chan->id); + + iowrite32(val, chan->chip->apb_regs + offset); +} /* Called in chan locked context */ static void axi_chan_block_xfer_start(struct axi_dma_chan *chan, struct axi_dma_desc *first) @@ -293,9 +356,26 @@ static void axi_chan_block_xfer_start(struct axi_dma_chan *chan, priority << CH_CFG_H_PRIORITY_POS | DWAXIDMAC_HS_SEL_HW << CH_CFG_H_HS_SEL_DST_POS | DWAXIDMAC_HS_SEL_HW << CH_CFG_H_HS_SEL_SRC_POS); + switch (chan->direction) { + case DMA_MEM_TO_DEV: + dw_axi_dma_set_byte_halfword(chan, true); + reg |= (chan->config.device_fc ? + DWAXIDMAC_TT_FC_MEM_TO_PER_DST : + DWAXIDMAC_TT_FC_MEM_TO_PER_DMAC) + << CH_CFG_H_TT_FC_POS; + break; + case DMA_DEV_TO_MEM: + reg |= (chan->config.device_fc ? + DWAXIDMAC_TT_FC_PER_TO_MEM_SRC : + DWAXIDMAC_TT_FC_PER_TO_MEM_DMAC) + << CH_CFG_H_TT_FC_POS; + break; + default: + break; + } axi_chan_iowrite32(chan, CH_CFG_H, reg); - write_chan_llp(chan, first->vd.tx.phys | lms); + write_chan_llp(chan, first->hw_desc[0].llp | lms); irq_mask = DWAXIDMAC_IRQ_DMA_TRF | DWAXIDMAC_IRQ_ALL_ERR; axi_chan_irq_sig_set(chan, irq_mask); @@ -333,6 +413,13 @@ static void dma_chan_issue_pending(struct dma_chan *dchan) spin_unlock_irqrestore(&chan->vc.lock, flags); } +static void dw_axi_dma_synchronize(struct dma_chan *dchan) +{ + struct axi_dma_chan *chan = dchan_to_axi_dma_chan(dchan); + + vchan_synchronize(&chan->vc); +} + static int dma_chan_alloc_chan_resources(struct dma_chan *dchan) { struct axi_dma_chan *chan = dchan_to_axi_dma_chan(dchan); @@ -344,6 +431,15 @@ static int dma_chan_alloc_chan_resources(struct dma_chan *dchan) return -EBUSY; } + /* LLI address must be aligned to a 64-byte boundary */ + chan->desc_pool = dma_pool_create(dev_name(chan2dev(chan)), + chan->chip->dev, + sizeof(struct axi_dma_lli), + 64, 0); + if (!chan->desc_pool) { + dev_err(chan2dev(chan), "No memory for descriptors\n"); + return -ENOMEM; + } dev_vdbg(dchan2dev(dchan), "%s: allocating\n", axi_chan_name(chan)); pm_runtime_get(chan->chip->dev); @@ -365,6 +461,8 @@ static void dma_chan_free_chan_resources(struct dma_chan *dchan) vchan_free_chan_resources(&chan->vc); + dma_pool_destroy(chan->desc_pool); + chan->desc_pool = NULL; dev_vdbg(dchan2dev(dchan), "%s: free resources, descriptor still allocated: %u\n", axi_chan_name(chan), atomic_read(&chan->descs_allocated)); @@ -372,73 +470,398 @@ static void dma_chan_free_chan_resources(struct dma_chan *dchan) pm_runtime_put(chan->chip->dev); } +static void dw_axi_dma_set_hw_channel(struct axi_dma_chip *chip, + u32 handshake_num, bool set) +{ + unsigned long start = 0; + unsigned long reg_value; + unsigned long reg_mask; + unsigned long reg_set; + unsigned long mask; + unsigned long val; + + if (!chip->apb_regs) { + dev_dbg(chip->dev, "apb_regs not initialized\n"); + return; + } + + /* + * An unused DMA channel has a default value of 0x3F. + * Lock the DMA channel by assign a handshake number to the channel. + * Unlock the DMA channel by assign 0x3F to the channel. + */ + if (set) { + reg_set = UNUSED_CHANNEL; + val = handshake_num; + } else { + reg_set = handshake_num; + val = UNUSED_CHANNEL; + } + + reg_value = lo_hi_readq(chip->apb_regs + DMAC_APB_HW_HS_SEL_0); + + for_each_set_clump8(start, reg_mask, ®_value, 64) { + if (reg_mask == reg_set) { + mask = GENMASK_ULL(start + 7, start); + reg_value &= ~mask; + reg_value |= rol64(val, start); + lo_hi_writeq(reg_value, + chip->apb_regs + DMAC_APB_HW_HS_SEL_0); + break; + } + } +} + /* * If DW_axi_dmac sees CHx_CTL.ShadowReg_Or_LLI_Last bit of the fetched LLI * as 1, it understands that the current block is the final block in the * transfer and completes the DMA transfer operation at the end of current * block transfer. */ -static void set_desc_last(struct axi_dma_desc *desc) +static void set_desc_last(struct axi_dma_hw_desc *desc) { u32 val; - val = le32_to_cpu(desc->lli.ctl_hi); + val = le32_to_cpu(desc->lli->ctl_hi); val |= CH_CTL_H_LLI_LAST; - desc->lli.ctl_hi = cpu_to_le32(val); + desc->lli->ctl_hi = cpu_to_le32(val); } -static void write_desc_sar(struct axi_dma_desc *desc, dma_addr_t adr) +static void write_desc_sar(struct axi_dma_hw_desc *desc, dma_addr_t adr) { - desc->lli.sar = cpu_to_le64(adr); + desc->lli->sar = cpu_to_le64(adr); } -static void write_desc_dar(struct axi_dma_desc *desc, dma_addr_t adr) +static void write_desc_dar(struct axi_dma_hw_desc *desc, dma_addr_t adr) { - desc->lli.dar = cpu_to_le64(adr); + desc->lli->dar = cpu_to_le64(adr); } -static void set_desc_src_master(struct axi_dma_desc *desc) +static void set_desc_src_master(struct axi_dma_hw_desc *desc) { u32 val; /* Select AXI0 for source master */ - val = le32_to_cpu(desc->lli.ctl_lo); + val = le32_to_cpu(desc->lli->ctl_lo); val &= ~CH_CTL_L_SRC_MAST; - desc->lli.ctl_lo = cpu_to_le32(val); + desc->lli->ctl_lo = cpu_to_le32(val); } -static void set_desc_dest_master(struct axi_dma_desc *desc) +static void set_desc_dest_master(struct axi_dma_hw_desc *hw_desc, + struct axi_dma_desc *desc) { u32 val; /* Select AXI1 for source master if available */ - val = le32_to_cpu(desc->lli.ctl_lo); + val = le32_to_cpu(hw_desc->lli->ctl_lo); if (desc->chan->chip->dw->hdata->nr_masters > 1) val |= CH_CTL_L_DST_MAST; else val &= ~CH_CTL_L_DST_MAST; - desc->lli.ctl_lo = cpu_to_le32(val); + hw_desc->lli->ctl_lo = cpu_to_le32(val); +} + +static int dw_axi_dma_set_hw_desc(struct axi_dma_chan *chan, + struct axi_dma_hw_desc *hw_desc, + dma_addr_t mem_addr, size_t len) +{ + unsigned int data_width = BIT(chan->chip->dw->hdata->m_data_width); + unsigned int reg_width; + unsigned int mem_width; + dma_addr_t device_addr; + size_t axi_block_ts; + size_t block_ts; + u32 ctllo, ctlhi; + u32 burst_len; + + axi_block_ts = chan->chip->dw->hdata->block_size[chan->id]; + + mem_width = __ffs(data_width | mem_addr | len); + if (mem_width > DWAXIDMAC_TRANS_WIDTH_32) + mem_width = DWAXIDMAC_TRANS_WIDTH_32; + + if (!IS_ALIGNED(mem_addr, 4)) { + dev_err(chan->chip->dev, "invalid buffer alignment\n"); + return -EINVAL; + } + + switch (chan->direction) { + case DMA_MEM_TO_DEV: + reg_width = __ffs(chan->config.dst_addr_width); + device_addr = chan->config.dst_addr; + ctllo = reg_width << CH_CTL_L_DST_WIDTH_POS | + mem_width << CH_CTL_L_SRC_WIDTH_POS | + DWAXIDMAC_CH_CTL_L_NOINC << CH_CTL_L_DST_INC_POS | + DWAXIDMAC_CH_CTL_L_INC << CH_CTL_L_SRC_INC_POS; + block_ts = len >> mem_width; + break; + case DMA_DEV_TO_MEM: + reg_width = __ffs(chan->config.src_addr_width); + device_addr = chan->config.src_addr; + ctllo = reg_width << CH_CTL_L_SRC_WIDTH_POS | + mem_width << CH_CTL_L_DST_WIDTH_POS | + DWAXIDMAC_CH_CTL_L_INC << CH_CTL_L_DST_INC_POS | + DWAXIDMAC_CH_CTL_L_NOINC << CH_CTL_L_SRC_INC_POS; + block_ts = len >> reg_width; + break; + default: + return -EINVAL; + } + + if (block_ts > axi_block_ts) + return -EINVAL; + + hw_desc->lli = axi_desc_get(chan, &hw_desc->llp); + if (unlikely(!hw_desc->lli)) + return -ENOMEM; + + ctlhi = CH_CTL_H_LLI_VALID; + + if (chan->chip->dw->hdata->restrict_axi_burst_len) { + burst_len = chan->chip->dw->hdata->axi_rw_burst_len; + ctlhi |= CH_CTL_H_ARLEN_EN | CH_CTL_H_AWLEN_EN | + burst_len << CH_CTL_H_ARLEN_POS | + burst_len << CH_CTL_H_AWLEN_POS; + } + + hw_desc->lli->ctl_hi = cpu_to_le32(ctlhi); + + if (chan->direction == DMA_MEM_TO_DEV) { + write_desc_sar(hw_desc, mem_addr); + write_desc_dar(hw_desc, device_addr); + } else { + write_desc_sar(hw_desc, device_addr); + write_desc_dar(hw_desc, mem_addr); + } + + hw_desc->lli->block_ts_lo = cpu_to_le32(block_ts - 1); + + ctllo |= DWAXIDMAC_BURST_TRANS_LEN_4 << CH_CTL_L_DST_MSIZE_POS | + DWAXIDMAC_BURST_TRANS_LEN_4 << CH_CTL_L_SRC_MSIZE_POS; + hw_desc->lli->ctl_lo = cpu_to_le32(ctllo); + + set_desc_src_master(hw_desc); + + hw_desc->len = len; + return 0; +} + +static size_t calculate_block_len(struct axi_dma_chan *chan, + dma_addr_t dma_addr, size_t buf_len, + enum dma_transfer_direction direction) +{ + u32 data_width, reg_width, mem_width; + size_t axi_block_ts, block_len; + + axi_block_ts = chan->chip->dw->hdata->block_size[chan->id]; + + switch (direction) { + case DMA_MEM_TO_DEV: + data_width = BIT(chan->chip->dw->hdata->m_data_width); + mem_width = __ffs(data_width | dma_addr | buf_len); + if (mem_width > DWAXIDMAC_TRANS_WIDTH_32) + mem_width = DWAXIDMAC_TRANS_WIDTH_32; + + block_len = axi_block_ts << mem_width; + break; + case DMA_DEV_TO_MEM: + reg_width = __ffs(chan->config.src_addr_width); + block_len = axi_block_ts << reg_width; + break; + default: + block_len = 0; + } + + return block_len; +} + +static struct dma_async_tx_descriptor * +dw_axi_dma_chan_prep_cyclic(struct dma_chan *dchan, dma_addr_t dma_addr, + size_t buf_len, size_t period_len, + enum dma_transfer_direction direction, + unsigned long flags) +{ + struct axi_dma_chan *chan = dchan_to_axi_dma_chan(dchan); + struct axi_dma_hw_desc *hw_desc = NULL; + struct axi_dma_desc *desc = NULL; + dma_addr_t src_addr = dma_addr; + u32 num_periods, num_segments; + size_t axi_block_len; + u32 total_segments; + u32 segment_len; + unsigned int i; + int status; + u64 llp = 0; + u8 lms = 0; /* Select AXI0 master for LLI fetching */ + + num_periods = buf_len / period_len; + + axi_block_len = calculate_block_len(chan, dma_addr, buf_len, direction); + if (axi_block_len == 0) + return NULL; + + num_segments = DIV_ROUND_UP(period_len, axi_block_len); + segment_len = DIV_ROUND_UP(period_len, num_segments); + + total_segments = num_periods * num_segments; + + desc = axi_desc_alloc(total_segments); + if (unlikely(!desc)) + goto err_desc_get; + + chan->direction = direction; + desc->chan = chan; + chan->cyclic = true; + desc->length = 0; + desc->period_len = period_len; + + for (i = 0; i < total_segments; i++) { + hw_desc = &desc->hw_desc[i]; + + status = dw_axi_dma_set_hw_desc(chan, hw_desc, src_addr, + segment_len); + if (status < 0) + goto err_desc_get; + + desc->length += hw_desc->len; + /* Set end-of-link to the linked descriptor, so that cyclic + * callback function can be triggered during interrupt. + */ + set_desc_last(hw_desc); + + src_addr += segment_len; + } + + llp = desc->hw_desc[0].llp; + + /* Managed transfer list */ + do { + hw_desc = &desc->hw_desc[--total_segments]; + write_desc_llp(hw_desc, llp | lms); + llp = hw_desc->llp; + } while (total_segments); + + dw_axi_dma_set_hw_channel(chan->chip, chan->hw_handshake_num, true); + + return vchan_tx_prep(&chan->vc, &desc->vd, flags); + +err_desc_get: + if (desc) + axi_desc_put(desc); + + return NULL; +} + +static struct dma_async_tx_descriptor * +dw_axi_dma_chan_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl, + unsigned int sg_len, + enum dma_transfer_direction direction, + unsigned long flags, void *context) +{ + struct axi_dma_chan *chan = dchan_to_axi_dma_chan(dchan); + struct axi_dma_hw_desc *hw_desc = NULL; + struct axi_dma_desc *desc = NULL; + u32 num_segments, segment_len; + unsigned int loop = 0; + struct scatterlist *sg; + size_t axi_block_len; + u32 len, num_sgs = 0; + unsigned int i; + dma_addr_t mem; + int status; + u64 llp = 0; + u8 lms = 0; /* Select AXI0 master for LLI fetching */ + + if (unlikely(!is_slave_direction(direction) || !sg_len)) + return NULL; + + mem = sg_dma_address(sgl); + len = sg_dma_len(sgl); + + axi_block_len = calculate_block_len(chan, mem, len, direction); + if (axi_block_len == 0) + return NULL; + + for_each_sg(sgl, sg, sg_len, i) + num_sgs += DIV_ROUND_UP(sg_dma_len(sg), axi_block_len); + + desc = axi_desc_alloc(num_sgs); + if (unlikely(!desc)) + goto err_desc_get; + + desc->chan = chan; + desc->length = 0; + chan->direction = direction; + + for_each_sg(sgl, sg, sg_len, i) { + mem = sg_dma_address(sg); + len = sg_dma_len(sg); + num_segments = DIV_ROUND_UP(sg_dma_len(sg), axi_block_len); + segment_len = DIV_ROUND_UP(sg_dma_len(sg), num_segments); + + do { + hw_desc = &desc->hw_desc[loop++]; + status = dw_axi_dma_set_hw_desc(chan, hw_desc, mem, segment_len); + if (status < 0) + goto err_desc_get; + + desc->length += hw_desc->len; + len -= segment_len; + mem += segment_len; + } while (len >= segment_len); + } + + /* Set end-of-link to the last link descriptor of list */ + set_desc_last(&desc->hw_desc[num_sgs - 1]); + + /* Managed transfer list */ + do { + hw_desc = &desc->hw_desc[--num_sgs]; + write_desc_llp(hw_desc, llp | lms); + llp = hw_desc->llp; + } while (num_sgs); + + dw_axi_dma_set_hw_channel(chan->chip, chan->hw_handshake_num, true); + + return vchan_tx_prep(&chan->vc, &desc->vd, flags); + +err_desc_get: + if (desc) + axi_desc_put(desc); + + return NULL; } static struct dma_async_tx_descriptor * dma_chan_prep_dma_memcpy(struct dma_chan *dchan, dma_addr_t dst_adr, dma_addr_t src_adr, size_t len, unsigned long flags) { - struct axi_dma_desc *first = NULL, *desc = NULL, *prev = NULL; struct axi_dma_chan *chan = dchan_to_axi_dma_chan(dchan); size_t block_ts, max_block_ts, xfer_len; - u32 xfer_width, reg; + struct axi_dma_hw_desc *hw_desc = NULL; + struct axi_dma_desc *desc = NULL; + u32 xfer_width, reg, num; + u64 llp = 0; u8 lms = 0; /* Select AXI0 master for LLI fetching */ dev_dbg(chan2dev(chan), "%s: memcpy: src: %pad dst: %pad length: %zd flags: %#lx", axi_chan_name(chan), &src_adr, &dst_adr, len, flags); max_block_ts = chan->chip->dw->hdata->block_size[chan->id]; + xfer_width = axi_chan_get_xfer_width(chan, src_adr, dst_adr, len); + num = DIV_ROUND_UP(len, max_block_ts << xfer_width); + desc = axi_desc_alloc(num); + if (unlikely(!desc)) + goto err_desc_get; + desc->chan = chan; + num = 0; + desc->length = 0; while (len) { xfer_len = len; + hw_desc = &desc->hw_desc[num]; /* * Take care for the alignment. * Actually source and destination widths can be different, but @@ -457,13 +880,13 @@ dma_chan_prep_dma_memcpy(struct dma_chan *dchan, dma_addr_t dst_adr, xfer_len = max_block_ts << xfer_width; } - desc = axi_desc_get(chan); - if (unlikely(!desc)) + hw_desc->lli = axi_desc_get(chan, &hw_desc->llp); + if (unlikely(!hw_desc->lli)) goto err_desc_get; - write_desc_sar(desc, src_adr); - write_desc_dar(desc, dst_adr); - desc->lli.block_ts_lo = cpu_to_le32(block_ts - 1); + write_desc_sar(hw_desc, src_adr); + write_desc_dar(hw_desc, dst_adr); + hw_desc->lli->block_ts_lo = cpu_to_le32(block_ts - 1); reg = CH_CTL_H_LLI_VALID; if (chan->chip->dw->hdata->restrict_axi_burst_len) { @@ -474,7 +897,7 @@ dma_chan_prep_dma_memcpy(struct dma_chan *dchan, dma_addr_t dst_adr, CH_CTL_H_AWLEN_EN | burst_len << CH_CTL_H_AWLEN_POS); } - desc->lli.ctl_hi = cpu_to_le32(reg); + hw_desc->lli->ctl_hi = cpu_to_le32(reg); reg = (DWAXIDMAC_BURST_TRANS_LEN_4 << CH_CTL_L_DST_MSIZE_POS | DWAXIDMAC_BURST_TRANS_LEN_4 << CH_CTL_L_SRC_MSIZE_POS | @@ -482,62 +905,68 @@ dma_chan_prep_dma_memcpy(struct dma_chan *dchan, dma_addr_t dst_adr, xfer_width << CH_CTL_L_SRC_WIDTH_POS | DWAXIDMAC_CH_CTL_L_INC << CH_CTL_L_DST_INC_POS | DWAXIDMAC_CH_CTL_L_INC << CH_CTL_L_SRC_INC_POS); - desc->lli.ctl_lo = cpu_to_le32(reg); + hw_desc->lli->ctl_lo = cpu_to_le32(reg); - set_desc_src_master(desc); - set_desc_dest_master(desc); - - /* Manage transfer list (xfer_list) */ - if (!first) { - first = desc; - } else { - list_add_tail(&desc->xfer_list, &first->xfer_list); - write_desc_llp(prev, desc->vd.tx.phys | lms); - } - prev = desc; + set_desc_src_master(hw_desc); + set_desc_dest_master(hw_desc, desc); + hw_desc->len = xfer_len; + desc->length += hw_desc->len; /* update the length and addresses for the next loop cycle */ len -= xfer_len; dst_adr += xfer_len; src_adr += xfer_len; + num++; } - /* Total len of src/dest sg == 0, so no descriptor were allocated */ - if (unlikely(!first)) - return NULL; - /* Set end-of-link to the last link descriptor of list */ - set_desc_last(desc); + set_desc_last(&desc->hw_desc[num - 1]); + /* Managed transfer list */ + do { + hw_desc = &desc->hw_desc[--num]; + write_desc_llp(hw_desc, llp | lms); + llp = hw_desc->llp; + } while (num); - return vchan_tx_prep(&chan->vc, &first->vd, flags); + return vchan_tx_prep(&chan->vc, &desc->vd, flags); err_desc_get: - if (first) - axi_desc_put(first); + if (desc) + axi_desc_put(desc); return NULL; } +static int dw_axi_dma_chan_slave_config(struct dma_chan *dchan, + struct dma_slave_config *config) +{ + struct axi_dma_chan *chan = dchan_to_axi_dma_chan(dchan); + + memcpy(&chan->config, config, sizeof(*config)); + + return 0; +} + static void axi_chan_dump_lli(struct axi_dma_chan *chan, - struct axi_dma_desc *desc) + struct axi_dma_hw_desc *desc) { dev_err(dchan2dev(&chan->vc.chan), "SAR: 0x%llx DAR: 0x%llx LLP: 0x%llx BTS 0x%x CTL: 0x%x:%08x", - le64_to_cpu(desc->lli.sar), - le64_to_cpu(desc->lli.dar), - le64_to_cpu(desc->lli.llp), - le32_to_cpu(desc->lli.block_ts_lo), - le32_to_cpu(desc->lli.ctl_hi), - le32_to_cpu(desc->lli.ctl_lo)); + le64_to_cpu(desc->lli->sar), + le64_to_cpu(desc->lli->dar), + le64_to_cpu(desc->lli->llp), + le32_to_cpu(desc->lli->block_ts_lo), + le32_to_cpu(desc->lli->ctl_hi), + le32_to_cpu(desc->lli->ctl_lo)); } static void axi_chan_list_dump_lli(struct axi_dma_chan *chan, struct axi_dma_desc *desc_head) { - struct axi_dma_desc *desc; + int count = atomic_read(&chan->descs_allocated); + int i; - axi_chan_dump_lli(chan, desc_head); - list_for_each_entry(desc, &desc_head->xfer_list, xfer_list) - axi_chan_dump_lli(chan, desc); + for (i = 0; i < count; i++) + axi_chan_dump_lli(chan, &desc_head->hw_desc[i]); } static noinline void axi_chan_handle_err(struct axi_dma_chan *chan, u32 status) @@ -570,8 +999,13 @@ static noinline void axi_chan_handle_err(struct axi_dma_chan *chan, u32 status) static void axi_chan_block_xfer_complete(struct axi_dma_chan *chan) { + int count = atomic_read(&chan->descs_allocated); + struct axi_dma_hw_desc *hw_desc; + struct axi_dma_desc *desc; struct virt_dma_desc *vd; unsigned long flags; + u64 llp; + int i; spin_lock_irqsave(&chan->vc.lock, flags); if (unlikely(axi_chan_is_hw_enable(chan))) { @@ -582,12 +1016,34 @@ static void axi_chan_block_xfer_complete(struct axi_dma_chan *chan) /* The completed descriptor currently is in the head of vc list */ vd = vchan_next_desc(&chan->vc); - /* Remove the completed descriptor from issued list before completing */ - list_del(&vd->node); - vchan_cookie_complete(vd); - /* Submit queued descriptors after processing the completed ones */ - axi_chan_start_first_queued(chan); + if (chan->cyclic) { + desc = vd_to_axi_desc(vd); + if (desc) { + llp = lo_hi_readq(chan->chan_regs + CH_LLP); + for (i = 0; i < count; i++) { + hw_desc = &desc->hw_desc[i]; + if (hw_desc->llp == llp) { + axi_chan_irq_clear(chan, hw_desc->lli->status_lo); + hw_desc->lli->ctl_hi |= CH_CTL_H_LLI_VALID; + desc->completed_blocks = i; + + if (((hw_desc->len * (i + 1)) % desc->period_len) == 0) + vchan_cyclic_callback(vd); + break; + } + } + + axi_chan_enable(chan); + } + } else { + /* Remove the completed descriptor from issued list before completing */ + list_del(&vd->node); + vchan_cookie_complete(vd); + + /* Submit queued descriptors after processing the completed ones */ + axi_chan_start_first_queued(chan); + } spin_unlock_irqrestore(&chan->vc.lock, flags); } @@ -627,15 +1083,31 @@ static irqreturn_t dw_axi_dma_interrupt(int irq, void *dev_id) static int dma_chan_terminate_all(struct dma_chan *dchan) { struct axi_dma_chan *chan = dchan_to_axi_dma_chan(dchan); + u32 chan_active = BIT(chan->id) << DMAC_CHAN_EN_SHIFT; unsigned long flags; + u32 val; + int ret; LIST_HEAD(head); - spin_lock_irqsave(&chan->vc.lock, flags); - axi_chan_disable(chan); + ret = readl_poll_timeout_atomic(chan->chip->regs + DMAC_CHEN, val, + !(val & chan_active), 1000, 10000); + if (ret == -ETIMEDOUT) + dev_warn(dchan2dev(dchan), + "%s failed to stop\n", axi_chan_name(chan)); + + if (chan->direction != DMA_MEM_TO_MEM) + dw_axi_dma_set_hw_channel(chan->chip, + chan->hw_handshake_num, false); + if (chan->direction == DMA_MEM_TO_DEV) + dw_axi_dma_set_byte_halfword(chan, false); + + spin_lock_irqsave(&chan->vc.lock, flags); + vchan_get_all_descriptors(&chan->vc, &head); + chan->cyclic = false; spin_unlock_irqrestore(&chan->vc.lock, flags); vchan_dma_desc_free_list(&chan->vc, &head); @@ -746,6 +1218,22 @@ static int __maybe_unused axi_dma_runtime_resume(struct device *dev) return axi_dma_resume(chip); } +static struct dma_chan *dw_axi_dma_of_xlate(struct of_phandle_args *dma_spec, + struct of_dma *ofdma) +{ + struct dw_axi_dma *dw = ofdma->of_dma_data; + struct axi_dma_chan *chan; + struct dma_chan *dchan; + + dchan = dma_get_any_slave_channel(&dw->dma); + if (!dchan) + return NULL; + + chan = dchan_to_axi_dma_chan(dchan); + chan->hw_handshake_num = dma_spec->args[0]; + return dchan; +} + static int parse_device_properties(struct axi_dma_chip *chip) { struct device *dev = chip->dev; @@ -816,6 +1304,7 @@ static int parse_device_properties(struct axi_dma_chip *chip) static int dw_probe(struct platform_device *pdev) { + struct device_node *node = pdev->dev.of_node; struct axi_dma_chip *chip; struct resource *mem; struct dw_axi_dma *dw; @@ -848,6 +1337,12 @@ static int dw_probe(struct platform_device *pdev) if (IS_ERR(chip->regs)) return PTR_ERR(chip->regs); + if (of_device_is_compatible(node, "intel,kmb-axi-dma")) { + chip->apb_regs = devm_platform_ioremap_resource(pdev, 1); + if (IS_ERR(chip->apb_regs)) + return PTR_ERR(chip->apb_regs); + } + chip->core_clk = devm_clk_get(chip->dev, "core-clk"); if (IS_ERR(chip->core_clk)) return PTR_ERR(chip->core_clk); @@ -870,13 +1365,6 @@ static int dw_probe(struct platform_device *pdev) if (ret) return ret; - /* Lli address must be aligned to a 64-byte boundary */ - dw->desc_pool = dmam_pool_create(KBUILD_MODNAME, chip->dev, - sizeof(struct axi_dma_desc), 64, 0); - if (!dw->desc_pool) { - dev_err(chip->dev, "No memory for descriptors dma pool\n"); - return -ENOMEM; - } INIT_LIST_HEAD(&dw->dma.channels); for (i = 0; i < hdata->nr_channels; i++) { @@ -893,13 +1381,16 @@ static int dw_probe(struct platform_device *pdev) /* Set capabilities */ dma_cap_set(DMA_MEMCPY, dw->dma.cap_mask); + dma_cap_set(DMA_SLAVE, dw->dma.cap_mask); + dma_cap_set(DMA_CYCLIC, dw->dma.cap_mask); /* DMA capabilities */ dw->dma.chancnt = hdata->nr_channels; dw->dma.src_addr_widths = AXI_DMA_BUSWIDTHS; dw->dma.dst_addr_widths = AXI_DMA_BUSWIDTHS; dw->dma.directions = BIT(DMA_MEM_TO_MEM); - dw->dma.residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR; + dw->dma.directions |= BIT(DMA_MEM_TO_DEV) | BIT(DMA_DEV_TO_MEM); + dw->dma.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; dw->dma.dev = chip->dev; dw->dma.device_tx_status = dma_chan_tx_status; @@ -912,7 +1403,18 @@ static int dw_probe(struct platform_device *pdev) dw->dma.device_free_chan_resources = dma_chan_free_chan_resources; dw->dma.device_prep_dma_memcpy = dma_chan_prep_dma_memcpy; + dw->dma.device_synchronize = dw_axi_dma_synchronize; + dw->dma.device_config = dw_axi_dma_chan_slave_config; + dw->dma.device_prep_slave_sg = dw_axi_dma_chan_prep_slave_sg; + dw->dma.device_prep_dma_cyclic = dw_axi_dma_chan_prep_cyclic; + /* + * Synopsis DesignWare AxiDMA datasheet mentioned Maximum + * supported blocks is 1024. Device register width is 4 bytes. + * Therefore, set constraint to 1024 * 4. + */ + dw->dma.dev->dma_parms = &dw->dma_parms; + dma_set_max_seg_size(&pdev->dev, MAX_BLOCK_SIZE); platform_set_drvdata(pdev, chip); pm_runtime_enable(chip->dev); @@ -935,6 +1437,13 @@ static int dw_probe(struct platform_device *pdev) if (ret) goto err_pm_disable; + /* Register with OF helpers for DMA lookups */ + ret = of_dma_controller_register(pdev->dev.of_node, + dw_axi_dma_of_xlate, dw); + if (ret < 0) + dev_warn(&pdev->dev, + "Failed to register OF DMA controller, fallback to MEM_TO_MEM mode\n"); + dev_info(chip->dev, "DesignWare AXI DMA Controller, %d channels\n", dw->hdata->nr_channels); @@ -968,6 +1477,8 @@ static int dw_remove(struct platform_device *pdev) devm_free_irq(chip->dev, chip->irq, chip); + of_dma_controller_free(chip->dev->of_node); + list_for_each_entry_safe(chan, _chan, &dw->dma.channels, vc.chan.device_node) { list_del(&chan->vc.chan.device_node); @@ -983,6 +1494,7 @@ static const struct dev_pm_ops dw_axi_dma_pm_ops = { static const struct of_device_id dw_dma_of_id_table[] = { { .compatible = "snps,axi-dma-1.01a" }, + { .compatible = "intel,kmb-axi-dma" }, {} }; MODULE_DEVICE_TABLE(of, dw_dma_of_id_table); diff --git a/drivers/dma/dw-axi-dmac/dw-axi-dmac.h b/drivers/dma/dw-axi-dmac/dw-axi-dmac.h index 18b6014cf9b40bde5ae17bff2a17e929467ab7e6..b69897887c76541d78ffcbb450dc20824749958c 100644 --- a/drivers/dma/dw-axi-dmac/dw-axi-dmac.h +++ b/drivers/dma/dw-axi-dmac/dw-axi-dmac.h @@ -37,10 +37,16 @@ struct axi_dma_chan { struct axi_dma_chip *chip; void __iomem *chan_regs; u8 id; + u8 hw_handshake_num; atomic_t descs_allocated; + struct dma_pool *desc_pool; struct virt_dma_chan vc; + struct axi_dma_desc *desc; + struct dma_slave_config config; + enum dma_transfer_direction direction; + bool cyclic; /* these other elements are all protected by vc.lock */ bool is_paused; }; @@ -48,7 +54,7 @@ struct axi_dma_chan { struct dw_axi_dma { struct dma_device dma; struct dw_axi_dma_hcfg *hdata; - struct dma_pool *desc_pool; + struct device_dma_parameters dma_parms; /* channels */ struct axi_dma_chan *chan; @@ -58,6 +64,7 @@ struct axi_dma_chip { struct device *dev; int irq; void __iomem *regs; + void __iomem *apb_regs; struct clk *core_clk; struct clk *cfgr_clk; struct dw_axi_dma *dw; @@ -80,12 +87,20 @@ struct __packed axi_dma_lli { __le32 reserved_hi; }; +struct axi_dma_hw_desc { + struct axi_dma_lli *lli; + dma_addr_t llp; + u32 len; +}; + struct axi_dma_desc { - struct axi_dma_lli lli; + struct axi_dma_hw_desc *hw_desc; struct virt_dma_desc vd; struct axi_dma_chan *chan; - struct list_head xfer_list; + u32 completed_blocks; + u32 length; + u32 period_len; }; static inline struct device *dchan2dev(struct dma_chan *dchan) @@ -157,6 +172,19 @@ static inline struct axi_dma_chan *dchan_to_axi_dma_chan(struct dma_chan *dchan) #define CH_INTSIGNAL_ENA 0x090 /* R/W Chan Interrupt Signal Enable */ #define CH_INTCLEAR 0x098 /* W Chan Interrupt Clear */ +/* These Apb registers are used by Intel KeemBay SoC */ +#define DMAC_APB_CFG 0x000 /* DMAC Apb Configuration Register */ +#define DMAC_APB_STAT 0x004 /* DMAC Apb Status Register */ +#define DMAC_APB_DEBUG_STAT_0 0x008 /* DMAC Apb Debug Status Register 0 */ +#define DMAC_APB_DEBUG_STAT_1 0x00C /* DMAC Apb Debug Status Register 1 */ +#define DMAC_APB_HW_HS_SEL_0 0x010 /* DMAC Apb HW HS register 0 */ +#define DMAC_APB_HW_HS_SEL_1 0x014 /* DMAC Apb HW HS register 1 */ +#define DMAC_APB_LPI 0x018 /* DMAC Apb Low Power Interface Reg */ +#define DMAC_APB_BYTE_WR_CH_EN 0x01C /* DMAC Apb Byte Write Enable */ +#define DMAC_APB_HALFWORD_WR_CH_EN 0x020 /* DMAC Halfword write enables */ + +#define UNUSED_CHANNEL 0x3F /* Set unused DMA channel to 0x3F */ +#define MAX_BLOCK_SIZE 0x1000 /* 1024 blocks * 4 bytes data width */ /* DMAC_CFG */ #define DMAC_EN_POS 0 diff --git a/drivers/dma/fsldma.c b/drivers/dma/fsldma.c index 0feb323bae1e3eb711888f07422bf4023dfdbe6d..f8459cc5315dfa49c079b578b94c1bf06acc5e8d 100644 --- a/drivers/dma/fsldma.c +++ b/drivers/dma/fsldma.c @@ -1214,6 +1214,7 @@ static int fsldma_of_probe(struct platform_device *op) { struct fsldma_device *fdev; struct device_node *child; + unsigned int i; int err; fdev = kzalloc(sizeof(*fdev), GFP_KERNEL); @@ -1292,6 +1293,10 @@ static int fsldma_of_probe(struct platform_device *op) return 0; out_free_fdev: + for (i = 0; i < FSL_DMA_MAX_CHANS_PER_DEVICE; i++) { + if (fdev->chan[i]) + fsl_dma_chan_remove(fdev->chan[i]); + } irq_dispose_mapping(fdev->irq); iounmap(fdev->regs); out_free: @@ -1314,6 +1319,7 @@ static int fsldma_of_remove(struct platform_device *op) if (fdev->chan[i]) fsl_dma_chan_remove(fdev->chan[i]); } + irq_dispose_mapping(fdev->irq); iounmap(fdev->regs); kfree(fdev); diff --git a/drivers/dma/hsu/pci.c b/drivers/dma/hsu/pci.c index 07cc7320a614f0e963c024a5bb98e8bc195c1550..9045a6f7f5893d38b7138ec0527d2a749f984c69 100644 --- a/drivers/dma/hsu/pci.c +++ b/drivers/dma/hsu/pci.c @@ -26,22 +26,12 @@ static irqreturn_t hsu_pci_irq(int irq, void *dev) { struct hsu_dma_chip *chip = dev; - struct pci_dev *pdev = to_pci_dev(chip->dev); u32 dmaisr; u32 status; unsigned short i; int ret = 0; int err; - /* - * On Intel Tangier B0 and Anniedale the interrupt line, disregarding - * to have different numbers, is shared between HSU DMA and UART IPs. - * Thus on such SoCs we are expecting that IRQ handler is called in - * UART driver only. - */ - if (pdev->device == PCI_DEVICE_ID_INTEL_MRFLD_HSU_DMA) - return IRQ_HANDLED; - dmaisr = readl(chip->regs + HSU_PCI_DMAISR); for (i = 0; i < chip->hsu->nr_channels; i++) { if (dmaisr & 0x1) { @@ -105,6 +95,17 @@ static int hsu_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (ret) goto err_register_irq; + /* + * On Intel Tangier B0 and Anniedale the interrupt line, disregarding + * to have different numbers, is shared between HSU DMA and UART IPs. + * Thus on such SoCs we are expecting that IRQ handler is called in + * UART driver only. Instead of handling the spurious interrupt + * from HSU DMA here and waste CPU time and delay HSU UART interrupt + * handling, disable the interrupt entirely. + */ + if (pdev->device == PCI_DEVICE_ID_INTEL_MRFLD_HSU_DMA) + disable_irq_nosync(chip->irq); + pci_set_drvdata(pdev, chip); return 0; diff --git a/drivers/dma/idxd/dma.c b/drivers/dma/idxd/dma.c index 71fd6e4c42cd7456cd61a81fd1198a0200d92752..a15e50126434e34e009da2cc4b33563dcc2f30d2 100644 --- a/drivers/dma/idxd/dma.c +++ b/drivers/dma/idxd/dma.c @@ -165,6 +165,7 @@ int idxd_register_dma_device(struct idxd_device *idxd) INIT_LIST_HEAD(&dma->channels); dma->dev = &idxd->pdev->dev; + dma_cap_set(DMA_PRIVATE, dma->cap_mask); dma_cap_set(DMA_COMPLETION_NO_ORDER, dma->cap_mask); dma->device_release = idxd_dma_release; diff --git a/drivers/dma/idxd/init.c b/drivers/dma/idxd/init.c index fa04acd5582a0a13b09e37933166eba68b35662b..085a0c3b62c68190d86ceb75f355ec7a6e3c910a 100644 --- a/drivers/dma/idxd/init.c +++ b/drivers/dma/idxd/init.c @@ -26,12 +26,16 @@ MODULE_VERSION(IDXD_DRIVER_VERSION); MODULE_LICENSE("GPL v2"); MODULE_AUTHOR("Intel Corporation"); +static bool sva = true; +module_param(sva, bool, 0644); +MODULE_PARM_DESC(sva, "Toggle SVA support on/off"); + #define DRV_NAME "idxd" bool support_enqcmd; static struct idr idxd_idrs[IDXD_TYPE_MAX]; -static struct mutex idxd_idr_lock; +static DEFINE_MUTEX(idxd_idr_lock); static struct pci_device_id idxd_pci_tbl[] = { /* DSA ver 1.0 platforms */ @@ -341,12 +345,14 @@ static int idxd_probe(struct idxd_device *idxd) dev_dbg(dev, "IDXD reset complete\n"); - if (IS_ENABLED(CONFIG_INTEL_IDXD_SVM)) { + if (IS_ENABLED(CONFIG_INTEL_IDXD_SVM) && sva) { rc = idxd_enable_system_pasid(idxd); if (rc < 0) dev_warn(dev, "Failed to enable PASID. No SVA support: %d\n", rc); else set_bit(IDXD_FLAG_PASID_ENABLED, &idxd->flags); + } else if (!sva) { + dev_warn(dev, "User forced SVA off via module param.\n"); } idxd_read_caps(idxd); @@ -547,7 +553,6 @@ static int __init idxd_init_module(void) else support_enqcmd = true; - mutex_init(&idxd_idr_lock); for (i = 0; i < IDXD_TYPE_MAX; i++) idr_init(&idxd_idrs[i]); diff --git a/drivers/dma/imx-sdma.c b/drivers/dma/imx-sdma.c index 41ba21eea7c8b2fee9125f27fa47b2fab7cc5cc7..d5590c08db51ec24d9bb25e92e2d455ccdc0e5fd 100644 --- a/drivers/dma/imx-sdma.c +++ b/drivers/dma/imx-sdma.c @@ -1952,8 +1952,6 @@ static struct dma_chan *sdma_xlate(struct of_phandle_args *dma_spec, static int sdma_probe(struct platform_device *pdev) { - const struct of_device_id *of_id = - of_match_device(sdma_dt_ids, &pdev->dev); struct device_node *np = pdev->dev.of_node; struct device_node *spba_bus; const char *fw_name; @@ -1961,17 +1959,9 @@ static int sdma_probe(struct platform_device *pdev) int irq; struct resource *iores; struct resource spba_res; - struct sdma_platform_data *pdata = dev_get_platdata(&pdev->dev); int i; struct sdma_engine *sdma; s32 *saddr_arr; - const struct sdma_driver_data *drvdata = NULL; - - drvdata = of_id->data; - if (!drvdata) { - dev_err(&pdev->dev, "unable to find driver data\n"); - return -EINVAL; - } ret = dma_coerce_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); if (ret) @@ -1984,7 +1974,7 @@ static int sdma_probe(struct platform_device *pdev) spin_lock_init(&sdma->channel_0_lock); sdma->dev = &pdev->dev; - sdma->drvdata = drvdata; + sdma->drvdata = of_device_get_match_data(sdma->dev); irq = platform_get_irq(pdev, 0); if (irq < 0) @@ -2063,8 +2053,6 @@ static int sdma_probe(struct platform_device *pdev) if (sdma->drvdata->script_addrs) sdma_add_scripts(sdma, sdma->drvdata->script_addrs); - if (pdata && pdata->script_addrs) - sdma_add_scripts(sdma, pdata->script_addrs); sdma->dma_device.dev = &pdev->dev; @@ -2110,30 +2098,18 @@ static int sdma_probe(struct platform_device *pdev) } /* - * Kick off firmware loading as the very last step: - * attempt to load firmware only if we're not on the error path, because - * the firmware callback requires a fully functional and allocated sdma - * instance. + * Because that device tree does not encode ROM script address, + * the RAM script in firmware is mandatory for device tree + * probe, otherwise it fails. */ - if (pdata) { - ret = sdma_get_firmware(sdma, pdata->fw_name); - if (ret) - dev_warn(&pdev->dev, "failed to get firmware from platform data\n"); + ret = of_property_read_string(np, "fsl,sdma-ram-script-name", + &fw_name); + if (ret) { + dev_warn(&pdev->dev, "failed to get firmware name\n"); } else { - /* - * Because that device tree does not encode ROM script address, - * the RAM script in firmware is mandatory for device tree - * probe, otherwise it fails. - */ - ret = of_property_read_string(np, "fsl,sdma-ram-script-name", - &fw_name); - if (ret) { - dev_warn(&pdev->dev, "failed to get firmware name\n"); - } else { - ret = sdma_get_firmware(sdma, fw_name); - if (ret) - dev_warn(&pdev->dev, "failed to get firmware from device tree\n"); - } + ret = sdma_get_firmware(sdma, fw_name); + if (ret) + dev_warn(&pdev->dev, "failed to get firmware from device tree\n"); } return 0; diff --git a/drivers/dma/lgm/Kconfig b/drivers/dma/lgm/Kconfig new file mode 100644 index 0000000000000000000000000000000000000000..9194330ed0f2534fe1e6902b7ab719b6bbfa0895 --- /dev/null +++ b/drivers/dma/lgm/Kconfig @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: GPL-2.0-only +config INTEL_LDMA + bool "Lightning Mountain centralized DMA controllers" + depends on X86 || COMPILE_TEST + select DMA_ENGINE + select DMA_VIRTUAL_CHANNELS + help + Enable support for Intel Lightning Mountain SOC DMA controllers. + These controllers provide DMA capabilities for a variety of on-chip + devices such as HSNAND and GSWIP (Gigabit Switch IP). diff --git a/drivers/dma/lgm/Makefile b/drivers/dma/lgm/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..f318a8eff464b53a0f0976bb876482a059571650 --- /dev/null +++ b/drivers/dma/lgm/Makefile @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_INTEL_LDMA) += lgm-dma.o diff --git a/drivers/dma/lgm/lgm-dma.c b/drivers/dma/lgm/lgm-dma.c new file mode 100644 index 0000000000000000000000000000000000000000..efe8bd3a0e2aa5c14a0f49fd9154d0930fd56196 --- /dev/null +++ b/drivers/dma/lgm/lgm-dma.c @@ -0,0 +1,1739 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Lightning Mountain centralized DMA controller driver + * + * Copyright (c) 2016 - 2020 Intel Corporation. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../dmaengine.h" +#include "../virt-dma.h" + +#define DRIVER_NAME "lgm-dma" + +#define DMA_ID 0x0008 +#define DMA_ID_REV GENMASK(7, 0) +#define DMA_ID_PNR GENMASK(19, 16) +#define DMA_ID_CHNR GENMASK(26, 20) +#define DMA_ID_DW_128B BIT(27) +#define DMA_ID_AW_36B BIT(28) +#define DMA_VER32 0x32 +#define DMA_VER31 0x31 +#define DMA_VER22 0x0A + +#define DMA_CTRL 0x0010 +#define DMA_CTRL_RST BIT(0) +#define DMA_CTRL_DSRAM_PATH BIT(1) +#define DMA_CTRL_DBURST_WR BIT(3) +#define DMA_CTRL_VLD_DF_ACK BIT(4) +#define DMA_CTRL_CH_FL BIT(6) +#define DMA_CTRL_DS_FOD BIT(7) +#define DMA_CTRL_DRB BIT(8) +#define DMA_CTRL_ENBE BIT(9) +#define DMA_CTRL_DESC_TMOUT_CNT_V31 GENMASK(27, 16) +#define DMA_CTRL_DESC_TMOUT_EN_V31 BIT(30) +#define DMA_CTRL_PKTARB BIT(31) + +#define DMA_CPOLL 0x0014 +#define DMA_CPOLL_CNT GENMASK(15, 4) +#define DMA_CPOLL_EN BIT(31) + +#define DMA_CS 0x0018 +#define DMA_CS_MASK GENMASK(5, 0) + +#define DMA_CCTRL 0x001C +#define DMA_CCTRL_ON BIT(0) +#define DMA_CCTRL_RST BIT(1) +#define DMA_CCTRL_CH_POLL_EN BIT(2) +#define DMA_CCTRL_CH_ABC BIT(3) /* Adaptive Burst Chop */ +#define DMA_CDBA_MSB GENMASK(7, 4) +#define DMA_CCTRL_DIR_TX BIT(8) +#define DMA_CCTRL_CLASS GENMASK(11, 9) +#define DMA_CCTRL_CLASSH GENMASK(19, 18) +#define DMA_CCTRL_WR_NP_EN BIT(21) +#define DMA_CCTRL_PDEN BIT(23) +#define DMA_MAX_CLASS (SZ_32 - 1) + +#define DMA_CDBA 0x0020 +#define DMA_CDLEN 0x0024 +#define DMA_CIS 0x0028 +#define DMA_CIE 0x002C +#define DMA_CI_EOP BIT(1) +#define DMA_CI_DUR BIT(2) +#define DMA_CI_DESCPT BIT(3) +#define DMA_CI_CHOFF BIT(4) +#define DMA_CI_RDERR BIT(5) +#define DMA_CI_ALL \ + (DMA_CI_EOP | DMA_CI_DUR | DMA_CI_DESCPT | DMA_CI_CHOFF | DMA_CI_RDERR) + +#define DMA_PS 0x0040 +#define DMA_PCTRL 0x0044 +#define DMA_PCTRL_RXBL16 BIT(0) +#define DMA_PCTRL_TXBL16 BIT(1) +#define DMA_PCTRL_RXBL GENMASK(3, 2) +#define DMA_PCTRL_RXBL_8 3 +#define DMA_PCTRL_TXBL GENMASK(5, 4) +#define DMA_PCTRL_TXBL_8 3 +#define DMA_PCTRL_PDEN BIT(6) +#define DMA_PCTRL_RXBL32 BIT(7) +#define DMA_PCTRL_RXENDI GENMASK(9, 8) +#define DMA_PCTRL_TXENDI GENMASK(11, 10) +#define DMA_PCTRL_TXBL32 BIT(15) +#define DMA_PCTRL_MEM_FLUSH BIT(16) + +#define DMA_IRNEN1 0x00E8 +#define DMA_IRNCR1 0x00EC +#define DMA_IRNEN 0x00F4 +#define DMA_IRNCR 0x00F8 +#define DMA_C_DP_TICK 0x100 +#define DMA_C_DP_TICK_TIKNARB GENMASK(15, 0) +#define DMA_C_DP_TICK_TIKARB GENMASK(31, 16) + +#define DMA_C_HDRM 0x110 +/* + * If header mode is set in DMA descriptor, + * If bit 30 is disabled, HDR_LEN must be configured according to channel + * requirement. + * If bit 30 is enabled(checksum with heade mode), HDR_LEN has no need to + * be configured. It will enable check sum for switch + * If header mode is not set in DMA descriptor, + * This register setting doesn't matter + */ +#define DMA_C_HDRM_HDR_SUM BIT(30) + +#define DMA_C_BOFF 0x120 +#define DMA_C_BOFF_BOF_LEN GENMASK(7, 0) +#define DMA_C_BOFF_EN BIT(31) + +#define DMA_ORRC 0x190 +#define DMA_ORRC_ORRCNT GENMASK(8, 4) +#define DMA_ORRC_EN BIT(31) + +#define DMA_C_ENDIAN 0x200 +#define DMA_C_END_DATAENDI GENMASK(1, 0) +#define DMA_C_END_DE_EN BIT(7) +#define DMA_C_END_DESENDI GENMASK(9, 8) +#define DMA_C_END_DES_EN BIT(16) + +/* DMA controller capability */ +#define DMA_ADDR_36BIT BIT(0) +#define DMA_DATA_128BIT BIT(1) +#define DMA_CHAN_FLOW_CTL BIT(2) +#define DMA_DESC_FOD BIT(3) +#define DMA_DESC_IN_SRAM BIT(4) +#define DMA_EN_BYTE_EN BIT(5) +#define DMA_DBURST_WR BIT(6) +#define DMA_VALID_DESC_FETCH_ACK BIT(7) +#define DMA_DFT_DRB BIT(8) + +#define DMA_ORRC_MAX_CNT (SZ_32 - 1) +#define DMA_DFT_POLL_CNT SZ_4 +#define DMA_DFT_BURST_V22 SZ_2 +#define DMA_BURSTL_8DW SZ_8 +#define DMA_BURSTL_16DW SZ_16 +#define DMA_BURSTL_32DW SZ_32 +#define DMA_DFT_BURST DMA_BURSTL_16DW +#define DMA_MAX_DESC_NUM (SZ_8K - 1) +#define DMA_CHAN_BOFF_MAX (SZ_256 - 1) +#define DMA_DFT_ENDIAN 0 + +#define DMA_DFT_DESC_TCNT 50 +#define DMA_HDR_LEN_MAX (SZ_16K - 1) + +/* DMA flags */ +#define DMA_TX_CH BIT(0) +#define DMA_RX_CH BIT(1) +#define DEVICE_ALLOC_DESC BIT(2) +#define CHAN_IN_USE BIT(3) +#define DMA_HW_DESC BIT(4) + +/* Descriptor fields */ +#define DESC_DATA_LEN GENMASK(15, 0) +#define DESC_BYTE_OFF GENMASK(25, 23) +#define DESC_EOP BIT(28) +#define DESC_SOP BIT(29) +#define DESC_C BIT(30) +#define DESC_OWN BIT(31) + +#define DMA_CHAN_RST 1 +#define DMA_MAX_SIZE (BIT(16) - 1) +#define MAX_LOWER_CHANS 32 +#define MASK_LOWER_CHANS GENMASK(4, 0) +#define DMA_OWN 1 +#define HIGH_4_BITS GENMASK(3, 0) +#define DMA_DFT_DESC_NUM 1 +#define DMA_PKT_DROP_DIS 0 + +enum ldma_chan_on_off { + DMA_CH_OFF = 0, + DMA_CH_ON = 1, +}; + +enum { + DMA_TYPE_TX = 0, + DMA_TYPE_RX, + DMA_TYPE_MCPY, +}; + +struct ldma_dev; +struct ldma_port; + +struct ldma_chan { + struct virt_dma_chan vchan; + struct ldma_port *port; /* back pointer */ + char name[8]; /* Channel name */ + int nr; /* Channel id in hardware */ + u32 flags; /* central way or channel based way */ + enum ldma_chan_on_off onoff; + dma_addr_t desc_phys; + void *desc_base; /* Virtual address */ + u32 desc_cnt; /* Number of descriptors */ + int rst; + u32 hdrm_len; + bool hdrm_csum; + u32 boff_len; + u32 data_endian; + u32 desc_endian; + bool pden; + bool desc_rx_np; + bool data_endian_en; + bool desc_endian_en; + bool abc_en; + bool desc_init; + struct dma_pool *desc_pool; /* Descriptors pool */ + u32 desc_num; + struct dw2_desc_sw *ds; + struct work_struct work; + struct dma_slave_config config; +}; + +struct ldma_port { + struct ldma_dev *ldev; /* back pointer */ + u32 portid; + u32 rxbl; + u32 txbl; + u32 rxendi; + u32 txendi; + u32 pkt_drop; +}; + +/* Instance specific data */ +struct ldma_inst_data { + bool desc_in_sram; + bool chan_fc; + bool desc_fod; /* Fetch On Demand */ + bool valid_desc_fetch_ack; + u32 orrc; /* Outstanding read count */ + const char *name; + u32 type; +}; + +struct ldma_dev { + struct device *dev; + void __iomem *base; + struct reset_control *rst; + struct clk *core_clk; + struct dma_device dma_dev; + u32 ver; + int irq; + struct ldma_port *ports; + struct ldma_chan *chans; /* channel list on this DMA or port */ + spinlock_t dev_lock; /* Controller register exclusive */ + u32 chan_nrs; + u32 port_nrs; + u32 channels_mask; + u32 flags; + u32 pollcnt; + const struct ldma_inst_data *inst; + struct workqueue_struct *wq; +}; + +struct dw2_desc { + u32 field; + u32 addr; +} __packed __aligned(8); + +struct dw2_desc_sw { + struct virt_dma_desc vdesc; + struct ldma_chan *chan; + dma_addr_t desc_phys; + size_t desc_cnt; + size_t size; + struct dw2_desc *desc_hw; +}; + +static inline void +ldma_update_bits(struct ldma_dev *d, u32 mask, u32 val, u32 ofs) +{ + u32 old_val, new_val; + + old_val = readl(d->base + ofs); + new_val = (old_val & ~mask) | (val & mask); + + if (new_val != old_val) + writel(new_val, d->base + ofs); +} + +static inline struct ldma_chan *to_ldma_chan(struct dma_chan *chan) +{ + return container_of(chan, struct ldma_chan, vchan.chan); +} + +static inline struct ldma_dev *to_ldma_dev(struct dma_device *dma_dev) +{ + return container_of(dma_dev, struct ldma_dev, dma_dev); +} + +static inline struct dw2_desc_sw *to_lgm_dma_desc(struct virt_dma_desc *vdesc) +{ + return container_of(vdesc, struct dw2_desc_sw, vdesc); +} + +static inline bool ldma_chan_tx(struct ldma_chan *c) +{ + return !!(c->flags & DMA_TX_CH); +} + +static inline bool ldma_chan_is_hw_desc(struct ldma_chan *c) +{ + return !!(c->flags & DMA_HW_DESC); +} + +static void ldma_dev_reset(struct ldma_dev *d) + +{ + unsigned long flags; + + spin_lock_irqsave(&d->dev_lock, flags); + ldma_update_bits(d, DMA_CTRL_RST, DMA_CTRL_RST, DMA_CTRL); + spin_unlock_irqrestore(&d->dev_lock, flags); +} + +static void ldma_dev_pkt_arb_cfg(struct ldma_dev *d, bool enable) +{ + unsigned long flags; + u32 mask = DMA_CTRL_PKTARB; + u32 val = enable ? DMA_CTRL_PKTARB : 0; + + spin_lock_irqsave(&d->dev_lock, flags); + ldma_update_bits(d, mask, val, DMA_CTRL); + spin_unlock_irqrestore(&d->dev_lock, flags); +} + +static void ldma_dev_sram_desc_cfg(struct ldma_dev *d, bool enable) +{ + unsigned long flags; + u32 mask = DMA_CTRL_DSRAM_PATH; + u32 val = enable ? DMA_CTRL_DSRAM_PATH : 0; + + spin_lock_irqsave(&d->dev_lock, flags); + ldma_update_bits(d, mask, val, DMA_CTRL); + spin_unlock_irqrestore(&d->dev_lock, flags); +} + +static void ldma_dev_chan_flow_ctl_cfg(struct ldma_dev *d, bool enable) +{ + unsigned long flags; + u32 mask, val; + + if (d->inst->type != DMA_TYPE_TX) + return; + + mask = DMA_CTRL_CH_FL; + val = enable ? DMA_CTRL_CH_FL : 0; + + spin_lock_irqsave(&d->dev_lock, flags); + ldma_update_bits(d, mask, val, DMA_CTRL); + spin_unlock_irqrestore(&d->dev_lock, flags); +} + +static void ldma_dev_global_polling_enable(struct ldma_dev *d) +{ + unsigned long flags; + u32 mask = DMA_CPOLL_EN | DMA_CPOLL_CNT; + u32 val = DMA_CPOLL_EN; + + val |= FIELD_PREP(DMA_CPOLL_CNT, d->pollcnt); + + spin_lock_irqsave(&d->dev_lock, flags); + ldma_update_bits(d, mask, val, DMA_CPOLL); + spin_unlock_irqrestore(&d->dev_lock, flags); +} + +static void ldma_dev_desc_fetch_on_demand_cfg(struct ldma_dev *d, bool enable) +{ + unsigned long flags; + u32 mask, val; + + if (d->inst->type == DMA_TYPE_MCPY) + return; + + mask = DMA_CTRL_DS_FOD; + val = enable ? DMA_CTRL_DS_FOD : 0; + + spin_lock_irqsave(&d->dev_lock, flags); + ldma_update_bits(d, mask, val, DMA_CTRL); + spin_unlock_irqrestore(&d->dev_lock, flags); +} + +static void ldma_dev_byte_enable_cfg(struct ldma_dev *d, bool enable) +{ + unsigned long flags; + u32 mask = DMA_CTRL_ENBE; + u32 val = enable ? DMA_CTRL_ENBE : 0; + + spin_lock_irqsave(&d->dev_lock, flags); + ldma_update_bits(d, mask, val, DMA_CTRL); + spin_unlock_irqrestore(&d->dev_lock, flags); +} + +static void ldma_dev_orrc_cfg(struct ldma_dev *d) +{ + unsigned long flags; + u32 val = 0; + u32 mask; + + if (d->inst->type == DMA_TYPE_RX) + return; + + mask = DMA_ORRC_EN | DMA_ORRC_ORRCNT; + if (d->inst->orrc > 0 && d->inst->orrc <= DMA_ORRC_MAX_CNT) + val = DMA_ORRC_EN | FIELD_PREP(DMA_ORRC_ORRCNT, d->inst->orrc); + + spin_lock_irqsave(&d->dev_lock, flags); + ldma_update_bits(d, mask, val, DMA_ORRC); + spin_unlock_irqrestore(&d->dev_lock, flags); +} + +static void ldma_dev_df_tout_cfg(struct ldma_dev *d, bool enable, int tcnt) +{ + u32 mask = DMA_CTRL_DESC_TMOUT_CNT_V31; + unsigned long flags; + u32 val; + + if (enable) + val = DMA_CTRL_DESC_TMOUT_EN_V31 | FIELD_PREP(DMA_CTRL_DESC_TMOUT_CNT_V31, tcnt); + else + val = 0; + + spin_lock_irqsave(&d->dev_lock, flags); + ldma_update_bits(d, mask, val, DMA_CTRL); + spin_unlock_irqrestore(&d->dev_lock, flags); +} + +static void ldma_dev_dburst_wr_cfg(struct ldma_dev *d, bool enable) +{ + unsigned long flags; + u32 mask, val; + + if (d->inst->type != DMA_TYPE_RX && d->inst->type != DMA_TYPE_MCPY) + return; + + mask = DMA_CTRL_DBURST_WR; + val = enable ? DMA_CTRL_DBURST_WR : 0; + + spin_lock_irqsave(&d->dev_lock, flags); + ldma_update_bits(d, mask, val, DMA_CTRL); + spin_unlock_irqrestore(&d->dev_lock, flags); +} + +static void ldma_dev_vld_fetch_ack_cfg(struct ldma_dev *d, bool enable) +{ + unsigned long flags; + u32 mask, val; + + if (d->inst->type != DMA_TYPE_TX) + return; + + mask = DMA_CTRL_VLD_DF_ACK; + val = enable ? DMA_CTRL_VLD_DF_ACK : 0; + + spin_lock_irqsave(&d->dev_lock, flags); + ldma_update_bits(d, mask, val, DMA_CTRL); + spin_unlock_irqrestore(&d->dev_lock, flags); +} + +static void ldma_dev_drb_cfg(struct ldma_dev *d, int enable) +{ + unsigned long flags; + u32 mask = DMA_CTRL_DRB; + u32 val = enable ? DMA_CTRL_DRB : 0; + + spin_lock_irqsave(&d->dev_lock, flags); + ldma_update_bits(d, mask, val, DMA_CTRL); + spin_unlock_irqrestore(&d->dev_lock, flags); +} + +static int ldma_dev_cfg(struct ldma_dev *d) +{ + bool enable; + + ldma_dev_pkt_arb_cfg(d, true); + ldma_dev_global_polling_enable(d); + + enable = !!(d->flags & DMA_DFT_DRB); + ldma_dev_drb_cfg(d, enable); + + enable = !!(d->flags & DMA_EN_BYTE_EN); + ldma_dev_byte_enable_cfg(d, enable); + + enable = !!(d->flags & DMA_CHAN_FLOW_CTL); + ldma_dev_chan_flow_ctl_cfg(d, enable); + + enable = !!(d->flags & DMA_DESC_FOD); + ldma_dev_desc_fetch_on_demand_cfg(d, enable); + + enable = !!(d->flags & DMA_DESC_IN_SRAM); + ldma_dev_sram_desc_cfg(d, enable); + + enable = !!(d->flags & DMA_DBURST_WR); + ldma_dev_dburst_wr_cfg(d, enable); + + enable = !!(d->flags & DMA_VALID_DESC_FETCH_ACK); + ldma_dev_vld_fetch_ack_cfg(d, enable); + + if (d->ver > DMA_VER22) { + ldma_dev_orrc_cfg(d); + ldma_dev_df_tout_cfg(d, true, DMA_DFT_DESC_TCNT); + } + + dev_dbg(d->dev, "%s Controller 0x%08x configuration done\n", + d->inst->name, readl(d->base + DMA_CTRL)); + + return 0; +} + +static int ldma_chan_cctrl_cfg(struct ldma_chan *c, u32 val) +{ + struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); + u32 class_low, class_high; + unsigned long flags; + u32 reg; + + spin_lock_irqsave(&d->dev_lock, flags); + ldma_update_bits(d, DMA_CS_MASK, c->nr, DMA_CS); + reg = readl(d->base + DMA_CCTRL); + /* Read from hardware */ + if (reg & DMA_CCTRL_DIR_TX) + c->flags |= DMA_TX_CH; + else + c->flags |= DMA_RX_CH; + + /* Keep the class value unchanged */ + class_low = FIELD_GET(DMA_CCTRL_CLASS, reg); + class_high = FIELD_GET(DMA_CCTRL_CLASSH, reg); + val &= ~DMA_CCTRL_CLASS; + val |= FIELD_PREP(DMA_CCTRL_CLASS, class_low); + val &= ~DMA_CCTRL_CLASSH; + val |= FIELD_PREP(DMA_CCTRL_CLASSH, class_high); + writel(val, d->base + DMA_CCTRL); + spin_unlock_irqrestore(&d->dev_lock, flags); + + return 0; +} + +static void ldma_chan_irq_init(struct ldma_chan *c) +{ + struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); + unsigned long flags; + u32 enofs, crofs; + u32 cn_bit; + + if (c->nr < MAX_LOWER_CHANS) { + enofs = DMA_IRNEN; + crofs = DMA_IRNCR; + } else { + enofs = DMA_IRNEN1; + crofs = DMA_IRNCR1; + } + + cn_bit = BIT(c->nr & MASK_LOWER_CHANS); + spin_lock_irqsave(&d->dev_lock, flags); + ldma_update_bits(d, DMA_CS_MASK, c->nr, DMA_CS); + + /* Clear all interrupts and disabled it */ + writel(0, d->base + DMA_CIE); + writel(DMA_CI_ALL, d->base + DMA_CIS); + + ldma_update_bits(d, cn_bit, 0, enofs); + writel(cn_bit, d->base + crofs); + spin_unlock_irqrestore(&d->dev_lock, flags); +} + +static void ldma_chan_set_class(struct ldma_chan *c, u32 val) +{ + struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); + u32 class_val; + + if (d->inst->type == DMA_TYPE_MCPY || val > DMA_MAX_CLASS) + return; + + /* 3 bits low */ + class_val = FIELD_PREP(DMA_CCTRL_CLASS, val & 0x7); + /* 2 bits high */ + class_val |= FIELD_PREP(DMA_CCTRL_CLASSH, (val >> 3) & 0x3); + + ldma_update_bits(d, DMA_CS_MASK, c->nr, DMA_CS); + ldma_update_bits(d, DMA_CCTRL_CLASS | DMA_CCTRL_CLASSH, class_val, + DMA_CCTRL); +} + +static int ldma_chan_on(struct ldma_chan *c) +{ + struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); + unsigned long flags; + + /* If descriptors not configured, not allow to turn on channel */ + if (WARN_ON(!c->desc_init)) + return -EINVAL; + + spin_lock_irqsave(&d->dev_lock, flags); + ldma_update_bits(d, DMA_CS_MASK, c->nr, DMA_CS); + ldma_update_bits(d, DMA_CCTRL_ON, DMA_CCTRL_ON, DMA_CCTRL); + spin_unlock_irqrestore(&d->dev_lock, flags); + + c->onoff = DMA_CH_ON; + + return 0; +} + +static int ldma_chan_off(struct ldma_chan *c) +{ + struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); + unsigned long flags; + u32 val; + int ret; + + spin_lock_irqsave(&d->dev_lock, flags); + ldma_update_bits(d, DMA_CS_MASK, c->nr, DMA_CS); + ldma_update_bits(d, DMA_CCTRL_ON, 0, DMA_CCTRL); + spin_unlock_irqrestore(&d->dev_lock, flags); + + ret = readl_poll_timeout_atomic(d->base + DMA_CCTRL, val, + !(val & DMA_CCTRL_ON), 0, 10000); + if (ret) + return ret; + + c->onoff = DMA_CH_OFF; + + return 0; +} + +static void ldma_chan_desc_hw_cfg(struct ldma_chan *c, dma_addr_t desc_base, + int desc_num) +{ + struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); + unsigned long flags; + + spin_lock_irqsave(&d->dev_lock, flags); + ldma_update_bits(d, DMA_CS_MASK, c->nr, DMA_CS); + writel(lower_32_bits(desc_base), d->base + DMA_CDBA); + + /* Higher 4 bits of 36 bit addressing */ + if (IS_ENABLED(CONFIG_64BIT)) { + u32 hi = upper_32_bits(desc_base) & HIGH_4_BITS; + + ldma_update_bits(d, DMA_CDBA_MSB, + FIELD_PREP(DMA_CDBA_MSB, hi), DMA_CCTRL); + } + writel(desc_num, d->base + DMA_CDLEN); + spin_unlock_irqrestore(&d->dev_lock, flags); + + c->desc_init = true; +} + +static struct dma_async_tx_descriptor * +ldma_chan_desc_cfg(struct dma_chan *chan, dma_addr_t desc_base, int desc_num) +{ + struct ldma_chan *c = to_ldma_chan(chan); + struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); + struct dma_async_tx_descriptor *tx; + struct dw2_desc_sw *ds; + + if (!desc_num) { + dev_err(d->dev, "Channel %d must allocate descriptor first\n", + c->nr); + return NULL; + } + + if (desc_num > DMA_MAX_DESC_NUM) { + dev_err(d->dev, "Channel %d descriptor number out of range %d\n", + c->nr, desc_num); + return NULL; + } + + ldma_chan_desc_hw_cfg(c, desc_base, desc_num); + + c->flags |= DMA_HW_DESC; + c->desc_cnt = desc_num; + c->desc_phys = desc_base; + + ds = kzalloc(sizeof(*ds), GFP_NOWAIT); + if (!ds) + return NULL; + + tx = &ds->vdesc.tx; + dma_async_tx_descriptor_init(tx, chan); + + return tx; +} + +static int ldma_chan_reset(struct ldma_chan *c) +{ + struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); + unsigned long flags; + u32 val; + int ret; + + ret = ldma_chan_off(c); + if (ret) + return ret; + + spin_lock_irqsave(&d->dev_lock, flags); + ldma_update_bits(d, DMA_CS_MASK, c->nr, DMA_CS); + ldma_update_bits(d, DMA_CCTRL_RST, DMA_CCTRL_RST, DMA_CCTRL); + spin_unlock_irqrestore(&d->dev_lock, flags); + + ret = readl_poll_timeout_atomic(d->base + DMA_CCTRL, val, + !(val & DMA_CCTRL_RST), 0, 10000); + if (ret) + return ret; + + c->rst = 1; + c->desc_init = false; + + return 0; +} + +static void ldma_chan_byte_offset_cfg(struct ldma_chan *c, u32 boff_len) +{ + struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); + u32 mask = DMA_C_BOFF_EN | DMA_C_BOFF_BOF_LEN; + u32 val; + + if (boff_len > 0 && boff_len <= DMA_CHAN_BOFF_MAX) + val = FIELD_PREP(DMA_C_BOFF_BOF_LEN, boff_len) | DMA_C_BOFF_EN; + else + val = 0; + + ldma_update_bits(d, DMA_CS_MASK, c->nr, DMA_CS); + ldma_update_bits(d, mask, val, DMA_C_BOFF); +} + +static void ldma_chan_data_endian_cfg(struct ldma_chan *c, bool enable, + u32 endian_type) +{ + struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); + u32 mask = DMA_C_END_DE_EN | DMA_C_END_DATAENDI; + u32 val; + + if (enable) + val = DMA_C_END_DE_EN | FIELD_PREP(DMA_C_END_DATAENDI, endian_type); + else + val = 0; + + ldma_update_bits(d, DMA_CS_MASK, c->nr, DMA_CS); + ldma_update_bits(d, mask, val, DMA_C_ENDIAN); +} + +static void ldma_chan_desc_endian_cfg(struct ldma_chan *c, bool enable, + u32 endian_type) +{ + struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); + u32 mask = DMA_C_END_DES_EN | DMA_C_END_DESENDI; + u32 val; + + if (enable) + val = DMA_C_END_DES_EN | FIELD_PREP(DMA_C_END_DESENDI, endian_type); + else + val = 0; + + ldma_update_bits(d, DMA_CS_MASK, c->nr, DMA_CS); + ldma_update_bits(d, mask, val, DMA_C_ENDIAN); +} + +static void ldma_chan_hdr_mode_cfg(struct ldma_chan *c, u32 hdr_len, bool csum) +{ + struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); + u32 mask, val; + + /* NB, csum disabled, hdr length must be provided */ + if (!csum && (!hdr_len || hdr_len > DMA_HDR_LEN_MAX)) + return; + + mask = DMA_C_HDRM_HDR_SUM; + val = DMA_C_HDRM_HDR_SUM; + + if (!csum && hdr_len) + val = hdr_len; + + ldma_update_bits(d, DMA_CS_MASK, c->nr, DMA_CS); + ldma_update_bits(d, mask, val, DMA_C_HDRM); +} + +static void ldma_chan_rxwr_np_cfg(struct ldma_chan *c, bool enable) +{ + struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); + u32 mask, val; + + /* Only valid for RX channel */ + if (ldma_chan_tx(c)) + return; + + mask = DMA_CCTRL_WR_NP_EN; + val = enable ? DMA_CCTRL_WR_NP_EN : 0; + + ldma_update_bits(d, DMA_CS_MASK, c->nr, DMA_CS); + ldma_update_bits(d, mask, val, DMA_CCTRL); +} + +static void ldma_chan_abc_cfg(struct ldma_chan *c, bool enable) +{ + struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); + u32 mask, val; + + if (d->ver < DMA_VER32 || ldma_chan_tx(c)) + return; + + mask = DMA_CCTRL_CH_ABC; + val = enable ? DMA_CCTRL_CH_ABC : 0; + + ldma_update_bits(d, DMA_CS_MASK, c->nr, DMA_CS); + ldma_update_bits(d, mask, val, DMA_CCTRL); +} + +static int ldma_port_cfg(struct ldma_port *p) +{ + unsigned long flags; + struct ldma_dev *d; + u32 reg; + + d = p->ldev; + reg = FIELD_PREP(DMA_PCTRL_TXENDI, p->txendi); + reg |= FIELD_PREP(DMA_PCTRL_RXENDI, p->rxendi); + + if (d->ver == DMA_VER22) { + reg |= FIELD_PREP(DMA_PCTRL_TXBL, p->txbl); + reg |= FIELD_PREP(DMA_PCTRL_RXBL, p->rxbl); + } else { + reg |= FIELD_PREP(DMA_PCTRL_PDEN, p->pkt_drop); + + if (p->txbl == DMA_BURSTL_32DW) + reg |= DMA_PCTRL_TXBL32; + else if (p->txbl == DMA_BURSTL_16DW) + reg |= DMA_PCTRL_TXBL16; + else + reg |= FIELD_PREP(DMA_PCTRL_TXBL, DMA_PCTRL_TXBL_8); + + if (p->rxbl == DMA_BURSTL_32DW) + reg |= DMA_PCTRL_RXBL32; + else if (p->rxbl == DMA_BURSTL_16DW) + reg |= DMA_PCTRL_RXBL16; + else + reg |= FIELD_PREP(DMA_PCTRL_RXBL, DMA_PCTRL_RXBL_8); + } + + spin_lock_irqsave(&d->dev_lock, flags); + writel(p->portid, d->base + DMA_PS); + writel(reg, d->base + DMA_PCTRL); + spin_unlock_irqrestore(&d->dev_lock, flags); + + reg = readl(d->base + DMA_PCTRL); /* read back */ + dev_dbg(d->dev, "Port Control 0x%08x configuration done\n", reg); + + return 0; +} + +static int ldma_chan_cfg(struct ldma_chan *c) +{ + struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); + unsigned long flags; + u32 reg; + + reg = c->pden ? DMA_CCTRL_PDEN : 0; + reg |= c->onoff ? DMA_CCTRL_ON : 0; + reg |= c->rst ? DMA_CCTRL_RST : 0; + + ldma_chan_cctrl_cfg(c, reg); + ldma_chan_irq_init(c); + + if (d->ver <= DMA_VER22) + return 0; + + spin_lock_irqsave(&d->dev_lock, flags); + ldma_chan_set_class(c, c->nr); + ldma_chan_byte_offset_cfg(c, c->boff_len); + ldma_chan_data_endian_cfg(c, c->data_endian_en, c->data_endian); + ldma_chan_desc_endian_cfg(c, c->desc_endian_en, c->desc_endian); + ldma_chan_hdr_mode_cfg(c, c->hdrm_len, c->hdrm_csum); + ldma_chan_rxwr_np_cfg(c, c->desc_rx_np); + ldma_chan_abc_cfg(c, c->abc_en); + spin_unlock_irqrestore(&d->dev_lock, flags); + + if (ldma_chan_is_hw_desc(c)) + ldma_chan_desc_hw_cfg(c, c->desc_phys, c->desc_cnt); + + return 0; +} + +static void ldma_dev_init(struct ldma_dev *d) +{ + unsigned long ch_mask = (unsigned long)d->channels_mask; + struct ldma_port *p; + struct ldma_chan *c; + int i; + u32 j; + + spin_lock_init(&d->dev_lock); + ldma_dev_reset(d); + ldma_dev_cfg(d); + + /* DMA port initialization */ + for (i = 0; i < d->port_nrs; i++) { + p = &d->ports[i]; + ldma_port_cfg(p); + } + + /* DMA channel initialization */ + for_each_set_bit(j, &ch_mask, d->chan_nrs) { + c = &d->chans[j]; + ldma_chan_cfg(c); + } +} + +static int ldma_cfg_init(struct ldma_dev *d) +{ + struct fwnode_handle *fwnode = dev_fwnode(d->dev); + struct ldma_port *p; + int i; + + if (fwnode_property_read_bool(fwnode, "intel,dma-byte-en")) + d->flags |= DMA_EN_BYTE_EN; + + if (fwnode_property_read_bool(fwnode, "intel,dma-dburst-wr")) + d->flags |= DMA_DBURST_WR; + + if (fwnode_property_read_bool(fwnode, "intel,dma-drb")) + d->flags |= DMA_DFT_DRB; + + if (fwnode_property_read_u32(fwnode, "intel,dma-poll-cnt", + &d->pollcnt)) + d->pollcnt = DMA_DFT_POLL_CNT; + + if (d->inst->chan_fc) + d->flags |= DMA_CHAN_FLOW_CTL; + + if (d->inst->desc_fod) + d->flags |= DMA_DESC_FOD; + + if (d->inst->desc_in_sram) + d->flags |= DMA_DESC_IN_SRAM; + + if (d->inst->valid_desc_fetch_ack) + d->flags |= DMA_VALID_DESC_FETCH_ACK; + + if (d->ver > DMA_VER22) { + if (!d->port_nrs) + return -EINVAL; + + for (i = 0; i < d->port_nrs; i++) { + p = &d->ports[i]; + p->rxendi = DMA_DFT_ENDIAN; + p->txendi = DMA_DFT_ENDIAN; + p->rxbl = DMA_DFT_BURST; + p->txbl = DMA_DFT_BURST; + p->pkt_drop = DMA_PKT_DROP_DIS; + } + } + + return 0; +} + +static void dma_free_desc_resource(struct virt_dma_desc *vdesc) +{ + struct dw2_desc_sw *ds = to_lgm_dma_desc(vdesc); + struct ldma_chan *c = ds->chan; + + dma_pool_free(c->desc_pool, ds->desc_hw, ds->desc_phys); + kfree(ds); +} + +static struct dw2_desc_sw * +dma_alloc_desc_resource(int num, struct ldma_chan *c) +{ + struct device *dev = c->vchan.chan.device->dev; + struct dw2_desc_sw *ds; + + if (num > c->desc_num) { + dev_err(dev, "sg num %d exceed max %d\n", num, c->desc_num); + return NULL; + } + + ds = kzalloc(sizeof(*ds), GFP_NOWAIT); + if (!ds) + return NULL; + + ds->chan = c; + ds->desc_hw = dma_pool_zalloc(c->desc_pool, GFP_ATOMIC, + &ds->desc_phys); + if (!ds->desc_hw) { + dev_dbg(dev, "out of memory for link descriptor\n"); + kfree(ds); + return NULL; + } + ds->desc_cnt = num; + + return ds; +} + +static void ldma_chan_irq_en(struct ldma_chan *c) +{ + struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); + unsigned long flags; + + spin_lock_irqsave(&d->dev_lock, flags); + writel(c->nr, d->base + DMA_CS); + writel(DMA_CI_EOP, d->base + DMA_CIE); + writel(BIT(c->nr), d->base + DMA_IRNEN); + spin_unlock_irqrestore(&d->dev_lock, flags); +} + +static void ldma_issue_pending(struct dma_chan *chan) +{ + struct ldma_chan *c = to_ldma_chan(chan); + struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); + unsigned long flags; + + if (d->ver == DMA_VER22) { + spin_lock_irqsave(&c->vchan.lock, flags); + if (vchan_issue_pending(&c->vchan)) { + struct virt_dma_desc *vdesc; + + /* Get the next descriptor */ + vdesc = vchan_next_desc(&c->vchan); + if (!vdesc) { + c->ds = NULL; + spin_unlock_irqrestore(&c->vchan.lock, flags); + return; + } + list_del(&vdesc->node); + c->ds = to_lgm_dma_desc(vdesc); + ldma_chan_desc_hw_cfg(c, c->ds->desc_phys, c->ds->desc_cnt); + ldma_chan_irq_en(c); + } + spin_unlock_irqrestore(&c->vchan.lock, flags); + } + ldma_chan_on(c); +} + +static void ldma_synchronize(struct dma_chan *chan) +{ + struct ldma_chan *c = to_ldma_chan(chan); + + /* + * clear any pending work if any. In that + * case the resource needs to be free here. + */ + cancel_work_sync(&c->work); + vchan_synchronize(&c->vchan); + if (c->ds) + dma_free_desc_resource(&c->ds->vdesc); +} + +static int ldma_terminate_all(struct dma_chan *chan) +{ + struct ldma_chan *c = to_ldma_chan(chan); + unsigned long flags; + LIST_HEAD(head); + + spin_lock_irqsave(&c->vchan.lock, flags); + vchan_get_all_descriptors(&c->vchan, &head); + spin_unlock_irqrestore(&c->vchan.lock, flags); + vchan_dma_desc_free_list(&c->vchan, &head); + + return ldma_chan_reset(c); +} + +static int ldma_resume_chan(struct dma_chan *chan) +{ + struct ldma_chan *c = to_ldma_chan(chan); + + ldma_chan_on(c); + + return 0; +} + +static int ldma_pause_chan(struct dma_chan *chan) +{ + struct ldma_chan *c = to_ldma_chan(chan); + + return ldma_chan_off(c); +} + +static enum dma_status +ldma_tx_status(struct dma_chan *chan, dma_cookie_t cookie, + struct dma_tx_state *txstate) +{ + struct ldma_chan *c = to_ldma_chan(chan); + struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); + enum dma_status status = DMA_COMPLETE; + + if (d->ver == DMA_VER22) + status = dma_cookie_status(chan, cookie, txstate); + + return status; +} + +static void dma_chan_irq(int irq, void *data) +{ + struct ldma_chan *c = data; + struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); + u32 stat; + + /* Disable channel interrupts */ + writel(c->nr, d->base + DMA_CS); + stat = readl(d->base + DMA_CIS); + if (!stat) + return; + + writel(readl(d->base + DMA_CIE) & ~DMA_CI_ALL, d->base + DMA_CIE); + writel(stat, d->base + DMA_CIS); + queue_work(d->wq, &c->work); +} + +static irqreturn_t dma_interrupt(int irq, void *dev_id) +{ + struct ldma_dev *d = dev_id; + struct ldma_chan *c; + unsigned long irncr; + u32 cid; + + irncr = readl(d->base + DMA_IRNCR); + if (!irncr) { + dev_err(d->dev, "dummy interrupt\n"); + return IRQ_NONE; + } + + for_each_set_bit(cid, &irncr, d->chan_nrs) { + /* Mask */ + writel(readl(d->base + DMA_IRNEN) & ~BIT(cid), d->base + DMA_IRNEN); + /* Ack */ + writel(readl(d->base + DMA_IRNCR) | BIT(cid), d->base + DMA_IRNCR); + + c = &d->chans[cid]; + dma_chan_irq(irq, c); + } + + return IRQ_HANDLED; +} + +static void prep_slave_burst_len(struct ldma_chan *c) +{ + struct ldma_port *p = c->port; + struct dma_slave_config *cfg = &c->config; + + if (cfg->dst_maxburst) + cfg->src_maxburst = cfg->dst_maxburst; + + /* TX and RX has the same burst length */ + p->txbl = ilog2(cfg->src_maxburst); + p->rxbl = p->txbl; +} + +static struct dma_async_tx_descriptor * +ldma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, + unsigned int sglen, enum dma_transfer_direction dir, + unsigned long flags, void *context) +{ + struct ldma_chan *c = to_ldma_chan(chan); + struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); + size_t len, avail, total = 0; + struct dw2_desc *hw_ds; + struct dw2_desc_sw *ds; + struct scatterlist *sg; + int num = sglen, i; + dma_addr_t addr; + + if (!sgl) + return NULL; + + if (d->ver > DMA_VER22) + return ldma_chan_desc_cfg(chan, sgl->dma_address, sglen); + + for_each_sg(sgl, sg, sglen, i) { + avail = sg_dma_len(sg); + if (avail > DMA_MAX_SIZE) + num += DIV_ROUND_UP(avail, DMA_MAX_SIZE) - 1; + } + + ds = dma_alloc_desc_resource(num, c); + if (!ds) + return NULL; + + c->ds = ds; + + num = 0; + /* sop and eop has to be handled nicely */ + for_each_sg(sgl, sg, sglen, i) { + addr = sg_dma_address(sg); + avail = sg_dma_len(sg); + total += avail; + + do { + len = min_t(size_t, avail, DMA_MAX_SIZE); + + hw_ds = &ds->desc_hw[num]; + switch (sglen) { + case 1: + hw_ds->field &= ~DESC_SOP; + hw_ds->field |= FIELD_PREP(DESC_SOP, 1); + + hw_ds->field &= ~DESC_EOP; + hw_ds->field |= FIELD_PREP(DESC_EOP, 1); + break; + default: + if (num == 0) { + hw_ds->field &= ~DESC_SOP; + hw_ds->field |= FIELD_PREP(DESC_SOP, 1); + + hw_ds->field &= ~DESC_EOP; + hw_ds->field |= FIELD_PREP(DESC_EOP, 0); + } else if (num == (sglen - 1)) { + hw_ds->field &= ~DESC_SOP; + hw_ds->field |= FIELD_PREP(DESC_SOP, 0); + hw_ds->field &= ~DESC_EOP; + hw_ds->field |= FIELD_PREP(DESC_EOP, 1); + } else { + hw_ds->field &= ~DESC_SOP; + hw_ds->field |= FIELD_PREP(DESC_SOP, 0); + + hw_ds->field &= ~DESC_EOP; + hw_ds->field |= FIELD_PREP(DESC_EOP, 0); + } + break; + } + /* Only 32 bit address supported */ + hw_ds->addr = (u32)addr; + + hw_ds->field &= ~DESC_DATA_LEN; + hw_ds->field |= FIELD_PREP(DESC_DATA_LEN, len); + + hw_ds->field &= ~DESC_C; + hw_ds->field |= FIELD_PREP(DESC_C, 0); + + hw_ds->field &= ~DESC_BYTE_OFF; + hw_ds->field |= FIELD_PREP(DESC_BYTE_OFF, addr & 0x3); + + /* Ensure data ready before ownership change */ + wmb(); + hw_ds->field &= ~DESC_OWN; + hw_ds->field |= FIELD_PREP(DESC_OWN, DMA_OWN); + + /* Ensure ownership changed before moving forward */ + wmb(); + num++; + addr += len; + avail -= len; + } while (avail); + } + + ds->size = total; + prep_slave_burst_len(c); + + return vchan_tx_prep(&c->vchan, &ds->vdesc, DMA_CTRL_ACK); +} + +static int +ldma_slave_config(struct dma_chan *chan, struct dma_slave_config *cfg) +{ + struct ldma_chan *c = to_ldma_chan(chan); + + memcpy(&c->config, cfg, sizeof(c->config)); + + return 0; +} + +static int ldma_alloc_chan_resources(struct dma_chan *chan) +{ + struct ldma_chan *c = to_ldma_chan(chan); + struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); + struct device *dev = c->vchan.chan.device->dev; + size_t desc_sz; + + if (d->ver > DMA_VER22) { + c->flags |= CHAN_IN_USE; + return 0; + } + + if (c->desc_pool) + return c->desc_num; + + desc_sz = c->desc_num * sizeof(struct dw2_desc); + c->desc_pool = dma_pool_create(c->name, dev, desc_sz, + __alignof__(struct dw2_desc), 0); + + if (!c->desc_pool) { + dev_err(dev, "unable to allocate descriptor pool\n"); + return -ENOMEM; + } + + return c->desc_num; +} + +static void ldma_free_chan_resources(struct dma_chan *chan) +{ + struct ldma_chan *c = to_ldma_chan(chan); + struct ldma_dev *d = to_ldma_dev(c->vchan.chan.device); + + if (d->ver == DMA_VER22) { + dma_pool_destroy(c->desc_pool); + c->desc_pool = NULL; + vchan_free_chan_resources(to_virt_chan(chan)); + ldma_chan_reset(c); + } else { + c->flags &= ~CHAN_IN_USE; + } +} + +static void dma_work(struct work_struct *work) +{ + struct ldma_chan *c = container_of(work, struct ldma_chan, work); + struct dma_async_tx_descriptor *tx = &c->ds->vdesc.tx; + struct virt_dma_chan *vc = &c->vchan; + struct dmaengine_desc_callback cb; + struct virt_dma_desc *vd, *_vd; + unsigned long flags; + LIST_HEAD(head); + + spin_lock_irqsave(&c->vchan.lock, flags); + list_splice_tail_init(&vc->desc_completed, &head); + spin_unlock_irqrestore(&c->vchan.lock, flags); + dmaengine_desc_get_callback(tx, &cb); + dma_cookie_complete(tx); + dmaengine_desc_callback_invoke(&cb, NULL); + + list_for_each_entry_safe(vd, _vd, &head, node) { + dmaengine_desc_get_callback(tx, &cb); + dma_cookie_complete(tx); + list_del(&vd->node); + dmaengine_desc_callback_invoke(&cb, NULL); + + vchan_vdesc_fini(vd); + } + c->ds = NULL; +} + +static void +update_burst_len_v22(struct ldma_chan *c, struct ldma_port *p, u32 burst) +{ + if (ldma_chan_tx(c)) + p->txbl = ilog2(burst); + else + p->rxbl = ilog2(burst); +} + +static void +update_burst_len_v3X(struct ldma_chan *c, struct ldma_port *p, u32 burst) +{ + if (ldma_chan_tx(c)) + p->txbl = burst; + else + p->rxbl = burst; +} + +static int +update_client_configs(struct of_dma *ofdma, struct of_phandle_args *spec) +{ + struct ldma_dev *d = ofdma->of_dma_data; + u32 chan_id = spec->args[0]; + u32 port_id = spec->args[1]; + u32 burst = spec->args[2]; + struct ldma_port *p; + struct ldma_chan *c; + + if (chan_id >= d->chan_nrs || port_id >= d->port_nrs) + return 0; + + p = &d->ports[port_id]; + c = &d->chans[chan_id]; + c->port = p; + + if (d->ver == DMA_VER22) + update_burst_len_v22(c, p, burst); + else + update_burst_len_v3X(c, p, burst); + + ldma_port_cfg(p); + + return 1; +} + +static struct dma_chan *ldma_xlate(struct of_phandle_args *spec, + struct of_dma *ofdma) +{ + struct ldma_dev *d = ofdma->of_dma_data; + u32 chan_id = spec->args[0]; + int ret; + + if (!spec->args_count) + return NULL; + + /* if args_count is 1 driver use default settings */ + if (spec->args_count > 1) { + ret = update_client_configs(ofdma, spec); + if (!ret) + return NULL; + } + + return dma_get_slave_channel(&d->chans[chan_id].vchan.chan); +} + +static void ldma_dma_init_v22(int i, struct ldma_dev *d) +{ + struct ldma_chan *c; + + c = &d->chans[i]; + c->nr = i; /* Real channel number */ + c->rst = DMA_CHAN_RST; + c->desc_num = DMA_DFT_DESC_NUM; + snprintf(c->name, sizeof(c->name), "chan%d", c->nr); + INIT_WORK(&c->work, dma_work); + c->vchan.desc_free = dma_free_desc_resource; + vchan_init(&c->vchan, &d->dma_dev); +} + +static void ldma_dma_init_v3X(int i, struct ldma_dev *d) +{ + struct ldma_chan *c; + + c = &d->chans[i]; + c->data_endian = DMA_DFT_ENDIAN; + c->desc_endian = DMA_DFT_ENDIAN; + c->data_endian_en = false; + c->desc_endian_en = false; + c->desc_rx_np = false; + c->flags |= DEVICE_ALLOC_DESC; + c->onoff = DMA_CH_OFF; + c->rst = DMA_CHAN_RST; + c->abc_en = true; + c->hdrm_csum = false; + c->boff_len = 0; + c->nr = i; + c->vchan.desc_free = dma_free_desc_resource; + vchan_init(&c->vchan, &d->dma_dev); +} + +static int ldma_init_v22(struct ldma_dev *d, struct platform_device *pdev) +{ + int ret; + + ret = device_property_read_u32(d->dev, "dma-channels", &d->chan_nrs); + if (ret < 0) { + dev_err(d->dev, "unable to read dma-channels property\n"); + return ret; + } + + d->irq = platform_get_irq(pdev, 0); + if (d->irq < 0) + return d->irq; + + ret = devm_request_irq(&pdev->dev, d->irq, dma_interrupt, 0, + DRIVER_NAME, d); + if (ret) + return ret; + + d->wq = alloc_ordered_workqueue("dma_wq", WQ_MEM_RECLAIM | + WQ_HIGHPRI); + if (!d->wq) + return -ENOMEM; + + return 0; +} + +static void ldma_clk_disable(void *data) +{ + struct ldma_dev *d = data; + + clk_disable_unprepare(d->core_clk); + reset_control_assert(d->rst); +} + +static const struct ldma_inst_data dma0 = { + .name = "dma0", + .chan_fc = false, + .desc_fod = false, + .desc_in_sram = false, + .valid_desc_fetch_ack = false, +}; + +static const struct ldma_inst_data dma2tx = { + .name = "dma2tx", + .type = DMA_TYPE_TX, + .orrc = 16, + .chan_fc = true, + .desc_fod = true, + .desc_in_sram = true, + .valid_desc_fetch_ack = true, +}; + +static const struct ldma_inst_data dma1rx = { + .name = "dma1rx", + .type = DMA_TYPE_RX, + .orrc = 16, + .chan_fc = false, + .desc_fod = true, + .desc_in_sram = true, + .valid_desc_fetch_ack = false, +}; + +static const struct ldma_inst_data dma1tx = { + .name = "dma1tx", + .type = DMA_TYPE_TX, + .orrc = 16, + .chan_fc = true, + .desc_fod = true, + .desc_in_sram = true, + .valid_desc_fetch_ack = true, +}; + +static const struct ldma_inst_data dma0tx = { + .name = "dma0tx", + .type = DMA_TYPE_TX, + .orrc = 16, + .chan_fc = true, + .desc_fod = true, + .desc_in_sram = true, + .valid_desc_fetch_ack = true, +}; + +static const struct ldma_inst_data dma3 = { + .name = "dma3", + .type = DMA_TYPE_MCPY, + .orrc = 16, + .chan_fc = false, + .desc_fod = false, + .desc_in_sram = true, + .valid_desc_fetch_ack = false, +}; + +static const struct ldma_inst_data toe_dma30 = { + .name = "toe_dma30", + .type = DMA_TYPE_MCPY, + .orrc = 16, + .chan_fc = false, + .desc_fod = false, + .desc_in_sram = true, + .valid_desc_fetch_ack = true, +}; + +static const struct ldma_inst_data toe_dma31 = { + .name = "toe_dma31", + .type = DMA_TYPE_MCPY, + .orrc = 16, + .chan_fc = false, + .desc_fod = false, + .desc_in_sram = true, + .valid_desc_fetch_ack = true, +}; + +static const struct of_device_id intel_ldma_match[] = { + { .compatible = "intel,lgm-cdma", .data = &dma0}, + { .compatible = "intel,lgm-dma2tx", .data = &dma2tx}, + { .compatible = "intel,lgm-dma1rx", .data = &dma1rx}, + { .compatible = "intel,lgm-dma1tx", .data = &dma1tx}, + { .compatible = "intel,lgm-dma0tx", .data = &dma0tx}, + { .compatible = "intel,lgm-dma3", .data = &dma3}, + { .compatible = "intel,lgm-toe-dma30", .data = &toe_dma30}, + { .compatible = "intel,lgm-toe-dma31", .data = &toe_dma31}, + {} +}; + +static int intel_ldma_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct dma_device *dma_dev; + unsigned long ch_mask; + struct ldma_chan *c; + struct ldma_port *p; + struct ldma_dev *d; + u32 id, bitn = 32, j; + int i, ret; + + d = devm_kzalloc(dev, sizeof(*d), GFP_KERNEL); + if (!d) + return -ENOMEM; + + /* Link controller to platform device */ + d->dev = &pdev->dev; + + d->inst = device_get_match_data(dev); + if (!d->inst) { + dev_err(dev, "No device match found\n"); + return -ENODEV; + } + + d->base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(d->base)) + return PTR_ERR(d->base); + + /* Power up and reset the dma engine, some DMAs always on?? */ + d->core_clk = devm_clk_get_optional(dev, NULL); + if (IS_ERR(d->core_clk)) + return PTR_ERR(d->core_clk); + clk_prepare_enable(d->core_clk); + + d->rst = devm_reset_control_get_optional(dev, NULL); + if (IS_ERR(d->rst)) + return PTR_ERR(d->rst); + reset_control_deassert(d->rst); + + ret = devm_add_action_or_reset(dev, ldma_clk_disable, d); + if (ret) { + dev_err(dev, "Failed to devm_add_action_or_reset, %d\n", ret); + return ret; + } + + id = readl(d->base + DMA_ID); + d->chan_nrs = FIELD_GET(DMA_ID_CHNR, id); + d->port_nrs = FIELD_GET(DMA_ID_PNR, id); + d->ver = FIELD_GET(DMA_ID_REV, id); + + if (id & DMA_ID_AW_36B) + d->flags |= DMA_ADDR_36BIT; + + if (IS_ENABLED(CONFIG_64BIT) && (id & DMA_ID_AW_36B)) + bitn = 36; + + if (id & DMA_ID_DW_128B) + d->flags |= DMA_DATA_128BIT; + + ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(bitn)); + if (ret) { + dev_err(dev, "No usable DMA configuration\n"); + return ret; + } + + if (d->ver == DMA_VER22) { + ret = ldma_init_v22(d, pdev); + if (ret) + return ret; + } + + ret = device_property_read_u32(dev, "dma-channel-mask", &d->channels_mask); + if (ret < 0) + d->channels_mask = GENMASK(d->chan_nrs - 1, 0); + + dma_dev = &d->dma_dev; + + dma_cap_zero(dma_dev->cap_mask); + dma_cap_set(DMA_SLAVE, dma_dev->cap_mask); + + /* Channel initializations */ + INIT_LIST_HEAD(&dma_dev->channels); + + /* Port Initializations */ + d->ports = devm_kcalloc(dev, d->port_nrs, sizeof(*p), GFP_KERNEL); + if (!d->ports) + return -ENOMEM; + + /* Channels Initializations */ + d->chans = devm_kcalloc(d->dev, d->chan_nrs, sizeof(*c), GFP_KERNEL); + if (!d->chans) + return -ENOMEM; + + for (i = 0; i < d->port_nrs; i++) { + p = &d->ports[i]; + p->portid = i; + p->ldev = d; + } + + ret = ldma_cfg_init(d); + if (ret) + return ret; + + dma_dev->dev = &pdev->dev; + + ch_mask = (unsigned long)d->channels_mask; + for_each_set_bit(j, &ch_mask, d->chan_nrs) { + if (d->ver == DMA_VER22) + ldma_dma_init_v22(j, d); + else + ldma_dma_init_v3X(j, d); + } + + dma_dev->device_alloc_chan_resources = ldma_alloc_chan_resources; + dma_dev->device_free_chan_resources = ldma_free_chan_resources; + dma_dev->device_terminate_all = ldma_terminate_all; + dma_dev->device_issue_pending = ldma_issue_pending; + dma_dev->device_tx_status = ldma_tx_status; + dma_dev->device_resume = ldma_resume_chan; + dma_dev->device_pause = ldma_pause_chan; + dma_dev->device_prep_slave_sg = ldma_prep_slave_sg; + + if (d->ver == DMA_VER22) { + dma_dev->device_config = ldma_slave_config; + dma_dev->device_synchronize = ldma_synchronize; + dma_dev->src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); + dma_dev->dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); + dma_dev->directions = BIT(DMA_MEM_TO_DEV) | + BIT(DMA_DEV_TO_MEM); + dma_dev->residue_granularity = + DMA_RESIDUE_GRANULARITY_DESCRIPTOR; + } + + platform_set_drvdata(pdev, d); + + ldma_dev_init(d); + + ret = dma_async_device_register(dma_dev); + if (ret) { + dev_err(dev, "Failed to register slave DMA engine device\n"); + return ret; + } + + ret = of_dma_controller_register(pdev->dev.of_node, ldma_xlate, d); + if (ret) { + dev_err(dev, "Failed to register of DMA controller\n"); + dma_async_device_unregister(dma_dev); + return ret; + } + + dev_info(dev, "Init done - rev: %x, ports: %d channels: %d\n", d->ver, + d->port_nrs, d->chan_nrs); + + return 0; +} + +static struct platform_driver intel_ldma_driver = { + .probe = intel_ldma_probe, + .driver = { + .name = DRIVER_NAME, + .of_match_table = intel_ldma_match, + }, +}; + +/* + * Perform this driver as device_initcall to make sure initialization happens + * before its DMA clients of some are platform specific and also to provide + * registered DMA channels and DMA capabilities to clients before their + * initialization. + */ +static int __init intel_ldma_init(void) +{ + return platform_driver_register(&intel_ldma_driver); +} + +device_initcall(intel_ldma_init); diff --git a/drivers/dma/mmp_pdma.c b/drivers/dma/mmp_pdma.c index b84303be8edf55d5ed1dfdf80e5705d8f30fc918..89f1814ff27a0825014130616044ee5f0e4167d1 100644 --- a/drivers/dma/mmp_pdma.c +++ b/drivers/dma/mmp_pdma.c @@ -18,7 +18,6 @@ #include #include #include -#include #include "dmaengine.h" @@ -1148,19 +1147,6 @@ static struct platform_driver mmp_pdma_driver = { .remove = mmp_pdma_remove, }; -bool mmp_pdma_filter_fn(struct dma_chan *chan, void *param) -{ - struct mmp_pdma_chan *c = to_mmp_pdma_chan(chan); - - if (chan->device->dev->driver != &mmp_pdma_driver.driver) - return false; - - c->drcmr = *(unsigned int *)param; - - return true; -} -EXPORT_SYMBOL_GPL(mmp_pdma_filter_fn); - module_platform_driver(mmp_pdma_driver); MODULE_DESCRIPTION("MARVELL MMP Peripheral DMA Driver"); diff --git a/drivers/dma/owl-dma.c b/drivers/dma/owl-dma.c index 9fede32641e9e3d29ce9c0193c01ae993cdbeaf8..1f0bbaed4643a434e9af41cdef4162be0bf7b92d 100644 --- a/drivers/dma/owl-dma.c +++ b/drivers/dma/owl-dma.c @@ -1080,8 +1080,9 @@ static struct dma_chan *owl_dma_of_xlate(struct of_phandle_args *dma_spec, } static const struct of_device_id owl_dma_match[] = { - { .compatible = "actions,s900-dma", .data = (void *)S900_DMA,}, + { .compatible = "actions,s500-dma", .data = (void *)S900_DMA,}, { .compatible = "actions,s700-dma", .data = (void *)S700_DMA,}, + { .compatible = "actions,s900-dma", .data = (void *)S900_DMA,}, { /* sentinel */ }, }; MODULE_DEVICE_TABLE(of, owl_dma_match); @@ -1245,6 +1246,7 @@ static int owl_dma_remove(struct platform_device *pdev) owl_dma_free(od); clk_disable_unprepare(od->clk); + dma_pool_destroy(od->lli_pool); return 0; } diff --git a/drivers/dma/pl330.c b/drivers/dma/pl330.c index bc0f66af0f1125c76740506f0ef32a5f74ac446b..fd8d2bc3be9f5196b2fb1676ef5a1863554fafb7 100644 --- a/drivers/dma/pl330.c +++ b/drivers/dma/pl330.c @@ -3195,7 +3195,7 @@ pl330_probe(struct amba_device *adev, const struct amba_id *id) return ret; } -static int pl330_remove(struct amba_device *adev) +static void pl330_remove(struct amba_device *adev) { struct pl330_dmac *pl330 = amba_get_drvdata(adev); struct dma_pl330_chan *pch, *_p; @@ -3235,7 +3235,6 @@ static int pl330_remove(struct amba_device *adev) if (pl330->rstc) reset_control_assert(pl330->rstc); - return 0; } static const struct amba_id pl330_ids[] = { diff --git a/drivers/dma/qcom/bam_dma.c b/drivers/dma/qcom/bam_dma.c index 88579857ca1d6c08a342c606001188a56157cf03..c8a77b428b528b8d3508fe15c3b64bc560cc6036 100644 --- a/drivers/dma/qcom/bam_dma.c +++ b/drivers/dma/qcom/bam_dma.c @@ -1270,13 +1270,13 @@ static int bam_dma_probe(struct platform_device *pdev) dev_err(bdev->dev, "num-ees unspecified in dt\n"); } - bdev->bamclk = devm_clk_get(bdev->dev, "bam_clk"); - if (IS_ERR(bdev->bamclk)) { - if (!bdev->controlled_remotely) - return PTR_ERR(bdev->bamclk); + if (bdev->controlled_remotely) + bdev->bamclk = devm_clk_get_optional(bdev->dev, "bam_clk"); + else + bdev->bamclk = devm_clk_get(bdev->dev, "bam_clk"); - bdev->bamclk = NULL; - } + if (IS_ERR(bdev->bamclk)) + return PTR_ERR(bdev->bamclk); ret = clk_prepare_enable(bdev->bamclk); if (ret) { @@ -1350,7 +1350,7 @@ static int bam_dma_probe(struct platform_device *pdev) if (ret) goto err_unregister_dma; - if (bdev->controlled_remotely) { + if (!bdev->bamclk) { pm_runtime_disable(&pdev->dev); return 0; } @@ -1438,10 +1438,10 @@ static int __maybe_unused bam_dma_suspend(struct device *dev) { struct bam_device *bdev = dev_get_drvdata(dev); - if (!bdev->controlled_remotely) + if (bdev->bamclk) { pm_runtime_force_suspend(dev); - - clk_unprepare(bdev->bamclk); + clk_unprepare(bdev->bamclk); + } return 0; } @@ -1451,12 +1451,13 @@ static int __maybe_unused bam_dma_resume(struct device *dev) struct bam_device *bdev = dev_get_drvdata(dev); int ret; - ret = clk_prepare(bdev->bamclk); - if (ret) - return ret; + if (bdev->bamclk) { + ret = clk_prepare(bdev->bamclk); + if (ret) + return ret; - if (!bdev->controlled_remotely) pm_runtime_force_resume(dev); + } return 0; } diff --git a/drivers/dma/qcom/gpi.c b/drivers/dma/qcom/gpi.c index 1a0bf6b0567a55e6d4ec009daae813cf83eb78dc..57f5ee4235c71b2b4424293b15fd8d7819156728 100644 --- a/drivers/dma/qcom/gpi.c +++ b/drivers/dma/qcom/gpi.c @@ -584,7 +584,7 @@ static inline void gpi_write_reg_field(struct gpii *gpii, void __iomem *addr, gpi_write_reg(gpii, addr, val); } -static inline void +static __always_inline void gpi_update_reg(struct gpii *gpii, u32 offset, u32 mask, u32 val) { void __iomem *addr = gpii->regs + offset; @@ -1700,7 +1700,7 @@ static int gpi_create_i2c_tre(struct gchan *chan, struct gpi_desc *desc, tre->dword[3] = u32_encode_bits(TRE_TYPE_DMA, TRE_FLAGS_TYPE); tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_IEOT); - }; + } for (i = 0; i < tre_idx; i++) dev_dbg(dev, "TRE:%d %x:%x:%x:%x\n", i, desc->tre[i].dword[0], diff --git a/drivers/dma/sh/rcar-dmac.c b/drivers/dma/sh/rcar-dmac.c index a57705356e8bb796a3a375540afe5e7e4030777c..d530c1bf11d977c69f3fda30b6cff3bc083bc926 100644 --- a/drivers/dma/sh/rcar-dmac.c +++ b/drivers/dma/sh/rcar-dmac.c @@ -189,7 +189,8 @@ struct rcar_dmac_chan { * struct rcar_dmac - R-Car Gen2 DMA Controller * @engine: base DMA engine object * @dev: the hardware device - * @iomem: remapped I/O memory base + * @dmac_base: remapped base register block + * @chan_base: remapped channel register block (optional) * @n_channels: number of available channels * @channels: array of DMAC channels * @channels_mask: bitfield of which DMA channels are managed by this driver @@ -198,7 +199,8 @@ struct rcar_dmac_chan { struct rcar_dmac { struct dma_device engine; struct device *dev; - void __iomem *iomem; + void __iomem *dmac_base; + void __iomem *chan_base; unsigned int n_channels; struct rcar_dmac_chan *channels; @@ -209,6 +211,10 @@ struct rcar_dmac { #define to_rcar_dmac(d) container_of(d, struct rcar_dmac, engine) +#define for_each_rcar_dmac_chan(i, dmac, chan) \ + for (i = 0, chan = &(dmac)->channels[0]; i < (dmac)->n_channels; i++, chan++) \ + if (!((dmac)->channels_mask & BIT(i))) continue; else + /* * struct rcar_dmac_of_data - This driver's OF data * @chan_offset_base: DMAC channels base offset @@ -230,7 +236,7 @@ struct rcar_dmac_of_data { #define RCAR_DMAOR_PRI_ROUND_ROBIN (3 << 8) #define RCAR_DMAOR_AE (1 << 2) #define RCAR_DMAOR_DME (1 << 0) -#define RCAR_DMACHCLR 0x0080 +#define RCAR_DMACHCLR 0x0080 /* Not on R-Car V3U */ #define RCAR_DMADPSEC 0x00a0 #define RCAR_DMASAR 0x0000 @@ -293,6 +299,9 @@ struct rcar_dmac_of_data { #define RCAR_DMAFIXDAR 0x0014 #define RCAR_DMAFIXDPBASE 0x0060 +/* For R-Car V3U */ +#define RCAR_V3U_DMACHCLR 0x0100 + /* Hardcode the MEMCPY transfer size to 4 bytes. */ #define RCAR_DMAC_MEMCPY_XFER_SIZE 4 @@ -303,17 +312,17 @@ struct rcar_dmac_of_data { static void rcar_dmac_write(struct rcar_dmac *dmac, u32 reg, u32 data) { if (reg == RCAR_DMAOR) - writew(data, dmac->iomem + reg); + writew(data, dmac->dmac_base + reg); else - writel(data, dmac->iomem + reg); + writel(data, dmac->dmac_base + reg); } static u32 rcar_dmac_read(struct rcar_dmac *dmac, u32 reg) { if (reg == RCAR_DMAOR) - return readw(dmac->iomem + reg); + return readw(dmac->dmac_base + reg); else - return readl(dmac->iomem + reg); + return readl(dmac->dmac_base + reg); } static u32 rcar_dmac_chan_read(struct rcar_dmac_chan *chan, u32 reg) @@ -332,6 +341,28 @@ static void rcar_dmac_chan_write(struct rcar_dmac_chan *chan, u32 reg, u32 data) writel(data, chan->iomem + reg); } +static void rcar_dmac_chan_clear(struct rcar_dmac *dmac, + struct rcar_dmac_chan *chan) +{ + if (dmac->chan_base) + rcar_dmac_chan_write(chan, RCAR_V3U_DMACHCLR, 1); + else + rcar_dmac_write(dmac, RCAR_DMACHCLR, BIT(chan->index)); +} + +static void rcar_dmac_chan_clear_all(struct rcar_dmac *dmac) +{ + struct rcar_dmac_chan *chan; + unsigned int i; + + if (dmac->chan_base) { + for_each_rcar_dmac_chan(i, dmac, chan) + rcar_dmac_chan_write(chan, RCAR_V3U_DMACHCLR, 1); + } else { + rcar_dmac_write(dmac, RCAR_DMACHCLR, dmac->channels_mask); + } +} + /* ----------------------------------------------------------------------------- * Initialization and configuration */ @@ -447,7 +478,7 @@ static int rcar_dmac_init(struct rcar_dmac *dmac) u16 dmaor; /* Clear all channels and enable the DMAC globally. */ - rcar_dmac_write(dmac, RCAR_DMACHCLR, dmac->channels_mask); + rcar_dmac_chan_clear_all(dmac); rcar_dmac_write(dmac, RCAR_DMAOR, RCAR_DMAOR_PRI_FIXED | RCAR_DMAOR_DME); @@ -817,15 +848,11 @@ static void rcar_dmac_chan_reinit(struct rcar_dmac_chan *chan) static void rcar_dmac_stop_all_chan(struct rcar_dmac *dmac) { + struct rcar_dmac_chan *chan; unsigned int i; /* Stop all channels. */ - for (i = 0; i < dmac->n_channels; ++i) { - struct rcar_dmac_chan *chan = &dmac->channels[i]; - - if (!(dmac->channels_mask & BIT(i))) - continue; - + for_each_rcar_dmac_chan(i, dmac, chan) { /* Stop and reinitialize the channel. */ spin_lock_irq(&chan->lock); rcar_dmac_chan_halt(chan); @@ -1566,7 +1593,7 @@ static irqreturn_t rcar_dmac_isr_channel(int irq, void *dev) * because channel is already stopped in error case. * We need to clear register and check DE bit as recovery. */ - rcar_dmac_write(dmac, RCAR_DMACHCLR, 1 << chan->index); + rcar_dmac_chan_clear(dmac, chan); rcar_dmac_chcr_de_barrier(chan); reinit = true; goto spin_lock_end; @@ -1732,9 +1759,7 @@ static const struct dev_pm_ops rcar_dmac_pm = { */ static int rcar_dmac_chan_probe(struct rcar_dmac *dmac, - struct rcar_dmac_chan *rchan, - const struct rcar_dmac_of_data *data, - unsigned int index) + struct rcar_dmac_chan *rchan) { struct platform_device *pdev = to_platform_device(dmac->dev); struct dma_chan *chan = &rchan->chan; @@ -1742,9 +1767,6 @@ static int rcar_dmac_chan_probe(struct rcar_dmac *dmac, char *irqname; int ret; - rchan->index = index; - rchan->iomem = dmac->iomem + data->chan_offset_base + - data->chan_offset_stride * index; rchan->mid_rid = -EINVAL; spin_lock_init(&rchan->lock); @@ -1756,13 +1778,13 @@ static int rcar_dmac_chan_probe(struct rcar_dmac *dmac, INIT_LIST_HEAD(&rchan->desc.wait); /* Request the channel interrupt. */ - sprintf(pdev_irqname, "ch%u", index); + sprintf(pdev_irqname, "ch%u", rchan->index); rchan->irq = platform_get_irq_byname(pdev, pdev_irqname); if (rchan->irq < 0) return -ENODEV; irqname = devm_kasprintf(dmac->dev, GFP_KERNEL, "%s:%u", - dev_name(dmac->dev), index); + dev_name(dmac->dev), rchan->index); if (!irqname) return -ENOMEM; @@ -1828,9 +1850,11 @@ static int rcar_dmac_probe(struct platform_device *pdev) DMA_SLAVE_BUSWIDTH_2_BYTES | DMA_SLAVE_BUSWIDTH_4_BYTES | DMA_SLAVE_BUSWIDTH_8_BYTES | DMA_SLAVE_BUSWIDTH_16_BYTES | DMA_SLAVE_BUSWIDTH_32_BYTES | DMA_SLAVE_BUSWIDTH_64_BYTES; + const struct rcar_dmac_of_data *data; + struct rcar_dmac_chan *chan; struct dma_device *engine; + void __iomem *chan_base; struct rcar_dmac *dmac; - const struct rcar_dmac_of_data *data; unsigned int i; int ret; @@ -1868,9 +1892,24 @@ static int rcar_dmac_probe(struct platform_device *pdev) return -ENOMEM; /* Request resources. */ - dmac->iomem = devm_platform_ioremap_resource(pdev, 0); - if (IS_ERR(dmac->iomem)) - return PTR_ERR(dmac->iomem); + dmac->dmac_base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(dmac->dmac_base)) + return PTR_ERR(dmac->dmac_base); + + if (!data->chan_offset_base) { + dmac->chan_base = devm_platform_ioremap_resource(pdev, 1); + if (IS_ERR(dmac->chan_base)) + return PTR_ERR(dmac->chan_base); + + chan_base = dmac->chan_base; + } else { + chan_base = dmac->dmac_base + data->chan_offset_base; + } + + for_each_rcar_dmac_chan(i, dmac, chan) { + chan->index = i; + chan->iomem = chan_base + i * data->chan_offset_stride; + } /* Enable runtime PM and initialize the device. */ pm_runtime_enable(&pdev->dev); @@ -1916,11 +1955,8 @@ static int rcar_dmac_probe(struct platform_device *pdev) INIT_LIST_HEAD(&engine->channels); - for (i = 0; i < dmac->n_channels; ++i) { - if (!(dmac->channels_mask & BIT(i))) - continue; - - ret = rcar_dmac_chan_probe(dmac, &dmac->channels[i], data, i); + for_each_rcar_dmac_chan(i, dmac, chan) { + ret = rcar_dmac_chan_probe(dmac, chan); if (ret < 0) goto error; } @@ -1968,14 +2004,22 @@ static void rcar_dmac_shutdown(struct platform_device *pdev) } static const struct rcar_dmac_of_data rcar_dmac_data = { - .chan_offset_base = 0x8000, - .chan_offset_stride = 0x80, + .chan_offset_base = 0x8000, + .chan_offset_stride = 0x80, +}; + +static const struct rcar_dmac_of_data rcar_v3u_dmac_data = { + .chan_offset_base = 0x0, + .chan_offset_stride = 0x1000, }; static const struct of_device_id rcar_dmac_of_ids[] = { { .compatible = "renesas,rcar-dmac", .data = &rcar_dmac_data, + }, { + .compatible = "renesas,dmac-r8a779a0", + .data = &rcar_v3u_dmac_data, }, { /* Sentinel */ } }; diff --git a/drivers/dma/sirf-dma.c b/drivers/dma/sirf-dma.c deleted file mode 100644 index a5c2843384fda42344277a346603246285a9088f..0000000000000000000000000000000000000000 --- a/drivers/dma/sirf-dma.c +++ /dev/null @@ -1,1170 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * DMA controller driver for CSR SiRFprimaII - * - * Copyright (c) 2011 Cambridge Silicon Radio Limited, a CSR plc group company. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "dmaengine.h" - -#define SIRFSOC_DMA_VER_A7V1 1 -#define SIRFSOC_DMA_VER_A7V2 2 -#define SIRFSOC_DMA_VER_A6 4 - -#define SIRFSOC_DMA_DESCRIPTORS 16 -#define SIRFSOC_DMA_CHANNELS 16 -#define SIRFSOC_DMA_TABLE_NUM 256 - -#define SIRFSOC_DMA_CH_ADDR 0x00 -#define SIRFSOC_DMA_CH_XLEN 0x04 -#define SIRFSOC_DMA_CH_YLEN 0x08 -#define SIRFSOC_DMA_CH_CTRL 0x0C - -#define SIRFSOC_DMA_WIDTH_0 0x100 -#define SIRFSOC_DMA_CH_VALID 0x140 -#define SIRFSOC_DMA_CH_INT 0x144 -#define SIRFSOC_DMA_INT_EN 0x148 -#define SIRFSOC_DMA_INT_EN_CLR 0x14C -#define SIRFSOC_DMA_CH_LOOP_CTRL 0x150 -#define SIRFSOC_DMA_CH_LOOP_CTRL_CLR 0x154 -#define SIRFSOC_DMA_WIDTH_ATLAS7 0x10 -#define SIRFSOC_DMA_VALID_ATLAS7 0x14 -#define SIRFSOC_DMA_INT_ATLAS7 0x18 -#define SIRFSOC_DMA_INT_EN_ATLAS7 0x1c -#define SIRFSOC_DMA_LOOP_CTRL_ATLAS7 0x20 -#define SIRFSOC_DMA_CUR_DATA_ADDR 0x34 -#define SIRFSOC_DMA_MUL_ATLAS7 0x38 -#define SIRFSOC_DMA_CH_LOOP_CTRL_ATLAS7 0x158 -#define SIRFSOC_DMA_CH_LOOP_CTRL_CLR_ATLAS7 0x15C -#define SIRFSOC_DMA_IOBG_SCMD_EN 0x800 -#define SIRFSOC_DMA_EARLY_RESP_SET 0x818 -#define SIRFSOC_DMA_EARLY_RESP_CLR 0x81C - -#define SIRFSOC_DMA_MODE_CTRL_BIT 4 -#define SIRFSOC_DMA_DIR_CTRL_BIT 5 -#define SIRFSOC_DMA_MODE_CTRL_BIT_ATLAS7 2 -#define SIRFSOC_DMA_CHAIN_CTRL_BIT_ATLAS7 3 -#define SIRFSOC_DMA_DIR_CTRL_BIT_ATLAS7 4 -#define SIRFSOC_DMA_TAB_NUM_ATLAS7 7 -#define SIRFSOC_DMA_CHAIN_INT_BIT_ATLAS7 5 -#define SIRFSOC_DMA_CHAIN_FLAG_SHIFT_ATLAS7 25 -#define SIRFSOC_DMA_CHAIN_ADDR_SHIFT 32 - -#define SIRFSOC_DMA_INT_FINI_INT_ATLAS7 BIT(0) -#define SIRFSOC_DMA_INT_CNT_INT_ATLAS7 BIT(1) -#define SIRFSOC_DMA_INT_PAU_INT_ATLAS7 BIT(2) -#define SIRFSOC_DMA_INT_LOOP_INT_ATLAS7 BIT(3) -#define SIRFSOC_DMA_INT_INV_INT_ATLAS7 BIT(4) -#define SIRFSOC_DMA_INT_END_INT_ATLAS7 BIT(5) -#define SIRFSOC_DMA_INT_ALL_ATLAS7 0x3F - -/* xlen and dma_width register is in 4 bytes boundary */ -#define SIRFSOC_DMA_WORD_LEN 4 -#define SIRFSOC_DMA_XLEN_MAX_V1 0x800 -#define SIRFSOC_DMA_XLEN_MAX_V2 0x1000 - -struct sirfsoc_dma_desc { - struct dma_async_tx_descriptor desc; - struct list_head node; - - /* SiRFprimaII 2D-DMA parameters */ - - int xlen; /* DMA xlen */ - int ylen; /* DMA ylen */ - int width; /* DMA width */ - int dir; - bool cyclic; /* is loop DMA? */ - bool chain; /* is chain DMA? */ - u32 addr; /* DMA buffer address */ - u64 chain_table[SIRFSOC_DMA_TABLE_NUM]; /* chain tbl */ -}; - -struct sirfsoc_dma_chan { - struct dma_chan chan; - struct list_head free; - struct list_head prepared; - struct list_head queued; - struct list_head active; - struct list_head completed; - unsigned long happened_cyclic; - unsigned long completed_cyclic; - - /* Lock for this structure */ - spinlock_t lock; - - int mode; -}; - -struct sirfsoc_dma_regs { - u32 ctrl[SIRFSOC_DMA_CHANNELS]; - u32 interrupt_en; -}; - -struct sirfsoc_dma { - struct dma_device dma; - struct tasklet_struct tasklet; - struct sirfsoc_dma_chan channels[SIRFSOC_DMA_CHANNELS]; - void __iomem *base; - int irq; - struct clk *clk; - int type; - void (*exec_desc)(struct sirfsoc_dma_desc *sdesc, - int cid, int burst_mode, void __iomem *base); - struct sirfsoc_dma_regs regs_save; -}; - -struct sirfsoc_dmadata { - void (*exec)(struct sirfsoc_dma_desc *sdesc, - int cid, int burst_mode, void __iomem *base); - int type; -}; - -enum sirfsoc_dma_chain_flag { - SIRFSOC_DMA_CHAIN_NORMAL = 0x01, - SIRFSOC_DMA_CHAIN_PAUSE = 0x02, - SIRFSOC_DMA_CHAIN_LOOP = 0x03, - SIRFSOC_DMA_CHAIN_END = 0x04 -}; - -#define DRV_NAME "sirfsoc_dma" - -static int sirfsoc_dma_runtime_suspend(struct device *dev); - -/* Convert struct dma_chan to struct sirfsoc_dma_chan */ -static inline -struct sirfsoc_dma_chan *dma_chan_to_sirfsoc_dma_chan(struct dma_chan *c) -{ - return container_of(c, struct sirfsoc_dma_chan, chan); -} - -/* Convert struct dma_chan to struct sirfsoc_dma */ -static inline struct sirfsoc_dma *dma_chan_to_sirfsoc_dma(struct dma_chan *c) -{ - struct sirfsoc_dma_chan *schan = dma_chan_to_sirfsoc_dma_chan(c); - return container_of(schan, struct sirfsoc_dma, channels[c->chan_id]); -} - -static void sirfsoc_dma_execute_hw_a7v2(struct sirfsoc_dma_desc *sdesc, - int cid, int burst_mode, void __iomem *base) -{ - if (sdesc->chain) { - /* DMA v2 HW chain mode */ - writel_relaxed((sdesc->dir << SIRFSOC_DMA_DIR_CTRL_BIT_ATLAS7) | - (sdesc->chain << - SIRFSOC_DMA_CHAIN_CTRL_BIT_ATLAS7) | - (0x8 << SIRFSOC_DMA_TAB_NUM_ATLAS7) | 0x3, - base + SIRFSOC_DMA_CH_CTRL); - } else { - /* DMA v2 legacy mode */ - writel_relaxed(sdesc->xlen, base + SIRFSOC_DMA_CH_XLEN); - writel_relaxed(sdesc->ylen, base + SIRFSOC_DMA_CH_YLEN); - writel_relaxed(sdesc->width, base + SIRFSOC_DMA_WIDTH_ATLAS7); - writel_relaxed((sdesc->width*((sdesc->ylen+1)>>1)), - base + SIRFSOC_DMA_MUL_ATLAS7); - writel_relaxed((sdesc->dir << SIRFSOC_DMA_DIR_CTRL_BIT_ATLAS7) | - (sdesc->chain << - SIRFSOC_DMA_CHAIN_CTRL_BIT_ATLAS7) | - 0x3, base + SIRFSOC_DMA_CH_CTRL); - } - writel_relaxed(sdesc->chain ? SIRFSOC_DMA_INT_END_INT_ATLAS7 : - (SIRFSOC_DMA_INT_FINI_INT_ATLAS7 | - SIRFSOC_DMA_INT_LOOP_INT_ATLAS7), - base + SIRFSOC_DMA_INT_EN_ATLAS7); - writel(sdesc->addr, base + SIRFSOC_DMA_CH_ADDR); - if (sdesc->cyclic) - writel(0x10001, base + SIRFSOC_DMA_LOOP_CTRL_ATLAS7); -} - -static void sirfsoc_dma_execute_hw_a7v1(struct sirfsoc_dma_desc *sdesc, - int cid, int burst_mode, void __iomem *base) -{ - writel_relaxed(1, base + SIRFSOC_DMA_IOBG_SCMD_EN); - writel_relaxed((1 << cid), base + SIRFSOC_DMA_EARLY_RESP_SET); - writel_relaxed(sdesc->width, base + SIRFSOC_DMA_WIDTH_0 + cid * 4); - writel_relaxed(cid | (burst_mode << SIRFSOC_DMA_MODE_CTRL_BIT) | - (sdesc->dir << SIRFSOC_DMA_DIR_CTRL_BIT), - base + cid * 0x10 + SIRFSOC_DMA_CH_CTRL); - writel_relaxed(sdesc->xlen, base + cid * 0x10 + SIRFSOC_DMA_CH_XLEN); - writel_relaxed(sdesc->ylen, base + cid * 0x10 + SIRFSOC_DMA_CH_YLEN); - writel_relaxed(readl_relaxed(base + SIRFSOC_DMA_INT_EN) | - (1 << cid), base + SIRFSOC_DMA_INT_EN); - writel(sdesc->addr >> 2, base + cid * 0x10 + SIRFSOC_DMA_CH_ADDR); - if (sdesc->cyclic) { - writel((1 << cid) | 1 << (cid + 16) | - readl_relaxed(base + SIRFSOC_DMA_CH_LOOP_CTRL_ATLAS7), - base + SIRFSOC_DMA_CH_LOOP_CTRL_ATLAS7); - } - -} - -static void sirfsoc_dma_execute_hw_a6(struct sirfsoc_dma_desc *sdesc, - int cid, int burst_mode, void __iomem *base) -{ - writel_relaxed(sdesc->width, base + SIRFSOC_DMA_WIDTH_0 + cid * 4); - writel_relaxed(cid | (burst_mode << SIRFSOC_DMA_MODE_CTRL_BIT) | - (sdesc->dir << SIRFSOC_DMA_DIR_CTRL_BIT), - base + cid * 0x10 + SIRFSOC_DMA_CH_CTRL); - writel_relaxed(sdesc->xlen, base + cid * 0x10 + SIRFSOC_DMA_CH_XLEN); - writel_relaxed(sdesc->ylen, base + cid * 0x10 + SIRFSOC_DMA_CH_YLEN); - writel_relaxed(readl_relaxed(base + SIRFSOC_DMA_INT_EN) | - (1 << cid), base + SIRFSOC_DMA_INT_EN); - writel(sdesc->addr >> 2, base + cid * 0x10 + SIRFSOC_DMA_CH_ADDR); - if (sdesc->cyclic) { - writel((1 << cid) | 1 << (cid + 16) | - readl_relaxed(base + SIRFSOC_DMA_CH_LOOP_CTRL), - base + SIRFSOC_DMA_CH_LOOP_CTRL); - } - -} - -/* Execute all queued DMA descriptors */ -static void sirfsoc_dma_execute(struct sirfsoc_dma_chan *schan) -{ - struct sirfsoc_dma *sdma = dma_chan_to_sirfsoc_dma(&schan->chan); - int cid = schan->chan.chan_id; - struct sirfsoc_dma_desc *sdesc = NULL; - void __iomem *base; - - /* - * lock has been held by functions calling this, so we don't hold - * lock again - */ - base = sdma->base; - sdesc = list_first_entry(&schan->queued, struct sirfsoc_dma_desc, - node); - /* Move the first queued descriptor to active list */ - list_move_tail(&sdesc->node, &schan->active); - - if (sdma->type == SIRFSOC_DMA_VER_A7V2) - cid = 0; - - /* Start the DMA transfer */ - sdma->exec_desc(sdesc, cid, schan->mode, base); - - if (sdesc->cyclic) - schan->happened_cyclic = schan->completed_cyclic = 0; -} - -/* Interrupt handler */ -static irqreturn_t sirfsoc_dma_irq(int irq, void *data) -{ - struct sirfsoc_dma *sdma = data; - struct sirfsoc_dma_chan *schan; - struct sirfsoc_dma_desc *sdesc = NULL; - u32 is; - bool chain; - int ch; - void __iomem *reg; - - switch (sdma->type) { - case SIRFSOC_DMA_VER_A6: - case SIRFSOC_DMA_VER_A7V1: - is = readl(sdma->base + SIRFSOC_DMA_CH_INT); - reg = sdma->base + SIRFSOC_DMA_CH_INT; - while ((ch = fls(is) - 1) >= 0) { - is &= ~(1 << ch); - writel_relaxed(1 << ch, reg); - schan = &sdma->channels[ch]; - spin_lock(&schan->lock); - sdesc = list_first_entry(&schan->active, - struct sirfsoc_dma_desc, node); - if (!sdesc->cyclic) { - /* Execute queued descriptors */ - list_splice_tail_init(&schan->active, - &schan->completed); - dma_cookie_complete(&sdesc->desc); - if (!list_empty(&schan->queued)) - sirfsoc_dma_execute(schan); - } else - schan->happened_cyclic++; - spin_unlock(&schan->lock); - } - break; - - case SIRFSOC_DMA_VER_A7V2: - is = readl(sdma->base + SIRFSOC_DMA_INT_ATLAS7); - - reg = sdma->base + SIRFSOC_DMA_INT_ATLAS7; - writel_relaxed(SIRFSOC_DMA_INT_ALL_ATLAS7, reg); - schan = &sdma->channels[0]; - spin_lock(&schan->lock); - sdesc = list_first_entry(&schan->active, - struct sirfsoc_dma_desc, node); - if (!sdesc->cyclic) { - chain = sdesc->chain; - if ((chain && (is & SIRFSOC_DMA_INT_END_INT_ATLAS7)) || - (!chain && - (is & SIRFSOC_DMA_INT_FINI_INT_ATLAS7))) { - /* Execute queued descriptors */ - list_splice_tail_init(&schan->active, - &schan->completed); - dma_cookie_complete(&sdesc->desc); - if (!list_empty(&schan->queued)) - sirfsoc_dma_execute(schan); - } - } else if (sdesc->cyclic && (is & - SIRFSOC_DMA_INT_LOOP_INT_ATLAS7)) - schan->happened_cyclic++; - - spin_unlock(&schan->lock); - break; - - default: - break; - } - - /* Schedule tasklet */ - tasklet_schedule(&sdma->tasklet); - - return IRQ_HANDLED; -} - -/* process completed descriptors */ -static void sirfsoc_dma_process_completed(struct sirfsoc_dma *sdma) -{ - dma_cookie_t last_cookie = 0; - struct sirfsoc_dma_chan *schan; - struct sirfsoc_dma_desc *sdesc; - struct dma_async_tx_descriptor *desc; - unsigned long flags; - unsigned long happened_cyclic; - LIST_HEAD(list); - int i; - - for (i = 0; i < sdma->dma.chancnt; i++) { - schan = &sdma->channels[i]; - - /* Get all completed descriptors */ - spin_lock_irqsave(&schan->lock, flags); - if (!list_empty(&schan->completed)) { - list_splice_tail_init(&schan->completed, &list); - spin_unlock_irqrestore(&schan->lock, flags); - - /* Execute callbacks and run dependencies */ - list_for_each_entry(sdesc, &list, node) { - desc = &sdesc->desc; - - dmaengine_desc_get_callback_invoke(desc, NULL); - last_cookie = desc->cookie; - dma_run_dependencies(desc); - } - - /* Free descriptors */ - spin_lock_irqsave(&schan->lock, flags); - list_splice_tail_init(&list, &schan->free); - schan->chan.completed_cookie = last_cookie; - spin_unlock_irqrestore(&schan->lock, flags); - } else { - if (list_empty(&schan->active)) { - spin_unlock_irqrestore(&schan->lock, flags); - continue; - } - - /* for cyclic channel, desc is always in active list */ - sdesc = list_first_entry(&schan->active, - struct sirfsoc_dma_desc, node); - - /* cyclic DMA */ - happened_cyclic = schan->happened_cyclic; - spin_unlock_irqrestore(&schan->lock, flags); - - desc = &sdesc->desc; - while (happened_cyclic != schan->completed_cyclic) { - dmaengine_desc_get_callback_invoke(desc, NULL); - schan->completed_cyclic++; - } - } - } -} - -/* DMA Tasklet */ -static void sirfsoc_dma_tasklet(struct tasklet_struct *t) -{ - struct sirfsoc_dma *sdma = from_tasklet(sdma, t, tasklet); - - sirfsoc_dma_process_completed(sdma); -} - -/* Submit descriptor to hardware */ -static dma_cookie_t sirfsoc_dma_tx_submit(struct dma_async_tx_descriptor *txd) -{ - struct sirfsoc_dma_chan *schan = dma_chan_to_sirfsoc_dma_chan(txd->chan); - struct sirfsoc_dma_desc *sdesc; - unsigned long flags; - dma_cookie_t cookie; - - sdesc = container_of(txd, struct sirfsoc_dma_desc, desc); - - spin_lock_irqsave(&schan->lock, flags); - - /* Move descriptor to queue */ - list_move_tail(&sdesc->node, &schan->queued); - - cookie = dma_cookie_assign(txd); - - spin_unlock_irqrestore(&schan->lock, flags); - - return cookie; -} - -static int sirfsoc_dma_slave_config(struct dma_chan *chan, - struct dma_slave_config *config) -{ - struct sirfsoc_dma_chan *schan = dma_chan_to_sirfsoc_dma_chan(chan); - unsigned long flags; - - if ((config->src_addr_width != DMA_SLAVE_BUSWIDTH_4_BYTES) || - (config->dst_addr_width != DMA_SLAVE_BUSWIDTH_4_BYTES)) - return -EINVAL; - - spin_lock_irqsave(&schan->lock, flags); - schan->mode = (config->src_maxburst == 4 ? 1 : 0); - spin_unlock_irqrestore(&schan->lock, flags); - - return 0; -} - -static int sirfsoc_dma_terminate_all(struct dma_chan *chan) -{ - struct sirfsoc_dma_chan *schan = dma_chan_to_sirfsoc_dma_chan(chan); - struct sirfsoc_dma *sdma = dma_chan_to_sirfsoc_dma(&schan->chan); - int cid = schan->chan.chan_id; - unsigned long flags; - - spin_lock_irqsave(&schan->lock, flags); - - switch (sdma->type) { - case SIRFSOC_DMA_VER_A7V1: - writel_relaxed(1 << cid, sdma->base + SIRFSOC_DMA_INT_EN_CLR); - writel_relaxed(1 << cid, sdma->base + SIRFSOC_DMA_CH_INT); - writel_relaxed((1 << cid) | 1 << (cid + 16), - sdma->base + - SIRFSOC_DMA_CH_LOOP_CTRL_CLR_ATLAS7); - writel_relaxed(1 << cid, sdma->base + SIRFSOC_DMA_CH_VALID); - break; - case SIRFSOC_DMA_VER_A7V2: - writel_relaxed(0, sdma->base + SIRFSOC_DMA_INT_EN_ATLAS7); - writel_relaxed(SIRFSOC_DMA_INT_ALL_ATLAS7, - sdma->base + SIRFSOC_DMA_INT_ATLAS7); - writel_relaxed(0, sdma->base + SIRFSOC_DMA_LOOP_CTRL_ATLAS7); - writel_relaxed(0, sdma->base + SIRFSOC_DMA_VALID_ATLAS7); - break; - case SIRFSOC_DMA_VER_A6: - writel_relaxed(readl_relaxed(sdma->base + SIRFSOC_DMA_INT_EN) & - ~(1 << cid), sdma->base + SIRFSOC_DMA_INT_EN); - writel_relaxed(readl_relaxed(sdma->base + - SIRFSOC_DMA_CH_LOOP_CTRL) & - ~((1 << cid) | 1 << (cid + 16)), - sdma->base + SIRFSOC_DMA_CH_LOOP_CTRL); - writel_relaxed(1 << cid, sdma->base + SIRFSOC_DMA_CH_VALID); - break; - default: - break; - } - - list_splice_tail_init(&schan->active, &schan->free); - list_splice_tail_init(&schan->queued, &schan->free); - - spin_unlock_irqrestore(&schan->lock, flags); - - return 0; -} - -static int sirfsoc_dma_pause_chan(struct dma_chan *chan) -{ - struct sirfsoc_dma_chan *schan = dma_chan_to_sirfsoc_dma_chan(chan); - struct sirfsoc_dma *sdma = dma_chan_to_sirfsoc_dma(&schan->chan); - int cid = schan->chan.chan_id; - unsigned long flags; - - spin_lock_irqsave(&schan->lock, flags); - - switch (sdma->type) { - case SIRFSOC_DMA_VER_A7V1: - writel_relaxed((1 << cid) | 1 << (cid + 16), - sdma->base + - SIRFSOC_DMA_CH_LOOP_CTRL_CLR_ATLAS7); - break; - case SIRFSOC_DMA_VER_A7V2: - writel_relaxed(0, sdma->base + SIRFSOC_DMA_LOOP_CTRL_ATLAS7); - break; - case SIRFSOC_DMA_VER_A6: - writel_relaxed(readl_relaxed(sdma->base + - SIRFSOC_DMA_CH_LOOP_CTRL) & - ~((1 << cid) | 1 << (cid + 16)), - sdma->base + SIRFSOC_DMA_CH_LOOP_CTRL); - break; - - default: - break; - } - - spin_unlock_irqrestore(&schan->lock, flags); - - return 0; -} - -static int sirfsoc_dma_resume_chan(struct dma_chan *chan) -{ - struct sirfsoc_dma_chan *schan = dma_chan_to_sirfsoc_dma_chan(chan); - struct sirfsoc_dma *sdma = dma_chan_to_sirfsoc_dma(&schan->chan); - int cid = schan->chan.chan_id; - unsigned long flags; - - spin_lock_irqsave(&schan->lock, flags); - switch (sdma->type) { - case SIRFSOC_DMA_VER_A7V1: - writel_relaxed((1 << cid) | 1 << (cid + 16), - sdma->base + SIRFSOC_DMA_CH_LOOP_CTRL_ATLAS7); - break; - case SIRFSOC_DMA_VER_A7V2: - writel_relaxed(0x10001, - sdma->base + SIRFSOC_DMA_LOOP_CTRL_ATLAS7); - break; - case SIRFSOC_DMA_VER_A6: - writel_relaxed(readl_relaxed(sdma->base + - SIRFSOC_DMA_CH_LOOP_CTRL) | - ((1 << cid) | 1 << (cid + 16)), - sdma->base + SIRFSOC_DMA_CH_LOOP_CTRL); - break; - - default: - break; - } - - spin_unlock_irqrestore(&schan->lock, flags); - - return 0; -} - -/* Alloc channel resources */ -static int sirfsoc_dma_alloc_chan_resources(struct dma_chan *chan) -{ - struct sirfsoc_dma *sdma = dma_chan_to_sirfsoc_dma(chan); - struct sirfsoc_dma_chan *schan = dma_chan_to_sirfsoc_dma_chan(chan); - struct sirfsoc_dma_desc *sdesc; - unsigned long flags; - LIST_HEAD(descs); - int i; - - pm_runtime_get_sync(sdma->dma.dev); - - /* Alloc descriptors for this channel */ - for (i = 0; i < SIRFSOC_DMA_DESCRIPTORS; i++) { - sdesc = kzalloc(sizeof(*sdesc), GFP_KERNEL); - if (!sdesc) { - dev_notice(sdma->dma.dev, "Memory allocation error. " - "Allocated only %u descriptors\n", i); - break; - } - - dma_async_tx_descriptor_init(&sdesc->desc, chan); - sdesc->desc.flags = DMA_CTRL_ACK; - sdesc->desc.tx_submit = sirfsoc_dma_tx_submit; - - list_add_tail(&sdesc->node, &descs); - } - - /* Return error only if no descriptors were allocated */ - if (i == 0) - return -ENOMEM; - - spin_lock_irqsave(&schan->lock, flags); - - list_splice_tail_init(&descs, &schan->free); - spin_unlock_irqrestore(&schan->lock, flags); - - return i; -} - -/* Free channel resources */ -static void sirfsoc_dma_free_chan_resources(struct dma_chan *chan) -{ - struct sirfsoc_dma_chan *schan = dma_chan_to_sirfsoc_dma_chan(chan); - struct sirfsoc_dma *sdma = dma_chan_to_sirfsoc_dma(chan); - struct sirfsoc_dma_desc *sdesc, *tmp; - unsigned long flags; - LIST_HEAD(descs); - - spin_lock_irqsave(&schan->lock, flags); - - /* Channel must be idle */ - BUG_ON(!list_empty(&schan->prepared)); - BUG_ON(!list_empty(&schan->queued)); - BUG_ON(!list_empty(&schan->active)); - BUG_ON(!list_empty(&schan->completed)); - - /* Move data */ - list_splice_tail_init(&schan->free, &descs); - - spin_unlock_irqrestore(&schan->lock, flags); - - /* Free descriptors */ - list_for_each_entry_safe(sdesc, tmp, &descs, node) - kfree(sdesc); - - pm_runtime_put(sdma->dma.dev); -} - -/* Send pending descriptor to hardware */ -static void sirfsoc_dma_issue_pending(struct dma_chan *chan) -{ - struct sirfsoc_dma_chan *schan = dma_chan_to_sirfsoc_dma_chan(chan); - unsigned long flags; - - spin_lock_irqsave(&schan->lock, flags); - - if (list_empty(&schan->active) && !list_empty(&schan->queued)) - sirfsoc_dma_execute(schan); - - spin_unlock_irqrestore(&schan->lock, flags); -} - -/* Check request completion status */ -static enum dma_status -sirfsoc_dma_tx_status(struct dma_chan *chan, dma_cookie_t cookie, - struct dma_tx_state *txstate) -{ - struct sirfsoc_dma *sdma = dma_chan_to_sirfsoc_dma(chan); - struct sirfsoc_dma_chan *schan = dma_chan_to_sirfsoc_dma_chan(chan); - unsigned long flags; - enum dma_status ret; - struct sirfsoc_dma_desc *sdesc; - int cid = schan->chan.chan_id; - unsigned long dma_pos; - unsigned long dma_request_bytes; - unsigned long residue; - - spin_lock_irqsave(&schan->lock, flags); - - if (list_empty(&schan->active)) { - ret = dma_cookie_status(chan, cookie, txstate); - dma_set_residue(txstate, 0); - spin_unlock_irqrestore(&schan->lock, flags); - return ret; - } - sdesc = list_first_entry(&schan->active, struct sirfsoc_dma_desc, node); - if (sdesc->cyclic) - dma_request_bytes = (sdesc->xlen + 1) * (sdesc->ylen + 1) * - (sdesc->width * SIRFSOC_DMA_WORD_LEN); - else - dma_request_bytes = sdesc->xlen * SIRFSOC_DMA_WORD_LEN; - - ret = dma_cookie_status(chan, cookie, txstate); - - if (sdma->type == SIRFSOC_DMA_VER_A7V2) - cid = 0; - - if (sdma->type == SIRFSOC_DMA_VER_A7V2) { - dma_pos = readl_relaxed(sdma->base + SIRFSOC_DMA_CUR_DATA_ADDR); - } else { - dma_pos = readl_relaxed( - sdma->base + cid * 0x10 + SIRFSOC_DMA_CH_ADDR) << 2; - } - - residue = dma_request_bytes - (dma_pos - sdesc->addr); - dma_set_residue(txstate, residue); - - spin_unlock_irqrestore(&schan->lock, flags); - - return ret; -} - -static struct dma_async_tx_descriptor *sirfsoc_dma_prep_interleaved( - struct dma_chan *chan, struct dma_interleaved_template *xt, - unsigned long flags) -{ - struct sirfsoc_dma *sdma = dma_chan_to_sirfsoc_dma(chan); - struct sirfsoc_dma_chan *schan = dma_chan_to_sirfsoc_dma_chan(chan); - struct sirfsoc_dma_desc *sdesc = NULL; - unsigned long iflags; - int ret; - - if ((xt->dir != DMA_MEM_TO_DEV) && (xt->dir != DMA_DEV_TO_MEM)) { - ret = -EINVAL; - goto err_dir; - } - - /* Get free descriptor */ - spin_lock_irqsave(&schan->lock, iflags); - if (!list_empty(&schan->free)) { - sdesc = list_first_entry(&schan->free, struct sirfsoc_dma_desc, - node); - list_del(&sdesc->node); - } - spin_unlock_irqrestore(&schan->lock, iflags); - - if (!sdesc) { - /* try to free completed descriptors */ - sirfsoc_dma_process_completed(sdma); - ret = 0; - goto no_desc; - } - - /* Place descriptor in prepared list */ - spin_lock_irqsave(&schan->lock, iflags); - - /* - * Number of chunks in a frame can only be 1 for prima2 - * and ylen (number of frame - 1) must be at least 0 - */ - if ((xt->frame_size == 1) && (xt->numf > 0)) { - sdesc->cyclic = 0; - sdesc->xlen = xt->sgl[0].size / SIRFSOC_DMA_WORD_LEN; - sdesc->width = (xt->sgl[0].size + xt->sgl[0].icg) / - SIRFSOC_DMA_WORD_LEN; - sdesc->ylen = xt->numf - 1; - if (xt->dir == DMA_MEM_TO_DEV) { - sdesc->addr = xt->src_start; - sdesc->dir = 1; - } else { - sdesc->addr = xt->dst_start; - sdesc->dir = 0; - } - - list_add_tail(&sdesc->node, &schan->prepared); - } else { - pr_err("sirfsoc DMA Invalid xfer\n"); - ret = -EINVAL; - goto err_xfer; - } - spin_unlock_irqrestore(&schan->lock, iflags); - - return &sdesc->desc; -err_xfer: - spin_unlock_irqrestore(&schan->lock, iflags); -no_desc: -err_dir: - return ERR_PTR(ret); -} - -static struct dma_async_tx_descriptor * -sirfsoc_dma_prep_cyclic(struct dma_chan *chan, dma_addr_t addr, - size_t buf_len, size_t period_len, - enum dma_transfer_direction direction, unsigned long flags) -{ - struct sirfsoc_dma_chan *schan = dma_chan_to_sirfsoc_dma_chan(chan); - struct sirfsoc_dma_desc *sdesc = NULL; - unsigned long iflags; - - /* - * we only support cycle transfer with 2 period - * If the X-length is set to 0, it would be the loop mode. - * The DMA address keeps increasing until reaching the end of a loop - * area whose size is defined by (DMA_WIDTH x (Y_LENGTH + 1)). Then - * the DMA address goes back to the beginning of this area. - * In loop mode, the DMA data region is divided into two parts, BUFA - * and BUFB. DMA controller generates interrupts twice in each loop: - * when the DMA address reaches the end of BUFA or the end of the - * BUFB - */ - if (buf_len != 2 * period_len) - return ERR_PTR(-EINVAL); - - /* Get free descriptor */ - spin_lock_irqsave(&schan->lock, iflags); - if (!list_empty(&schan->free)) { - sdesc = list_first_entry(&schan->free, struct sirfsoc_dma_desc, - node); - list_del(&sdesc->node); - } - spin_unlock_irqrestore(&schan->lock, iflags); - - if (!sdesc) - return NULL; - - /* Place descriptor in prepared list */ - spin_lock_irqsave(&schan->lock, iflags); - sdesc->addr = addr; - sdesc->cyclic = 1; - sdesc->xlen = 0; - sdesc->ylen = buf_len / SIRFSOC_DMA_WORD_LEN - 1; - sdesc->width = 1; - list_add_tail(&sdesc->node, &schan->prepared); - spin_unlock_irqrestore(&schan->lock, iflags); - - return &sdesc->desc; -} - -/* - * The DMA controller consists of 16 independent DMA channels. - * Each channel is allocated to a different function - */ -bool sirfsoc_dma_filter_id(struct dma_chan *chan, void *chan_id) -{ - unsigned int ch_nr = (unsigned int) chan_id; - - if (ch_nr == chan->chan_id + - chan->device->dev_id * SIRFSOC_DMA_CHANNELS) - return true; - - return false; -} -EXPORT_SYMBOL(sirfsoc_dma_filter_id); - -#define SIRFSOC_DMA_BUSWIDTHS \ - (BIT(DMA_SLAVE_BUSWIDTH_UNDEFINED) | \ - BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \ - BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \ - BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | \ - BIT(DMA_SLAVE_BUSWIDTH_8_BYTES)) - -static struct dma_chan *of_dma_sirfsoc_xlate(struct of_phandle_args *dma_spec, - struct of_dma *ofdma) -{ - struct sirfsoc_dma *sdma = ofdma->of_dma_data; - unsigned int request = dma_spec->args[0]; - - if (request >= SIRFSOC_DMA_CHANNELS) - return NULL; - - return dma_get_slave_channel(&sdma->channels[request].chan); -} - -static int sirfsoc_dma_probe(struct platform_device *op) -{ - struct device_node *dn = op->dev.of_node; - struct device *dev = &op->dev; - struct dma_device *dma; - struct sirfsoc_dma *sdma; - struct sirfsoc_dma_chan *schan; - struct sirfsoc_dmadata *data; - struct resource res; - ulong regs_start, regs_size; - u32 id; - int ret, i; - - sdma = devm_kzalloc(dev, sizeof(*sdma), GFP_KERNEL); - if (!sdma) - return -ENOMEM; - - data = (struct sirfsoc_dmadata *) - (of_match_device(op->dev.driver->of_match_table, - &op->dev)->data); - sdma->exec_desc = data->exec; - sdma->type = data->type; - - if (of_property_read_u32(dn, "cell-index", &id)) { - dev_err(dev, "Fail to get DMAC index\n"); - return -ENODEV; - } - - sdma->irq = irq_of_parse_and_map(dn, 0); - if (!sdma->irq) { - dev_err(dev, "Error mapping IRQ!\n"); - return -EINVAL; - } - - sdma->clk = devm_clk_get(dev, NULL); - if (IS_ERR(sdma->clk)) { - dev_err(dev, "failed to get a clock.\n"); - return PTR_ERR(sdma->clk); - } - - ret = of_address_to_resource(dn, 0, &res); - if (ret) { - dev_err(dev, "Error parsing memory region!\n"); - goto irq_dispose; - } - - regs_start = res.start; - regs_size = resource_size(&res); - - sdma->base = devm_ioremap(dev, regs_start, regs_size); - if (!sdma->base) { - dev_err(dev, "Error mapping memory region!\n"); - ret = -ENOMEM; - goto irq_dispose; - } - - ret = request_irq(sdma->irq, &sirfsoc_dma_irq, 0, DRV_NAME, sdma); - if (ret) { - dev_err(dev, "Error requesting IRQ!\n"); - ret = -EINVAL; - goto irq_dispose; - } - - dma = &sdma->dma; - dma->dev = dev; - - dma->device_alloc_chan_resources = sirfsoc_dma_alloc_chan_resources; - dma->device_free_chan_resources = sirfsoc_dma_free_chan_resources; - dma->device_issue_pending = sirfsoc_dma_issue_pending; - dma->device_config = sirfsoc_dma_slave_config; - dma->device_pause = sirfsoc_dma_pause_chan; - dma->device_resume = sirfsoc_dma_resume_chan; - dma->device_terminate_all = sirfsoc_dma_terminate_all; - dma->device_tx_status = sirfsoc_dma_tx_status; - dma->device_prep_interleaved_dma = sirfsoc_dma_prep_interleaved; - dma->device_prep_dma_cyclic = sirfsoc_dma_prep_cyclic; - dma->src_addr_widths = SIRFSOC_DMA_BUSWIDTHS; - dma->dst_addr_widths = SIRFSOC_DMA_BUSWIDTHS; - dma->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); - - INIT_LIST_HEAD(&dma->channels); - dma_cap_set(DMA_SLAVE, dma->cap_mask); - dma_cap_set(DMA_CYCLIC, dma->cap_mask); - dma_cap_set(DMA_INTERLEAVE, dma->cap_mask); - dma_cap_set(DMA_PRIVATE, dma->cap_mask); - - for (i = 0; i < SIRFSOC_DMA_CHANNELS; i++) { - schan = &sdma->channels[i]; - - schan->chan.device = dma; - dma_cookie_init(&schan->chan); - - INIT_LIST_HEAD(&schan->free); - INIT_LIST_HEAD(&schan->prepared); - INIT_LIST_HEAD(&schan->queued); - INIT_LIST_HEAD(&schan->active); - INIT_LIST_HEAD(&schan->completed); - - spin_lock_init(&schan->lock); - list_add_tail(&schan->chan.device_node, &dma->channels); - } - - tasklet_setup(&sdma->tasklet, sirfsoc_dma_tasklet); - - /* Register DMA engine */ - dev_set_drvdata(dev, sdma); - - ret = dma_async_device_register(dma); - if (ret) - goto free_irq; - - /* Device-tree DMA controller registration */ - ret = of_dma_controller_register(dn, of_dma_sirfsoc_xlate, sdma); - if (ret) { - dev_err(dev, "failed to register DMA controller\n"); - goto unreg_dma_dev; - } - - pm_runtime_enable(&op->dev); - dev_info(dev, "initialized SIRFSOC DMAC driver\n"); - - return 0; - -unreg_dma_dev: - dma_async_device_unregister(dma); -free_irq: - free_irq(sdma->irq, sdma); -irq_dispose: - irq_dispose_mapping(sdma->irq); - return ret; -} - -static int sirfsoc_dma_remove(struct platform_device *op) -{ - struct device *dev = &op->dev; - struct sirfsoc_dma *sdma = dev_get_drvdata(dev); - - of_dma_controller_free(op->dev.of_node); - dma_async_device_unregister(&sdma->dma); - free_irq(sdma->irq, sdma); - tasklet_kill(&sdma->tasklet); - irq_dispose_mapping(sdma->irq); - pm_runtime_disable(&op->dev); - if (!pm_runtime_status_suspended(&op->dev)) - sirfsoc_dma_runtime_suspend(&op->dev); - - return 0; -} - -static int __maybe_unused sirfsoc_dma_runtime_suspend(struct device *dev) -{ - struct sirfsoc_dma *sdma = dev_get_drvdata(dev); - - clk_disable_unprepare(sdma->clk); - return 0; -} - -static int __maybe_unused sirfsoc_dma_runtime_resume(struct device *dev) -{ - struct sirfsoc_dma *sdma = dev_get_drvdata(dev); - int ret; - - ret = clk_prepare_enable(sdma->clk); - if (ret < 0) { - dev_err(dev, "clk_enable failed: %d\n", ret); - return ret; - } - return 0; -} - -static int __maybe_unused sirfsoc_dma_pm_suspend(struct device *dev) -{ - struct sirfsoc_dma *sdma = dev_get_drvdata(dev); - struct sirfsoc_dma_regs *save = &sdma->regs_save; - struct sirfsoc_dma_chan *schan; - int ch; - int ret; - int count; - u32 int_offset; - - /* - * if we were runtime-suspended before, resume to enable clock - * before accessing register - */ - if (pm_runtime_status_suspended(dev)) { - ret = sirfsoc_dma_runtime_resume(dev); - if (ret < 0) - return ret; - } - - if (sdma->type == SIRFSOC_DMA_VER_A7V2) { - count = 1; - int_offset = SIRFSOC_DMA_INT_EN_ATLAS7; - } else { - count = SIRFSOC_DMA_CHANNELS; - int_offset = SIRFSOC_DMA_INT_EN; - } - - /* - * DMA controller will lose all registers while suspending - * so we need to save registers for active channels - */ - for (ch = 0; ch < count; ch++) { - schan = &sdma->channels[ch]; - if (list_empty(&schan->active)) - continue; - save->ctrl[ch] = readl_relaxed(sdma->base + - ch * 0x10 + SIRFSOC_DMA_CH_CTRL); - } - save->interrupt_en = readl_relaxed(sdma->base + int_offset); - - /* Disable clock */ - sirfsoc_dma_runtime_suspend(dev); - - return 0; -} - -static int __maybe_unused sirfsoc_dma_pm_resume(struct device *dev) -{ - struct sirfsoc_dma *sdma = dev_get_drvdata(dev); - struct sirfsoc_dma_regs *save = &sdma->regs_save; - struct sirfsoc_dma_desc *sdesc; - struct sirfsoc_dma_chan *schan; - int ch; - int ret; - int count; - u32 int_offset; - u32 width_offset; - - /* Enable clock before accessing register */ - ret = sirfsoc_dma_runtime_resume(dev); - if (ret < 0) - return ret; - - if (sdma->type == SIRFSOC_DMA_VER_A7V2) { - count = 1; - int_offset = SIRFSOC_DMA_INT_EN_ATLAS7; - width_offset = SIRFSOC_DMA_WIDTH_ATLAS7; - } else { - count = SIRFSOC_DMA_CHANNELS; - int_offset = SIRFSOC_DMA_INT_EN; - width_offset = SIRFSOC_DMA_WIDTH_0; - } - - writel_relaxed(save->interrupt_en, sdma->base + int_offset); - for (ch = 0; ch < count; ch++) { - schan = &sdma->channels[ch]; - if (list_empty(&schan->active)) - continue; - sdesc = list_first_entry(&schan->active, - struct sirfsoc_dma_desc, - node); - writel_relaxed(sdesc->width, - sdma->base + width_offset + ch * 4); - writel_relaxed(sdesc->xlen, - sdma->base + ch * 0x10 + SIRFSOC_DMA_CH_XLEN); - writel_relaxed(sdesc->ylen, - sdma->base + ch * 0x10 + SIRFSOC_DMA_CH_YLEN); - writel_relaxed(save->ctrl[ch], - sdma->base + ch * 0x10 + SIRFSOC_DMA_CH_CTRL); - if (sdma->type == SIRFSOC_DMA_VER_A7V2) { - writel_relaxed(sdesc->addr, - sdma->base + SIRFSOC_DMA_CH_ADDR); - } else { - writel_relaxed(sdesc->addr >> 2, - sdma->base + ch * 0x10 + SIRFSOC_DMA_CH_ADDR); - - } - } - - /* if we were runtime-suspended before, suspend again */ - if (pm_runtime_status_suspended(dev)) - sirfsoc_dma_runtime_suspend(dev); - - return 0; -} - -static const struct dev_pm_ops sirfsoc_dma_pm_ops = { - SET_RUNTIME_PM_OPS(sirfsoc_dma_runtime_suspend, sirfsoc_dma_runtime_resume, NULL) - SET_SYSTEM_SLEEP_PM_OPS(sirfsoc_dma_pm_suspend, sirfsoc_dma_pm_resume) -}; - -static struct sirfsoc_dmadata sirfsoc_dmadata_a6 = { - .exec = sirfsoc_dma_execute_hw_a6, - .type = SIRFSOC_DMA_VER_A6, -}; - -static struct sirfsoc_dmadata sirfsoc_dmadata_a7v1 = { - .exec = sirfsoc_dma_execute_hw_a7v1, - .type = SIRFSOC_DMA_VER_A7V1, -}; - -static struct sirfsoc_dmadata sirfsoc_dmadata_a7v2 = { - .exec = sirfsoc_dma_execute_hw_a7v2, - .type = SIRFSOC_DMA_VER_A7V2, -}; - -static const struct of_device_id sirfsoc_dma_match[] = { - { .compatible = "sirf,prima2-dmac", .data = &sirfsoc_dmadata_a6,}, - { .compatible = "sirf,atlas7-dmac", .data = &sirfsoc_dmadata_a7v1,}, - { .compatible = "sirf,atlas7-dmac-v2", .data = &sirfsoc_dmadata_a7v2,}, - {}, -}; -MODULE_DEVICE_TABLE(of, sirfsoc_dma_match); - -static struct platform_driver sirfsoc_dma_driver = { - .probe = sirfsoc_dma_probe, - .remove = sirfsoc_dma_remove, - .driver = { - .name = DRV_NAME, - .pm = &sirfsoc_dma_pm_ops, - .of_match_table = sirfsoc_dma_match, - }, -}; - -static __init int sirfsoc_dma_init(void) -{ - return platform_driver_register(&sirfsoc_dma_driver); -} - -static void __exit sirfsoc_dma_exit(void) -{ - platform_driver_unregister(&sirfsoc_dma_driver); -} - -subsys_initcall(sirfsoc_dma_init); -module_exit(sirfsoc_dma_exit); - -MODULE_AUTHOR("Rongjun Ying "); -MODULE_AUTHOR("Barry Song "); -MODULE_DESCRIPTION("SIRFSOC DMA control driver"); -MODULE_LICENSE("GPL v2"); diff --git a/drivers/dma/ste_dma40.c b/drivers/dma/ste_dma40.c index 4256e55bbf25ff15ad7027796aa98738b4f62138..265d7c07b348e96b8c9cfd3b2123591522a784eb 100644 --- a/drivers/dma/ste_dma40.c +++ b/drivers/dma/ste_dma40.c @@ -78,7 +78,7 @@ static int dma40_memcpy_channels[] = { DB8500_DMA_MEMCPY_EV_5, }; -/* Default configuration for physcial memcpy */ +/* Default configuration for physical memcpy */ static const struct stedma40_chan_cfg dma40_memcpy_conf_phy = { .mode = STEDMA40_MODE_PHYSICAL, .dir = DMA_MEM_TO_MEM, diff --git a/drivers/dma/ti/k3-udma.c b/drivers/dma/ti/k3-udma.c index f474a1232335450df6b4410c8cb298432a61f1ee..96ad21869ba720ba6c630893a8bfd7acb8e289d2 100644 --- a/drivers/dma/ti/k3-udma.c +++ b/drivers/dma/ti/k3-udma.c @@ -121,6 +121,11 @@ struct udma_oes_offsets { #define UDMA_FLAG_PDMA_ACC32 BIT(0) #define UDMA_FLAG_PDMA_BURST BIT(1) #define UDMA_FLAG_TDTYPE BIT(2) +#define UDMA_FLAG_BURST_SIZE BIT(3) +#define UDMA_FLAGS_J7_CLASS (UDMA_FLAG_PDMA_ACC32 | \ + UDMA_FLAG_PDMA_BURST | \ + UDMA_FLAG_TDTYPE | \ + UDMA_FLAG_BURST_SIZE) struct udma_match_data { enum k3_dma_type type; @@ -128,6 +133,7 @@ struct udma_match_data { bool enable_memcpy_support; u32 flags; u32 statictr_z_mask; + u8 burst_size[3]; }; struct udma_soc_data { @@ -436,6 +442,18 @@ static void k3_configure_chan_coherency(struct dma_chan *chan, u32 asel) } } +static u8 udma_get_chan_tpl_index(struct udma_tpl *tpl_map, int chan_id) +{ + int i; + + for (i = 0; i < tpl_map->levels; i++) { + if (chan_id >= tpl_map->start_idx[i]) + return i; + } + + return 0; +} + static void udma_reset_uchan(struct udma_chan *uc) { memset(&uc->config, 0, sizeof(uc->config)); @@ -1811,13 +1829,21 @@ static int udma_tisci_m2m_channel_config(struct udma_chan *uc) const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops; struct udma_tchan *tchan = uc->tchan; struct udma_rchan *rchan = uc->rchan; - int ret = 0; + u8 burst_size = 0; + int ret; + u8 tpl; /* Non synchronized - mem to mem type of transfer */ int tc_ring = k3_ringacc_get_ring_id(tchan->tc_ring); struct ti_sci_msg_rm_udmap_tx_ch_cfg req_tx = { 0 }; struct ti_sci_msg_rm_udmap_rx_ch_cfg req_rx = { 0 }; + if (ud->match_data->flags & UDMA_FLAG_BURST_SIZE) { + tpl = udma_get_chan_tpl_index(&ud->tchan_tpl, tchan->id); + + burst_size = ud->match_data->burst_size[tpl]; + } + req_tx.valid_params = TISCI_UDMA_TCHAN_VALID_PARAMS; req_tx.nav_id = tisci_rm->tisci_dev_id; req_tx.index = tchan->id; @@ -1825,6 +1851,10 @@ static int udma_tisci_m2m_channel_config(struct udma_chan *uc) req_tx.tx_fetch_size = sizeof(struct cppi5_desc_hdr_t) >> 2; req_tx.txcq_qnum = tc_ring; req_tx.tx_atype = ud->atype; + if (burst_size) { + req_tx.valid_params |= TI_SCI_MSG_VALUE_RM_UDMAP_CH_BURST_SIZE_VALID; + req_tx.tx_burst_size = burst_size; + } ret = tisci_ops->tx_ch_cfg(tisci_rm->tisci, &req_tx); if (ret) { @@ -1839,6 +1869,10 @@ static int udma_tisci_m2m_channel_config(struct udma_chan *uc) req_rx.rxcq_qnum = tc_ring; req_rx.rx_chan_type = TI_SCI_RM_UDMAP_CHAN_TYPE_3RDP_BCOPY_PBRR; req_rx.rx_atype = ud->atype; + if (burst_size) { + req_rx.valid_params |= TI_SCI_MSG_VALUE_RM_UDMAP_CH_BURST_SIZE_VALID; + req_rx.rx_burst_size = burst_size; + } ret = tisci_ops->rx_ch_cfg(tisci_rm->tisci, &req_rx); if (ret) @@ -1854,12 +1888,24 @@ static int bcdma_tisci_m2m_channel_config(struct udma_chan *uc) const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops; struct ti_sci_msg_rm_udmap_tx_ch_cfg req_tx = { 0 }; struct udma_bchan *bchan = uc->bchan; - int ret = 0; + u8 burst_size = 0; + int ret; + u8 tpl; + + if (ud->match_data->flags & UDMA_FLAG_BURST_SIZE) { + tpl = udma_get_chan_tpl_index(&ud->bchan_tpl, bchan->id); + + burst_size = ud->match_data->burst_size[tpl]; + } req_tx.valid_params = TISCI_BCDMA_BCHAN_VALID_PARAMS; req_tx.nav_id = tisci_rm->tisci_dev_id; req_tx.extended_ch_type = TI_SCI_RM_BCDMA_EXTENDED_CH_TYPE_BCHAN; req_tx.index = bchan->id; + if (burst_size) { + req_tx.valid_params |= TI_SCI_MSG_VALUE_RM_UDMAP_CH_BURST_SIZE_VALID; + req_tx.tx_burst_size = burst_size; + } ret = tisci_ops->tx_ch_cfg(tisci_rm->tisci, &req_tx); if (ret) @@ -1877,7 +1923,7 @@ static int udma_tisci_tx_channel_config(struct udma_chan *uc) int tc_ring = k3_ringacc_get_ring_id(tchan->tc_ring); struct ti_sci_msg_rm_udmap_tx_ch_cfg req_tx = { 0 }; u32 mode, fetch_size; - int ret = 0; + int ret; if (uc->config.pkt_mode) { mode = TI_SCI_RM_UDMAP_CHAN_TYPE_PKT_PBRR; @@ -1918,7 +1964,7 @@ static int bcdma_tisci_tx_channel_config(struct udma_chan *uc) const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops; struct udma_tchan *tchan = uc->tchan; struct ti_sci_msg_rm_udmap_tx_ch_cfg req_tx = { 0 }; - int ret = 0; + int ret; req_tx.valid_params = TISCI_BCDMA_TCHAN_VALID_PARAMS; req_tx.nav_id = tisci_rm->tisci_dev_id; @@ -1951,7 +1997,7 @@ static int udma_tisci_rx_channel_config(struct udma_chan *uc) struct ti_sci_msg_rm_udmap_rx_ch_cfg req_rx = { 0 }; struct ti_sci_msg_rm_udmap_flow_cfg flow_req = { 0 }; u32 mode, fetch_size; - int ret = 0; + int ret; if (uc->config.pkt_mode) { mode = TI_SCI_RM_UDMAP_CHAN_TYPE_PKT_PBRR; @@ -2028,7 +2074,7 @@ static int bcdma_tisci_rx_channel_config(struct udma_chan *uc) const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops; struct udma_rchan *rchan = uc->rchan; struct ti_sci_msg_rm_udmap_rx_ch_cfg req_rx = { 0 }; - int ret = 0; + int ret; req_rx.valid_params = TISCI_BCDMA_RCHAN_VALID_PARAMS; req_rx.nav_id = tisci_rm->tisci_dev_id; @@ -2048,7 +2094,7 @@ static int pktdma_tisci_rx_channel_config(struct udma_chan *uc) const struct ti_sci_rm_udmap_ops *tisci_ops = tisci_rm->tisci_udmap_ops; struct ti_sci_msg_rm_udmap_rx_ch_cfg req_rx = { 0 }; struct ti_sci_msg_rm_udmap_flow_cfg flow_req = { 0 }; - int ret = 0; + int ret; req_rx.valid_params = TISCI_BCDMA_RCHAN_VALID_PARAMS; req_rx.nav_id = tisci_rm->tisci_dev_id; @@ -4168,6 +4214,11 @@ static struct udma_match_data am654_main_data = { .psil_base = 0x1000, .enable_memcpy_support = true, .statictr_z_mask = GENMASK(11, 0), + .burst_size = { + TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* Normal Channels */ + TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* H Channels */ + 0, /* No UH Channels */ + }, }; static struct udma_match_data am654_mcu_data = { @@ -4175,38 +4226,63 @@ static struct udma_match_data am654_mcu_data = { .psil_base = 0x6000, .enable_memcpy_support = false, .statictr_z_mask = GENMASK(11, 0), + .burst_size = { + TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* Normal Channels */ + TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* H Channels */ + 0, /* No UH Channels */ + }, }; static struct udma_match_data j721e_main_data = { .type = DMA_TYPE_UDMA, .psil_base = 0x1000, .enable_memcpy_support = true, - .flags = UDMA_FLAG_PDMA_ACC32 | UDMA_FLAG_PDMA_BURST | UDMA_FLAG_TDTYPE, + .flags = UDMA_FLAGS_J7_CLASS, .statictr_z_mask = GENMASK(23, 0), + .burst_size = { + TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* Normal Channels */ + TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_256_BYTES, /* H Channels */ + TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_256_BYTES, /* UH Channels */ + }, }; static struct udma_match_data j721e_mcu_data = { .type = DMA_TYPE_UDMA, .psil_base = 0x6000, .enable_memcpy_support = false, /* MEM_TO_MEM is slow via MCU UDMA */ - .flags = UDMA_FLAG_PDMA_ACC32 | UDMA_FLAG_PDMA_BURST | UDMA_FLAG_TDTYPE, + .flags = UDMA_FLAGS_J7_CLASS, .statictr_z_mask = GENMASK(23, 0), + .burst_size = { + TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* Normal Channels */ + TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_128_BYTES, /* H Channels */ + 0, /* No UH Channels */ + }, }; static struct udma_match_data am64_bcdma_data = { .type = DMA_TYPE_BCDMA, .psil_base = 0x2000, /* for tchan and rchan, not applicable to bchan */ .enable_memcpy_support = true, /* Supported via bchan */ - .flags = UDMA_FLAG_PDMA_ACC32 | UDMA_FLAG_PDMA_BURST | UDMA_FLAG_TDTYPE, + .flags = UDMA_FLAGS_J7_CLASS, .statictr_z_mask = GENMASK(23, 0), + .burst_size = { + TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* Normal Channels */ + 0, /* No H Channels */ + 0, /* No UH Channels */ + }, }; static struct udma_match_data am64_pktdma_data = { .type = DMA_TYPE_PKTDMA, .psil_base = 0x1000, .enable_memcpy_support = false, /* PKTDMA does not support MEM_TO_MEM */ - .flags = UDMA_FLAG_PDMA_ACC32 | UDMA_FLAG_PDMA_BURST | UDMA_FLAG_TDTYPE, + .flags = UDMA_FLAGS_J7_CLASS, .statictr_z_mask = GENMASK(23, 0), + .burst_size = { + TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* Normal Channels */ + 0, /* No H Channels */ + 0, /* No UH Channels */ + }, }; static const struct of_device_id udma_of_match[] = { @@ -4306,6 +4382,7 @@ static int udma_get_mmrs(struct platform_device *pdev, struct udma_dev *ud) ud->bchan_cnt = BCDMA_CAP2_BCHAN_CNT(cap2); ud->tchan_cnt = BCDMA_CAP2_TCHAN_CNT(cap2); ud->rchan_cnt = BCDMA_CAP2_RCHAN_CNT(cap2); + ud->rflow_cnt = ud->rchan_cnt; break; case DMA_TYPE_PKTDMA: cap4 = udma_read(ud->mmrs[MMR_GCFG], 0x30); @@ -5046,6 +5123,34 @@ static void udma_dbg_summary_show(struct seq_file *s, } #endif /* CONFIG_DEBUG_FS */ +static enum dmaengine_alignment udma_get_copy_align(struct udma_dev *ud) +{ + const struct udma_match_data *match_data = ud->match_data; + u8 tpl; + + if (!match_data->enable_memcpy_support) + return DMAENGINE_ALIGN_8_BYTES; + + /* Get the highest TPL level the device supports for memcpy */ + if (ud->bchan_cnt) + tpl = udma_get_chan_tpl_index(&ud->bchan_tpl, 0); + else if (ud->tchan_cnt) + tpl = udma_get_chan_tpl_index(&ud->tchan_tpl, 0); + else + return DMAENGINE_ALIGN_8_BYTES; + + switch (match_data->burst_size[tpl]) { + case TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_256_BYTES: + return DMAENGINE_ALIGN_256_BYTES; + case TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_128_BYTES: + return DMAENGINE_ALIGN_128_BYTES; + case TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES: + fallthrough; + default: + return DMAENGINE_ALIGN_64_BYTES; + } +} + #define TI_UDMAC_BUSWIDTHS (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \ BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \ BIT(DMA_SLAVE_BUSWIDTH_3_BYTES) | \ @@ -5202,7 +5307,6 @@ static int udma_probe(struct platform_device *pdev) ud->ddev.dst_addr_widths = TI_UDMAC_BUSWIDTHS; ud->ddev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); ud->ddev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; - ud->ddev.copy_align = DMAENGINE_ALIGN_8_BYTES; ud->ddev.desc_metadata_modes = DESC_METADATA_CLIENT | DESC_METADATA_ENGINE; if (ud->match_data->enable_memcpy_support && @@ -5284,6 +5388,9 @@ static int udma_probe(struct platform_device *pdev) INIT_DELAYED_WORK(&uc->tx_drain.work, udma_check_tx_completion); } + /* Configure the copy_align to the maximum burst size the device supports */ + ud->ddev.copy_align = udma_get_copy_align(ud); + ret = dma_async_device_register(&ud->ddev); if (ret) { dev_err(dev, "failed to register slave DMA engine: %d\n", ret); diff --git a/drivers/dma/xilinx/xilinx_dma.c b/drivers/dma/xilinx/xilinx_dma.c index 79777550a6ffc2e8db8e6fbb472f4b8fa55f9c12..3aded7861fef85bcc2b38de15dd4bbc2b42dedb1 100644 --- a/drivers/dma/xilinx/xilinx_dma.c +++ b/drivers/dma/xilinx/xilinx_dma.c @@ -800,7 +800,7 @@ xilinx_dma_alloc_tx_descriptor(struct xilinx_dma_chan *chan) { struct xilinx_dma_tx_descriptor *desc; - desc = kzalloc(sizeof(*desc), GFP_KERNEL); + desc = kzalloc(sizeof(*desc), GFP_NOWAIT); if (!desc) return NULL; diff --git a/drivers/dma/zx_dma.c b/drivers/dma/zx_dma.c deleted file mode 100644 index b057582b2facb76709823aec2412e8b15c4f62a1..0000000000000000000000000000000000000000 --- a/drivers/dma/zx_dma.c +++ /dev/null @@ -1,941 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Copyright 2015 Linaro. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "virt-dma.h" - -#define DRIVER_NAME "zx-dma" -#define DMA_ALIGN 4 -#define DMA_MAX_SIZE (0x10000 - 512) -#define LLI_BLOCK_SIZE (4 * PAGE_SIZE) - -#define REG_ZX_SRC_ADDR 0x00 -#define REG_ZX_DST_ADDR 0x04 -#define REG_ZX_TX_X_COUNT 0x08 -#define REG_ZX_TX_ZY_COUNT 0x0c -#define REG_ZX_SRC_ZY_STEP 0x10 -#define REG_ZX_DST_ZY_STEP 0x14 -#define REG_ZX_LLI_ADDR 0x1c -#define REG_ZX_CTRL 0x20 -#define REG_ZX_TC_IRQ 0x800 -#define REG_ZX_SRC_ERR_IRQ 0x804 -#define REG_ZX_DST_ERR_IRQ 0x808 -#define REG_ZX_CFG_ERR_IRQ 0x80c -#define REG_ZX_TC_IRQ_RAW 0x810 -#define REG_ZX_SRC_ERR_IRQ_RAW 0x814 -#define REG_ZX_DST_ERR_IRQ_RAW 0x818 -#define REG_ZX_CFG_ERR_IRQ_RAW 0x81c -#define REG_ZX_STATUS 0x820 -#define REG_ZX_DMA_GRP_PRIO 0x824 -#define REG_ZX_DMA_ARB 0x828 - -#define ZX_FORCE_CLOSE BIT(31) -#define ZX_DST_BURST_WIDTH(x) (((x) & 0x7) << 13) -#define ZX_MAX_BURST_LEN 16 -#define ZX_SRC_BURST_LEN(x) (((x) & 0xf) << 9) -#define ZX_SRC_BURST_WIDTH(x) (((x) & 0x7) << 6) -#define ZX_IRQ_ENABLE_ALL (3 << 4) -#define ZX_DST_FIFO_MODE BIT(3) -#define ZX_SRC_FIFO_MODE BIT(2) -#define ZX_SOFT_REQ BIT(1) -#define ZX_CH_ENABLE BIT(0) - -#define ZX_DMA_BUSWIDTHS \ - (BIT(DMA_SLAVE_BUSWIDTH_UNDEFINED) | \ - BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \ - BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \ - BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | \ - BIT(DMA_SLAVE_BUSWIDTH_8_BYTES)) - -enum zx_dma_burst_width { - ZX_DMA_WIDTH_8BIT = 0, - ZX_DMA_WIDTH_16BIT = 1, - ZX_DMA_WIDTH_32BIT = 2, - ZX_DMA_WIDTH_64BIT = 3, -}; - -struct zx_desc_hw { - u32 saddr; - u32 daddr; - u32 src_x; - u32 src_zy; - u32 src_zy_step; - u32 dst_zy_step; - u32 reserved1; - u32 lli; - u32 ctr; - u32 reserved[7]; /* pack as hardware registers region size */ -} __aligned(32); - -struct zx_dma_desc_sw { - struct virt_dma_desc vd; - dma_addr_t desc_hw_lli; - size_t desc_num; - size_t size; - struct zx_desc_hw *desc_hw; -}; - -struct zx_dma_phy; - -struct zx_dma_chan { - struct dma_slave_config slave_cfg; - int id; /* Request phy chan id */ - u32 ccfg; - u32 cyclic; - struct virt_dma_chan vc; - struct zx_dma_phy *phy; - struct list_head node; - dma_addr_t dev_addr; - enum dma_status status; -}; - -struct zx_dma_phy { - u32 idx; - void __iomem *base; - struct zx_dma_chan *vchan; - struct zx_dma_desc_sw *ds_run; - struct zx_dma_desc_sw *ds_done; -}; - -struct zx_dma_dev { - struct dma_device slave; - void __iomem *base; - spinlock_t lock; /* lock for ch and phy */ - struct list_head chan_pending; - struct zx_dma_phy *phy; - struct zx_dma_chan *chans; - struct clk *clk; - struct dma_pool *pool; - u32 dma_channels; - u32 dma_requests; - int irq; -}; - -#define to_zx_dma(dmadev) container_of(dmadev, struct zx_dma_dev, slave) - -static struct zx_dma_chan *to_zx_chan(struct dma_chan *chan) -{ - return container_of(chan, struct zx_dma_chan, vc.chan); -} - -static void zx_dma_terminate_chan(struct zx_dma_phy *phy, struct zx_dma_dev *d) -{ - u32 val = 0; - - val = readl_relaxed(phy->base + REG_ZX_CTRL); - val &= ~ZX_CH_ENABLE; - val |= ZX_FORCE_CLOSE; - writel_relaxed(val, phy->base + REG_ZX_CTRL); - - val = 0x1 << phy->idx; - writel_relaxed(val, d->base + REG_ZX_TC_IRQ_RAW); - writel_relaxed(val, d->base + REG_ZX_SRC_ERR_IRQ_RAW); - writel_relaxed(val, d->base + REG_ZX_DST_ERR_IRQ_RAW); - writel_relaxed(val, d->base + REG_ZX_CFG_ERR_IRQ_RAW); -} - -static void zx_dma_set_desc(struct zx_dma_phy *phy, struct zx_desc_hw *hw) -{ - writel_relaxed(hw->saddr, phy->base + REG_ZX_SRC_ADDR); - writel_relaxed(hw->daddr, phy->base + REG_ZX_DST_ADDR); - writel_relaxed(hw->src_x, phy->base + REG_ZX_TX_X_COUNT); - writel_relaxed(0, phy->base + REG_ZX_TX_ZY_COUNT); - writel_relaxed(0, phy->base + REG_ZX_SRC_ZY_STEP); - writel_relaxed(0, phy->base + REG_ZX_DST_ZY_STEP); - writel_relaxed(hw->lli, phy->base + REG_ZX_LLI_ADDR); - writel_relaxed(hw->ctr, phy->base + REG_ZX_CTRL); -} - -static u32 zx_dma_get_curr_lli(struct zx_dma_phy *phy) -{ - return readl_relaxed(phy->base + REG_ZX_LLI_ADDR); -} - -static u32 zx_dma_get_chan_stat(struct zx_dma_dev *d) -{ - return readl_relaxed(d->base + REG_ZX_STATUS); -} - -static void zx_dma_init_state(struct zx_dma_dev *d) -{ - /* set same priority */ - writel_relaxed(0x0, d->base + REG_ZX_DMA_ARB); - /* clear all irq */ - writel_relaxed(0xffffffff, d->base + REG_ZX_TC_IRQ_RAW); - writel_relaxed(0xffffffff, d->base + REG_ZX_SRC_ERR_IRQ_RAW); - writel_relaxed(0xffffffff, d->base + REG_ZX_DST_ERR_IRQ_RAW); - writel_relaxed(0xffffffff, d->base + REG_ZX_CFG_ERR_IRQ_RAW); -} - -static int zx_dma_start_txd(struct zx_dma_chan *c) -{ - struct zx_dma_dev *d = to_zx_dma(c->vc.chan.device); - struct virt_dma_desc *vd = vchan_next_desc(&c->vc); - - if (!c->phy) - return -EAGAIN; - - if (BIT(c->phy->idx) & zx_dma_get_chan_stat(d)) - return -EAGAIN; - - if (vd) { - struct zx_dma_desc_sw *ds = - container_of(vd, struct zx_dma_desc_sw, vd); - /* - * fetch and remove request from vc->desc_issued - * so vc->desc_issued only contains desc pending - */ - list_del(&ds->vd.node); - c->phy->ds_run = ds; - c->phy->ds_done = NULL; - /* start dma */ - zx_dma_set_desc(c->phy, ds->desc_hw); - return 0; - } - c->phy->ds_done = NULL; - c->phy->ds_run = NULL; - return -EAGAIN; -} - -static void zx_dma_task(struct zx_dma_dev *d) -{ - struct zx_dma_phy *p; - struct zx_dma_chan *c, *cn; - unsigned pch, pch_alloc = 0; - unsigned long flags; - - /* check new dma request of running channel in vc->desc_issued */ - list_for_each_entry_safe(c, cn, &d->slave.channels, - vc.chan.device_node) { - spin_lock_irqsave(&c->vc.lock, flags); - p = c->phy; - if (p && p->ds_done && zx_dma_start_txd(c)) { - /* No current txd associated with this channel */ - dev_dbg(d->slave.dev, "pchan %u: free\n", p->idx); - /* Mark this channel free */ - c->phy = NULL; - p->vchan = NULL; - } - spin_unlock_irqrestore(&c->vc.lock, flags); - } - - /* check new channel request in d->chan_pending */ - spin_lock_irqsave(&d->lock, flags); - while (!list_empty(&d->chan_pending)) { - c = list_first_entry(&d->chan_pending, - struct zx_dma_chan, node); - p = &d->phy[c->id]; - if (!p->vchan) { - /* remove from d->chan_pending */ - list_del_init(&c->node); - pch_alloc |= 1 << c->id; - /* Mark this channel allocated */ - p->vchan = c; - c->phy = p; - } else { - dev_dbg(d->slave.dev, "pchan %u: busy!\n", c->id); - } - } - spin_unlock_irqrestore(&d->lock, flags); - - for (pch = 0; pch < d->dma_channels; pch++) { - if (pch_alloc & (1 << pch)) { - p = &d->phy[pch]; - c = p->vchan; - if (c) { - spin_lock_irqsave(&c->vc.lock, flags); - zx_dma_start_txd(c); - spin_unlock_irqrestore(&c->vc.lock, flags); - } - } - } -} - -static irqreturn_t zx_dma_int_handler(int irq, void *dev_id) -{ - struct zx_dma_dev *d = (struct zx_dma_dev *)dev_id; - struct zx_dma_phy *p; - struct zx_dma_chan *c; - u32 tc = readl_relaxed(d->base + REG_ZX_TC_IRQ); - u32 serr = readl_relaxed(d->base + REG_ZX_SRC_ERR_IRQ); - u32 derr = readl_relaxed(d->base + REG_ZX_DST_ERR_IRQ); - u32 cfg = readl_relaxed(d->base + REG_ZX_CFG_ERR_IRQ); - u32 i, irq_chan = 0, task = 0; - - while (tc) { - i = __ffs(tc); - tc &= ~BIT(i); - p = &d->phy[i]; - c = p->vchan; - if (c) { - spin_lock(&c->vc.lock); - if (c->cyclic) { - vchan_cyclic_callback(&p->ds_run->vd); - } else { - vchan_cookie_complete(&p->ds_run->vd); - p->ds_done = p->ds_run; - task = 1; - } - spin_unlock(&c->vc.lock); - irq_chan |= BIT(i); - } - } - - if (serr || derr || cfg) - dev_warn(d->slave.dev, "DMA ERR src 0x%x, dst 0x%x, cfg 0x%x\n", - serr, derr, cfg); - - writel_relaxed(irq_chan, d->base + REG_ZX_TC_IRQ_RAW); - writel_relaxed(serr, d->base + REG_ZX_SRC_ERR_IRQ_RAW); - writel_relaxed(derr, d->base + REG_ZX_DST_ERR_IRQ_RAW); - writel_relaxed(cfg, d->base + REG_ZX_CFG_ERR_IRQ_RAW); - - if (task) - zx_dma_task(d); - return IRQ_HANDLED; -} - -static void zx_dma_free_chan_resources(struct dma_chan *chan) -{ - struct zx_dma_chan *c = to_zx_chan(chan); - struct zx_dma_dev *d = to_zx_dma(chan->device); - unsigned long flags; - - spin_lock_irqsave(&d->lock, flags); - list_del_init(&c->node); - spin_unlock_irqrestore(&d->lock, flags); - - vchan_free_chan_resources(&c->vc); - c->ccfg = 0; -} - -static enum dma_status zx_dma_tx_status(struct dma_chan *chan, - dma_cookie_t cookie, - struct dma_tx_state *state) -{ - struct zx_dma_chan *c = to_zx_chan(chan); - struct zx_dma_phy *p; - struct virt_dma_desc *vd; - unsigned long flags; - enum dma_status ret; - size_t bytes = 0; - - ret = dma_cookie_status(&c->vc.chan, cookie, state); - if (ret == DMA_COMPLETE || !state) - return ret; - - spin_lock_irqsave(&c->vc.lock, flags); - p = c->phy; - ret = c->status; - - /* - * If the cookie is on our issue queue, then the residue is - * its total size. - */ - vd = vchan_find_desc(&c->vc, cookie); - if (vd) { - bytes = container_of(vd, struct zx_dma_desc_sw, vd)->size; - } else if ((!p) || (!p->ds_run)) { - bytes = 0; - } else { - struct zx_dma_desc_sw *ds = p->ds_run; - u32 clli = 0, index = 0; - - bytes = 0; - clli = zx_dma_get_curr_lli(p); - index = (clli - ds->desc_hw_lli) / - sizeof(struct zx_desc_hw) + 1; - for (; index < ds->desc_num; index++) { - bytes += ds->desc_hw[index].src_x; - /* end of lli */ - if (!ds->desc_hw[index].lli) - break; - } - } - spin_unlock_irqrestore(&c->vc.lock, flags); - dma_set_residue(state, bytes); - return ret; -} - -static void zx_dma_issue_pending(struct dma_chan *chan) -{ - struct zx_dma_chan *c = to_zx_chan(chan); - struct zx_dma_dev *d = to_zx_dma(chan->device); - unsigned long flags; - int issue = 0; - - spin_lock_irqsave(&c->vc.lock, flags); - /* add request to vc->desc_issued */ - if (vchan_issue_pending(&c->vc)) { - spin_lock(&d->lock); - if (!c->phy && list_empty(&c->node)) { - /* if new channel, add chan_pending */ - list_add_tail(&c->node, &d->chan_pending); - issue = 1; - dev_dbg(d->slave.dev, "vchan %p: issued\n", &c->vc); - } - spin_unlock(&d->lock); - } else { - dev_dbg(d->slave.dev, "vchan %p: nothing to issue\n", &c->vc); - } - spin_unlock_irqrestore(&c->vc.lock, flags); - - if (issue) - zx_dma_task(d); -} - -static void zx_dma_fill_desc(struct zx_dma_desc_sw *ds, dma_addr_t dst, - dma_addr_t src, size_t len, u32 num, u32 ccfg) -{ - if ((num + 1) < ds->desc_num) - ds->desc_hw[num].lli = ds->desc_hw_lli + (num + 1) * - sizeof(struct zx_desc_hw); - ds->desc_hw[num].saddr = src; - ds->desc_hw[num].daddr = dst; - ds->desc_hw[num].src_x = len; - ds->desc_hw[num].ctr = ccfg; -} - -static struct zx_dma_desc_sw *zx_alloc_desc_resource(int num, - struct dma_chan *chan) -{ - struct zx_dma_chan *c = to_zx_chan(chan); - struct zx_dma_desc_sw *ds; - struct zx_dma_dev *d = to_zx_dma(chan->device); - int lli_limit = LLI_BLOCK_SIZE / sizeof(struct zx_desc_hw); - - if (num > lli_limit) { - dev_dbg(chan->device->dev, "vch %p: sg num %d exceed max %d\n", - &c->vc, num, lli_limit); - return NULL; - } - - ds = kzalloc(sizeof(*ds), GFP_ATOMIC); - if (!ds) - return NULL; - - ds->desc_hw = dma_pool_zalloc(d->pool, GFP_NOWAIT, &ds->desc_hw_lli); - if (!ds->desc_hw) { - dev_dbg(chan->device->dev, "vch %p: dma alloc fail\n", &c->vc); - kfree(ds); - return NULL; - } - ds->desc_num = num; - return ds; -} - -static enum zx_dma_burst_width zx_dma_burst_width(enum dma_slave_buswidth width) -{ - switch (width) { - case DMA_SLAVE_BUSWIDTH_1_BYTE: - case DMA_SLAVE_BUSWIDTH_2_BYTES: - case DMA_SLAVE_BUSWIDTH_4_BYTES: - case DMA_SLAVE_BUSWIDTH_8_BYTES: - return ffs(width) - 1; - default: - return ZX_DMA_WIDTH_32BIT; - } -} - -static int zx_pre_config(struct zx_dma_chan *c, enum dma_transfer_direction dir) -{ - struct dma_slave_config *cfg = &c->slave_cfg; - enum zx_dma_burst_width src_width; - enum zx_dma_burst_width dst_width; - u32 maxburst = 0; - - switch (dir) { - case DMA_MEM_TO_MEM: - c->ccfg = ZX_CH_ENABLE | ZX_SOFT_REQ - | ZX_SRC_BURST_LEN(ZX_MAX_BURST_LEN - 1) - | ZX_SRC_BURST_WIDTH(ZX_DMA_WIDTH_32BIT) - | ZX_DST_BURST_WIDTH(ZX_DMA_WIDTH_32BIT); - break; - case DMA_MEM_TO_DEV: - c->dev_addr = cfg->dst_addr; - /* dst len is calculated from src width, len and dst width. - * We need make sure dst len not exceed MAX LEN. - * Trailing single transaction that does not fill a full - * burst also require identical src/dst data width. - */ - dst_width = zx_dma_burst_width(cfg->dst_addr_width); - maxburst = cfg->dst_maxburst; - maxburst = maxburst < ZX_MAX_BURST_LEN ? - maxburst : ZX_MAX_BURST_LEN; - c->ccfg = ZX_DST_FIFO_MODE | ZX_CH_ENABLE - | ZX_SRC_BURST_LEN(maxburst - 1) - | ZX_SRC_BURST_WIDTH(dst_width) - | ZX_DST_BURST_WIDTH(dst_width); - break; - case DMA_DEV_TO_MEM: - c->dev_addr = cfg->src_addr; - src_width = zx_dma_burst_width(cfg->src_addr_width); - maxburst = cfg->src_maxburst; - maxburst = maxburst < ZX_MAX_BURST_LEN ? - maxburst : ZX_MAX_BURST_LEN; - c->ccfg = ZX_SRC_FIFO_MODE | ZX_CH_ENABLE - | ZX_SRC_BURST_LEN(maxburst - 1) - | ZX_SRC_BURST_WIDTH(src_width) - | ZX_DST_BURST_WIDTH(src_width); - break; - default: - return -EINVAL; - } - return 0; -} - -static struct dma_async_tx_descriptor *zx_dma_prep_memcpy( - struct dma_chan *chan, dma_addr_t dst, dma_addr_t src, - size_t len, unsigned long flags) -{ - struct zx_dma_chan *c = to_zx_chan(chan); - struct zx_dma_desc_sw *ds; - size_t copy = 0; - int num = 0; - - if (!len) - return NULL; - - if (zx_pre_config(c, DMA_MEM_TO_MEM)) - return NULL; - - num = DIV_ROUND_UP(len, DMA_MAX_SIZE); - - ds = zx_alloc_desc_resource(num, chan); - if (!ds) - return NULL; - - ds->size = len; - num = 0; - - do { - copy = min_t(size_t, len, DMA_MAX_SIZE); - zx_dma_fill_desc(ds, dst, src, copy, num++, c->ccfg); - - src += copy; - dst += copy; - len -= copy; - } while (len); - - c->cyclic = 0; - ds->desc_hw[num - 1].lli = 0; /* end of link */ - ds->desc_hw[num - 1].ctr |= ZX_IRQ_ENABLE_ALL; - return vchan_tx_prep(&c->vc, &ds->vd, flags); -} - -static struct dma_async_tx_descriptor *zx_dma_prep_slave_sg( - struct dma_chan *chan, struct scatterlist *sgl, unsigned int sglen, - enum dma_transfer_direction dir, unsigned long flags, void *context) -{ - struct zx_dma_chan *c = to_zx_chan(chan); - struct zx_dma_desc_sw *ds; - size_t len, avail, total = 0; - struct scatterlist *sg; - dma_addr_t addr, src = 0, dst = 0; - int num = sglen, i; - - if (!sgl) - return NULL; - - if (zx_pre_config(c, dir)) - return NULL; - - for_each_sg(sgl, sg, sglen, i) { - avail = sg_dma_len(sg); - if (avail > DMA_MAX_SIZE) - num += DIV_ROUND_UP(avail, DMA_MAX_SIZE) - 1; - } - - ds = zx_alloc_desc_resource(num, chan); - if (!ds) - return NULL; - - c->cyclic = 0; - num = 0; - for_each_sg(sgl, sg, sglen, i) { - addr = sg_dma_address(sg); - avail = sg_dma_len(sg); - total += avail; - - do { - len = min_t(size_t, avail, DMA_MAX_SIZE); - - if (dir == DMA_MEM_TO_DEV) { - src = addr; - dst = c->dev_addr; - } else if (dir == DMA_DEV_TO_MEM) { - src = c->dev_addr; - dst = addr; - } - - zx_dma_fill_desc(ds, dst, src, len, num++, c->ccfg); - - addr += len; - avail -= len; - } while (avail); - } - - ds->desc_hw[num - 1].lli = 0; /* end of link */ - ds->desc_hw[num - 1].ctr |= ZX_IRQ_ENABLE_ALL; - ds->size = total; - return vchan_tx_prep(&c->vc, &ds->vd, flags); -} - -static struct dma_async_tx_descriptor *zx_dma_prep_dma_cyclic( - struct dma_chan *chan, dma_addr_t dma_addr, size_t buf_len, - size_t period_len, enum dma_transfer_direction dir, - unsigned long flags) -{ - struct zx_dma_chan *c = to_zx_chan(chan); - struct zx_dma_desc_sw *ds; - dma_addr_t src = 0, dst = 0; - int num_periods = buf_len / period_len; - int buf = 0, num = 0; - - if (period_len > DMA_MAX_SIZE) { - dev_err(chan->device->dev, "maximum period size exceeded\n"); - return NULL; - } - - if (zx_pre_config(c, dir)) - return NULL; - - ds = zx_alloc_desc_resource(num_periods, chan); - if (!ds) - return NULL; - c->cyclic = 1; - - while (buf < buf_len) { - if (dir == DMA_MEM_TO_DEV) { - src = dma_addr; - dst = c->dev_addr; - } else if (dir == DMA_DEV_TO_MEM) { - src = c->dev_addr; - dst = dma_addr; - } - zx_dma_fill_desc(ds, dst, src, period_len, num++, - c->ccfg | ZX_IRQ_ENABLE_ALL); - dma_addr += period_len; - buf += period_len; - } - - ds->desc_hw[num - 1].lli = ds->desc_hw_lli; - ds->size = buf_len; - return vchan_tx_prep(&c->vc, &ds->vd, flags); -} - -static int zx_dma_config(struct dma_chan *chan, - struct dma_slave_config *cfg) -{ - struct zx_dma_chan *c = to_zx_chan(chan); - - if (!cfg) - return -EINVAL; - - memcpy(&c->slave_cfg, cfg, sizeof(*cfg)); - - return 0; -} - -static int zx_dma_terminate_all(struct dma_chan *chan) -{ - struct zx_dma_chan *c = to_zx_chan(chan); - struct zx_dma_dev *d = to_zx_dma(chan->device); - struct zx_dma_phy *p = c->phy; - unsigned long flags; - LIST_HEAD(head); - - dev_dbg(d->slave.dev, "vchan %p: terminate all\n", &c->vc); - - /* Prevent this channel being scheduled */ - spin_lock(&d->lock); - list_del_init(&c->node); - spin_unlock(&d->lock); - - /* Clear the tx descriptor lists */ - spin_lock_irqsave(&c->vc.lock, flags); - vchan_get_all_descriptors(&c->vc, &head); - if (p) { - /* vchan is assigned to a pchan - stop the channel */ - zx_dma_terminate_chan(p, d); - c->phy = NULL; - p->vchan = NULL; - p->ds_run = NULL; - p->ds_done = NULL; - } - spin_unlock_irqrestore(&c->vc.lock, flags); - vchan_dma_desc_free_list(&c->vc, &head); - - return 0; -} - -static int zx_dma_transfer_pause(struct dma_chan *chan) -{ - struct zx_dma_chan *c = to_zx_chan(chan); - u32 val = 0; - - val = readl_relaxed(c->phy->base + REG_ZX_CTRL); - val &= ~ZX_CH_ENABLE; - writel_relaxed(val, c->phy->base + REG_ZX_CTRL); - - return 0; -} - -static int zx_dma_transfer_resume(struct dma_chan *chan) -{ - struct zx_dma_chan *c = to_zx_chan(chan); - u32 val = 0; - - val = readl_relaxed(c->phy->base + REG_ZX_CTRL); - val |= ZX_CH_ENABLE; - writel_relaxed(val, c->phy->base + REG_ZX_CTRL); - - return 0; -} - -static void zx_dma_free_desc(struct virt_dma_desc *vd) -{ - struct zx_dma_desc_sw *ds = - container_of(vd, struct zx_dma_desc_sw, vd); - struct zx_dma_dev *d = to_zx_dma(vd->tx.chan->device); - - dma_pool_free(d->pool, ds->desc_hw, ds->desc_hw_lli); - kfree(ds); -} - -static const struct of_device_id zx6702_dma_dt_ids[] = { - { .compatible = "zte,zx296702-dma", }, - {} -}; -MODULE_DEVICE_TABLE(of, zx6702_dma_dt_ids); - -static struct dma_chan *zx_of_dma_simple_xlate(struct of_phandle_args *dma_spec, - struct of_dma *ofdma) -{ - struct zx_dma_dev *d = ofdma->of_dma_data; - unsigned int request = dma_spec->args[0]; - struct dma_chan *chan; - struct zx_dma_chan *c; - - if (request >= d->dma_requests) - return NULL; - - chan = dma_get_any_slave_channel(&d->slave); - if (!chan) { - dev_err(d->slave.dev, "get channel fail in %s.\n", __func__); - return NULL; - } - c = to_zx_chan(chan); - c->id = request; - dev_info(d->slave.dev, "zx_dma: pchan %u: alloc vchan %p\n", - c->id, &c->vc); - return chan; -} - -static int zx_dma_probe(struct platform_device *op) -{ - struct zx_dma_dev *d; - int i, ret = 0; - - d = devm_kzalloc(&op->dev, sizeof(*d), GFP_KERNEL); - if (!d) - return -ENOMEM; - - d->base = devm_platform_ioremap_resource(op, 0); - if (IS_ERR(d->base)) - return PTR_ERR(d->base); - - of_property_read_u32((&op->dev)->of_node, - "dma-channels", &d->dma_channels); - of_property_read_u32((&op->dev)->of_node, - "dma-requests", &d->dma_requests); - if (!d->dma_requests || !d->dma_channels) - return -EINVAL; - - d->clk = devm_clk_get(&op->dev, NULL); - if (IS_ERR(d->clk)) { - dev_err(&op->dev, "no dma clk\n"); - return PTR_ERR(d->clk); - } - - d->irq = platform_get_irq(op, 0); - ret = devm_request_irq(&op->dev, d->irq, zx_dma_int_handler, - 0, DRIVER_NAME, d); - if (ret) - return ret; - - /* A DMA memory pool for LLIs, align on 32-byte boundary */ - d->pool = dmam_pool_create(DRIVER_NAME, &op->dev, - LLI_BLOCK_SIZE, 32, 0); - if (!d->pool) - return -ENOMEM; - - /* init phy channel */ - d->phy = devm_kcalloc(&op->dev, - d->dma_channels, sizeof(struct zx_dma_phy), GFP_KERNEL); - if (!d->phy) - return -ENOMEM; - - for (i = 0; i < d->dma_channels; i++) { - struct zx_dma_phy *p = &d->phy[i]; - - p->idx = i; - p->base = d->base + i * 0x40; - } - - INIT_LIST_HEAD(&d->slave.channels); - dma_cap_set(DMA_SLAVE, d->slave.cap_mask); - dma_cap_set(DMA_MEMCPY, d->slave.cap_mask); - dma_cap_set(DMA_CYCLIC, d->slave.cap_mask); - dma_cap_set(DMA_PRIVATE, d->slave.cap_mask); - d->slave.dev = &op->dev; - d->slave.device_free_chan_resources = zx_dma_free_chan_resources; - d->slave.device_tx_status = zx_dma_tx_status; - d->slave.device_prep_dma_memcpy = zx_dma_prep_memcpy; - d->slave.device_prep_slave_sg = zx_dma_prep_slave_sg; - d->slave.device_prep_dma_cyclic = zx_dma_prep_dma_cyclic; - d->slave.device_issue_pending = zx_dma_issue_pending; - d->slave.device_config = zx_dma_config; - d->slave.device_terminate_all = zx_dma_terminate_all; - d->slave.device_pause = zx_dma_transfer_pause; - d->slave.device_resume = zx_dma_transfer_resume; - d->slave.copy_align = DMA_ALIGN; - d->slave.src_addr_widths = ZX_DMA_BUSWIDTHS; - d->slave.dst_addr_widths = ZX_DMA_BUSWIDTHS; - d->slave.directions = BIT(DMA_MEM_TO_MEM) | BIT(DMA_MEM_TO_DEV) - | BIT(DMA_DEV_TO_MEM); - d->slave.residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT; - - /* init virtual channel */ - d->chans = devm_kcalloc(&op->dev, - d->dma_requests, sizeof(struct zx_dma_chan), GFP_KERNEL); - if (!d->chans) - return -ENOMEM; - - for (i = 0; i < d->dma_requests; i++) { - struct zx_dma_chan *c = &d->chans[i]; - - c->status = DMA_IN_PROGRESS; - INIT_LIST_HEAD(&c->node); - c->vc.desc_free = zx_dma_free_desc; - vchan_init(&c->vc, &d->slave); - } - - /* Enable clock before accessing registers */ - ret = clk_prepare_enable(d->clk); - if (ret < 0) { - dev_err(&op->dev, "clk_prepare_enable failed: %d\n", ret); - goto zx_dma_out; - } - - zx_dma_init_state(d); - - spin_lock_init(&d->lock); - INIT_LIST_HEAD(&d->chan_pending); - platform_set_drvdata(op, d); - - ret = dma_async_device_register(&d->slave); - if (ret) - goto clk_dis; - - ret = of_dma_controller_register((&op->dev)->of_node, - zx_of_dma_simple_xlate, d); - if (ret) - goto of_dma_register_fail; - - dev_info(&op->dev, "initialized\n"); - return 0; - -of_dma_register_fail: - dma_async_device_unregister(&d->slave); -clk_dis: - clk_disable_unprepare(d->clk); -zx_dma_out: - return ret; -} - -static int zx_dma_remove(struct platform_device *op) -{ - struct zx_dma_chan *c, *cn; - struct zx_dma_dev *d = platform_get_drvdata(op); - - /* explictly free the irq */ - devm_free_irq(&op->dev, d->irq, d); - - dma_async_device_unregister(&d->slave); - of_dma_controller_free((&op->dev)->of_node); - - list_for_each_entry_safe(c, cn, &d->slave.channels, - vc.chan.device_node) { - list_del(&c->vc.chan.device_node); - } - clk_disable_unprepare(d->clk); - - return 0; -} - -#ifdef CONFIG_PM_SLEEP -static int zx_dma_suspend_dev(struct device *dev) -{ - struct zx_dma_dev *d = dev_get_drvdata(dev); - u32 stat = 0; - - stat = zx_dma_get_chan_stat(d); - if (stat) { - dev_warn(d->slave.dev, - "chan %d is running fail to suspend\n", stat); - return -1; - } - clk_disable_unprepare(d->clk); - return 0; -} - -static int zx_dma_resume_dev(struct device *dev) -{ - struct zx_dma_dev *d = dev_get_drvdata(dev); - int ret = 0; - - ret = clk_prepare_enable(d->clk); - if (ret < 0) { - dev_err(d->slave.dev, "clk_prepare_enable failed: %d\n", ret); - return ret; - } - zx_dma_init_state(d); - return 0; -} -#endif - -static SIMPLE_DEV_PM_OPS(zx_dma_pmops, zx_dma_suspend_dev, zx_dma_resume_dev); - -static struct platform_driver zx_pdma_driver = { - .driver = { - .name = DRIVER_NAME, - .pm = &zx_dma_pmops, - .of_match_table = zx6702_dma_dt_ids, - }, - .probe = zx_dma_probe, - .remove = zx_dma_remove, -}; - -module_platform_driver(zx_pdma_driver); - -MODULE_DESCRIPTION("ZTE ZX296702 DMA Driver"); -MODULE_AUTHOR("Jun Nie jun.nie@linaro.org"); -MODULE_LICENSE("GPL v2"); diff --git a/drivers/firewire/core-device.c b/drivers/firewire/core-device.c index 80db43a22069802992f0b01214df6564acbd662a..68216988391f754314c4a1608ae481512eda7d22 100644 --- a/drivers/firewire/core-device.c +++ b/drivers/firewire/core-device.c @@ -192,7 +192,9 @@ static int fw_unit_remove(struct device *dev) struct fw_driver *driver = container_of(dev->driver, struct fw_driver, driver); - return driver->remove(fw_unit(dev)), 0; + driver->remove(fw_unit(dev)); + + return 0; } static int get_modalias(struct fw_unit *unit, char *buffer, size_t buffer_size) diff --git a/drivers/firmware/efi/libstub/Makefile b/drivers/firmware/efi/libstub/Makefile index 8a94388e38b33ec580de7bbf35e62f4ad7cdbe15..c23466e05e60dff3fd1a6cab9f7dd84c68bc4391 100644 --- a/drivers/firmware/efi/libstub/Makefile +++ b/drivers/firmware/efi/libstub/Makefile @@ -38,6 +38,8 @@ KBUILD_CFLAGS := $(cflags-y) -Os -DDISABLE_BRANCH_PROFILING \ # remove SCS flags from all objects in this directory KBUILD_CFLAGS := $(filter-out $(CC_FLAGS_SCS), $(KBUILD_CFLAGS)) +# disable LTO +KBUILD_CFLAGS := $(filter-out $(CC_FLAGS_LTO), $(KBUILD_CFLAGS)) GCOV_PROFILE := n # Sanitizer runtimes are unavailable and cannot be linked here. diff --git a/drivers/firmware/google/coreboot_table.c b/drivers/firmware/google/coreboot_table.c index 0205987a4fd4f08614e386b294e0bb9330b33ffc..dc83ea118c670d4c99fd234b32b25c1d4fef6a2c 100644 --- a/drivers/firmware/google/coreboot_table.c +++ b/drivers/firmware/google/coreboot_table.c @@ -46,14 +46,13 @@ static int coreboot_bus_probe(struct device *dev) static int coreboot_bus_remove(struct device *dev) { - int ret = 0; struct coreboot_device *device = CB_DEV(dev); struct coreboot_driver *driver = CB_DRV(dev->driver); if (driver->remove) - ret = driver->remove(device); + driver->remove(device); - return ret; + return 0; } static struct bus_type coreboot_bus_type = { diff --git a/drivers/firmware/google/coreboot_table.h b/drivers/firmware/google/coreboot_table.h index 7b7b4a6eeddae359d571fe4b4731bfa275502a15..beb778674acdc0601248158d46a18d0bb6c1ce74 100644 --- a/drivers/firmware/google/coreboot_table.h +++ b/drivers/firmware/google/coreboot_table.h @@ -72,7 +72,7 @@ struct coreboot_device { /* A driver for handling devices described in coreboot tables. */ struct coreboot_driver { int (*probe)(struct coreboot_device *); - int (*remove)(struct coreboot_device *); + void (*remove)(struct coreboot_device *); struct device_driver drv; u32 tag; }; diff --git a/drivers/firmware/google/framebuffer-coreboot.c b/drivers/firmware/google/framebuffer-coreboot.c index 916f26adc5955b8c021e607001558ec4028146b1..c6dcc1ef93acfde32a92e043ddb5be6b8c61e52a 100644 --- a/drivers/firmware/google/framebuffer-coreboot.c +++ b/drivers/firmware/google/framebuffer-coreboot.c @@ -72,13 +72,11 @@ static int framebuffer_probe(struct coreboot_device *dev) return PTR_ERR_OR_ZERO(pdev); } -static int framebuffer_remove(struct coreboot_device *dev) +static void framebuffer_remove(struct coreboot_device *dev) { struct platform_device *pdev = dev_get_drvdata(&dev->dev); platform_device_unregister(pdev); - - return 0; } static struct coreboot_driver framebuffer_driver = { diff --git a/drivers/firmware/google/memconsole-coreboot.c b/drivers/firmware/google/memconsole-coreboot.c index d17e4d6ac9bc42d36803460cdd3f104e34b0746f..74b5286518eefbc2df08baffc62f150f4329ffd0 100644 --- a/drivers/firmware/google/memconsole-coreboot.c +++ b/drivers/firmware/google/memconsole-coreboot.c @@ -91,11 +91,9 @@ static int memconsole_probe(struct coreboot_device *dev) return memconsole_sysfs_init(); } -static int memconsole_remove(struct coreboot_device *dev) +static void memconsole_remove(struct coreboot_device *dev) { memconsole_exit(); - - return 0; } static struct coreboot_driver memconsole_driver = { diff --git a/drivers/firmware/google/vpd.c b/drivers/firmware/google/vpd.c index d23c5c69ab5298313973eb4317c363dc99dab24d..ee6e08c0592b5492d208829191a4321bd29f6f5e 100644 --- a/drivers/firmware/google/vpd.c +++ b/drivers/firmware/google/vpd.c @@ -298,14 +298,12 @@ static int vpd_probe(struct coreboot_device *dev) return 0; } -static int vpd_remove(struct coreboot_device *dev) +static void vpd_remove(struct coreboot_device *dev) { vpd_section_destroy(&ro_vpd); vpd_section_destroy(&rw_vpd); kobject_put(vpd_kobj); - - return 0; } static struct coreboot_driver vpd_driver = { diff --git a/drivers/firmware/qcom_scm.c b/drivers/firmware/qcom_scm.c index 7be48c1bec96dd2a26188f9e79197cf6115768a7..f57779fc7ee932712c644e38c7fca3648892f311 100644 --- a/drivers/firmware/qcom_scm.c +++ b/drivers/firmware/qcom_scm.c @@ -965,8 +965,11 @@ EXPORT_SYMBOL(qcom_scm_ice_available); * qcom_scm_ice_invalidate_key() - Invalidate an inline encryption key * @index: the keyslot to invalidate * - * The UFSHCI standard defines a standard way to do this, but it doesn't work on - * these SoCs; only this SCM call does. + * The UFSHCI and eMMC standards define a standard way to do this, but it + * doesn't work on these SoCs; only this SCM call does. + * + * It is assumed that the SoC has only one ICE instance being used, as this SCM + * call doesn't specify which ICE instance the keyslot belongs to. * * Return: 0 on success; -errno on failure. */ @@ -995,10 +998,13 @@ EXPORT_SYMBOL(qcom_scm_ice_invalidate_key); * units, e.g. 1 = 512 bytes, 8 = 4096 bytes, etc. * * Program a key into a keyslot of Qualcomm ICE (Inline Crypto Engine), where it - * can then be used to encrypt/decrypt UFS I/O requests inline. + * can then be used to encrypt/decrypt UFS or eMMC I/O requests inline. + * + * The UFSHCI and eMMC standards define a standard way to do this, but it + * doesn't work on these SoCs; only this SCM call does. * - * The UFSHCI standard defines a standard way to do this, but it doesn't work on - * these SoCs; only this SCM call does. + * It is assumed that the SoC has only one ICE instance being used, as this SCM + * call doesn't specify which ICE instance the keyslot belongs to. * * Return: 0 on success; -errno on failure. */ diff --git a/drivers/fpga/Kconfig b/drivers/fpga/Kconfig index 5645226ca3ce07245af876fc6e89c53088bf9e51..5ff9438b7b4611803aedf34ebc3e004a0be0de60 100644 --- a/drivers/fpga/Kconfig +++ b/drivers/fpga/Kconfig @@ -192,6 +192,17 @@ config FPGA_DFL_AFU to the FPGA infrastructure via a Port. There may be more than one Port/AFU per DFL based FPGA device. +config FPGA_DFL_NIOS_INTEL_PAC_N3000 + tristate "FPGA DFL NIOS Driver for Intel PAC N3000" + depends on FPGA_DFL + select REGMAP + help + This is the driver for the N3000 Nios private feature on Intel + PAC (Programmable Acceleration Card) N3000. It communicates + with the embedded Nios processor to configure the retimers on + the card. It also instantiates the SPI master (spi-altera) for + the card's BMC (Board Management Controller). + config FPGA_DFL_PCI tristate "FPGA DFL PCIe Device Driver" depends on PCI && FPGA_DFL diff --git a/drivers/fpga/Makefile b/drivers/fpga/Makefile index d8e21dfc677877a13aa6de5d150d28affbe0594e..18dc9885883a29fd92d05d4fff86642656485e96 100644 --- a/drivers/fpga/Makefile +++ b/drivers/fpga/Makefile @@ -44,5 +44,7 @@ dfl-fme-objs += dfl-fme-perf.o dfl-afu-objs := dfl-afu-main.o dfl-afu-region.o dfl-afu-dma-region.o dfl-afu-objs += dfl-afu-error.o +obj-$(CONFIG_FPGA_DFL_NIOS_INTEL_PAC_N3000) += dfl-n3000-nios.o + # Drivers for FPGAs which implement DFL obj-$(CONFIG_FPGA_DFL_PCI) += dfl-pci.o diff --git a/drivers/fpga/dfl-fme-perf.c b/drivers/fpga/dfl-fme-perf.c index 531266287eeee01766f76e5bf663d95ddfa60ad7..4299145ef347e7e46a051354e23b6802389bfcb3 100644 --- a/drivers/fpga/dfl-fme-perf.c +++ b/drivers/fpga/dfl-fme-perf.c @@ -192,7 +192,7 @@ static struct attribute *fme_perf_cpumask_attrs[] = { NULL, }; -static struct attribute_group fme_perf_cpumask_group = { +static const struct attribute_group fme_perf_cpumask_group = { .attrs = fme_perf_cpumask_attrs, }; @@ -225,7 +225,7 @@ static struct attribute *fme_perf_format_attrs[] = { NULL, }; -static struct attribute_group fme_perf_format_group = { +static const struct attribute_group fme_perf_format_group = { .name = "format", .attrs = fme_perf_format_attrs, }; @@ -239,7 +239,7 @@ static struct attribute *fme_perf_events_attrs_empty[] = { NULL, }; -static struct attribute_group fme_perf_events_group = { +static const struct attribute_group fme_perf_events_group = { .name = "events", .attrs = fme_perf_events_attrs_empty, }; diff --git a/drivers/fpga/dfl-n3000-nios.c b/drivers/fpga/dfl-n3000-nios.c new file mode 100644 index 0000000000000000000000000000000000000000..7a95366f6516f87ddb93be06e514e290d893c2ee --- /dev/null +++ b/drivers/fpga/dfl-n3000-nios.c @@ -0,0 +1,588 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * DFL device driver for Nios private feature on Intel PAC (Programmable + * Acceleration Card) N3000 + * + * Copyright (C) 2019-2020 Intel Corporation, Inc. + * + * Authors: + * Wu Hao + * Xu Yilun + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * N3000 Nios private feature registers, named as NIOS_SPI_XX on spec. + * NS is the abbreviation of NIOS_SPI. + */ +#define N3000_NS_PARAM 0x8 +#define N3000_NS_PARAM_SHIFT_MODE_MSK BIT_ULL(1) +#define N3000_NS_PARAM_SHIFT_MODE_MSB 0 +#define N3000_NS_PARAM_SHIFT_MODE_LSB 1 +#define N3000_NS_PARAM_DATA_WIDTH GENMASK_ULL(7, 2) +#define N3000_NS_PARAM_NUM_CS GENMASK_ULL(13, 8) +#define N3000_NS_PARAM_CLK_POL BIT_ULL(14) +#define N3000_NS_PARAM_CLK_PHASE BIT_ULL(15) +#define N3000_NS_PARAM_PERIPHERAL_ID GENMASK_ULL(47, 32) + +#define N3000_NS_CTRL 0x10 +#define N3000_NS_CTRL_WR_DATA GENMASK_ULL(31, 0) +#define N3000_NS_CTRL_ADDR GENMASK_ULL(44, 32) +#define N3000_NS_CTRL_CMD_MSK GENMASK_ULL(63, 62) +#define N3000_NS_CTRL_CMD_NOP 0 +#define N3000_NS_CTRL_CMD_RD 1 +#define N3000_NS_CTRL_CMD_WR 2 + +#define N3000_NS_STAT 0x18 +#define N3000_NS_STAT_RD_DATA GENMASK_ULL(31, 0) +#define N3000_NS_STAT_RW_VAL BIT_ULL(32) + +/* Nios handshake registers, indirect access */ +#define N3000_NIOS_INIT 0x1000 +#define N3000_NIOS_INIT_DONE BIT(0) +#define N3000_NIOS_INIT_START BIT(1) +/* Mode for retimer A, link 0, the same below */ +#define N3000_NIOS_INIT_REQ_FEC_MODE_A0_MSK GENMASK(9, 8) +#define N3000_NIOS_INIT_REQ_FEC_MODE_A1_MSK GENMASK(11, 10) +#define N3000_NIOS_INIT_REQ_FEC_MODE_A2_MSK GENMASK(13, 12) +#define N3000_NIOS_INIT_REQ_FEC_MODE_A3_MSK GENMASK(15, 14) +#define N3000_NIOS_INIT_REQ_FEC_MODE_B0_MSK GENMASK(17, 16) +#define N3000_NIOS_INIT_REQ_FEC_MODE_B1_MSK GENMASK(19, 18) +#define N3000_NIOS_INIT_REQ_FEC_MODE_B2_MSK GENMASK(21, 20) +#define N3000_NIOS_INIT_REQ_FEC_MODE_B3_MSK GENMASK(23, 22) +#define N3000_NIOS_INIT_REQ_FEC_MODE_NO 0x0 +#define N3000_NIOS_INIT_REQ_FEC_MODE_KR 0x1 +#define N3000_NIOS_INIT_REQ_FEC_MODE_RS 0x2 + +#define N3000_NIOS_FW_VERSION 0x1004 +#define N3000_NIOS_FW_VERSION_PATCH GENMASK(23, 20) +#define N3000_NIOS_FW_VERSION_MINOR GENMASK(27, 24) +#define N3000_NIOS_FW_VERSION_MAJOR GENMASK(31, 28) + +/* The retimers we use on Intel PAC N3000 is Parkvale, abbreviated to PKVL */ +#define N3000_NIOS_PKVL_A_MODE_STS 0x1020 +#define N3000_NIOS_PKVL_B_MODE_STS 0x1024 +#define N3000_NIOS_PKVL_MODE_STS_GROUP_MSK GENMASK(15, 8) +#define N3000_NIOS_PKVL_MODE_STS_GROUP_OK 0x0 +#define N3000_NIOS_PKVL_MODE_STS_ID_MSK GENMASK(7, 0) +/* When GROUP MASK field == GROUP_OK */ +#define N3000_NIOS_PKVL_MODE_ID_RESET 0x0 +#define N3000_NIOS_PKVL_MODE_ID_4X10G 0x1 +#define N3000_NIOS_PKVL_MODE_ID_4X25G 0x2 +#define N3000_NIOS_PKVL_MODE_ID_2X25G 0x3 +#define N3000_NIOS_PKVL_MODE_ID_2X25G_2X10G 0x4 +#define N3000_NIOS_PKVL_MODE_ID_1X25G 0x5 + +#define N3000_NIOS_REGBUS_RETRY_COUNT 10000 /* loop count */ + +#define N3000_NIOS_INIT_TIMEOUT 10000000 /* usec */ +#define N3000_NIOS_INIT_TIME_INTV 100000 /* usec */ + +#define N3000_NIOS_INIT_REQ_FEC_MODE_MSK_ALL \ + (N3000_NIOS_INIT_REQ_FEC_MODE_A0_MSK | \ + N3000_NIOS_INIT_REQ_FEC_MODE_A1_MSK | \ + N3000_NIOS_INIT_REQ_FEC_MODE_A2_MSK | \ + N3000_NIOS_INIT_REQ_FEC_MODE_A3_MSK | \ + N3000_NIOS_INIT_REQ_FEC_MODE_B0_MSK | \ + N3000_NIOS_INIT_REQ_FEC_MODE_B1_MSK | \ + N3000_NIOS_INIT_REQ_FEC_MODE_B2_MSK | \ + N3000_NIOS_INIT_REQ_FEC_MODE_B3_MSK) + +#define N3000_NIOS_INIT_REQ_FEC_MODE_NO_ALL \ + (FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_A0_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_NO) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_A1_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_NO) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_A2_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_NO) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_A3_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_NO) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_B0_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_NO) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_B1_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_NO) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_B2_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_NO) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_B3_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_NO)) + +#define N3000_NIOS_INIT_REQ_FEC_MODE_KR_ALL \ + (FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_A0_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_KR) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_A1_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_KR) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_A2_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_KR) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_A3_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_KR) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_B0_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_KR) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_B1_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_KR) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_B2_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_KR) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_B3_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_KR)) + +#define N3000_NIOS_INIT_REQ_FEC_MODE_RS_ALL \ + (FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_A0_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_RS) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_A1_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_RS) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_A2_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_RS) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_A3_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_RS) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_B0_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_RS) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_B1_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_RS) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_B2_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_RS) | \ + FIELD_PREP(N3000_NIOS_INIT_REQ_FEC_MODE_B3_MSK, \ + N3000_NIOS_INIT_REQ_FEC_MODE_RS)) + +struct n3000_nios { + void __iomem *base; + struct regmap *regmap; + struct device *dev; + struct platform_device *altera_spi; +}; + +static ssize_t nios_fw_version_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct n3000_nios *nn = dev_get_drvdata(dev); + unsigned int val; + int ret; + + ret = regmap_read(nn->regmap, N3000_NIOS_FW_VERSION, &val); + if (ret) + return ret; + + return sysfs_emit(buf, "%x.%x.%x\n", + (u8)FIELD_GET(N3000_NIOS_FW_VERSION_MAJOR, val), + (u8)FIELD_GET(N3000_NIOS_FW_VERSION_MINOR, val), + (u8)FIELD_GET(N3000_NIOS_FW_VERSION_PATCH, val)); +} +static DEVICE_ATTR_RO(nios_fw_version); + +#define IS_MODE_STATUS_OK(mode_stat) \ + (FIELD_GET(N3000_NIOS_PKVL_MODE_STS_GROUP_MSK, (mode_stat)) == \ + N3000_NIOS_PKVL_MODE_STS_GROUP_OK) + +#define IS_RETIMER_FEC_SUPPORTED(retimer_mode) \ + ((retimer_mode) != N3000_NIOS_PKVL_MODE_ID_RESET && \ + (retimer_mode) != N3000_NIOS_PKVL_MODE_ID_4X10G) + +static int get_retimer_mode(struct n3000_nios *nn, unsigned int mode_stat_reg, + unsigned int *retimer_mode) +{ + unsigned int val; + int ret; + + ret = regmap_read(nn->regmap, mode_stat_reg, &val); + if (ret) + return ret; + + if (!IS_MODE_STATUS_OK(val)) + return -EFAULT; + + *retimer_mode = FIELD_GET(N3000_NIOS_PKVL_MODE_STS_ID_MSK, val); + + return 0; +} + +static ssize_t retimer_A_mode_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct n3000_nios *nn = dev_get_drvdata(dev); + unsigned int mode; + int ret; + + ret = get_retimer_mode(nn, N3000_NIOS_PKVL_A_MODE_STS, &mode); + if (ret) + return ret; + + return sysfs_emit(buf, "0x%x\n", mode); +} +static DEVICE_ATTR_RO(retimer_A_mode); + +static ssize_t retimer_B_mode_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct n3000_nios *nn = dev_get_drvdata(dev); + unsigned int mode; + int ret; + + ret = get_retimer_mode(nn, N3000_NIOS_PKVL_B_MODE_STS, &mode); + if (ret) + return ret; + + return sysfs_emit(buf, "0x%x\n", mode); +} +static DEVICE_ATTR_RO(retimer_B_mode); + +static ssize_t fec_mode_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + unsigned int val, retimer_a_mode, retimer_b_mode, fec_modes; + struct n3000_nios *nn = dev_get_drvdata(dev); + int ret; + + /* FEC mode setting is not supported in early FW versions */ + ret = regmap_read(nn->regmap, N3000_NIOS_FW_VERSION, &val); + if (ret) + return ret; + + if (FIELD_GET(N3000_NIOS_FW_VERSION_MAJOR, val) < 3) + return sysfs_emit(buf, "not supported\n"); + + /* If no 25G links, FEC mode setting is not supported either */ + ret = get_retimer_mode(nn, N3000_NIOS_PKVL_A_MODE_STS, &retimer_a_mode); + if (ret) + return ret; + + ret = get_retimer_mode(nn, N3000_NIOS_PKVL_B_MODE_STS, &retimer_b_mode); + if (ret) + return ret; + + if (!IS_RETIMER_FEC_SUPPORTED(retimer_a_mode) && + !IS_RETIMER_FEC_SUPPORTED(retimer_b_mode)) + return sysfs_emit(buf, "not supported\n"); + + /* get the valid FEC mode for 25G links */ + ret = regmap_read(nn->regmap, N3000_NIOS_INIT, &val); + if (ret) + return ret; + + /* + * FEC mode should always be the same for all links, as we set them + * in this way. + */ + fec_modes = (val & N3000_NIOS_INIT_REQ_FEC_MODE_MSK_ALL); + if (fec_modes == N3000_NIOS_INIT_REQ_FEC_MODE_NO_ALL) + return sysfs_emit(buf, "no\n"); + else if (fec_modes == N3000_NIOS_INIT_REQ_FEC_MODE_KR_ALL) + return sysfs_emit(buf, "kr\n"); + else if (fec_modes == N3000_NIOS_INIT_REQ_FEC_MODE_RS_ALL) + return sysfs_emit(buf, "rs\n"); + + return -EFAULT; +} +static DEVICE_ATTR_RO(fec_mode); + +static struct attribute *n3000_nios_attrs[] = { + &dev_attr_nios_fw_version.attr, + &dev_attr_retimer_A_mode.attr, + &dev_attr_retimer_B_mode.attr, + &dev_attr_fec_mode.attr, + NULL, +}; +ATTRIBUTE_GROUPS(n3000_nios); + +static int n3000_nios_init_done_check(struct n3000_nios *nn) +{ + unsigned int val, state_a, state_b; + struct device *dev = nn->dev; + int ret, ret2; + + /* + * The SPI is shared by the Nios core inside the FPGA, Nios will use + * this SPI master to do some one time initialization after power up, + * and then release the control to OS. The driver needs to poll on + * INIT_DONE to see when driver could take the control. + * + * Please note that after Nios firmware version 3.0.0, INIT_START is + * introduced, so driver needs to trigger START firstly and then check + * INIT_DONE. + */ + + ret = regmap_read(nn->regmap, N3000_NIOS_FW_VERSION, &val); + if (ret) + return ret; + + /* + * If Nios version register is totally uninitialized(== 0x0), then the + * Nios firmware is missing. So host could take control of SPI master + * safely, but initialization work for Nios is not done. To restore the + * card, we need to reprogram a new Nios firmware via the BMC chip on + * SPI bus. So the driver doesn't error out, it continues to create the + * spi controller device and spi_board_info for BMC. + */ + if (val == 0) { + dev_err(dev, "Nios version reg = 0x%x, skip INIT_DONE check, but the retimer may be uninitialized\n", + val); + return 0; + } + + if (FIELD_GET(N3000_NIOS_FW_VERSION_MAJOR, val) >= 3) { + /* read NIOS_INIT to check if retimer initialization is done */ + ret = regmap_read(nn->regmap, N3000_NIOS_INIT, &val); + if (ret) + return ret; + + /* check if retimers are initialized already */ + if (val & (N3000_NIOS_INIT_DONE | N3000_NIOS_INIT_START)) + goto nios_init_done; + + /* configure FEC mode per module param */ + val = N3000_NIOS_INIT_START; + + /* + * When the retimer is to be set to 10G mode, there is no FEC + * mode setting, so the REQ_FEC_MODE field will be ignored by + * Nios firmware in this case. But we should still fill the FEC + * mode field cause host could not get the retimer working mode + * until the Nios init is done. + * + * For now the driver doesn't support the retimer FEC mode + * switching per user's request. It is always set to Reed + * Solomon FEC. + * + * The driver will set the same FEC mode for all links. + */ + val |= N3000_NIOS_INIT_REQ_FEC_MODE_RS_ALL; + + ret = regmap_write(nn->regmap, N3000_NIOS_INIT, val); + if (ret) + return ret; + } + +nios_init_done: + /* polls on NIOS_INIT_DONE */ + ret = regmap_read_poll_timeout(nn->regmap, N3000_NIOS_INIT, val, + val & N3000_NIOS_INIT_DONE, + N3000_NIOS_INIT_TIME_INTV, + N3000_NIOS_INIT_TIMEOUT); + if (ret) + dev_err(dev, "NIOS_INIT_DONE %s\n", + (ret == -ETIMEDOUT) ? "timed out" : "check error"); + + ret2 = regmap_read(nn->regmap, N3000_NIOS_PKVL_A_MODE_STS, &state_a); + if (ret2) + return ret2; + + ret2 = regmap_read(nn->regmap, N3000_NIOS_PKVL_B_MODE_STS, &state_b); + if (ret2) + return ret2; + + if (!ret) { + /* + * After INIT_DONE is detected, it still needs to check if the + * Nios firmware reports any error during the retimer + * configuration. + */ + if (IS_MODE_STATUS_OK(state_a) && IS_MODE_STATUS_OK(state_b)) + return 0; + + /* + * If the retimer configuration is failed, the Nios firmware + * will still release the spi controller for host to + * communicate with the BMC. It makes possible for people to + * reprogram a new Nios firmware and restore the card. So the + * driver doesn't error out, it continues to create the spi + * controller device and spi_board_info for BMC. + */ + dev_err(dev, "NIOS_INIT_DONE OK, but err on retimer init\n"); + } + + dev_err(nn->dev, "PKVL_A_MODE_STS 0x%x\n", state_a); + dev_err(nn->dev, "PKVL_B_MODE_STS 0x%x\n", state_b); + + return ret; +} + +static struct spi_board_info m10_n3000_info = { + .modalias = "m10-n3000", + .max_speed_hz = 12500000, + .bus_num = 0, + .chip_select = 0, +}; + +static int create_altera_spi_controller(struct n3000_nios *nn) +{ + struct altera_spi_platform_data pdata = { 0 }; + struct platform_device_info pdevinfo = { 0 }; + void __iomem *base = nn->base; + u64 v; + + v = readq(base + N3000_NS_PARAM); + + pdata.mode_bits = SPI_CS_HIGH; + if (FIELD_GET(N3000_NS_PARAM_CLK_POL, v)) + pdata.mode_bits |= SPI_CPOL; + if (FIELD_GET(N3000_NS_PARAM_CLK_PHASE, v)) + pdata.mode_bits |= SPI_CPHA; + + pdata.num_chipselect = FIELD_GET(N3000_NS_PARAM_NUM_CS, v); + pdata.bits_per_word_mask = + SPI_BPW_RANGE_MASK(1, FIELD_GET(N3000_NS_PARAM_DATA_WIDTH, v)); + + pdata.num_devices = 1; + pdata.devices = &m10_n3000_info; + + dev_dbg(nn->dev, "%s cs %u bpm 0x%x mode 0x%x\n", __func__, + pdata.num_chipselect, pdata.bits_per_word_mask, + pdata.mode_bits); + + pdevinfo.name = "subdev_spi_altera"; + pdevinfo.id = PLATFORM_DEVID_AUTO; + pdevinfo.parent = nn->dev; + pdevinfo.data = &pdata; + pdevinfo.size_data = sizeof(pdata); + + nn->altera_spi = platform_device_register_full(&pdevinfo); + return PTR_ERR_OR_ZERO(nn->altera_spi); +} + +static void destroy_altera_spi_controller(struct n3000_nios *nn) +{ + platform_device_unregister(nn->altera_spi); +} + +static int n3000_nios_poll_stat_timeout(void __iomem *base, u64 *v) +{ + int loops; + + /* + * We don't use the time based timeout here for performance. + * + * The regbus read/write is on the critical path of Intel PAC N3000 + * image programing. The time based timeout checking will add too much + * overhead on it. Usually the state changes in 1 or 2 loops on the + * test server, and we set 10000 times loop here for safety. + */ + for (loops = N3000_NIOS_REGBUS_RETRY_COUNT; loops > 0 ; loops--) { + *v = readq(base + N3000_NS_STAT); + if (*v & N3000_NS_STAT_RW_VAL) + break; + cpu_relax(); + } + + return (loops > 0) ? 0 : -ETIMEDOUT; +} + +static int n3000_nios_reg_write(void *context, unsigned int reg, unsigned int val) +{ + struct n3000_nios *nn = context; + u64 v; + int ret; + + v = FIELD_PREP(N3000_NS_CTRL_CMD_MSK, N3000_NS_CTRL_CMD_WR) | + FIELD_PREP(N3000_NS_CTRL_ADDR, reg) | + FIELD_PREP(N3000_NS_CTRL_WR_DATA, val); + writeq(v, nn->base + N3000_NS_CTRL); + + ret = n3000_nios_poll_stat_timeout(nn->base, &v); + if (ret) + dev_err(nn->dev, "fail to write reg 0x%x val 0x%x: %d\n", + reg, val, ret); + + return ret; +} + +static int n3000_nios_reg_read(void *context, unsigned int reg, unsigned int *val) +{ + struct n3000_nios *nn = context; + u64 v; + int ret; + + v = FIELD_PREP(N3000_NS_CTRL_CMD_MSK, N3000_NS_CTRL_CMD_RD) | + FIELD_PREP(N3000_NS_CTRL_ADDR, reg); + writeq(v, nn->base + N3000_NS_CTRL); + + ret = n3000_nios_poll_stat_timeout(nn->base, &v); + if (ret) + dev_err(nn->dev, "fail to read reg 0x%x: %d\n", reg, ret); + else + *val = FIELD_GET(N3000_NS_STAT_RD_DATA, v); + + return ret; +} + +static const struct regmap_config n3000_nios_regbus_cfg = { + .reg_bits = 32, + .reg_stride = 4, + .val_bits = 32, + .fast_io = true, + + .reg_write = n3000_nios_reg_write, + .reg_read = n3000_nios_reg_read, +}; + +static int n3000_nios_probe(struct dfl_device *ddev) +{ + struct device *dev = &ddev->dev; + struct n3000_nios *nn; + int ret; + + nn = devm_kzalloc(dev, sizeof(*nn), GFP_KERNEL); + if (!nn) + return -ENOMEM; + + dev_set_drvdata(&ddev->dev, nn); + + nn->dev = dev; + + nn->base = devm_ioremap_resource(&ddev->dev, &ddev->mmio_res); + if (IS_ERR(nn->base)) + return PTR_ERR(nn->base); + + nn->regmap = devm_regmap_init(dev, NULL, nn, &n3000_nios_regbus_cfg); + if (IS_ERR(nn->regmap)) + return PTR_ERR(nn->regmap); + + ret = n3000_nios_init_done_check(nn); + if (ret) + return ret; + + ret = create_altera_spi_controller(nn); + if (ret) + dev_err(dev, "altera spi controller create failed: %d\n", ret); + + return ret; +} + +static void n3000_nios_remove(struct dfl_device *ddev) +{ + struct n3000_nios *nn = dev_get_drvdata(&ddev->dev); + + destroy_altera_spi_controller(nn); +} + +#define FME_FEATURE_ID_N3000_NIOS 0xd + +static const struct dfl_device_id n3000_nios_ids[] = { + { FME_ID, FME_FEATURE_ID_N3000_NIOS }, + { } +}; +MODULE_DEVICE_TABLE(dfl, n3000_nios_ids); + +static struct dfl_driver n3000_nios_driver = { + .drv = { + .name = "dfl-n3000-nios", + .dev_groups = n3000_nios_groups, + }, + .id_table = n3000_nios_ids, + .probe = n3000_nios_probe, + .remove = n3000_nios_remove, +}; + +module_dfl_driver(n3000_nios_driver); + +MODULE_DESCRIPTION("Driver for Nios private feature on Intel PAC N3000"); +MODULE_AUTHOR("Intel Corporation"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/fpga/dfl-pci.c b/drivers/fpga/dfl-pci.c index a2203d03c9e2b156622b5528ffa95cb380615c61..04e47e266f26d0641e36d3d1b44e7fd1e1f8a61b 100644 --- a/drivers/fpga/dfl-pci.c +++ b/drivers/fpga/dfl-pci.c @@ -27,6 +27,14 @@ #define DRV_VERSION "0.8" #define DRV_NAME "dfl-pci" +#define PCI_VSEC_ID_INTEL_DFLS 0x43 + +#define PCI_VNDR_DFLS_CNT 0x8 +#define PCI_VNDR_DFLS_RES 0xc + +#define PCI_VNDR_DFLS_RES_BAR_MASK GENMASK(2, 0) +#define PCI_VNDR_DFLS_RES_OFF_MASK GENMASK(31, 3) + struct cci_drvdata { struct dfl_fpga_cdev *cdev; /* container device */ }; @@ -119,49 +127,94 @@ static int *cci_pci_create_irq_table(struct pci_dev *pcidev, unsigned int nvec) return table; } -/* enumerate feature devices under pci device */ -static int cci_enumerate_feature_devs(struct pci_dev *pcidev) +static int find_dfls_by_vsec(struct pci_dev *pcidev, struct dfl_fpga_enum_info *info) { - struct cci_drvdata *drvdata = pci_get_drvdata(pcidev); - int port_num, bar, i, nvec, ret = 0; - struct dfl_fpga_enum_info *info; - struct dfl_fpga_cdev *cdev; + u32 bir, offset, vndr_hdr, dfl_cnt, dfl_res; + int dfl_res_off, i, bars, voff = 0; resource_size_t start, len; - void __iomem *base; - int *irq_table; - u32 offset; - u64 v; - /* allocate enumeration info via pci_dev */ - info = dfl_fpga_enum_info_alloc(&pcidev->dev); - if (!info) - return -ENOMEM; + while ((voff = pci_find_next_ext_capability(pcidev, voff, PCI_EXT_CAP_ID_VNDR))) { + vndr_hdr = 0; + pci_read_config_dword(pcidev, voff + PCI_VNDR_HEADER, &vndr_hdr); - /* add irq info for enumeration if the device support irq */ - nvec = cci_pci_alloc_irq(pcidev); - if (nvec < 0) { - dev_err(&pcidev->dev, "Fail to alloc irq %d.\n", nvec); - ret = nvec; - goto enum_info_free_exit; - } else if (nvec) { - irq_table = cci_pci_create_irq_table(pcidev, nvec); - if (!irq_table) { - ret = -ENOMEM; - goto irq_free_exit; + if (PCI_VNDR_HEADER_ID(vndr_hdr) == PCI_VSEC_ID_INTEL_DFLS && + pcidev->vendor == PCI_VENDOR_ID_INTEL) + break; + } + + if (!voff) { + dev_dbg(&pcidev->dev, "%s no DFL VSEC found\n", __func__); + return -ENODEV; + } + + dfl_cnt = 0; + pci_read_config_dword(pcidev, voff + PCI_VNDR_DFLS_CNT, &dfl_cnt); + if (dfl_cnt > PCI_STD_NUM_BARS) { + dev_err(&pcidev->dev, "%s too many DFLs %d > %d\n", + __func__, dfl_cnt, PCI_STD_NUM_BARS); + return -EINVAL; + } + + dfl_res_off = voff + PCI_VNDR_DFLS_RES; + if (dfl_res_off + (dfl_cnt * sizeof(u32)) > PCI_CFG_SPACE_EXP_SIZE) { + dev_err(&pcidev->dev, "%s DFL VSEC too big for PCIe config space\n", + __func__); + return -EINVAL; + } + + for (i = 0, bars = 0; i < dfl_cnt; i++, dfl_res_off += sizeof(u32)) { + dfl_res = GENMASK(31, 0); + pci_read_config_dword(pcidev, dfl_res_off, &dfl_res); + + bir = dfl_res & PCI_VNDR_DFLS_RES_BAR_MASK; + if (bir >= PCI_STD_NUM_BARS) { + dev_err(&pcidev->dev, "%s bad bir number %d\n", + __func__, bir); + return -EINVAL; } - ret = dfl_fpga_enum_info_add_irq(info, nvec, irq_table); - kfree(irq_table); - if (ret) - goto irq_free_exit; + if (bars & BIT(bir)) { + dev_err(&pcidev->dev, "%s DFL for BAR %d already specified\n", + __func__, bir); + return -EINVAL; + } + + bars |= BIT(bir); + + len = pci_resource_len(pcidev, bir); + offset = dfl_res & PCI_VNDR_DFLS_RES_OFF_MASK; + if (offset >= len) { + dev_err(&pcidev->dev, "%s bad offset %u >= %pa\n", + __func__, offset, &len); + return -EINVAL; + } + + dev_dbg(&pcidev->dev, "%s BAR %d offset 0x%x\n", __func__, bir, offset); + + len -= offset; + + start = pci_resource_start(pcidev, bir) + offset; + + dfl_fpga_enum_info_add_dfl(info, start, len); } - /* start to find Device Feature List in Bar 0 */ + return 0; +} + +/* default method of finding dfls starting at offset 0 of bar 0 */ +static int find_dfls_by_default(struct pci_dev *pcidev, + struct dfl_fpga_enum_info *info) +{ + int port_num, bar, i, ret = 0; + resource_size_t start, len; + void __iomem *base; + u32 offset; + u64 v; + + /* start to find Device Feature List from Bar 0 */ base = cci_pci_ioremap_bar0(pcidev); - if (!base) { - ret = -ENOMEM; - goto irq_free_exit; - } + if (!base) + return -ENOMEM; /* * PF device has FME and Ports/AFUs, and VF device only has one @@ -208,12 +261,54 @@ static int cci_enumerate_feature_devs(struct pci_dev *pcidev) dfl_fpga_enum_info_add_dfl(info, start, len); } else { ret = -ENODEV; - goto irq_free_exit; } /* release I/O mappings for next step enumeration */ pcim_iounmap_regions(pcidev, BIT(0)); + return ret; +} + +/* enumerate feature devices under pci device */ +static int cci_enumerate_feature_devs(struct pci_dev *pcidev) +{ + struct cci_drvdata *drvdata = pci_get_drvdata(pcidev); + struct dfl_fpga_enum_info *info; + struct dfl_fpga_cdev *cdev; + int nvec, ret = 0; + int *irq_table; + + /* allocate enumeration info via pci_dev */ + info = dfl_fpga_enum_info_alloc(&pcidev->dev); + if (!info) + return -ENOMEM; + + /* add irq info for enumeration if the device support irq */ + nvec = cci_pci_alloc_irq(pcidev); + if (nvec < 0) { + dev_err(&pcidev->dev, "Fail to alloc irq %d.\n", nvec); + ret = nvec; + goto enum_info_free_exit; + } else if (nvec) { + irq_table = cci_pci_create_irq_table(pcidev, nvec); + if (!irq_table) { + ret = -ENOMEM; + goto irq_free_exit; + } + + ret = dfl_fpga_enum_info_add_irq(info, nvec, irq_table); + kfree(irq_table); + if (ret) + goto irq_free_exit; + } + + ret = find_dfls_by_vsec(pcidev, info); + if (ret == -ENODEV) + ret = find_dfls_by_default(pcidev, info); + + if (ret) + goto irq_free_exit; + /* start enumeration with prepared enumeration information */ cdev = dfl_fpga_feature_devs_enumerate(info); if (IS_ERR(cdev)) { diff --git a/drivers/fpga/dfl.c b/drivers/fpga/dfl.c index b450870b75ed8d3076bea9d198e5a8a01fd047e1..511b20ff35a336c3f2a80107ebf8323359a86a0f 100644 --- a/drivers/fpga/dfl.c +++ b/drivers/fpga/dfl.c @@ -10,6 +10,7 @@ * Wu Hao * Xiao Guangrong */ +#include #include #include #include @@ -298,8 +299,7 @@ static int dfl_bus_uevent(struct device *dev, struct kobj_uevent_env *env) { struct dfl_device *ddev = to_dfl_dev(dev); - /* The type has 4 valid bits and feature_id has 12 valid bits */ - return add_uevent_var(env, "MODALIAS=dfl:t%01Xf%03X", + return add_uevent_var(env, "MODALIAS=dfl:t%04Xf%04X", ddev->type, ddev->feature_id); } diff --git a/drivers/fpga/dfl.h b/drivers/fpga/dfl.h index 5dc758f655b7dd732da656914758d22f1905a922..2b82c96ba56c76df11c71a4561f2dcc0f9be773e 100644 --- a/drivers/fpga/dfl.h +++ b/drivers/fpga/dfl.h @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -516,88 +517,4 @@ long dfl_feature_ioctl_set_irq(struct platform_device *pdev, struct dfl_feature *feature, unsigned long arg); -/** - * enum dfl_id_type - define the DFL FIU types - */ -enum dfl_id_type { - FME_ID, - PORT_ID, - DFL_ID_MAX, -}; - -/** - * struct dfl_device_id - dfl device identifier - * @type: contains 4 bits DFL FIU type of the device. See enum dfl_id_type. - * @feature_id: contains 12 bits feature identifier local to its DFL FIU type. - * @driver_data: driver specific data. - */ -struct dfl_device_id { - u8 type; - u16 feature_id; - unsigned long driver_data; -}; - -/** - * struct dfl_device - represent an dfl device on dfl bus - * - * @dev: generic device interface. - * @id: id of the dfl device. - * @type: type of DFL FIU of the device. See enum dfl_id_type. - * @feature_id: 16 bits feature identifier local to its DFL FIU type. - * @mmio_res: mmio resource of this dfl device. - * @irqs: list of Linux IRQ numbers of this dfl device. - * @num_irqs: number of IRQs supported by this dfl device. - * @cdev: pointer to DFL FPGA container device this dfl device belongs to. - * @id_entry: matched id entry in dfl driver's id table. - */ -struct dfl_device { - struct device dev; - int id; - u8 type; - u16 feature_id; - struct resource mmio_res; - int *irqs; - unsigned int num_irqs; - struct dfl_fpga_cdev *cdev; - const struct dfl_device_id *id_entry; -}; - -/** - * struct dfl_driver - represent an dfl device driver - * - * @drv: driver model structure. - * @id_table: pointer to table of device IDs the driver is interested in. - * { } member terminated. - * @probe: mandatory callback for device binding. - * @remove: callback for device unbinding. - */ -struct dfl_driver { - struct device_driver drv; - const struct dfl_device_id *id_table; - - int (*probe)(struct dfl_device *dfl_dev); - void (*remove)(struct dfl_device *dfl_dev); -}; - -#define to_dfl_dev(d) container_of(d, struct dfl_device, dev) -#define to_dfl_drv(d) container_of(d, struct dfl_driver, drv) - -/* - * use a macro to avoid include chaining to get THIS_MODULE. - */ -#define dfl_driver_register(drv) \ - __dfl_driver_register(drv, THIS_MODULE) -int __dfl_driver_register(struct dfl_driver *dfl_drv, struct module *owner); -void dfl_driver_unregister(struct dfl_driver *dfl_drv); - -/* - * module_dfl_driver() - Helper macro for drivers that don't do - * anything special in module init/exit. This eliminates a lot of - * boilerplate. Each module may only use this macro once, and - * calling it replaces module_init() and module_exit(). - */ -#define module_dfl_driver(__dfl_driver) \ - module_driver(__dfl_driver, dfl_driver_register, \ - dfl_driver_unregister) - #endif /* __FPGA_DFL_H */ diff --git a/drivers/fpga/fpga-bridge.c b/drivers/fpga/fpga-bridge.c index 2deccacc3aa759ad8dc3761d404a38cceb1034e0..e9266b2a357f6029cd63f0edf8ea6722eb1649cd 100644 --- a/drivers/fpga/fpga-bridge.c +++ b/drivers/fpga/fpga-bridge.c @@ -17,7 +17,7 @@ static DEFINE_IDA(fpga_bridge_ida); static struct class *fpga_bridge_class; /* Lock for adding/removing bridges to linked lists*/ -static spinlock_t bridge_list_lock; +static DEFINE_SPINLOCK(bridge_list_lock); /** * fpga_bridge_enable - Enable transactions on the bridge @@ -479,8 +479,6 @@ static void fpga_bridge_dev_release(struct device *dev) static int __init fpga_bridge_dev_init(void) { - spin_lock_init(&bridge_list_lock); - fpga_bridge_class = class_create(THIS_MODULE, "fpga_bridge"); if (IS_ERR(fpga_bridge_class)) return PTR_ERR(fpga_bridge_class); diff --git a/drivers/gpio/Kconfig b/drivers/gpio/Kconfig index fa225175e68de04824105b298fec33af2fadbebb..e3607ec4c2e837927b5b3b9198801b21d824b564 100644 --- a/drivers/gpio/Kconfig +++ b/drivers/gpio/Kconfig @@ -487,11 +487,11 @@ config GPIO_PXA Say yes here to support the PXA GPIO device config GPIO_RCAR - tristate "Renesas R-Car GPIO" + tristate "Renesas R-Car and RZ/G GPIO support" depends on ARCH_RENESAS || COMPILE_TEST select GPIOLIB_IRQCHIP help - Say yes here to support GPIO on Renesas R-Car SoCs. + Say yes here to support GPIO on Renesas R-Car or RZ/G SoCs. config GPIO_RDA bool "RDA Micro GPIO controller support" @@ -595,7 +595,7 @@ config GPIO_TB10X select OF_GPIO config GPIO_TEGRA - bool "NVIDIA Tegra GPIO support" + tristate "NVIDIA Tegra GPIO support" default ARCH_TEGRA depends on ARCH_TEGRA || COMPILE_TEST depends on OF_GPIO @@ -648,6 +648,16 @@ config GPIO_VF610 help Say yes here to support Vybrid vf610 GPIOs. +config GPIO_VISCONTI + tristate "Toshiba Visconti GPIO support" + depends on ARCH_VISCONTI || COMPILE_TEST + depends on OF_GPIO + select GPIOLIB_IRQCHIP + select GPIO_GENERIC + select IRQ_DOMAIN_HIERARCHY + help + Say yes here to support GPIO on Tohisba Visconti. + config GPIO_VR41XX tristate "NEC VR4100 series General-purpose I/O Uint support" depends on CPU_VR41XX @@ -670,7 +680,7 @@ config GPIO_WCD934X tristate "Qualcomm Technologies Inc WCD9340/WCD9341 gpio controller driver" depends on MFD_WCD934X && OF_GPIO help - This driver is to supprot GPIO block found on the Qualcomm Technologies + This driver is to support GPIO block found on the Qualcomm Technologies Inc WCD9340/WCD9341 Audio Codec. config GPIO_XGENE @@ -694,6 +704,8 @@ config GPIO_XGENE_SB config GPIO_XILINX tristate "Xilinx GPIO support" + select GPIOLIB_IRQCHIP + depends on OF_GPIO help Say yes here to support the Xilinx FPGA GPIO device @@ -731,13 +743,6 @@ config GPIO_ZYNQ help Say yes here to support Xilinx Zynq GPIO controller. -config GPIO_ZX - bool "ZTE ZX GPIO support" - depends on ARCH_ZX || COMPILE_TEST - select GPIOLIB_IRQCHIP - help - Say yes here to support the GPIO device on ZTE ZX SoCs. - config GPIO_LOONGSON1 tristate "Loongson1 GPIO support" depends on MACH_LOONGSON32 @@ -1253,13 +1258,6 @@ config GPIO_MAX77650 GPIO driver for MAX77650/77651 PMIC from Maxim Semiconductor. These chips have a single pin that can be configured as GPIO. -config GPIO_MSIC - bool "Intel MSIC mixed signal gpio support" - depends on (X86 || COMPILE_TEST) && MFD_INTEL_MSIC - help - Enable support for GPIO on intel MSIC controllers found in - intel MID devices - config GPIO_PALMAS bool "TI PALMAS series PMICs GPIO" depends on MFD_PALMAS @@ -1455,13 +1453,6 @@ config GPIO_BT8XX If unsure, say N. -config GPIO_INTEL_MID - bool "Intel MID GPIO support" - depends on X86_INTEL_MID - select GPIOLIB_IRQCHIP - help - Say Y here to support Intel MID GPIO. - config GPIO_MERRIFIELD tristate "Intel Merrifield GPIO support" depends on X86_INTEL_MID @@ -1637,8 +1628,7 @@ config GPIO_MOCKUP select IRQ_SIM help This enables GPIO Testing driver, which provides a way to test GPIO - subsystem through sysfs(or char device) and debugfs. GPIO_SYSFS - must be selected for this test. + subsystem through sysfs (or char device) and debugfs. User could use it through the script in tools/testing/selftests/gpio/gpio-mockup.sh. Reference the usage in it. diff --git a/drivers/gpio/Makefile b/drivers/gpio/Makefile index 35e3b60266650eb53fe4dae915fb19042894efb4..c58a90a3c3b116a76abe717437feaf0a8db5b0f9 100644 --- a/drivers/gpio/Makefile +++ b/drivers/gpio/Makefile @@ -67,7 +67,6 @@ obj-$(CONFIG_GPIO_HISI) += gpio-hisi.o obj-$(CONFIG_GPIO_HLWD) += gpio-hlwd.o obj-$(CONFIG_HTC_EGPIO) += gpio-htc-egpio.o obj-$(CONFIG_GPIO_ICH) += gpio-ich.o -obj-$(CONFIG_GPIO_INTEL_MID) += gpio-intel-mid.o obj-$(CONFIG_GPIO_IOP) += gpio-iop.o obj-$(CONFIG_GPIO_IT87) += gpio-it87.o obj-$(CONFIG_GPIO_IXP4XX) += gpio-ixp4xx.o @@ -103,7 +102,6 @@ obj-$(CONFIG_GPIO_MOXTET) += gpio-moxtet.o obj-$(CONFIG_GPIO_MPC5200) += gpio-mpc5200.o obj-$(CONFIG_GPIO_MPC8XXX) += gpio-mpc8xxx.o obj-$(CONFIG_GPIO_MSC313) += gpio-msc313.o -obj-$(CONFIG_GPIO_MSIC) += gpio-msic.o obj-$(CONFIG_GPIO_MT7621) += gpio-mt7621.o obj-$(CONFIG_GPIO_MVEBU) += gpio-mvebu.o obj-$(CONFIG_GPIO_MXC) += gpio-mxc.o @@ -164,6 +162,7 @@ obj-$(CONFIG_GPIO_UCB1400) += gpio-ucb1400.o obj-$(CONFIG_GPIO_UNIPHIER) += gpio-uniphier.o obj-$(CONFIG_GPIO_VF610) += gpio-vf610.o obj-$(CONFIG_GPIO_VIPERBOARD) += gpio-viperboard.o +obj-$(CONFIG_GPIO_VISCONTI) += gpio-visconti.o obj-$(CONFIG_GPIO_VR41XX) += gpio-vr41xx.o obj-$(CONFIG_GPIO_VX855) += gpio-vx855.o obj-$(CONFIG_GPIO_WCD934X) += gpio-wcd934x.o @@ -180,5 +179,4 @@ obj-$(CONFIG_GPIO_XLP) += gpio-xlp.o obj-$(CONFIG_GPIO_XRA1403) += gpio-xra1403.o obj-$(CONFIG_GPIO_XTENSA) += gpio-xtensa.o obj-$(CONFIG_GPIO_ZEVIO) += gpio-zevio.o -obj-$(CONFIG_GPIO_ZX) += gpio-zx.o obj-$(CONFIG_GPIO_ZYNQ) += gpio-zynq.o diff --git a/drivers/gpio/TODO b/drivers/gpio/TODO index 0229fa79499e3d0734f119f37bda7b4e38b3dc2d..b8b1473a5b1e9670c0d19027965ac89e440c6c85 100644 --- a/drivers/gpio/TODO +++ b/drivers/gpio/TODO @@ -101,7 +101,7 @@ for a few GPIOs. Those should stay where they are. At the same time it makes sense to get rid of code duplication in existing or new coming drivers. For example, gpio-ml-ioh should be incorporated into -gpio-pch. In similar way gpio-intel-mid into gpio-pxa. +gpio-pch. Generic MMIO GPIO diff --git a/drivers/gpio/gpio-aggregator.c b/drivers/gpio/gpio-aggregator.c index dfd8a4876a27ac4baa5cac2aaffdbfb0223aab54..08171431bb8f6162e92e0feff1f276c4f465712b 100644 --- a/drivers/gpio/gpio-aggregator.c +++ b/drivers/gpio/gpio-aggregator.c @@ -62,34 +62,6 @@ static char *get_arg(char **args) return start; } -static bool isrange(const char *s) -{ - size_t n; - - if (IS_ERR_OR_NULL(s)) - return false; - - while (1) { - n = strspn(s, "0123456789"); - if (!n) - return false; - - s += n; - - switch (*s++) { - case '\0': - return true; - - case '-': - case ',': - break; - - default: - return false; - } - } -} - static int aggr_add_gpio(struct gpio_aggregator *aggr, const char *key, int hwnum, unsigned int *n) { @@ -100,8 +72,7 @@ static int aggr_add_gpio(struct gpio_aggregator *aggr, const char *key, if (!lookups) return -ENOMEM; - lookups->table[*n] = - (struct gpiod_lookup)GPIO_LOOKUP_IDX(key, hwnum, NULL, *n, 0); + lookups->table[*n] = GPIO_LOOKUP_IDX(key, hwnum, NULL, *n, 0); (*n)++; memset(&lookups->table[*n], 0, sizeof(lookups->table[*n])); @@ -112,10 +83,10 @@ static int aggr_add_gpio(struct gpio_aggregator *aggr, const char *key, static int aggr_parse(struct gpio_aggregator *aggr) { + char *name, *offsets, *p; char *args = aggr->args; unsigned long *bitmap; unsigned int i, n = 0; - char *name, *offsets; int error = 0; bitmap = bitmap_alloc(ARCH_NR_GPIOS, GFP_KERNEL); @@ -130,7 +101,8 @@ static int aggr_parse(struct gpio_aggregator *aggr) goto free_bitmap; } - if (!isrange(offsets)) { + p = get_options(offsets, 0, &error); + if (error == 0 || *p) { /* Named GPIO line */ error = aggr_add_gpio(aggr, name, U16_MAX, &n); if (error) @@ -271,7 +243,7 @@ static DRIVER_ATTR_WO(delete_device); static struct attribute *gpio_aggregator_attrs[] = { &driver_attr_new_device.attr, &driver_attr_delete_device.attr, - NULL, + NULL }; ATTRIBUTE_GROUPS(gpio_aggregator); @@ -545,7 +517,7 @@ static const struct of_device_id gpio_aggregator_dt_ids[] = { * Add GPIO-operated devices controlled from userspace below, * or use "driver_override" in sysfs */ - {}, + {} }; MODULE_DEVICE_TABLE(of, gpio_aggregator_dt_ids); #endif diff --git a/drivers/gpio/gpio-bd70528.c b/drivers/gpio/gpio-bd70528.c index 45b3da8da336f50cc30c3fbd9c2bae3dbbcb3960..397a50d6bc654049e7c81cf8110c72c1397da3d2 100644 --- a/drivers/gpio/gpio-bd70528.c +++ b/drivers/gpio/gpio-bd70528.c @@ -12,7 +12,8 @@ #define GPIO_OUT_REG(offset) (BD70528_REG_GPIO1_OUT + (offset) * 2) struct bd70528_gpio { - struct rohm_regmap_dev chip; + struct regmap *regmap; + struct device *dev; struct gpio_chip gpio; }; @@ -35,11 +36,11 @@ static int bd70528_set_debounce(struct bd70528_gpio *bdgpio, val = BD70528_DEBOUNCE_50MS; break; default: - dev_err(bdgpio->chip.dev, + dev_err(bdgpio->dev, "Invalid debounce value %u\n", debounce); return -EINVAL; } - return regmap_update_bits(bdgpio->chip.regmap, GPIO_IN_REG(offset), + return regmap_update_bits(bdgpio->regmap, GPIO_IN_REG(offset), BD70528_DEBOUNCE_MASK, val); } @@ -49,9 +50,9 @@ static int bd70528_get_direction(struct gpio_chip *chip, unsigned int offset) int val, ret; /* Do we need to do something to IRQs here? */ - ret = regmap_read(bdgpio->chip.regmap, GPIO_OUT_REG(offset), &val); + ret = regmap_read(bdgpio->regmap, GPIO_OUT_REG(offset), &val); if (ret) { - dev_err(bdgpio->chip.dev, "Could not read gpio direction\n"); + dev_err(bdgpio->dev, "Could not read gpio direction\n"); return ret; } if (val & BD70528_GPIO_OUT_EN_MASK) @@ -67,13 +68,13 @@ static int bd70528_gpio_set_config(struct gpio_chip *chip, unsigned int offset, switch (pinconf_to_config_param(config)) { case PIN_CONFIG_DRIVE_OPEN_DRAIN: - return regmap_update_bits(bdgpio->chip.regmap, + return regmap_update_bits(bdgpio->regmap, GPIO_OUT_REG(offset), BD70528_GPIO_DRIVE_MASK, BD70528_GPIO_OPEN_DRAIN); break; case PIN_CONFIG_DRIVE_PUSH_PULL: - return regmap_update_bits(bdgpio->chip.regmap, + return regmap_update_bits(bdgpio->regmap, GPIO_OUT_REG(offset), BD70528_GPIO_DRIVE_MASK, BD70528_GPIO_PUSH_PULL); @@ -93,7 +94,7 @@ static int bd70528_direction_input(struct gpio_chip *chip, unsigned int offset) struct bd70528_gpio *bdgpio = gpiochip_get_data(chip); /* Do we need to do something to IRQs here? */ - return regmap_update_bits(bdgpio->chip.regmap, GPIO_OUT_REG(offset), + return regmap_update_bits(bdgpio->regmap, GPIO_OUT_REG(offset), BD70528_GPIO_OUT_EN_MASK, BD70528_GPIO_OUT_DISABLE); } @@ -105,10 +106,10 @@ static void bd70528_gpio_set(struct gpio_chip *chip, unsigned int offset, struct bd70528_gpio *bdgpio = gpiochip_get_data(chip); u8 val = (value) ? BD70528_GPIO_OUT_HI : BD70528_GPIO_OUT_LO; - ret = regmap_update_bits(bdgpio->chip.regmap, GPIO_OUT_REG(offset), + ret = regmap_update_bits(bdgpio->regmap, GPIO_OUT_REG(offset), BD70528_GPIO_OUT_MASK, val); if (ret) - dev_err(bdgpio->chip.dev, "Could not set gpio to %d\n", value); + dev_err(bdgpio->dev, "Could not set gpio to %d\n", value); } static int bd70528_direction_output(struct gpio_chip *chip, unsigned int offset, @@ -117,7 +118,7 @@ static int bd70528_direction_output(struct gpio_chip *chip, unsigned int offset, struct bd70528_gpio *bdgpio = gpiochip_get_data(chip); bd70528_gpio_set(chip, offset, value); - return regmap_update_bits(bdgpio->chip.regmap, GPIO_OUT_REG(offset), + return regmap_update_bits(bdgpio->regmap, GPIO_OUT_REG(offset), BD70528_GPIO_OUT_EN_MASK, BD70528_GPIO_OUT_ENABLE); } @@ -129,11 +130,11 @@ static int bd70528_gpio_get_o(struct bd70528_gpio *bdgpio, unsigned int offset) int ret; unsigned int val; - ret = regmap_read(bdgpio->chip.regmap, GPIO_OUT_REG(offset), &val); + ret = regmap_read(bdgpio->regmap, GPIO_OUT_REG(offset), &val); if (!ret) ret = !!(val & BD70528_GPIO_OUT_MASK); else - dev_err(bdgpio->chip.dev, "GPIO (out) state read failed\n"); + dev_err(bdgpio->dev, "GPIO (out) state read failed\n"); return ret; } @@ -143,12 +144,12 @@ static int bd70528_gpio_get_i(struct bd70528_gpio *bdgpio, unsigned int offset) unsigned int val; int ret; - ret = regmap_read(bdgpio->chip.regmap, BD70528_REG_GPIO_STATE, &val); + ret = regmap_read(bdgpio->regmap, BD70528_REG_GPIO_STATE, &val); if (!ret) ret = !(val & GPIO_IN_STATE_MASK(offset)); else - dev_err(bdgpio->chip.dev, "GPIO (in) state read failed\n"); + dev_err(bdgpio->dev, "GPIO (in) state read failed\n"); return ret; } @@ -173,29 +174,22 @@ static int bd70528_gpio_get(struct gpio_chip *chip, unsigned int offset) else if (ret == GPIO_LINE_DIRECTION_IN) ret = bd70528_gpio_get_i(bdgpio, offset); else - dev_err(bdgpio->chip.dev, "failed to read GPIO direction\n"); + dev_err(bdgpio->dev, "failed to read GPIO direction\n"); return ret; } static int bd70528_probe(struct platform_device *pdev) { + struct device *dev = &pdev->dev; struct bd70528_gpio *bdgpio; - struct rohm_regmap_dev *bd70528; int ret; - bd70528 = dev_get_drvdata(pdev->dev.parent); - if (!bd70528) { - dev_err(&pdev->dev, "No MFD driver data\n"); - return -EINVAL; - } - - bdgpio = devm_kzalloc(&pdev->dev, sizeof(*bdgpio), - GFP_KERNEL); + bdgpio = devm_kzalloc(dev, sizeof(*bdgpio), GFP_KERNEL); if (!bdgpio) return -ENOMEM; - bdgpio->chip.dev = &pdev->dev; - bdgpio->gpio.parent = pdev->dev.parent; + bdgpio->dev = dev; + bdgpio->gpio.parent = dev->parent; bdgpio->gpio.label = "bd70528-gpio"; bdgpio->gpio.owner = THIS_MODULE; bdgpio->gpio.get_direction = bd70528_get_direction; @@ -208,14 +202,15 @@ static int bd70528_probe(struct platform_device *pdev) bdgpio->gpio.ngpio = 4; bdgpio->gpio.base = -1; #ifdef CONFIG_OF_GPIO - bdgpio->gpio.of_node = pdev->dev.parent->of_node; + bdgpio->gpio.of_node = dev->parent->of_node; #endif - bdgpio->chip.regmap = bd70528->regmap; + bdgpio->regmap = dev_get_regmap(dev->parent, NULL); + if (!bdgpio->regmap) + return -ENODEV; - ret = devm_gpiochip_add_data(&pdev->dev, &bdgpio->gpio, - bdgpio); + ret = devm_gpiochip_add_data(dev, &bdgpio->gpio, bdgpio); if (ret) - dev_err(&pdev->dev, "gpio_init: Failed to add bd70528-gpio\n"); + dev_err(dev, "gpio_init: Failed to add bd70528-gpio\n"); return ret; } diff --git a/drivers/gpio/gpio-bd71828.c b/drivers/gpio/gpio-bd71828.c index 3dbbc638e9a911bfa8119e8b615736ef243d7d80..c8e382b53f2f67333002a5a33fed109e9840dea7 100644 --- a/drivers/gpio/gpio-bd71828.c +++ b/drivers/gpio/gpio-bd71828.c @@ -11,7 +11,8 @@ #define HALL_GPIO_OFFSET 3 struct bd71828_gpio { - struct rohm_regmap_dev chip; + struct regmap *regmap; + struct device *dev; struct gpio_chip gpio; }; @@ -29,10 +30,10 @@ static void bd71828_gpio_set(struct gpio_chip *chip, unsigned int offset, if (offset == HALL_GPIO_OFFSET) return; - ret = regmap_update_bits(bdgpio->chip.regmap, GPIO_OUT_REG(offset), + ret = regmap_update_bits(bdgpio->regmap, GPIO_OUT_REG(offset), BD71828_GPIO_OUT_MASK, val); if (ret) - dev_err(bdgpio->chip.dev, "Could not set gpio to %d\n", value); + dev_err(bdgpio->dev, "Could not set gpio to %d\n", value); } static int bd71828_gpio_get(struct gpio_chip *chip, unsigned int offset) @@ -42,10 +43,10 @@ static int bd71828_gpio_get(struct gpio_chip *chip, unsigned int offset) struct bd71828_gpio *bdgpio = gpiochip_get_data(chip); if (offset == HALL_GPIO_OFFSET) - ret = regmap_read(bdgpio->chip.regmap, BD71828_REG_IO_STAT, + ret = regmap_read(bdgpio->regmap, BD71828_REG_IO_STAT, &val); else - ret = regmap_read(bdgpio->chip.regmap, GPIO_OUT_REG(offset), + ret = regmap_read(bdgpio->regmap, GPIO_OUT_REG(offset), &val); if (!ret) ret = (val & BD71828_GPIO_OUT_MASK); @@ -63,12 +64,12 @@ static int bd71828_gpio_set_config(struct gpio_chip *chip, unsigned int offset, switch (pinconf_to_config_param(config)) { case PIN_CONFIG_DRIVE_OPEN_DRAIN: - return regmap_update_bits(bdgpio->chip.regmap, + return regmap_update_bits(bdgpio->regmap, GPIO_OUT_REG(offset), BD71828_GPIO_DRIVE_MASK, BD71828_GPIO_OPEN_DRAIN); case PIN_CONFIG_DRIVE_PUSH_PULL: - return regmap_update_bits(bdgpio->chip.regmap, + return regmap_update_bits(bdgpio->regmap, GPIO_OUT_REG(offset), BD71828_GPIO_DRIVE_MASK, BD71828_GPIO_PUSH_PULL); @@ -96,22 +97,15 @@ static int bd71828_get_direction(struct gpio_chip *chip, unsigned int offset) static int bd71828_probe(struct platform_device *pdev) { + struct device *dev = &pdev->dev; struct bd71828_gpio *bdgpio; - struct rohm_regmap_dev *bd71828; - bd71828 = dev_get_drvdata(pdev->dev.parent); - if (!bd71828) { - dev_err(&pdev->dev, "No MFD driver data\n"); - return -EINVAL; - } - - bdgpio = devm_kzalloc(&pdev->dev, sizeof(*bdgpio), - GFP_KERNEL); + bdgpio = devm_kzalloc(dev, sizeof(*bdgpio), GFP_KERNEL); if (!bdgpio) return -ENOMEM; - bdgpio->chip.dev = &pdev->dev; - bdgpio->gpio.parent = pdev->dev.parent; + bdgpio->dev = dev; + bdgpio->gpio.parent = dev->parent; bdgpio->gpio.label = "bd71828-gpio"; bdgpio->gpio.owner = THIS_MODULE; bdgpio->gpio.get_direction = bd71828_get_direction; @@ -127,11 +121,12 @@ static int bd71828_probe(struct platform_device *pdev) * "gpio-reserved-ranges" and exclude them from control */ bdgpio->gpio.ngpio = 4; - bdgpio->gpio.of_node = pdev->dev.parent->of_node; - bdgpio->chip.regmap = bd71828->regmap; + bdgpio->gpio.of_node = dev->parent->of_node; + bdgpio->regmap = dev_get_regmap(dev->parent, NULL); + if (!bdgpio->regmap) + return -ENODEV; - return devm_gpiochip_add_data(&pdev->dev, &bdgpio->gpio, - bdgpio); + return devm_gpiochip_add_data(dev, &bdgpio->gpio, bdgpio); } static struct platform_driver bd71828_gpio = { diff --git a/drivers/gpio/gpio-bd9571mwv.c b/drivers/gpio/gpio-bd9571mwv.c index c0abc9c6851b61c55f65f188df70e062e928c8f5..df6102b57734ff563ad9e7b7b694bc690caf169f 100644 --- a/drivers/gpio/gpio-bd9571mwv.c +++ b/drivers/gpio/gpio-bd9571mwv.c @@ -1,31 +1,24 @@ +// SPDX-License-Identifier: GPL-2.0-only /* - * ROHM BD9571MWV-M GPIO driver + * ROHM BD9571MWV-M and BD9574MWF-M GPIO driver * * Copyright (C) 2017 Marek Vasut * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether expressed or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License version 2 for more details. - * * Based on the TPS65086 driver * * NOTE: Interrupts are not supported yet. */ #include +#include #include #include #include struct bd9571mwv_gpio { + struct regmap *regmap; struct gpio_chip chip; - struct bd9571mwv *bd; }; static int bd9571mwv_gpio_get_direction(struct gpio_chip *chip, @@ -34,7 +27,7 @@ static int bd9571mwv_gpio_get_direction(struct gpio_chip *chip, struct bd9571mwv_gpio *gpio = gpiochip_get_data(chip); int ret, val; - ret = regmap_read(gpio->bd->regmap, BD9571MWV_GPIO_DIR, &val); + ret = regmap_read(gpio->regmap, BD9571MWV_GPIO_DIR, &val); if (ret < 0) return ret; if (val & BIT(offset)) @@ -48,8 +41,7 @@ static int bd9571mwv_gpio_direction_input(struct gpio_chip *chip, { struct bd9571mwv_gpio *gpio = gpiochip_get_data(chip); - regmap_update_bits(gpio->bd->regmap, BD9571MWV_GPIO_DIR, - BIT(offset), 0); + regmap_update_bits(gpio->regmap, BD9571MWV_GPIO_DIR, BIT(offset), 0); return 0; } @@ -60,9 +52,9 @@ static int bd9571mwv_gpio_direction_output(struct gpio_chip *chip, struct bd9571mwv_gpio *gpio = gpiochip_get_data(chip); /* Set the initial value */ - regmap_update_bits(gpio->bd->regmap, BD9571MWV_GPIO_OUT, + regmap_update_bits(gpio->regmap, BD9571MWV_GPIO_OUT, BIT(offset), value ? BIT(offset) : 0); - regmap_update_bits(gpio->bd->regmap, BD9571MWV_GPIO_DIR, + regmap_update_bits(gpio->regmap, BD9571MWV_GPIO_DIR, BIT(offset), BIT(offset)); return 0; @@ -73,7 +65,7 @@ static int bd9571mwv_gpio_get(struct gpio_chip *chip, unsigned int offset) struct bd9571mwv_gpio *gpio = gpiochip_get_data(chip); int ret, val; - ret = regmap_read(gpio->bd->regmap, BD9571MWV_GPIO_IN, &val); + ret = regmap_read(gpio->regmap, BD9571MWV_GPIO_IN, &val); if (ret < 0) return ret; @@ -85,7 +77,7 @@ static void bd9571mwv_gpio_set(struct gpio_chip *chip, unsigned int offset, { struct bd9571mwv_gpio *gpio = gpiochip_get_data(chip); - regmap_update_bits(gpio->bd->regmap, BD9571MWV_GPIO_OUT, + regmap_update_bits(gpio->regmap, BD9571MWV_GPIO_OUT, BIT(offset), value ? BIT(offset) : 0); } @@ -113,9 +105,9 @@ static int bd9571mwv_gpio_probe(struct platform_device *pdev) platform_set_drvdata(pdev, gpio); - gpio->bd = dev_get_drvdata(pdev->dev.parent); + gpio->regmap = dev_get_regmap(pdev->dev.parent, NULL); gpio->chip = template_chip; - gpio->chip.parent = gpio->bd->dev; + gpio->chip.parent = pdev->dev.parent; ret = devm_gpiochip_add_data(&pdev->dev, &gpio->chip, gpio); if (ret < 0) { @@ -127,7 +119,8 @@ static int bd9571mwv_gpio_probe(struct platform_device *pdev) } static const struct platform_device_id bd9571mwv_gpio_id_table[] = { - { "bd9571mwv-gpio", }, + { "bd9571mwv-gpio", ROHM_CHIP_TYPE_BD9571 }, + { "bd9574mwf-gpio", ROHM_CHIP_TYPE_BD9574 }, { /* sentinel */ } }; MODULE_DEVICE_TABLE(platform, bd9571mwv_gpio_id_table); diff --git a/drivers/gpio/gpio-ep93xx.c b/drivers/gpio/gpio-ep93xx.c index 94d9fa0d6aa70d1d6b94d30b34bec6cdea9b3f16..ef148b26b587694a3e298d3a6123933f163c1cf3 100644 --- a/drivers/gpio/gpio-ep93xx.c +++ b/drivers/gpio/gpio-ep93xx.c @@ -31,6 +31,8 @@ /* Maximum value for irq capable line identifiers */ #define EP93XX_GPIO_LINE_MAX_IRQ 23 +#define EP93XX_GPIO_A_IRQ_BASE 64 +#define EP93XX_GPIO_B_IRQ_BASE 72 /* * Static mapping of GPIO bank F IRQS: * F0..F7 (16..24) to irq 80..87. @@ -292,14 +294,14 @@ struct ep93xx_gpio_bank { static struct ep93xx_gpio_bank ep93xx_gpio_banks[] = { /* Bank A has 8 IRQs */ - EP93XX_GPIO_BANK("A", 0x00, 0x10, 0x90, 0, true, false, 64), + EP93XX_GPIO_BANK("A", 0x00, 0x10, 0x90, 0, true, false, EP93XX_GPIO_A_IRQ_BASE), /* Bank B has 8 IRQs */ - EP93XX_GPIO_BANK("B", 0x04, 0x14, 0xac, 8, true, false, 72), + EP93XX_GPIO_BANK("B", 0x04, 0x14, 0xac, 8, true, false, EP93XX_GPIO_B_IRQ_BASE), EP93XX_GPIO_BANK("C", 0x08, 0x18, 0x00, 40, false, false, 0), EP93XX_GPIO_BANK("D", 0x0c, 0x1c, 0x00, 24, false, false, 0), EP93XX_GPIO_BANK("E", 0x20, 0x24, 0x00, 32, false, false, 0), /* Bank F has 8 IRQs */ - EP93XX_GPIO_BANK("F", 0x30, 0x34, 0x4c, 16, false, true, 0), + EP93XX_GPIO_BANK("F", 0x30, 0x34, 0x4c, 16, false, true, EP93XX_GPIO_F_IRQ_BASE), EP93XX_GPIO_BANK("G", 0x38, 0x3c, 0x00, 48, false, false, 0), EP93XX_GPIO_BANK("H", 0x40, 0x44, 0x00, 56, false, false, 0), }; @@ -318,11 +320,6 @@ static int ep93xx_gpio_set_config(struct gpio_chip *gc, unsigned offset, return 0; } -static int ep93xx_gpio_f_to_irq(struct gpio_chip *gc, unsigned offset) -{ - return EP93XX_GPIO_F_IRQ_BASE + offset; -} - static void ep93xx_init_irq_chip(struct device *dev, struct irq_chip *ic) { ic->irq_ack = ep93xx_gpio_irq_ack; @@ -375,7 +372,7 @@ static int ep93xx_gpio_add_bank(struct ep93xx_gpio_chip *egc, girq->parent_handler = ep93xx_gpio_ab_irq_handler; girq->num_parents = 1; - girq->parents = devm_kcalloc(dev, 1, + girq->parents = devm_kcalloc(dev, girq->num_parents, sizeof(*girq->parents), GFP_KERNEL); if (!girq->parents) @@ -393,20 +390,19 @@ static int ep93xx_gpio_add_bank(struct ep93xx_gpio_chip *egc, /* * FIXME: convert this to use hierarchical IRQ support! - * this requires fixing the root irqchip to be hierarchial. + * this requires fixing the root irqchip to be hierarchical. */ girq->parent_handler = ep93xx_gpio_f_irq_handler; girq->num_parents = 8; - girq->parents = devm_kcalloc(dev, 8, + girq->parents = devm_kcalloc(dev, girq->num_parents, sizeof(*girq->parents), GFP_KERNEL); if (!girq->parents) return -ENOMEM; /* Pick resources 1..8 for these IRQs */ - for (i = 1; i <= 8; i++) - girq->parents[i - 1] = platform_get_irq(pdev, i); - for (i = 0; i < 8; i++) { - gpio_irq = EP93XX_GPIO_F_IRQ_BASE + i; + for (i = 0; i < girq->num_parents; i++) { + girq->parents[i] = platform_get_irq(pdev, i + 1); + gpio_irq = bank->irq_base + i; irq_set_chip_data(gpio_irq, &epg->gc[5]); irq_set_chip_and_handler(gpio_irq, girq->chip, @@ -415,7 +411,7 @@ static int ep93xx_gpio_add_bank(struct ep93xx_gpio_chip *egc, } girq->default_type = IRQ_TYPE_NONE; girq->handler = handle_level_irq; - gc->to_irq = ep93xx_gpio_f_to_irq; + girq->first = bank->irq_base; } return devm_gpiochip_add_data(dev, gc, epg); diff --git a/drivers/gpio/gpio-intel-mid.c b/drivers/gpio/gpio-intel-mid.c deleted file mode 100644 index 86a10c808ef658c84a1218ccbd7b11e3712197fb..0000000000000000000000000000000000000000 --- a/drivers/gpio/gpio-intel-mid.c +++ /dev/null @@ -1,414 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * Intel MID GPIO driver - * - * Copyright (c) 2008-2014,2016 Intel Corporation. - */ - -/* Supports: - * Moorestown platform Langwell chip. - * Medfield platform Penwell chip. - * Clovertrail platform Cloverview chip. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define INTEL_MID_IRQ_TYPE_EDGE (1 << 0) -#define INTEL_MID_IRQ_TYPE_LEVEL (1 << 1) - -/* - * Langwell chip has 64 pins and thus there are 2 32bit registers to control - * each feature, while Penwell chip has 96 pins for each block, and need 3 32bit - * registers to control them, so we only define the order here instead of a - * structure, to get a bit offset for a pin (use GPDR as an example): - * - * nreg = ngpio / 32; - * reg = offset / 32; - * bit = offset % 32; - * reg_addr = reg_base + GPDR * nreg * 4 + reg * 4; - * - * so the bit of reg_addr is to control pin offset's GPDR feature -*/ - -enum GPIO_REG { - GPLR = 0, /* pin level read-only */ - GPDR, /* pin direction */ - GPSR, /* pin set */ - GPCR, /* pin clear */ - GRER, /* rising edge detect */ - GFER, /* falling edge detect */ - GEDR, /* edge detect result */ - GAFR, /* alt function */ -}; - -/* intel_mid gpio driver data */ -struct intel_mid_gpio_ddata { - u16 ngpio; /* number of gpio pins */ - u32 chip_irq_type; /* chip interrupt type */ -}; - -struct intel_mid_gpio { - struct gpio_chip chip; - void __iomem *reg_base; - spinlock_t lock; - struct pci_dev *pdev; -}; - -static void __iomem *gpio_reg(struct gpio_chip *chip, unsigned offset, - enum GPIO_REG reg_type) -{ - struct intel_mid_gpio *priv = gpiochip_get_data(chip); - unsigned nreg = chip->ngpio / 32; - u8 reg = offset / 32; - - return priv->reg_base + reg_type * nreg * 4 + reg * 4; -} - -static void __iomem *gpio_reg_2bit(struct gpio_chip *chip, unsigned offset, - enum GPIO_REG reg_type) -{ - struct intel_mid_gpio *priv = gpiochip_get_data(chip); - unsigned nreg = chip->ngpio / 32; - u8 reg = offset / 16; - - return priv->reg_base + reg_type * nreg * 4 + reg * 4; -} - -static int intel_gpio_request(struct gpio_chip *chip, unsigned offset) -{ - void __iomem *gafr = gpio_reg_2bit(chip, offset, GAFR); - u32 value = readl(gafr); - int shift = (offset % 16) << 1, af = (value >> shift) & 3; - - if (af) { - value &= ~(3 << shift); - writel(value, gafr); - } - return 0; -} - -static int intel_gpio_get(struct gpio_chip *chip, unsigned offset) -{ - void __iomem *gplr = gpio_reg(chip, offset, GPLR); - - return !!(readl(gplr) & BIT(offset % 32)); -} - -static void intel_gpio_set(struct gpio_chip *chip, unsigned offset, int value) -{ - void __iomem *gpsr, *gpcr; - - if (value) { - gpsr = gpio_reg(chip, offset, GPSR); - writel(BIT(offset % 32), gpsr); - } else { - gpcr = gpio_reg(chip, offset, GPCR); - writel(BIT(offset % 32), gpcr); - } -} - -static int intel_gpio_direction_input(struct gpio_chip *chip, unsigned offset) -{ - struct intel_mid_gpio *priv = gpiochip_get_data(chip); - void __iomem *gpdr = gpio_reg(chip, offset, GPDR); - u32 value; - unsigned long flags; - - if (priv->pdev) - pm_runtime_get(&priv->pdev->dev); - - spin_lock_irqsave(&priv->lock, flags); - value = readl(gpdr); - value &= ~BIT(offset % 32); - writel(value, gpdr); - spin_unlock_irqrestore(&priv->lock, flags); - - if (priv->pdev) - pm_runtime_put(&priv->pdev->dev); - - return 0; -} - -static int intel_gpio_direction_output(struct gpio_chip *chip, - unsigned offset, int value) -{ - struct intel_mid_gpio *priv = gpiochip_get_data(chip); - void __iomem *gpdr = gpio_reg(chip, offset, GPDR); - unsigned long flags; - - intel_gpio_set(chip, offset, value); - - if (priv->pdev) - pm_runtime_get(&priv->pdev->dev); - - spin_lock_irqsave(&priv->lock, flags); - value = readl(gpdr); - value |= BIT(offset % 32); - writel(value, gpdr); - spin_unlock_irqrestore(&priv->lock, flags); - - if (priv->pdev) - pm_runtime_put(&priv->pdev->dev); - - return 0; -} - -static int intel_mid_irq_type(struct irq_data *d, unsigned type) -{ - struct gpio_chip *gc = irq_data_get_irq_chip_data(d); - struct intel_mid_gpio *priv = gpiochip_get_data(gc); - u32 gpio = irqd_to_hwirq(d); - unsigned long flags; - u32 value; - void __iomem *grer = gpio_reg(&priv->chip, gpio, GRER); - void __iomem *gfer = gpio_reg(&priv->chip, gpio, GFER); - - if (gpio >= priv->chip.ngpio) - return -EINVAL; - - if (priv->pdev) - pm_runtime_get(&priv->pdev->dev); - - spin_lock_irqsave(&priv->lock, flags); - if (type & IRQ_TYPE_EDGE_RISING) - value = readl(grer) | BIT(gpio % 32); - else - value = readl(grer) & (~BIT(gpio % 32)); - writel(value, grer); - - if (type & IRQ_TYPE_EDGE_FALLING) - value = readl(gfer) | BIT(gpio % 32); - else - value = readl(gfer) & (~BIT(gpio % 32)); - writel(value, gfer); - spin_unlock_irqrestore(&priv->lock, flags); - - if (priv->pdev) - pm_runtime_put(&priv->pdev->dev); - - return 0; -} - -static void intel_mid_irq_unmask(struct irq_data *d) -{ -} - -static void intel_mid_irq_mask(struct irq_data *d) -{ -} - -static struct irq_chip intel_mid_irqchip = { - .name = "INTEL_MID-GPIO", - .irq_mask = intel_mid_irq_mask, - .irq_unmask = intel_mid_irq_unmask, - .irq_set_type = intel_mid_irq_type, -}; - -static const struct intel_mid_gpio_ddata gpio_lincroft = { - .ngpio = 64, -}; - -static const struct intel_mid_gpio_ddata gpio_penwell_aon = { - .ngpio = 96, - .chip_irq_type = INTEL_MID_IRQ_TYPE_EDGE, -}; - -static const struct intel_mid_gpio_ddata gpio_penwell_core = { - .ngpio = 96, - .chip_irq_type = INTEL_MID_IRQ_TYPE_EDGE, -}; - -static const struct intel_mid_gpio_ddata gpio_cloverview_aon = { - .ngpio = 96, - .chip_irq_type = INTEL_MID_IRQ_TYPE_EDGE | INTEL_MID_IRQ_TYPE_LEVEL, -}; - -static const struct intel_mid_gpio_ddata gpio_cloverview_core = { - .ngpio = 96, - .chip_irq_type = INTEL_MID_IRQ_TYPE_EDGE, -}; - -static const struct pci_device_id intel_gpio_ids[] = { - { - /* Lincroft */ - PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x080f), - .driver_data = (kernel_ulong_t)&gpio_lincroft, - }, - { - /* Penwell AON */ - PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x081f), - .driver_data = (kernel_ulong_t)&gpio_penwell_aon, - }, - { - /* Penwell Core */ - PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x081a), - .driver_data = (kernel_ulong_t)&gpio_penwell_core, - }, - { - /* Cloverview Aon */ - PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x08eb), - .driver_data = (kernel_ulong_t)&gpio_cloverview_aon, - }, - { - /* Cloverview Core */ - PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x08f7), - .driver_data = (kernel_ulong_t)&gpio_cloverview_core, - }, - { } -}; - -static void intel_mid_irq_handler(struct irq_desc *desc) -{ - struct gpio_chip *gc = irq_desc_get_handler_data(desc); - struct intel_mid_gpio *priv = gpiochip_get_data(gc); - struct irq_data *data = irq_desc_get_irq_data(desc); - struct irq_chip *chip = irq_data_get_irq_chip(data); - u32 base, gpio, mask; - unsigned long pending; - void __iomem *gedr; - - /* check GPIO controller to check which pin triggered the interrupt */ - for (base = 0; base < priv->chip.ngpio; base += 32) { - gedr = gpio_reg(&priv->chip, base, GEDR); - while ((pending = readl(gedr))) { - gpio = __ffs(pending); - mask = BIT(gpio); - /* Clear before handling so we can't lose an edge */ - writel(mask, gedr); - generic_handle_irq(irq_find_mapping(gc->irq.domain, - base + gpio)); - } - } - - chip->irq_eoi(data); -} - -static int intel_mid_irq_init_hw(struct gpio_chip *chip) -{ - struct intel_mid_gpio *priv = gpiochip_get_data(chip); - void __iomem *reg; - unsigned base; - - for (base = 0; base < priv->chip.ngpio; base += 32) { - /* Clear the rising-edge detect register */ - reg = gpio_reg(&priv->chip, base, GRER); - writel(0, reg); - /* Clear the falling-edge detect register */ - reg = gpio_reg(&priv->chip, base, GFER); - writel(0, reg); - /* Clear the edge detect status register */ - reg = gpio_reg(&priv->chip, base, GEDR); - writel(~0, reg); - } - - return 0; -} - -static int __maybe_unused intel_gpio_runtime_idle(struct device *dev) -{ - int err = pm_schedule_suspend(dev, 500); - return err ?: -EBUSY; -} - -static const struct dev_pm_ops intel_gpio_pm_ops = { - SET_RUNTIME_PM_OPS(NULL, NULL, intel_gpio_runtime_idle) -}; - -static int intel_gpio_probe(struct pci_dev *pdev, - const struct pci_device_id *id) -{ - void __iomem *base; - struct intel_mid_gpio *priv; - u32 gpio_base; - u32 irq_base; - int retval; - struct gpio_irq_chip *girq; - struct intel_mid_gpio_ddata *ddata = - (struct intel_mid_gpio_ddata *)id->driver_data; - - retval = pcim_enable_device(pdev); - if (retval) - return retval; - - retval = pcim_iomap_regions(pdev, 1 << 0 | 1 << 1, pci_name(pdev)); - if (retval) { - dev_err(&pdev->dev, "I/O memory mapping error\n"); - return retval; - } - - base = pcim_iomap_table(pdev)[1]; - - irq_base = readl(base); - gpio_base = readl(sizeof(u32) + base); - - /* release the IO mapping, since we already get the info from bar1 */ - pcim_iounmap_regions(pdev, 1 << 1); - - priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL); - if (!priv) - return -ENOMEM; - - priv->reg_base = pcim_iomap_table(pdev)[0]; - priv->chip.label = dev_name(&pdev->dev); - priv->chip.parent = &pdev->dev; - priv->chip.request = intel_gpio_request; - priv->chip.direction_input = intel_gpio_direction_input; - priv->chip.direction_output = intel_gpio_direction_output; - priv->chip.get = intel_gpio_get; - priv->chip.set = intel_gpio_set; - priv->chip.base = gpio_base; - priv->chip.ngpio = ddata->ngpio; - priv->chip.can_sleep = false; - priv->pdev = pdev; - - spin_lock_init(&priv->lock); - - girq = &priv->chip.irq; - girq->chip = &intel_mid_irqchip; - girq->init_hw = intel_mid_irq_init_hw; - girq->parent_handler = intel_mid_irq_handler; - girq->num_parents = 1; - girq->parents = devm_kcalloc(&pdev->dev, girq->num_parents, - sizeof(*girq->parents), - GFP_KERNEL); - if (!girq->parents) - return -ENOMEM; - girq->parents[0] = pdev->irq; - girq->first = irq_base; - girq->default_type = IRQ_TYPE_NONE; - girq->handler = handle_simple_irq; - - pci_set_drvdata(pdev, priv); - - retval = devm_gpiochip_add_data(&pdev->dev, &priv->chip, priv); - if (retval) { - dev_err(&pdev->dev, "gpiochip_add error %d\n", retval); - return retval; - } - - pm_runtime_put_noidle(&pdev->dev); - pm_runtime_allow(&pdev->dev); - - return 0; -} - -static struct pci_driver intel_gpio_driver = { - .name = "intel_mid_gpio", - .id_table = intel_gpio_ids, - .probe = intel_gpio_probe, - .driver = { - .pm = &intel_gpio_pm_ops, - }, -}; - -builtin_pci_driver(intel_gpio_driver); diff --git a/drivers/gpio/gpio-max77620.c b/drivers/gpio/gpio-max77620.c index 7c0a9ef0b5000b9fc2c7743d33cb2befea7fa639..82b3a913005df3467534d82e4643b4fe68e18403 100644 --- a/drivers/gpio/gpio-max77620.c +++ b/drivers/gpio/gpio-max77620.c @@ -325,7 +325,7 @@ static int max77620_gpio_probe(struct platform_device *pdev) girq->parents = NULL; girq->default_type = IRQ_TYPE_NONE; girq->handler = handle_edge_irq; - girq->init_hw = max77620_gpio_irq_init_hw, + girq->init_hw = max77620_gpio_irq_init_hw; girq->threaded = true; platform_set_drvdata(pdev, mgpio); diff --git a/drivers/gpio/gpio-merrifield.c b/drivers/gpio/gpio-merrifield.c index 706687fab634e7545e7ed8e75f1f74bd9a644c49..22f3ce218f5d8c90a40a1144b52d1c978754d37b 100644 --- a/drivers/gpio/gpio-merrifield.c +++ b/drivers/gpio/gpio-merrifield.c @@ -194,6 +194,11 @@ static int mrfld_gpio_set_config(struct gpio_chip *chip, unsigned int offset, { u32 debounce; + if ((pinconf_to_config_param(config) == PIN_CONFIG_BIAS_DISABLE) || + (pinconf_to_config_param(config) == PIN_CONFIG_BIAS_PULL_UP) || + (pinconf_to_config_param(config) == PIN_CONFIG_BIAS_PULL_DOWN)) + return gpiochip_generic_config(chip, offset, config); + if (pinconf_to_config_param(config) != PIN_CONFIG_INPUT_DEBOUNCE) return -ENOTSUPP; diff --git a/drivers/gpio/gpio-msic.c b/drivers/gpio/gpio-msic.c deleted file mode 100644 index 7e3c96e4ab2c47c5e00db40090c4d2facc7b2e55..0000000000000000000000000000000000000000 --- a/drivers/gpio/gpio-msic.c +++ /dev/null @@ -1,314 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * Intel Medfield MSIC GPIO driver> - * Copyright (c) 2011, Intel Corporation. - * - * Author: Mathias Nyman - * Based on intel_pmic_gpio.c - */ - -#include -#include -#include -#include -#include -#include -#include - -/* the offset for the mapping of global gpio pin to irq */ -#define MSIC_GPIO_IRQ_OFFSET 0x100 - -#define MSIC_GPIO_DIR_IN 0 -#define MSIC_GPIO_DIR_OUT BIT(5) -#define MSIC_GPIO_TRIG_FALL BIT(1) -#define MSIC_GPIO_TRIG_RISE BIT(2) - -/* masks for msic gpio output GPIOxxxxCTLO registers */ -#define MSIC_GPIO_DIR_MASK BIT(5) -#define MSIC_GPIO_DRV_MASK BIT(4) -#define MSIC_GPIO_REN_MASK BIT(3) -#define MSIC_GPIO_RVAL_MASK (BIT(2) | BIT(1)) -#define MSIC_GPIO_DOUT_MASK BIT(0) - -/* masks for msic gpio input GPIOxxxxCTLI registers */ -#define MSIC_GPIO_GLBYP_MASK BIT(5) -#define MSIC_GPIO_DBNC_MASK (BIT(4) | BIT(3)) -#define MSIC_GPIO_INTCNT_MASK (BIT(2) | BIT(1)) -#define MSIC_GPIO_DIN_MASK BIT(0) - -#define MSIC_NUM_GPIO 24 - -struct msic_gpio { - struct platform_device *pdev; - struct mutex buslock; - struct gpio_chip chip; - int irq; - unsigned irq_base; - unsigned long trig_change_mask; - unsigned trig_type; -}; - -/* - * MSIC has 24 gpios, 16 low voltage (1.2-1.8v) and 8 high voltage (3v). - * Both the high and low voltage gpios are divided in two banks. - * GPIOs are numbered with GPIO0LV0 as gpio_base in the following order: - * GPIO0LV0..GPIO0LV7: low voltage, bank 0, gpio_base - * GPIO1LV0..GPIO1LV7: low voltage, bank 1, gpio_base + 8 - * GPIO0HV0..GPIO0HV3: high voltage, bank 0, gpio_base + 16 - * GPIO1HV0..GPIO1HV3: high voltage, bank 1, gpio_base + 20 - */ - -static int msic_gpio_to_ireg(unsigned offset) -{ - if (offset >= MSIC_NUM_GPIO) - return -EINVAL; - - if (offset < 8) - return INTEL_MSIC_GPIO0LV0CTLI - offset; - if (offset < 16) - return INTEL_MSIC_GPIO1LV0CTLI - offset + 8; - if (offset < 20) - return INTEL_MSIC_GPIO0HV0CTLI - offset + 16; - - return INTEL_MSIC_GPIO1HV0CTLI - offset + 20; -} - -static int msic_gpio_to_oreg(unsigned offset) -{ - if (offset >= MSIC_NUM_GPIO) - return -EINVAL; - - if (offset < 8) - return INTEL_MSIC_GPIO0LV0CTLO - offset; - if (offset < 16) - return INTEL_MSIC_GPIO1LV0CTLO - offset + 8; - if (offset < 20) - return INTEL_MSIC_GPIO0HV0CTLO - offset + 16; - - return INTEL_MSIC_GPIO1HV0CTLO - offset + 20; -} - -static int msic_gpio_direction_input(struct gpio_chip *chip, unsigned offset) -{ - int reg; - - reg = msic_gpio_to_oreg(offset); - if (reg < 0) - return reg; - - return intel_msic_reg_update(reg, MSIC_GPIO_DIR_IN, MSIC_GPIO_DIR_MASK); -} - -static int msic_gpio_direction_output(struct gpio_chip *chip, - unsigned offset, int value) -{ - int reg; - unsigned mask; - - value = (!!value) | MSIC_GPIO_DIR_OUT; - mask = MSIC_GPIO_DIR_MASK | MSIC_GPIO_DOUT_MASK; - - reg = msic_gpio_to_oreg(offset); - if (reg < 0) - return reg; - - return intel_msic_reg_update(reg, value, mask); -} - -static int msic_gpio_get(struct gpio_chip *chip, unsigned offset) -{ - u8 r; - int ret; - int reg; - - reg = msic_gpio_to_ireg(offset); - if (reg < 0) - return reg; - - ret = intel_msic_reg_read(reg, &r); - if (ret < 0) - return ret; - - return !!(r & MSIC_GPIO_DIN_MASK); -} - -static void msic_gpio_set(struct gpio_chip *chip, unsigned offset, int value) -{ - int reg; - - reg = msic_gpio_to_oreg(offset); - if (reg < 0) - return; - - intel_msic_reg_update(reg, !!value , MSIC_GPIO_DOUT_MASK); -} - -/* - * This is called from genirq with mg->buslock locked and - * irq_desc->lock held. We can not access the scu bus here, so we - * store the change and update in the bus_sync_unlock() function below - */ -static int msic_irq_type(struct irq_data *data, unsigned type) -{ - struct msic_gpio *mg = irq_data_get_irq_chip_data(data); - u32 gpio = data->irq - mg->irq_base; - - if (gpio >= mg->chip.ngpio) - return -EINVAL; - - /* mark for which gpio the trigger changed, protected by buslock */ - mg->trig_change_mask |= (1 << gpio); - mg->trig_type = type; - - return 0; -} - -static int msic_gpio_to_irq(struct gpio_chip *chip, unsigned offset) -{ - struct msic_gpio *mg = gpiochip_get_data(chip); - return mg->irq_base + offset; -} - -static void msic_bus_lock(struct irq_data *data) -{ - struct msic_gpio *mg = irq_data_get_irq_chip_data(data); - mutex_lock(&mg->buslock); -} - -static void msic_bus_sync_unlock(struct irq_data *data) -{ - struct msic_gpio *mg = irq_data_get_irq_chip_data(data); - int offset; - int reg; - u8 trig = 0; - - /* We can only get one change at a time as the buslock covers the - entire transaction. The irq_desc->lock is dropped before we are - called but that is fine */ - if (mg->trig_change_mask) { - offset = __ffs(mg->trig_change_mask); - - reg = msic_gpio_to_ireg(offset); - if (reg < 0) - goto out; - - if (mg->trig_type & IRQ_TYPE_EDGE_RISING) - trig |= MSIC_GPIO_TRIG_RISE; - if (mg->trig_type & IRQ_TYPE_EDGE_FALLING) - trig |= MSIC_GPIO_TRIG_FALL; - - intel_msic_reg_update(reg, trig, MSIC_GPIO_INTCNT_MASK); - mg->trig_change_mask = 0; - } -out: - mutex_unlock(&mg->buslock); -} - -/* Firmware does all the masking and unmasking for us, no masking here. */ -static void msic_irq_unmask(struct irq_data *data) { } - -static void msic_irq_mask(struct irq_data *data) { } - -static struct irq_chip msic_irqchip = { - .name = "MSIC-GPIO", - .irq_mask = msic_irq_mask, - .irq_unmask = msic_irq_unmask, - .irq_set_type = msic_irq_type, - .irq_bus_lock = msic_bus_lock, - .irq_bus_sync_unlock = msic_bus_sync_unlock, -}; - -static void msic_gpio_irq_handler(struct irq_desc *desc) -{ - struct irq_data *data = irq_desc_get_irq_data(desc); - struct msic_gpio *mg = irq_data_get_irq_handler_data(data); - struct irq_chip *chip = irq_data_get_irq_chip(data); - struct intel_msic *msic = pdev_to_intel_msic(mg->pdev); - unsigned long pending; - int i; - int bitnr; - u8 pin; - - for (i = 0; i < (mg->chip.ngpio / BITS_PER_BYTE); i++) { - intel_msic_irq_read(msic, INTEL_MSIC_GPIO0LVIRQ + i, &pin); - pending = pin; - - for_each_set_bit(bitnr, &pending, BITS_PER_BYTE) - generic_handle_irq(mg->irq_base + i * BITS_PER_BYTE + bitnr); - } - chip->irq_eoi(data); -} - -static int platform_msic_gpio_probe(struct platform_device *pdev) -{ - struct device *dev = &pdev->dev; - struct intel_msic_gpio_pdata *pdata = dev_get_platdata(dev); - struct msic_gpio *mg; - int irq = platform_get_irq(pdev, 0); - int retval; - int i; - - if (irq < 0) { - dev_err(dev, "no IRQ line: %d\n", irq); - return irq; - } - - if (!pdata || !pdata->gpio_base) { - dev_err(dev, "incorrect or missing platform data\n"); - return -EINVAL; - } - - mg = kzalloc(sizeof(*mg), GFP_KERNEL); - if (!mg) - return -ENOMEM; - - dev_set_drvdata(dev, mg); - - mg->pdev = pdev; - mg->irq = irq; - mg->irq_base = pdata->gpio_base + MSIC_GPIO_IRQ_OFFSET; - mg->chip.label = "msic_gpio"; - mg->chip.direction_input = msic_gpio_direction_input; - mg->chip.direction_output = msic_gpio_direction_output; - mg->chip.get = msic_gpio_get; - mg->chip.set = msic_gpio_set; - mg->chip.to_irq = msic_gpio_to_irq; - mg->chip.base = pdata->gpio_base; - mg->chip.ngpio = MSIC_NUM_GPIO; - mg->chip.can_sleep = true; - mg->chip.parent = dev; - - mutex_init(&mg->buslock); - - retval = gpiochip_add_data(&mg->chip, mg); - if (retval) { - dev_err(dev, "Adding MSIC gpio chip failed\n"); - goto err; - } - - for (i = 0; i < mg->chip.ngpio; i++) { - irq_set_chip_data(i + mg->irq_base, mg); - irq_set_chip_and_handler(i + mg->irq_base, - &msic_irqchip, - handle_simple_irq); - } - irq_set_chained_handler_and_data(mg->irq, msic_gpio_irq_handler, mg); - - return 0; -err: - kfree(mg); - return retval; -} - -static struct platform_driver platform_msic_gpio_driver = { - .driver = { - .name = "msic_gpio", - }, - .probe = platform_msic_gpio_probe, -}; - -static int __init platform_msic_gpio_init(void) -{ - return platform_driver_register(&platform_msic_gpio_driver); -} -subsys_initcall(platform_msic_gpio_init); diff --git a/drivers/gpio/gpio-mvebu.c b/drivers/gpio/gpio-mvebu.c index a912a8fed197ac32bb393a87ba17c63421f12184..8f429d9f36616f9aa89519bcedec8da531022ff9 100644 --- a/drivers/gpio/gpio-mvebu.c +++ b/drivers/gpio/gpio-mvebu.c @@ -70,7 +70,12 @@ */ #define PWM_BLINK_ON_DURATION_OFF 0x0 #define PWM_BLINK_OFF_DURATION_OFF 0x4 +#define PWM_BLINK_COUNTER_B_OFF 0x8 +/* Armada 8k variant gpios register offsets */ +#define AP80X_GPIO0_OFF_A8K 0x1040 +#define CP11X_GPIO0_OFF_A8K 0x100 +#define CP11X_GPIO1_OFF_A8K 0x140 /* The MV78200 has per-CPU registers for edge mask and level mask */ #define GPIO_EDGE_MASK_MV78200_OFF(cpu) ((cpu) ? 0x30 : 0x18) @@ -93,6 +98,7 @@ struct mvebu_pwm { struct regmap *regs; + u32 offset; unsigned long clk_rate; struct gpio_desc *gpiod; struct pwm_chip chip; @@ -283,12 +289,12 @@ mvebu_gpio_write_level_mask(struct mvebu_gpio_chip *mvchip, u32 val) */ static unsigned int mvebu_pwmreg_blink_on_duration(struct mvebu_pwm *mvpwm) { - return PWM_BLINK_ON_DURATION_OFF; + return mvpwm->offset + PWM_BLINK_ON_DURATION_OFF; } static unsigned int mvebu_pwmreg_blink_off_duration(struct mvebu_pwm *mvpwm) { - return PWM_BLINK_OFF_DURATION_OFF; + return mvpwm->offset + PWM_BLINK_OFF_DURATION_OFF; } /* @@ -667,26 +673,21 @@ static void mvebu_pwm_get_state(struct pwm_chip *chip, spin_lock_irqsave(&mvpwm->lock, flags); regmap_read(mvpwm->regs, mvebu_pwmreg_blink_on_duration(mvpwm), &u); - val = (unsigned long long) u * NSEC_PER_SEC; - do_div(val, mvpwm->clk_rate); - if (val > UINT_MAX) - state->duty_cycle = UINT_MAX; - else if (val) - state->duty_cycle = val; + /* Hardware treats zero as 2^32. See mvebu_pwm_apply(). */ + if (u > 0) + val = u; else - state->duty_cycle = 1; + val = UINT_MAX + 1ULL; + state->duty_cycle = DIV_ROUND_UP_ULL(val * NSEC_PER_SEC, + mvpwm->clk_rate); - val = (unsigned long long) u; /* on duration */ regmap_read(mvpwm->regs, mvebu_pwmreg_blink_off_duration(mvpwm), &u); - val += (unsigned long long) u; /* period = on + off duration */ - val *= NSEC_PER_SEC; - do_div(val, mvpwm->clk_rate); - if (val > UINT_MAX) - state->period = UINT_MAX; - else if (val) - state->period = val; + /* period = on + off duration */ + if (u > 0) + val += u; else - state->period = 1; + val += UINT_MAX + 1ULL; + state->period = DIV_ROUND_UP_ULL(val * NSEC_PER_SEC, mvpwm->clk_rate); regmap_read(mvchip->regs, GPIO_BLINK_EN_OFF + mvchip->offset, &u); if (u) @@ -708,19 +709,27 @@ static int mvebu_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, val = (unsigned long long) mvpwm->clk_rate * state->duty_cycle; do_div(val, NSEC_PER_SEC); - if (val > UINT_MAX) + if (val > UINT_MAX + 1ULL) return -EINVAL; - if (val) + /* + * Zero on/off values don't work as expected. Experimentation shows + * that zero value is treated as 2^32. This behavior is not documented. + */ + if (val == UINT_MAX + 1ULL) + on = 0; + else if (val) on = val; else on = 1; - val = (unsigned long long) mvpwm->clk_rate * - (state->period - state->duty_cycle); + val = (unsigned long long) mvpwm->clk_rate * state->period; do_div(val, NSEC_PER_SEC); - if (val > UINT_MAX) + val -= on; + if (val > UINT_MAX + 1ULL) return -EINVAL; - if (val) + if (val == UINT_MAX + 1ULL) + off = 0; + else if (val) off = val; else off = 1; @@ -778,51 +787,80 @@ static int mvebu_pwm_probe(struct platform_device *pdev, struct device *dev = &pdev->dev; struct mvebu_pwm *mvpwm; void __iomem *base; + u32 offset; u32 set; - if (!of_device_is_compatible(mvchip->chip.of_node, - "marvell,armada-370-gpio")) - return 0; - - /* - * There are only two sets of PWM configuration registers for - * all the GPIO lines on those SoCs which this driver reserves - * for the first two GPIO chips. So if the resource is missing - * we can't treat it as an error. - */ - if (!platform_get_resource_byname(pdev, IORESOURCE_MEM, "pwm")) + if (of_device_is_compatible(mvchip->chip.of_node, + "marvell,armada-370-gpio")) { + /* + * There are only two sets of PWM configuration registers for + * all the GPIO lines on those SoCs which this driver reserves + * for the first two GPIO chips. So if the resource is missing + * we can't treat it as an error. + */ + if (!platform_get_resource_byname(pdev, IORESOURCE_MEM, "pwm")) + return 0; + offset = 0; + } else if (mvchip->soc_variant == MVEBU_GPIO_SOC_VARIANT_A8K) { + int ret = of_property_read_u32(dev->of_node, + "marvell,pwm-offset", &offset); + if (ret < 0) + return 0; + } else { return 0; + } if (IS_ERR(mvchip->clk)) return PTR_ERR(mvchip->clk); - /* - * Use set A for lines of GPIO chip with id 0, B for GPIO chip - * with id 1. Don't allow further GPIO chips to be used for PWM. - */ - if (id == 0) - set = 0; - else if (id == 1) - set = U32_MAX; - else - return -EINVAL; - regmap_write(mvchip->regs, - GPIO_BLINK_CNT_SELECT_OFF + mvchip->offset, set); - mvpwm = devm_kzalloc(dev, sizeof(struct mvebu_pwm), GFP_KERNEL); if (!mvpwm) return -ENOMEM; mvchip->mvpwm = mvpwm; mvpwm->mvchip = mvchip; + mvpwm->offset = offset; + + if (mvchip->soc_variant == MVEBU_GPIO_SOC_VARIANT_A8K) { + mvpwm->regs = mvchip->regs; + + switch (mvchip->offset) { + case AP80X_GPIO0_OFF_A8K: + case CP11X_GPIO0_OFF_A8K: + /* Blink counter A */ + set = 0; + break; + case CP11X_GPIO1_OFF_A8K: + /* Blink counter B */ + set = U32_MAX; + mvpwm->offset += PWM_BLINK_COUNTER_B_OFF; + break; + default: + return -EINVAL; + } + } else { + base = devm_platform_ioremap_resource_byname(pdev, "pwm"); + if (IS_ERR(base)) + return PTR_ERR(base); - base = devm_platform_ioremap_resource_byname(pdev, "pwm"); - if (IS_ERR(base)) - return PTR_ERR(base); + mvpwm->regs = devm_regmap_init_mmio(&pdev->dev, base, + &mvebu_gpio_regmap_config); + if (IS_ERR(mvpwm->regs)) + return PTR_ERR(mvpwm->regs); - mvpwm->regs = devm_regmap_init_mmio(&pdev->dev, base, - &mvebu_gpio_regmap_config); - if (IS_ERR(mvpwm->regs)) - return PTR_ERR(mvpwm->regs); + /* + * Use set A for lines of GPIO chip with id 0, B for GPIO chip + * with id 1. Don't allow further GPIO chips to be used for PWM. + */ + if (id == 0) + set = 0; + else if (id == 1) + set = U32_MAX; + else + return -EINVAL; + } + + regmap_write(mvchip->regs, + GPIO_BLINK_CNT_SELECT_OFF + mvchip->offset, set); mvpwm->clk_rate = clk_get_rate(mvchip->clk); if (!mvpwm->clk_rate) { diff --git a/drivers/gpio/gpio-pca953x.c b/drivers/gpio/gpio-pca953x.c index 825b362eb4b7de2edb5cc5ba777e93c8d673efbc..5ea09fd01544482aaadda083a0dae7de0f54a924 100644 --- a/drivers/gpio/gpio-pca953x.c +++ b/drivers/gpio/gpio-pca953x.c @@ -73,6 +73,7 @@ static const struct i2c_device_id pca953x_id[] = { { "pca6416", 16 | PCA953X_TYPE | PCA_INT, }, { "pca9505", 40 | PCA953X_TYPE | PCA_INT, }, + { "pca9506", 40 | PCA953X_TYPE | PCA_INT, }, { "pca9534", 8 | PCA953X_TYPE | PCA_INT, }, { "pca9535", 16 | PCA953X_TYPE | PCA_INT, }, { "pca9536", 4 | PCA953X_TYPE, }, @@ -1236,6 +1237,7 @@ static int pca953x_resume(struct device *dev) static const struct of_device_id pca953x_dt_ids[] = { { .compatible = "nxp,pca6416", .data = OF_953X(16, PCA_INT), }, { .compatible = "nxp,pca9505", .data = OF_953X(40, PCA_INT), }, + { .compatible = "nxp,pca9506", .data = OF_953X(40, PCA_INT), }, { .compatible = "nxp,pca9534", .data = OF_953X( 8, PCA_INT), }, { .compatible = "nxp,pca9535", .data = OF_953X(16, PCA_INT), }, { .compatible = "nxp,pca9536", .data = OF_953X( 4, 0), }, diff --git a/drivers/gpio/gpio-pcf857x.c b/drivers/gpio/gpio-pcf857x.c index a2a8d155c75e35e99f59e6a05e5185e7982a6c73..b7568ee33696db11a0e49cb17aca5dbe48e00f29 100644 --- a/drivers/gpio/gpio-pcf857x.c +++ b/drivers/gpio/gpio-pcf857x.c @@ -332,7 +332,7 @@ static int pcf857x_probe(struct i2c_client *client, * reset state. Otherwise it flags pins to be driven low. */ gpio->out = ~n_latch; - gpio->status = gpio->out; + gpio->status = gpio->read(gpio->client); /* Enable irqchip if we have an interrupt */ if (client->irq) { diff --git a/drivers/gpio/gpio-rcar.c b/drivers/gpio/gpio-rcar.c index 0b572dbc4a36801dc052cd2ca4db604fe6483b0e..e7092d5fe700d2cec396e94d35e07999c91d1aa3 100644 --- a/drivers/gpio/gpio-rcar.c +++ b/drivers/gpio/gpio-rcar.c @@ -35,6 +35,8 @@ struct gpio_rcar_bank_info { struct gpio_rcar_info { bool has_outdtsel; bool has_both_edge_trigger; + bool has_always_in; + bool has_inen; }; struct gpio_rcar_priv { @@ -62,6 +64,7 @@ struct gpio_rcar_priv { #define FILONOFF 0x28 /* Chattering Prevention On/Off Register */ #define OUTDTSEL 0x40 /* Output Data Select Register */ #define BOTHEDGE 0x4c /* One Edge/Both Edge Select Register */ +#define INEN 0x50 /* General Input Enable Register */ #define RCAR_MAX_GPIO_PER_BANK 32 @@ -302,9 +305,11 @@ static int gpio_rcar_get(struct gpio_chip *chip, unsigned offset) struct gpio_rcar_priv *p = gpiochip_get_data(chip); u32 bit = BIT(offset); - /* testing on r8a7790 shows that INDT does not show correct pin state - * when configured as output, so use OUTDT in case of output pins */ - if (gpio_rcar_read(p, INOUTSEL) & bit) + /* + * Before R-Car Gen3, INDT does not show correct pin state when + * configured as output, so use OUTDT in case of output pins + */ + if (!p->info.has_always_in && (gpio_rcar_read(p, INOUTSEL) & bit)) return !!(gpio_rcar_read(p, OUTDT) & bit); else return !!(gpio_rcar_read(p, INDT) & bit); @@ -324,6 +329,11 @@ static int gpio_rcar_get_multiple(struct gpio_chip *chip, unsigned long *mask, if (!bankmask) return 0; + if (p->info.has_always_in) { + bits[0] = gpio_rcar_read(p, INDT) & bankmask; + return 0; + } + spin_lock_irqsave(&p->lock, flags); outputs = gpio_rcar_read(p, INOUTSEL); m = outputs & bankmask; @@ -383,41 +393,35 @@ static int gpio_rcar_direction_output(struct gpio_chip *chip, unsigned offset, static const struct gpio_rcar_info gpio_rcar_info_gen1 = { .has_outdtsel = false, .has_both_edge_trigger = false, + .has_always_in = false, + .has_inen = false, }; static const struct gpio_rcar_info gpio_rcar_info_gen2 = { .has_outdtsel = true, .has_both_edge_trigger = true, + .has_always_in = false, + .has_inen = false, +}; + +static const struct gpio_rcar_info gpio_rcar_info_gen3 = { + .has_outdtsel = true, + .has_both_edge_trigger = true, + .has_always_in = true, + .has_inen = false, +}; + +static const struct gpio_rcar_info gpio_rcar_info_v3u = { + .has_outdtsel = true, + .has_both_edge_trigger = true, + .has_always_in = true, + .has_inen = true, }; static const struct of_device_id gpio_rcar_of_table[] = { { - .compatible = "renesas,gpio-r8a7743", - /* RZ/G1 GPIO is identical to R-Car Gen2. */ - .data = &gpio_rcar_info_gen2, - }, { - .compatible = "renesas,gpio-r8a7790", - .data = &gpio_rcar_info_gen2, - }, { - .compatible = "renesas,gpio-r8a7791", - .data = &gpio_rcar_info_gen2, - }, { - .compatible = "renesas,gpio-r8a7792", - .data = &gpio_rcar_info_gen2, - }, { - .compatible = "renesas,gpio-r8a7793", - .data = &gpio_rcar_info_gen2, - }, { - .compatible = "renesas,gpio-r8a7794", - .data = &gpio_rcar_info_gen2, - }, { - .compatible = "renesas,gpio-r8a7795", - /* Gen3 GPIO is identical to Gen2. */ - .data = &gpio_rcar_info_gen2, - }, { - .compatible = "renesas,gpio-r8a7796", - /* Gen3 GPIO is identical to Gen2. */ - .data = &gpio_rcar_info_gen2, + .compatible = "renesas,gpio-r8a779a0", + .data = &gpio_rcar_info_v3u, }, { .compatible = "renesas,rcar-gen1-gpio", .data = &gpio_rcar_info_gen1, @@ -426,8 +430,7 @@ static const struct of_device_id gpio_rcar_of_table[] = { .data = &gpio_rcar_info_gen2, }, { .compatible = "renesas,rcar-gen3-gpio", - /* Gen3 GPIO is identical to Gen2. */ - .data = &gpio_rcar_info_gen2, + .data = &gpio_rcar_info_gen3, }, { .compatible = "renesas,gpio-rcar", .data = &gpio_rcar_info_gen1, @@ -460,6 +463,17 @@ static int gpio_rcar_parse_dt(struct gpio_rcar_priv *p, unsigned int *npins) return 0; } +static void gpio_rcar_enable_inputs(struct gpio_rcar_priv *p) +{ + u32 mask = GENMASK(p->gpio_chip.ngpio - 1, 0); + + /* Select "Input Enable" in INEN */ + if (p->gpio_chip.valid_mask) + mask &= p->gpio_chip.valid_mask[0]; + if (mask) + gpio_rcar_write(p, INEN, gpio_rcar_read(p, INEN) | mask); +} + static int gpio_rcar_probe(struct platform_device *pdev) { struct gpio_rcar_priv *p; @@ -549,6 +563,12 @@ static int gpio_rcar_probe(struct platform_device *pdev) goto err1; } + if (p->info.has_inen) { + pm_runtime_get_sync(p->dev); + gpio_rcar_enable_inputs(p); + pm_runtime_put(p->dev); + } + dev_info(dev, "driving %d GPIOs\n", npins); return 0; @@ -624,6 +644,9 @@ static int gpio_rcar_resume(struct device *dev) } } + if (p->info.has_inen) + gpio_rcar_enable_inputs(p); + return 0; } #endif /* CONFIG_PM_SLEEP*/ diff --git a/drivers/gpio/gpio-sl28cpld.c b/drivers/gpio/gpio-sl28cpld.c index 889b8f5622c202b465bd362b1c59915cd249dc54..52404736ac861bb16ea8e9bc7ef2bfb70f54bd56 100644 --- a/drivers/gpio/gpio-sl28cpld.c +++ b/drivers/gpio/gpio-sl28cpld.c @@ -65,13 +65,13 @@ static int sl28cpld_gpio_irq_init(struct platform_device *pdev, if (!irq_chip) return -ENOMEM; - irq_chip->name = "sl28cpld-gpio-irq", + irq_chip->name = "sl28cpld-gpio-irq"; irq_chip->irqs = sl28cpld_gpio_irqs; irq_chip->num_irqs = ARRAY_SIZE(sl28cpld_gpio_irqs); irq_chip->num_regs = 1; irq_chip->status_base = base + GPIO_REG_IP; irq_chip->mask_base = base + GPIO_REG_IE; - irq_chip->mask_invert = true, + irq_chip->mask_invert = true; irq_chip->ack_base = base + GPIO_REG_IP; ret = devm_regmap_add_irq_chip_fwnode(dev, dev_fwnode(dev), diff --git a/drivers/gpio/gpio-tegra.c b/drivers/gpio/gpio-tegra.c index e19ebff6018cc911c594c5df27b11f4a36803b66..0025f613d9b31da12e9892a9099373b77bc5a813 100644 --- a/drivers/gpio/gpio-tegra.c +++ b/drivers/gpio/gpio-tegra.c @@ -60,7 +60,6 @@ struct tegra_gpio_info; struct tegra_gpio_bank { unsigned int bank; - unsigned int irq; /* * IRQ-core code uses raw locking, and thus, nested locking also @@ -81,7 +80,6 @@ struct tegra_gpio_bank { u32 dbc_enb[4]; #endif u32 dbc_cnt[4]; - struct tegra_gpio_info *tgi; }; struct tegra_gpio_soc_config { @@ -93,12 +91,12 @@ struct tegra_gpio_soc_config { struct tegra_gpio_info { struct device *dev; void __iomem *regs; - struct irq_domain *irq_domain; struct tegra_gpio_bank *bank_info; const struct tegra_gpio_soc_config *soc; struct gpio_chip gc; struct irq_chip ic; u32 bank_count; + unsigned int *irqs; }; static inline void tegra_gpio_writel(struct tegra_gpio_info *tgi, @@ -274,17 +272,10 @@ static int tegra_gpio_set_config(struct gpio_chip *chip, unsigned int offset, return tegra_gpio_set_debounce(chip, offset, debounce); } -static int tegra_gpio_to_irq(struct gpio_chip *chip, unsigned int offset) -{ - struct tegra_gpio_info *tgi = gpiochip_get_data(chip); - - return irq_find_mapping(tgi->irq_domain, offset); -} - static void tegra_gpio_irq_ack(struct irq_data *d) { - struct tegra_gpio_bank *bank = irq_data_get_irq_chip_data(d); - struct tegra_gpio_info *tgi = bank->tgi; + struct gpio_chip *chip = irq_data_get_irq_chip_data(d); + struct tegra_gpio_info *tgi = gpiochip_get_data(chip); unsigned int gpio = d->hwirq; tegra_gpio_writel(tgi, 1 << GPIO_BIT(gpio), GPIO_INT_CLR(tgi, gpio)); @@ -292,8 +283,8 @@ static void tegra_gpio_irq_ack(struct irq_data *d) static void tegra_gpio_irq_mask(struct irq_data *d) { - struct tegra_gpio_bank *bank = irq_data_get_irq_chip_data(d); - struct tegra_gpio_info *tgi = bank->tgi; + struct gpio_chip *chip = irq_data_get_irq_chip_data(d); + struct tegra_gpio_info *tgi = gpiochip_get_data(chip); unsigned int gpio = d->hwirq; tegra_gpio_mask_write(tgi, GPIO_MSK_INT_ENB(tgi, gpio), gpio, 0); @@ -301,8 +292,8 @@ static void tegra_gpio_irq_mask(struct irq_data *d) static void tegra_gpio_irq_unmask(struct irq_data *d) { - struct tegra_gpio_bank *bank = irq_data_get_irq_chip_data(d); - struct tegra_gpio_info *tgi = bank->tgi; + struct gpio_chip *chip = irq_data_get_irq_chip_data(d); + struct tegra_gpio_info *tgi = gpiochip_get_data(chip); unsigned int gpio = d->hwirq; tegra_gpio_mask_write(tgi, GPIO_MSK_INT_ENB(tgi, gpio), gpio, 1); @@ -311,11 +302,14 @@ static void tegra_gpio_irq_unmask(struct irq_data *d) static int tegra_gpio_irq_set_type(struct irq_data *d, unsigned int type) { unsigned int gpio = d->hwirq, port = GPIO_PORT(gpio), lvl_type; - struct tegra_gpio_bank *bank = irq_data_get_irq_chip_data(d); - struct tegra_gpio_info *tgi = bank->tgi; + struct gpio_chip *chip = irq_data_get_irq_chip_data(d); + struct tegra_gpio_info *tgi = gpiochip_get_data(chip); + struct tegra_gpio_bank *bank; unsigned long flags; - u32 val; int ret; + u32 val; + + bank = &tgi->bank_info[GPIO_BANK(d->hwirq)]; switch (type & IRQ_TYPE_SENSE_MASK) { case IRQ_TYPE_EDGE_RISING: @@ -367,13 +361,16 @@ static int tegra_gpio_irq_set_type(struct irq_data *d, unsigned int type) else if (type & (IRQ_TYPE_EDGE_FALLING | IRQ_TYPE_EDGE_RISING)) irq_set_handler_locked(d, handle_edge_irq); - return 0; + if (d->parent_data) + ret = irq_chip_set_type_parent(d, type); + + return ret; } static void tegra_gpio_irq_shutdown(struct irq_data *d) { - struct tegra_gpio_bank *bank = irq_data_get_irq_chip_data(d); - struct tegra_gpio_info *tgi = bank->tgi; + struct gpio_chip *chip = irq_data_get_irq_chip_data(d); + struct tegra_gpio_info *tgi = gpiochip_get_data(chip); unsigned int gpio = d->hwirq; tegra_gpio_irq_mask(d); @@ -382,13 +379,25 @@ static void tegra_gpio_irq_shutdown(struct irq_data *d) static void tegra_gpio_irq_handler(struct irq_desc *desc) { - unsigned int port, pin, gpio; + struct tegra_gpio_info *tgi = irq_desc_get_handler_data(desc); + struct irq_chip *chip = irq_desc_get_chip(desc); + struct irq_domain *domain = tgi->gc.irq.domain; + unsigned int irq = irq_desc_get_irq(desc); + struct tegra_gpio_bank *bank = NULL; + unsigned int port, pin, gpio, i; bool unmasked = false; - u32 lvl; unsigned long sta; - struct irq_chip *chip = irq_desc_get_chip(desc); - struct tegra_gpio_bank *bank = irq_desc_get_handler_data(desc); - struct tegra_gpio_info *tgi = bank->tgi; + u32 lvl; + + for (i = 0; i < tgi->bank_count; i++) { + if (tgi->irqs[i] == irq) { + bank = &tgi->bank_info[i]; + break; + } + } + + if (WARN_ON(bank == NULL)) + return; chained_irq_enter(chip, desc); @@ -411,14 +420,47 @@ static void tegra_gpio_irq_handler(struct irq_desc *desc) chained_irq_exit(chip, desc); } - generic_handle_irq(irq_find_mapping(tgi->irq_domain, - gpio + pin)); + irq = irq_find_mapping(domain, gpio + pin); + if (WARN_ON(irq == 0)) + continue; + + generic_handle_irq(irq); } } if (!unmasked) chained_irq_exit(chip, desc); +} + +static int tegra_gpio_child_to_parent_hwirq(struct gpio_chip *chip, + unsigned int hwirq, + unsigned int type, + unsigned int *parent_hwirq, + unsigned int *parent_type) +{ + *parent_hwirq = chip->irq.child_offset_to_irq(chip, hwirq); + *parent_type = type; + + return 0; +} + +static void *tegra_gpio_populate_parent_fwspec(struct gpio_chip *chip, + unsigned int parent_hwirq, + unsigned int parent_type) +{ + struct irq_fwspec *fwspec; + + fwspec = kmalloc(sizeof(*fwspec), GFP_KERNEL); + if (!fwspec) + return NULL; + fwspec->fwnode = chip->irq.parent_domain->fwnode; + fwspec->param_count = 3; + fwspec->param[0] = 0; + fwspec->param[1] = parent_hwirq; + fwspec->param[2] = parent_type; + + return fwspec; } #ifdef CONFIG_PM_SLEEP @@ -497,19 +539,31 @@ static int tegra_gpio_suspend(struct device *dev) static int tegra_gpio_irq_set_wake(struct irq_data *d, unsigned int enable) { - struct tegra_gpio_bank *bank = irq_data_get_irq_chip_data(d); + struct gpio_chip *chip = irq_data_get_irq_chip_data(d); + struct tegra_gpio_info *tgi = gpiochip_get_data(chip); + struct tegra_gpio_bank *bank; unsigned int gpio = d->hwirq; u32 port, bit, mask; int err; - err = irq_set_irq_wake(bank->irq, enable); - if (err) - return err; + bank = &tgi->bank_info[GPIO_BANK(d->hwirq)]; port = GPIO_PORT(gpio); bit = GPIO_BIT(gpio); mask = BIT(bit); + err = irq_set_irq_wake(tgi->irqs[bank->bank], enable); + if (err) + return err; + + if (d->parent_data) { + err = irq_chip_set_wake_parent(d, enable); + if (err) { + irq_set_irq_wake(tgi->irqs[bank->bank], !enable); + return err; + } + } + if (enable) bank->wake_enb[port] |= mask; else @@ -519,6 +573,35 @@ static int tegra_gpio_irq_set_wake(struct irq_data *d, unsigned int enable) } #endif +static int tegra_gpio_irq_set_affinity(struct irq_data *data, + const struct cpumask *dest, + bool force) +{ + if (data->parent_data) + return irq_chip_set_affinity_parent(data, dest, force); + + return -EINVAL; +} + +static int tegra_gpio_irq_request_resources(struct irq_data *d) +{ + struct gpio_chip *chip = irq_data_get_irq_chip_data(d); + struct tegra_gpio_info *tgi = gpiochip_get_data(chip); + + tegra_gpio_enable(tgi, d->hwirq); + + return gpiochip_reqres_irq(chip, d->hwirq); +} + +static void tegra_gpio_irq_release_resources(struct irq_data *d) +{ + struct gpio_chip *chip = irq_data_get_irq_chip_data(d); + struct tegra_gpio_info *tgi = gpiochip_get_data(chip); + + gpiochip_relres_irq(chip, d->hwirq); + tegra_gpio_enable(tgi, d->hwirq); +} + #ifdef CONFIG_DEBUG_FS #include @@ -526,7 +609,7 @@ static int tegra_gpio_irq_set_wake(struct irq_data *d, unsigned int enable) static int tegra_dbg_gpio_show(struct seq_file *s, void *unused) { - struct tegra_gpio_info *tgi = s->private; + struct tegra_gpio_info *tgi = dev_get_drvdata(s->private); unsigned int i, j; for (i = 0; i < tgi->bank_count; i++) { @@ -548,12 +631,10 @@ static int tegra_dbg_gpio_show(struct seq_file *s, void *unused) return 0; } -DEFINE_SHOW_ATTRIBUTE(tegra_dbg_gpio); - static void tegra_gpio_debuginit(struct tegra_gpio_info *tgi) { - debugfs_create_file("tegra_gpio", 0444, NULL, tgi, - &tegra_dbg_gpio_fops); + debugfs_create_devm_seqfile(tgi->dev, "tegra_gpio", NULL, + tegra_dbg_gpio_show); } #else @@ -568,14 +649,18 @@ static const struct dev_pm_ops tegra_gpio_pm_ops = { SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(tegra_gpio_suspend, tegra_gpio_resume) }; -static struct lock_class_key gpio_lock_class; -static struct lock_class_key gpio_request_class; +static const struct of_device_id tegra_pmc_of_match[] = { + { .compatible = "nvidia,tegra210-pmc", }, + { /* sentinel */ }, +}; static int tegra_gpio_probe(struct platform_device *pdev) { - struct tegra_gpio_info *tgi; struct tegra_gpio_bank *bank; - unsigned int gpio, i, j; + struct tegra_gpio_info *tgi; + struct gpio_irq_chip *irq; + struct device_node *np; + unsigned int i, j; int ret; tgi = devm_kzalloc(&pdev->dev, sizeof(*tgi), GFP_KERNEL); @@ -604,7 +689,6 @@ static int tegra_gpio_probe(struct platform_device *pdev) tgi->gc.direction_output = tegra_gpio_direction_output; tgi->gc.set = tegra_gpio_set; tgi->gc.get_direction = tegra_gpio_get_direction; - tgi->gc.to_irq = tegra_gpio_to_irq; tgi->gc.base = 0; tgi->gc.ngpio = tgi->bank_count * 32; tgi->gc.parent = &pdev->dev; @@ -619,6 +703,8 @@ static int tegra_gpio_probe(struct platform_device *pdev) #ifdef CONFIG_PM_SLEEP tgi->ic.irq_set_wake = tegra_gpio_irq_set_wake; #endif + tgi->ic.irq_request_resources = tegra_gpio_irq_request_resources; + tgi->ic.irq_release_resources = tegra_gpio_irq_release_resources; platform_set_drvdata(pdev, tgi); @@ -630,11 +716,10 @@ static int tegra_gpio_probe(struct platform_device *pdev) if (!tgi->bank_info) return -ENOMEM; - tgi->irq_domain = irq_domain_add_linear(pdev->dev.of_node, - tgi->gc.ngpio, - &irq_domain_simple_ops, NULL); - if (!tgi->irq_domain) - return -ENODEV; + tgi->irqs = devm_kcalloc(&pdev->dev, tgi->bank_count, + sizeof(*tgi->irqs), GFP_KERNEL); + if (!tgi->irqs) + return -ENOMEM; for (i = 0; i < tgi->bank_count; i++) { ret = platform_get_irq(pdev, i); @@ -643,8 +728,36 @@ static int tegra_gpio_probe(struct platform_device *pdev) bank = &tgi->bank_info[i]; bank->bank = i; - bank->irq = ret; - bank->tgi = tgi; + + tgi->irqs[i] = ret; + + for (j = 0; j < 4; j++) { + raw_spin_lock_init(&bank->lvl_lock[j]); + spin_lock_init(&bank->dbc_lock[j]); + } + } + + irq = &tgi->gc.irq; + irq->chip = &tgi->ic; + irq->fwnode = of_node_to_fwnode(pdev->dev.of_node); + irq->child_to_parent_hwirq = tegra_gpio_child_to_parent_hwirq; + irq->populate_parent_alloc_arg = tegra_gpio_populate_parent_fwspec; + irq->handler = handle_simple_irq; + irq->default_type = IRQ_TYPE_NONE; + irq->parent_handler = tegra_gpio_irq_handler; + irq->parent_handler_data = tgi; + irq->num_parents = tgi->bank_count; + irq->parents = tgi->irqs; + + np = of_find_matching_node(NULL, tegra_pmc_of_match); + if (np) { + irq->parent_domain = irq_find_host(np); + of_node_put(np); + + if (!irq->parent_domain) + return -EPROBE_DEFER; + + tgi->ic.irq_set_affinity = tegra_gpio_irq_set_affinity; } tgi->regs = devm_platform_ioremap_resource(pdev, 0); @@ -660,33 +773,8 @@ static int tegra_gpio_probe(struct platform_device *pdev) } ret = devm_gpiochip_add_data(&pdev->dev, &tgi->gc, tgi); - if (ret < 0) { - irq_domain_remove(tgi->irq_domain); + if (ret < 0) return ret; - } - - for (gpio = 0; gpio < tgi->gc.ngpio; gpio++) { - int irq = irq_create_mapping(tgi->irq_domain, gpio); - /* No validity check; all Tegra GPIOs are valid IRQs */ - - bank = &tgi->bank_info[GPIO_BANK(gpio)]; - - irq_set_chip_data(irq, bank); - irq_set_lockdep_class(irq, &gpio_lock_class, &gpio_request_class); - irq_set_chip_and_handler(irq, &tgi->ic, handle_simple_irq); - } - - for (i = 0; i < tgi->bank_count; i++) { - bank = &tgi->bank_info[i]; - - irq_set_chained_handler_and_data(bank->irq, - tegra_gpio_irq_handler, bank); - - for (j = 0; j < 4; j++) { - raw_spin_lock_init(&bank->lvl_lock[j]); - spin_lock_init(&bank->dbc_lock[j]); - } - } tegra_gpio_debuginit(tgi); @@ -715,18 +803,21 @@ static const struct of_device_id tegra_gpio_of_match[] = { { .compatible = "nvidia,tegra20-gpio", .data = &tegra20_gpio_config }, { }, }; +MODULE_DEVICE_TABLE(of, tegra_gpio_of_match); static struct platform_driver tegra_gpio_driver = { - .driver = { - .name = "tegra-gpio", - .pm = &tegra_gpio_pm_ops, + .driver = { + .name = "tegra-gpio", + .pm = &tegra_gpio_pm_ops, .of_match_table = tegra_gpio_of_match, }, - .probe = tegra_gpio_probe, + .probe = tegra_gpio_probe, }; - -static int __init tegra_gpio_init(void) -{ - return platform_driver_register(&tegra_gpio_driver); -} -subsys_initcall(tegra_gpio_init); +module_platform_driver(tegra_gpio_driver); + +MODULE_DESCRIPTION("NVIDIA Tegra GPIO controller driver"); +MODULE_AUTHOR("Laxman Dewangan "); +MODULE_AUTHOR("Stephen Warren "); +MODULE_AUTHOR("Thierry Reding "); +MODULE_AUTHOR("Erik Gilling "); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/gpio/gpio-tegra186.c b/drivers/gpio/gpio-tegra186.c index 286e0b1f46e4820700d336f2c45f54189d553143..1bd9e44df718484bd38318142c7a1bf7a1baa3bc 100644 --- a/drivers/gpio/gpio-tegra186.c +++ b/drivers/gpio/gpio-tegra186.c @@ -657,7 +657,7 @@ static int tegra186_gpio_probe(struct platform_device *pdev) gpio->gpio.get_direction = tegra186_gpio_get_direction; gpio->gpio.direction_input = tegra186_gpio_direction_input; gpio->gpio.direction_output = tegra186_gpio_direction_output; - gpio->gpio.get = tegra186_gpio_get, + gpio->gpio.get = tegra186_gpio_get; gpio->gpio.set = tegra186_gpio_set; gpio->gpio.set_config = tegra186_gpio_set_config; gpio->gpio.add_pin_ranges = tegra186_gpio_add_pin_ranges; diff --git a/drivers/gpio/gpio-visconti.c b/drivers/gpio/gpio-visconti.c new file mode 100644 index 0000000000000000000000000000000000000000..0e3d19828eb11b08803a030760ef91303565d7c8 --- /dev/null +++ b/drivers/gpio/gpio-visconti.c @@ -0,0 +1,218 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Toshiba Visconti GPIO Support + * + * (C) Copyright 2020 Toshiba Electronic Devices & Storage Corporation + * (C) Copyright 2020 TOSHIBA CORPORATION + * + * Nobuhiro Iwamatsu + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +/* register offset */ +#define GPIO_DIR 0x00 +#define GPIO_IDATA 0x08 +#define GPIO_ODATA 0x10 +#define GPIO_OSET 0x18 +#define GPIO_OCLR 0x20 +#define GPIO_INTMODE 0x30 + +#define BASE_HW_IRQ 24 + +struct visconti_gpio { + void __iomem *base; + spinlock_t lock; /* protect gpio register */ + struct gpio_chip gpio_chip; + struct irq_chip irq_chip; +}; + +static int visconti_gpio_irq_set_type(struct irq_data *d, unsigned int type) +{ + struct gpio_chip *gc = irq_data_get_irq_chip_data(d); + struct visconti_gpio *priv = gpiochip_get_data(gc); + u32 offset = irqd_to_hwirq(d); + u32 bit = BIT(offset); + u32 intc_type = IRQ_TYPE_EDGE_RISING; + u32 intmode, odata; + int ret = 0; + unsigned long flags; + + spin_lock_irqsave(&priv->lock, flags); + + odata = readl(priv->base + GPIO_ODATA); + intmode = readl(priv->base + GPIO_INTMODE); + + switch (type) { + case IRQ_TYPE_EDGE_RISING: + odata &= ~bit; + intmode &= ~bit; + break; + case IRQ_TYPE_EDGE_FALLING: + odata |= bit; + intmode &= ~bit; + break; + case IRQ_TYPE_EDGE_BOTH: + intmode |= bit; + break; + case IRQ_TYPE_LEVEL_HIGH: + intc_type = IRQ_TYPE_LEVEL_HIGH; + odata &= ~bit; + intmode &= ~bit; + break; + case IRQ_TYPE_LEVEL_LOW: + intc_type = IRQ_TYPE_LEVEL_HIGH; + odata |= bit; + intmode &= ~bit; + break; + default: + ret = -EINVAL; + goto err; + } + + writel(odata, priv->base + GPIO_ODATA); + writel(intmode, priv->base + GPIO_INTMODE); + irq_set_irq_type(offset, intc_type); + + ret = irq_chip_set_type_parent(d, type); +err: + spin_unlock_irqrestore(&priv->lock, flags); + return ret; +} + +static int visconti_gpio_child_to_parent_hwirq(struct gpio_chip *gc, + unsigned int child, + unsigned int child_type, + unsigned int *parent, + unsigned int *parent_type) +{ + /* Interrupts 0..15 mapped to interrupts 24..39 on the GIC */ + if (child < 16) { + /* All these interrupts are level high in the CPU */ + *parent_type = IRQ_TYPE_LEVEL_HIGH; + *parent = child + BASE_HW_IRQ; + return 0; + } + return -EINVAL; +} + +static void *visconti_gpio_populate_parent_fwspec(struct gpio_chip *chip, + unsigned int parent_hwirq, + unsigned int parent_type) +{ + struct irq_fwspec *fwspec; + + fwspec = kmalloc(sizeof(*fwspec), GFP_KERNEL); + if (!fwspec) + return NULL; + + fwspec->fwnode = chip->irq.parent_domain->fwnode; + fwspec->param_count = 3; + fwspec->param[0] = 0; + fwspec->param[1] = parent_hwirq; + fwspec->param[2] = parent_type; + + return fwspec; +} + +static int visconti_gpio_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct visconti_gpio *priv; + struct irq_chip *irq_chip; + struct gpio_irq_chip *girq; + struct irq_domain *parent; + struct device_node *irq_parent; + struct fwnode_handle *fwnode; + int ret; + + priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + + spin_lock_init(&priv->lock); + + priv->base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(priv->base)) + return PTR_ERR(priv->base); + + irq_parent = of_irq_find_parent(dev->of_node); + if (!irq_parent) { + dev_err(dev, "No IRQ parent node\n"); + return -ENODEV; + } + + parent = irq_find_host(irq_parent); + if (!parent) { + dev_err(dev, "No IRQ parent domain\n"); + return -ENODEV; + } + + fwnode = of_node_to_fwnode(irq_parent); + of_node_put(irq_parent); + + ret = bgpio_init(&priv->gpio_chip, dev, 4, + priv->base + GPIO_IDATA, + priv->base + GPIO_OSET, + priv->base + GPIO_OCLR, + priv->base + GPIO_DIR, + NULL, + 0); + if (ret) { + dev_err(dev, "unable to init generic GPIO\n"); + return ret; + } + + irq_chip = &priv->irq_chip; + irq_chip->name = dev_name(dev); + irq_chip->irq_mask = irq_chip_mask_parent; + irq_chip->irq_unmask = irq_chip_unmask_parent; + irq_chip->irq_eoi = irq_chip_eoi_parent; + irq_chip->irq_set_type = visconti_gpio_irq_set_type; + irq_chip->flags = IRQCHIP_SET_TYPE_MASKED | IRQCHIP_MASK_ON_SUSPEND; + + girq = &priv->gpio_chip.irq; + girq->chip = irq_chip; + girq->fwnode = fwnode; + girq->parent_domain = parent; + girq->child_to_parent_hwirq = visconti_gpio_child_to_parent_hwirq; + girq->populate_parent_alloc_arg = visconti_gpio_populate_parent_fwspec; + girq->default_type = IRQ_TYPE_NONE; + girq->handler = handle_level_irq; + + ret = devm_gpiochip_add_data(dev, &priv->gpio_chip, priv); + if (ret) { + dev_err(dev, "failed to add GPIO chip\n"); + return ret; + } + + platform_set_drvdata(pdev, priv); + + return ret; +} + +static const struct of_device_id visconti_gpio_of_match[] = { + { .compatible = "toshiba,gpio-tmpv7708", }, + { /* end of table */ } +}; +MODULE_DEVICE_TABLE(of, visconti_gpio_of_match); + +static struct platform_driver visconti_gpio_driver = { + .probe = visconti_gpio_probe, + .driver = { + .name = "visconti_gpio", + .of_match_table = of_match_ptr(visconti_gpio_of_match), + } +}; +module_platform_driver(visconti_gpio_driver); + +MODULE_AUTHOR("Nobuhiro Iwamatsu "); +MODULE_DESCRIPTION("Toshiba Visconti GPIO Driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/gpio/gpio-vx855.c b/drivers/gpio/gpio-vx855.c index 3bf397b8dfbc7d7bfb80695fe47703a28e8cd6be..69713fd5485bc68bc45cad7a6a9aadfb01e44313 100644 --- a/drivers/gpio/gpio-vx855.c +++ b/drivers/gpio/gpio-vx855.c @@ -216,7 +216,7 @@ static void vx855gpio_gpio_setup(struct vx855_gpio *vg) c->direction_output = vx855gpio_direction_output; c->get = vx855gpio_get; c->set = vx855gpio_set; - c->set_config = vx855gpio_set_config, + c->set_config = vx855gpio_set_config; c->dbg_show = NULL; c->base = 0; c->ngpio = NR_VX855_GP; diff --git a/drivers/gpio/gpio-wcove.c b/drivers/gpio/gpio-wcove.c index b5fbba5a783af2402a3fd3a86e3fc8f1a127ac92..a19eeef6cf1ecb883bdc063b1514f266cb8fc0b5 100644 --- a/drivers/gpio/gpio-wcove.c +++ b/drivers/gpio/gpio-wcove.c @@ -73,6 +73,8 @@ enum ctrl_register { CTRL_IN, CTRL_OUT, + IRQ_STATUS, + IRQ_MASK, }; /* @@ -112,22 +114,29 @@ static inline int to_reg(int gpio, enum ctrl_register reg_type) return reg; } -static void wcove_update_irq_mask(struct wcove_gpio *wg, int gpio) +static inline int to_ireg(int gpio, enum ctrl_register type, unsigned int *mask) { - unsigned int reg, mask; + unsigned int reg = type == IRQ_STATUS ? IRQ_STATUS_BASE : IRQ_MASK_BASE; if (gpio < GROUP0_NR_IRQS) { - reg = IRQ_MASK_BASE; - mask = BIT(gpio % GROUP0_NR_IRQS); + reg += 0; + *mask = BIT(gpio); } else { - reg = IRQ_MASK_BASE + 1; - mask = BIT((gpio - GROUP0_NR_IRQS) % GROUP1_NR_IRQS); + reg += 1; + *mask = BIT(gpio - GROUP0_NR_IRQS); } + return reg; +} + +static void wcove_update_irq_mask(struct wcove_gpio *wg, int gpio) +{ + unsigned int mask, reg = to_ireg(gpio, IRQ_MASK, &mask); + if (wg->set_irq_mask) - regmap_update_bits(wg->regmap, reg, mask, mask); + regmap_set_bits(wg->regmap, reg, mask); else - regmap_update_bits(wg->regmap, reg, mask, 0); + regmap_clear_bits(wg->regmap, reg, mask); } static void wcove_update_irq_ctrl(struct wcove_gpio *wg, int gpio) @@ -207,9 +216,9 @@ static void wcove_gpio_set(struct gpio_chip *chip, unsigned int gpio, int value) return; if (value) - regmap_update_bits(wg->regmap, reg, 1, 1); + regmap_set_bits(wg->regmap, reg, 1); else - regmap_update_bits(wg->regmap, reg, 1, 0); + regmap_clear_bits(wg->regmap, reg, 1); } static int wcove_gpio_set_config(struct gpio_chip *chip, unsigned int gpio, @@ -324,7 +333,8 @@ static struct irq_chip wcove_irqchip = { static irqreturn_t wcove_gpio_irq_handler(int irq, void *data) { struct wcove_gpio *wg = (struct wcove_gpio *)data; - unsigned int pending, virq, gpio, mask, offset; + unsigned int virq, gpio; + unsigned long pending; u8 p[2]; if (regmap_bulk_read(wg->regmap, IRQ_STATUS_BASE, p, 2)) { @@ -339,15 +349,12 @@ static irqreturn_t wcove_gpio_irq_handler(int irq, void *data) /* Iterate until no interrupt is pending */ while (pending) { /* One iteration is for all pending bits */ - for_each_set_bit(gpio, (const unsigned long *)&pending, - WCOVE_GPIO_NUM) { - offset = (gpio > GROUP0_NR_IRQS) ? 1 : 0; - mask = (offset == 1) ? BIT(gpio - GROUP0_NR_IRQS) : - BIT(gpio); + for_each_set_bit(gpio, &pending, WCOVE_GPIO_NUM) { + unsigned int mask, reg = to_ireg(gpio, IRQ_STATUS, &mask); + virq = irq_find_mapping(wg->chip.irq.domain, gpio); handle_nested_irq(virq); - regmap_update_bits(wg->regmap, IRQ_STATUS_BASE + offset, - mask, mask); + regmap_set_bits(wg->regmap, reg, mask); } /* Next iteration */ @@ -367,30 +374,26 @@ static void wcove_gpio_dbg_show(struct seq_file *s, { unsigned int ctlo, ctli, irq_mask, irq_status; struct wcove_gpio *wg = gpiochip_get_data(chip); - int gpio, offset, group, ret = 0; + int gpio, mask, ret = 0; for (gpio = 0; gpio < WCOVE_GPIO_NUM; gpio++) { - group = gpio < GROUP0_NR_IRQS ? 0 : 1; ret += regmap_read(wg->regmap, to_reg(gpio, CTRL_OUT), &ctlo); ret += regmap_read(wg->regmap, to_reg(gpio, CTRL_IN), &ctli); - ret += regmap_read(wg->regmap, IRQ_MASK_BASE + group, - &irq_mask); - ret += regmap_read(wg->regmap, IRQ_STATUS_BASE + group, - &irq_status); + ret += regmap_read(wg->regmap, to_ireg(gpio, IRQ_MASK, &mask), &irq_mask); + ret += regmap_read(wg->regmap, to_ireg(gpio, IRQ_STATUS, &mask), &irq_status); if (ret) { pr_err("Failed to read registers: ctrl out/in or irq status/mask\n"); break; } - offset = gpio % 8; seq_printf(s, " gpio-%-2d %s %s %s %s ctlo=%2x,%s %s\n", gpio, ctlo & CTLO_DIR_OUT ? "out" : "in ", ctli & 0x1 ? "hi" : "lo", ctli & CTLI_INTCNT_NE ? "fall" : " ", ctli & CTLI_INTCNT_PE ? "rise" : " ", ctlo, - irq_mask & BIT(offset) ? "mask " : "unmask", - irq_status & BIT(offset) ? "pending" : " "); + irq_mask & mask ? "mask " : "unmask", + irq_status & mask ? "pending" : " "); } } @@ -434,7 +437,7 @@ static int wcove_gpio_probe(struct platform_device *pdev) wg->chip.get_direction = wcove_gpio_get_direction; wg->chip.get = wcove_gpio_get; wg->chip.set = wcove_gpio_set; - wg->chip.set_config = wcove_gpio_set_config, + wg->chip.set_config = wcove_gpio_set_config; wg->chip.base = -1; wg->chip.ngpio = WCOVE_VGPIO_NUM; wg->chip.can_sleep = true; @@ -473,14 +476,12 @@ static int wcove_gpio_probe(struct platform_device *pdev) } /* Enable GPIO0 interrupts */ - ret = regmap_update_bits(wg->regmap, IRQ_MASK_BASE, GPIO_IRQ0_MASK, - 0x00); + ret = regmap_clear_bits(wg->regmap, IRQ_MASK_BASE + 0, GPIO_IRQ0_MASK); if (ret) return ret; /* Enable GPIO1 interrupts */ - ret = regmap_update_bits(wg->regmap, IRQ_MASK_BASE + 1, GPIO_IRQ1_MASK, - 0x00); + ret = regmap_clear_bits(wg->regmap, IRQ_MASK_BASE + 1, GPIO_IRQ1_MASK); if (ret) return ret; diff --git a/drivers/gpio/gpio-xilinx.c b/drivers/gpio/gpio-xilinx.c index be539381fd82eab94183783e9ea38a54b99e1501..b411d3156e0b687c39244e82ee492d6dfa3d1c54 100644 --- a/drivers/gpio/gpio-xilinx.c +++ b/drivers/gpio/gpio-xilinx.c @@ -10,10 +10,13 @@ #include #include #include +#include #include +#include #include #include #include +#include #include /* Register Offset Definitions */ @@ -22,6 +25,11 @@ #define XGPIO_CHANNEL_OFFSET 0x8 +#define XGPIO_GIER_OFFSET 0x11c /* Global Interrupt Enable */ +#define XGPIO_GIER_IE BIT(31) +#define XGPIO_IPISR_OFFSET 0x120 /* IP Interrupt Status */ +#define XGPIO_IPIER_OFFSET 0x128 /* IP Interrupt Enable */ + /* Read/Write access to the GPIO registers */ #if defined(CONFIG_ARCH_ZYNQ) || defined(CONFIG_X86) # define xgpio_readreg(offset) readl(offset) @@ -36,9 +44,15 @@ * @gc: GPIO chip * @regs: register block * @gpio_width: GPIO width for every channel - * @gpio_state: GPIO state shadow register + * @gpio_state: GPIO write state shadow register + * @gpio_last_irq_read: GPIO read state register from last interrupt * @gpio_dir: GPIO direction shadow register * @gpio_lock: Lock used for synchronization + * @irq: IRQ used by GPIO device + * @irqchip: IRQ chip + * @irq_enable: GPIO IRQ enable/disable bitfield + * @irq_rising_edge: GPIO IRQ rising edge enable/disable bitfield + * @irq_falling_edge: GPIO IRQ falling edge enable/disable bitfield * @clk: clock resource for this driver */ struct xgpio_instance { @@ -46,8 +60,14 @@ struct xgpio_instance { void __iomem *regs; unsigned int gpio_width[2]; u32 gpio_state[2]; + u32 gpio_last_irq_read[2]; u32 gpio_dir[2]; - spinlock_t gpio_lock[2]; + spinlock_t gpio_lock; /* For serializing operations */ + int irq; + struct irq_chip irqchip; + u32 irq_enable[2]; + u32 irq_rising_edge[2]; + u32 irq_falling_edge[2]; struct clk *clk; }; @@ -113,7 +133,7 @@ static void xgpio_set(struct gpio_chip *gc, unsigned int gpio, int val) int index = xgpio_index(chip, gpio); int offset = xgpio_offset(chip, gpio); - spin_lock_irqsave(&chip->gpio_lock[index], flags); + spin_lock_irqsave(&chip->gpio_lock, flags); /* Write to GPIO signal and set its direction to output */ if (val) @@ -124,7 +144,7 @@ static void xgpio_set(struct gpio_chip *gc, unsigned int gpio, int val) xgpio_writereg(chip->regs + XGPIO_DATA_OFFSET + xgpio_regoffset(chip, gpio), chip->gpio_state[index]); - spin_unlock_irqrestore(&chip->gpio_lock[index], flags); + spin_unlock_irqrestore(&chip->gpio_lock, flags); } /** @@ -144,7 +164,7 @@ static void xgpio_set_multiple(struct gpio_chip *gc, unsigned long *mask, int index = xgpio_index(chip, 0); int offset, i; - spin_lock_irqsave(&chip->gpio_lock[index], flags); + spin_lock_irqsave(&chip->gpio_lock, flags); /* Write to GPIO signals */ for (i = 0; i < gc->ngpio; i++) { @@ -155,9 +175,9 @@ static void xgpio_set_multiple(struct gpio_chip *gc, unsigned long *mask, xgpio_writereg(chip->regs + XGPIO_DATA_OFFSET + index * XGPIO_CHANNEL_OFFSET, chip->gpio_state[index]); - spin_unlock_irqrestore(&chip->gpio_lock[index], flags); + spin_unlock_irqrestore(&chip->gpio_lock, flags); index = xgpio_index(chip, i); - spin_lock_irqsave(&chip->gpio_lock[index], flags); + spin_lock_irqsave(&chip->gpio_lock, flags); } if (__test_and_clear_bit(i, mask)) { offset = xgpio_offset(chip, i); @@ -171,7 +191,7 @@ static void xgpio_set_multiple(struct gpio_chip *gc, unsigned long *mask, xgpio_writereg(chip->regs + XGPIO_DATA_OFFSET + index * XGPIO_CHANNEL_OFFSET, chip->gpio_state[index]); - spin_unlock_irqrestore(&chip->gpio_lock[index], flags); + spin_unlock_irqrestore(&chip->gpio_lock, flags); } /** @@ -190,14 +210,14 @@ static int xgpio_dir_in(struct gpio_chip *gc, unsigned int gpio) int index = xgpio_index(chip, gpio); int offset = xgpio_offset(chip, gpio); - spin_lock_irqsave(&chip->gpio_lock[index], flags); + spin_lock_irqsave(&chip->gpio_lock, flags); /* Set the GPIO bit in shadow register and set direction as input */ chip->gpio_dir[index] |= BIT(offset); xgpio_writereg(chip->regs + XGPIO_TRI_OFFSET + xgpio_regoffset(chip, gpio), chip->gpio_dir[index]); - spin_unlock_irqrestore(&chip->gpio_lock[index], flags); + spin_unlock_irqrestore(&chip->gpio_lock, flags); return 0; } @@ -221,7 +241,7 @@ static int xgpio_dir_out(struct gpio_chip *gc, unsigned int gpio, int val) int index = xgpio_index(chip, gpio); int offset = xgpio_offset(chip, gpio); - spin_lock_irqsave(&chip->gpio_lock[index], flags); + spin_lock_irqsave(&chip->gpio_lock, flags); /* Write state of GPIO signal */ if (val) @@ -236,7 +256,7 @@ static int xgpio_dir_out(struct gpio_chip *gc, unsigned int gpio, int val) xgpio_writereg(chip->regs + XGPIO_TRI_OFFSET + xgpio_regoffset(chip, gpio), chip->gpio_dir[index]); - spin_unlock_irqrestore(&chip->gpio_lock[index], flags); + spin_unlock_irqrestore(&chip->gpio_lock, flags); return 0; } @@ -259,6 +279,39 @@ static void xgpio_save_regs(struct xgpio_instance *chip) chip->gpio_dir[1]); } +static int xgpio_request(struct gpio_chip *chip, unsigned int offset) +{ + int ret; + + ret = pm_runtime_get_sync(chip->parent); + /* + * If the device is already active pm_runtime_get() will return 1 on + * success, but gpio_request still needs to return 0. + */ + return ret < 0 ? ret : 0; +} + +static void xgpio_free(struct gpio_chip *chip, unsigned int offset) +{ + pm_runtime_put(chip->parent); +} + +static int __maybe_unused xgpio_suspend(struct device *dev) +{ + struct xgpio_instance *gpio = dev_get_drvdata(dev); + struct irq_data *data = irq_get_irq_data(gpio->irq); + + if (!data) { + dev_err(dev, "irq_get_irq_data() failed\n"); + return -EINVAL; + } + + if (!irqd_is_wakeup_set(data)) + return pm_runtime_force_suspend(dev); + + return 0; +} + /** * xgpio_remove - Remove method for the GPIO device. * @pdev: pointer to the platform device @@ -271,11 +324,223 @@ static int xgpio_remove(struct platform_device *pdev) { struct xgpio_instance *gpio = platform_get_drvdata(pdev); + pm_runtime_get_sync(&pdev->dev); + pm_runtime_put_noidle(&pdev->dev); + pm_runtime_disable(&pdev->dev); clk_disable_unprepare(gpio->clk); return 0; } +/** + * xgpio_irq_ack - Acknowledge a child GPIO interrupt. + * @irq_data: per IRQ and chip data passed down to chip functions + * This currently does nothing, but irq_ack is unconditionally called by + * handle_edge_irq and therefore must be defined. + */ +static void xgpio_irq_ack(struct irq_data *irq_data) +{ +} + +static int __maybe_unused xgpio_resume(struct device *dev) +{ + struct xgpio_instance *gpio = dev_get_drvdata(dev); + struct irq_data *data = irq_get_irq_data(gpio->irq); + + if (!data) { + dev_err(dev, "irq_get_irq_data() failed\n"); + return -EINVAL; + } + + if (!irqd_is_wakeup_set(data)) + return pm_runtime_force_resume(dev); + + return 0; +} + +static int __maybe_unused xgpio_runtime_suspend(struct device *dev) +{ + struct platform_device *pdev = to_platform_device(dev); + struct xgpio_instance *gpio = platform_get_drvdata(pdev); + + clk_disable(gpio->clk); + + return 0; +} + +static int __maybe_unused xgpio_runtime_resume(struct device *dev) +{ + struct platform_device *pdev = to_platform_device(dev); + struct xgpio_instance *gpio = platform_get_drvdata(pdev); + + return clk_enable(gpio->clk); +} + +static const struct dev_pm_ops xgpio_dev_pm_ops = { + SET_SYSTEM_SLEEP_PM_OPS(xgpio_suspend, xgpio_resume) + SET_RUNTIME_PM_OPS(xgpio_runtime_suspend, + xgpio_runtime_resume, NULL) +}; + +/** + * xgpio_irq_mask - Write the specified signal of the GPIO device. + * @irq_data: per IRQ and chip data passed down to chip functions + */ +static void xgpio_irq_mask(struct irq_data *irq_data) +{ + unsigned long flags; + struct xgpio_instance *chip = irq_data_get_irq_chip_data(irq_data); + int irq_offset = irqd_to_hwirq(irq_data); + int index = xgpio_index(chip, irq_offset); + int offset = xgpio_offset(chip, irq_offset); + + spin_lock_irqsave(&chip->gpio_lock, flags); + + chip->irq_enable[index] &= ~BIT(offset); + + if (!chip->irq_enable[index]) { + /* Disable per channel interrupt */ + u32 temp = xgpio_readreg(chip->regs + XGPIO_IPIER_OFFSET); + + temp &= ~BIT(index); + xgpio_writereg(chip->regs + XGPIO_IPIER_OFFSET, temp); + } + spin_unlock_irqrestore(&chip->gpio_lock, flags); +} + +/** + * xgpio_irq_unmask - Write the specified signal of the GPIO device. + * @irq_data: per IRQ and chip data passed down to chip functions + */ +static void xgpio_irq_unmask(struct irq_data *irq_data) +{ + unsigned long flags; + struct xgpio_instance *chip = irq_data_get_irq_chip_data(irq_data); + int irq_offset = irqd_to_hwirq(irq_data); + int index = xgpio_index(chip, irq_offset); + int offset = xgpio_offset(chip, irq_offset); + u32 old_enable = chip->irq_enable[index]; + + spin_lock_irqsave(&chip->gpio_lock, flags); + + chip->irq_enable[index] |= BIT(offset); + + if (!old_enable) { + /* Clear any existing per-channel interrupts */ + u32 val = xgpio_readreg(chip->regs + XGPIO_IPISR_OFFSET) & + BIT(index); + + if (val) + xgpio_writereg(chip->regs + XGPIO_IPISR_OFFSET, val); + + /* Update GPIO IRQ read data before enabling interrupt*/ + val = xgpio_readreg(chip->regs + XGPIO_DATA_OFFSET + + index * XGPIO_CHANNEL_OFFSET); + chip->gpio_last_irq_read[index] = val; + + /* Enable per channel interrupt */ + val = xgpio_readreg(chip->regs + XGPIO_IPIER_OFFSET); + val |= BIT(index); + xgpio_writereg(chip->regs + XGPIO_IPIER_OFFSET, val); + } + + spin_unlock_irqrestore(&chip->gpio_lock, flags); +} + +/** + * xgpio_set_irq_type - Write the specified signal of the GPIO device. + * @irq_data: Per IRQ and chip data passed down to chip functions + * @type: Interrupt type that is to be set for the gpio pin + * + * Return: + * 0 if interrupt type is supported otherwise -EINVAL + */ +static int xgpio_set_irq_type(struct irq_data *irq_data, unsigned int type) +{ + struct xgpio_instance *chip = irq_data_get_irq_chip_data(irq_data); + int irq_offset = irqd_to_hwirq(irq_data); + int index = xgpio_index(chip, irq_offset); + int offset = xgpio_offset(chip, irq_offset); + + /* + * The Xilinx GPIO hardware provides a single interrupt status + * indication for any state change in a given GPIO channel (bank). + * Therefore, only rising edge or falling edge triggers are + * supported. + */ + switch (type & IRQ_TYPE_SENSE_MASK) { + case IRQ_TYPE_EDGE_BOTH: + chip->irq_rising_edge[index] |= BIT(offset); + chip->irq_falling_edge[index] |= BIT(offset); + break; + case IRQ_TYPE_EDGE_RISING: + chip->irq_rising_edge[index] |= BIT(offset); + chip->irq_falling_edge[index] &= ~BIT(offset); + break; + case IRQ_TYPE_EDGE_FALLING: + chip->irq_rising_edge[index] &= ~BIT(offset); + chip->irq_falling_edge[index] |= BIT(offset); + break; + default: + return -EINVAL; + } + + irq_set_handler_locked(irq_data, handle_edge_irq); + return 0; +} + +/** + * xgpio_irqhandler - Gpio interrupt service routine + * @desc: Pointer to interrupt description + */ +static void xgpio_irqhandler(struct irq_desc *desc) +{ + struct xgpio_instance *chip = irq_desc_get_handler_data(desc); + struct irq_chip *irqchip = irq_desc_get_chip(desc); + u32 num_channels = chip->gpio_width[1] ? 2 : 1; + u32 offset = 0, index; + u32 status = xgpio_readreg(chip->regs + XGPIO_IPISR_OFFSET); + + xgpio_writereg(chip->regs + XGPIO_IPISR_OFFSET, status); + + chained_irq_enter(irqchip, desc); + for (index = 0; index < num_channels; index++) { + if ((status & BIT(index))) { + unsigned long rising_events, falling_events, all_events; + unsigned long flags; + u32 data, bit; + unsigned int irq; + + spin_lock_irqsave(&chip->gpio_lock, flags); + data = xgpio_readreg(chip->regs + XGPIO_DATA_OFFSET + + index * XGPIO_CHANNEL_OFFSET); + rising_events = data & + ~chip->gpio_last_irq_read[index] & + chip->irq_enable[index] & + chip->irq_rising_edge[index]; + falling_events = ~data & + chip->gpio_last_irq_read[index] & + chip->irq_enable[index] & + chip->irq_falling_edge[index]; + dev_dbg(chip->gc.parent, + "IRQ chan %u rising 0x%lx falling 0x%lx\n", + index, rising_events, falling_events); + all_events = rising_events | falling_events; + chip->gpio_last_irq_read[index] = data; + spin_unlock_irqrestore(&chip->gpio_lock, flags); + + for_each_set_bit(bit, &all_events, 32) { + irq = irq_find_mapping(chip->gc.irq.domain, + offset + bit); + generic_handle_irq(irq); + } + } + offset += chip->gpio_width[index]; + } + + chained_irq_exit(irqchip, desc); +} + /** * xgpio_of_probe - Probe method for the GPIO device. * @pdev: pointer to the platform device @@ -289,7 +554,10 @@ static int xgpio_probe(struct platform_device *pdev) struct xgpio_instance *chip; int status = 0; struct device_node *np = pdev->dev.of_node; - u32 is_dual; + u32 is_dual = 0; + u32 cells = 2; + struct gpio_irq_chip *girq; + u32 temp; chip = devm_kzalloc(&pdev->dev, sizeof(*chip), GFP_KERNEL); if (!chip) @@ -305,6 +573,15 @@ static int xgpio_probe(struct platform_device *pdev) if (of_property_read_u32(np, "xlnx,tri-default", &chip->gpio_dir[0])) chip->gpio_dir[0] = 0xFFFFFFFF; + /* Update cells with gpio-cells value */ + if (of_property_read_u32(np, "#gpio-cells", &cells)) + dev_dbg(&pdev->dev, "Missing gpio-cells property\n"); + + if (cells != 2) { + dev_err(&pdev->dev, "#gpio-cells mismatch\n"); + return -EINVAL; + } + /* * Check device node and parent device node for device width * and assume default width of 32 @@ -312,7 +589,10 @@ static int xgpio_probe(struct platform_device *pdev) if (of_property_read_u32(np, "xlnx,gpio-width", &chip->gpio_width[0])) chip->gpio_width[0] = 32; - spin_lock_init(&chip->gpio_lock[0]); + if (chip->gpio_width[0] > 32) + return -EINVAL; + + spin_lock_init(&chip->gpio_lock); if (of_property_read_u32(np, "xlnx,is-dual", &is_dual)) is_dual = 0; @@ -336,7 +616,8 @@ static int xgpio_probe(struct platform_device *pdev) &chip->gpio_width[1])) chip->gpio_width[1] = 32; - spin_lock_init(&chip->gpio_lock[1]); + if (chip->gpio_width[1] > 32) + return -EINVAL; } chip->gc.base = -1; @@ -344,8 +625,11 @@ static int xgpio_probe(struct platform_device *pdev) chip->gc.parent = &pdev->dev; chip->gc.direction_input = xgpio_dir_in; chip->gc.direction_output = xgpio_dir_out; + chip->gc.of_gpio_n_cells = cells; chip->gc.get = xgpio_get; chip->gc.set = xgpio_set; + chip->gc.request = xgpio_request; + chip->gc.free = xgpio_free; chip->gc.set_multiple = xgpio_set_multiple; chip->gc.label = dev_name(&pdev->dev); @@ -357,28 +641,68 @@ static int xgpio_probe(struct platform_device *pdev) } chip->clk = devm_clk_get_optional(&pdev->dev, NULL); - if (IS_ERR(chip->clk)) { - if (PTR_ERR(chip->clk) != -EPROBE_DEFER) - dev_dbg(&pdev->dev, "Input clock not found\n"); - return PTR_ERR(chip->clk); - } + if (IS_ERR(chip->clk)) + return dev_err_probe(&pdev->dev, PTR_ERR(chip->clk), "input clock not found.\n"); status = clk_prepare_enable(chip->clk); if (status < 0) { dev_err(&pdev->dev, "Failed to prepare clk\n"); return status; } + pm_runtime_get_noresume(&pdev->dev); + pm_runtime_set_active(&pdev->dev); + pm_runtime_enable(&pdev->dev); xgpio_save_regs(chip); + chip->irq = platform_get_irq_optional(pdev, 0); + if (chip->irq <= 0) + goto skip_irq; + + chip->irqchip.name = "gpio-xilinx"; + chip->irqchip.irq_ack = xgpio_irq_ack; + chip->irqchip.irq_mask = xgpio_irq_mask; + chip->irqchip.irq_unmask = xgpio_irq_unmask; + chip->irqchip.irq_set_type = xgpio_set_irq_type; + + /* Disable per-channel interrupts */ + xgpio_writereg(chip->regs + XGPIO_IPIER_OFFSET, 0); + /* Clear any existing per-channel interrupts */ + temp = xgpio_readreg(chip->regs + XGPIO_IPISR_OFFSET); + xgpio_writereg(chip->regs + XGPIO_IPISR_OFFSET, temp); + /* Enable global interrupts */ + xgpio_writereg(chip->regs + XGPIO_GIER_OFFSET, XGPIO_GIER_IE); + + girq = &chip->gc.irq; + girq->chip = &chip->irqchip; + girq->parent_handler = xgpio_irqhandler; + girq->num_parents = 1; + girq->parents = devm_kcalloc(&pdev->dev, 1, + sizeof(*girq->parents), + GFP_KERNEL); + if (!girq->parents) { + status = -ENOMEM; + goto err_pm_put; + } + girq->parents[0] = chip->irq; + girq->default_type = IRQ_TYPE_NONE; + girq->handler = handle_bad_irq; + +skip_irq: status = devm_gpiochip_add_data(&pdev->dev, &chip->gc, chip); if (status) { dev_err(&pdev->dev, "failed to add GPIO chip\n"); - clk_disable_unprepare(chip->clk); - return status; + goto err_pm_put; } + pm_runtime_put(&pdev->dev); return 0; + +err_pm_put: + pm_runtime_disable(&pdev->dev); + pm_runtime_put_noidle(&pdev->dev); + clk_disable_unprepare(chip->clk); + return status; } static const struct of_device_id xgpio_of_match[] = { @@ -394,6 +718,7 @@ static struct platform_driver xgpio_plat_driver = { .driver = { .name = "gpio-xilinx", .of_match_table = xgpio_of_match, + .pm = &xgpio_dev_pm_ops, }, }; diff --git a/drivers/gpio/gpio-zx.c b/drivers/gpio/gpio-zx.c deleted file mode 100644 index 64bfb722756aa645ebafad9aa7c19be866ab02c7..0000000000000000000000000000000000000000 --- a/drivers/gpio/gpio-zx.c +++ /dev/null @@ -1,289 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * ZTE ZX296702 GPIO driver - * - * Author: Jun Nie - * - * Copyright (C) 2015 Linaro Ltd. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define ZX_GPIO_DIR 0x00 -#define ZX_GPIO_IVE 0x04 -#define ZX_GPIO_IV 0x08 -#define ZX_GPIO_IEP 0x0C -#define ZX_GPIO_IEN 0x10 -#define ZX_GPIO_DI 0x14 -#define ZX_GPIO_DO1 0x18 -#define ZX_GPIO_DO0 0x1C -#define ZX_GPIO_DO 0x20 - -#define ZX_GPIO_IM 0x28 -#define ZX_GPIO_IE 0x2C - -#define ZX_GPIO_MIS 0x30 -#define ZX_GPIO_IC 0x34 - -#define ZX_GPIO_NR 16 - -struct zx_gpio { - raw_spinlock_t lock; - - void __iomem *base; - struct gpio_chip gc; -}; - -static int zx_direction_input(struct gpio_chip *gc, unsigned offset) -{ - struct zx_gpio *chip = gpiochip_get_data(gc); - unsigned long flags; - u16 gpiodir; - - if (offset >= gc->ngpio) - return -EINVAL; - - raw_spin_lock_irqsave(&chip->lock, flags); - gpiodir = readw_relaxed(chip->base + ZX_GPIO_DIR); - gpiodir &= ~BIT(offset); - writew_relaxed(gpiodir, chip->base + ZX_GPIO_DIR); - raw_spin_unlock_irqrestore(&chip->lock, flags); - - return 0; -} - -static int zx_direction_output(struct gpio_chip *gc, unsigned offset, - int value) -{ - struct zx_gpio *chip = gpiochip_get_data(gc); - unsigned long flags; - u16 gpiodir; - - if (offset >= gc->ngpio) - return -EINVAL; - - raw_spin_lock_irqsave(&chip->lock, flags); - gpiodir = readw_relaxed(chip->base + ZX_GPIO_DIR); - gpiodir |= BIT(offset); - writew_relaxed(gpiodir, chip->base + ZX_GPIO_DIR); - - if (value) - writew_relaxed(BIT(offset), chip->base + ZX_GPIO_DO1); - else - writew_relaxed(BIT(offset), chip->base + ZX_GPIO_DO0); - raw_spin_unlock_irqrestore(&chip->lock, flags); - - return 0; -} - -static int zx_get_value(struct gpio_chip *gc, unsigned offset) -{ - struct zx_gpio *chip = gpiochip_get_data(gc); - - return !!(readw_relaxed(chip->base + ZX_GPIO_DI) & BIT(offset)); -} - -static void zx_set_value(struct gpio_chip *gc, unsigned offset, int value) -{ - struct zx_gpio *chip = gpiochip_get_data(gc); - - if (value) - writew_relaxed(BIT(offset), chip->base + ZX_GPIO_DO1); - else - writew_relaxed(BIT(offset), chip->base + ZX_GPIO_DO0); -} - -static int zx_irq_type(struct irq_data *d, unsigned trigger) -{ - struct gpio_chip *gc = irq_data_get_irq_chip_data(d); - struct zx_gpio *chip = gpiochip_get_data(gc); - int offset = irqd_to_hwirq(d); - unsigned long flags; - u16 gpiois, gpioi_epos, gpioi_eneg, gpioiev; - u16 bit = BIT(offset); - - if (offset < 0 || offset >= ZX_GPIO_NR) - return -EINVAL; - - raw_spin_lock_irqsave(&chip->lock, flags); - - gpioiev = readw_relaxed(chip->base + ZX_GPIO_IV); - gpiois = readw_relaxed(chip->base + ZX_GPIO_IVE); - gpioi_epos = readw_relaxed(chip->base + ZX_GPIO_IEP); - gpioi_eneg = readw_relaxed(chip->base + ZX_GPIO_IEN); - - if (trigger & (IRQ_TYPE_LEVEL_HIGH | IRQ_TYPE_LEVEL_LOW)) { - gpiois |= bit; - if (trigger & IRQ_TYPE_LEVEL_HIGH) - gpioiev |= bit; - else - gpioiev &= ~bit; - } else - gpiois &= ~bit; - - if ((trigger & IRQ_TYPE_EDGE_BOTH) == IRQ_TYPE_EDGE_BOTH) { - gpioi_epos |= bit; - gpioi_eneg |= bit; - } else { - if (trigger & IRQ_TYPE_EDGE_RISING) { - gpioi_epos |= bit; - gpioi_eneg &= ~bit; - } else if (trigger & IRQ_TYPE_EDGE_FALLING) { - gpioi_eneg |= bit; - gpioi_epos &= ~bit; - } - } - - writew_relaxed(gpiois, chip->base + ZX_GPIO_IVE); - writew_relaxed(gpioi_epos, chip->base + ZX_GPIO_IEP); - writew_relaxed(gpioi_eneg, chip->base + ZX_GPIO_IEN); - writew_relaxed(gpioiev, chip->base + ZX_GPIO_IV); - raw_spin_unlock_irqrestore(&chip->lock, flags); - - return 0; -} - -static void zx_irq_handler(struct irq_desc *desc) -{ - unsigned long pending; - int offset; - struct gpio_chip *gc = irq_desc_get_handler_data(desc); - struct zx_gpio *chip = gpiochip_get_data(gc); - struct irq_chip *irqchip = irq_desc_get_chip(desc); - - chained_irq_enter(irqchip, desc); - - pending = readw_relaxed(chip->base + ZX_GPIO_MIS); - writew_relaxed(pending, chip->base + ZX_GPIO_IC); - if (pending) { - for_each_set_bit(offset, &pending, ZX_GPIO_NR) - generic_handle_irq(irq_find_mapping(gc->irq.domain, - offset)); - } - - chained_irq_exit(irqchip, desc); -} - -static void zx_irq_mask(struct irq_data *d) -{ - struct gpio_chip *gc = irq_data_get_irq_chip_data(d); - struct zx_gpio *chip = gpiochip_get_data(gc); - u16 mask = BIT(irqd_to_hwirq(d) % ZX_GPIO_NR); - u16 gpioie; - - raw_spin_lock(&chip->lock); - gpioie = readw_relaxed(chip->base + ZX_GPIO_IM) | mask; - writew_relaxed(gpioie, chip->base + ZX_GPIO_IM); - gpioie = readw_relaxed(chip->base + ZX_GPIO_IE) & ~mask; - writew_relaxed(gpioie, chip->base + ZX_GPIO_IE); - raw_spin_unlock(&chip->lock); -} - -static void zx_irq_unmask(struct irq_data *d) -{ - struct gpio_chip *gc = irq_data_get_irq_chip_data(d); - struct zx_gpio *chip = gpiochip_get_data(gc); - u16 mask = BIT(irqd_to_hwirq(d) % ZX_GPIO_NR); - u16 gpioie; - - raw_spin_lock(&chip->lock); - gpioie = readw_relaxed(chip->base + ZX_GPIO_IM) & ~mask; - writew_relaxed(gpioie, chip->base + ZX_GPIO_IM); - gpioie = readw_relaxed(chip->base + ZX_GPIO_IE) | mask; - writew_relaxed(gpioie, chip->base + ZX_GPIO_IE); - raw_spin_unlock(&chip->lock); -} - -static struct irq_chip zx_irqchip = { - .name = "zx-gpio", - .irq_mask = zx_irq_mask, - .irq_unmask = zx_irq_unmask, - .irq_set_type = zx_irq_type, -}; - -static int zx_gpio_probe(struct platform_device *pdev) -{ - struct device *dev = &pdev->dev; - struct zx_gpio *chip; - struct gpio_irq_chip *girq; - int irq, id, ret; - - chip = devm_kzalloc(dev, sizeof(*chip), GFP_KERNEL); - if (!chip) - return -ENOMEM; - - chip->base = devm_platform_ioremap_resource(pdev, 0); - if (IS_ERR(chip->base)) - return PTR_ERR(chip->base); - - id = of_alias_get_id(dev->of_node, "gpio"); - - raw_spin_lock_init(&chip->lock); - chip->gc.request = gpiochip_generic_request; - chip->gc.free = gpiochip_generic_free; - chip->gc.direction_input = zx_direction_input; - chip->gc.direction_output = zx_direction_output; - chip->gc.get = zx_get_value; - chip->gc.set = zx_set_value; - chip->gc.base = ZX_GPIO_NR * id; - chip->gc.ngpio = ZX_GPIO_NR; - chip->gc.label = dev_name(dev); - chip->gc.parent = dev; - chip->gc.owner = THIS_MODULE; - - /* - * irq_chip support - */ - writew_relaxed(0xffff, chip->base + ZX_GPIO_IM); - writew_relaxed(0, chip->base + ZX_GPIO_IE); - irq = platform_get_irq(pdev, 0); - if (irq < 0) - return irq; - girq = &chip->gc.irq; - girq->chip = &zx_irqchip; - girq->parent_handler = zx_irq_handler; - girq->num_parents = 1; - girq->parents = devm_kcalloc(&pdev->dev, 1, - sizeof(*girq->parents), - GFP_KERNEL); - if (!girq->parents) - return -ENOMEM; - girq->parents[0] = irq; - girq->default_type = IRQ_TYPE_NONE; - girq->handler = handle_simple_irq; - - ret = gpiochip_add_data(&chip->gc, chip); - if (ret) - return ret; - - platform_set_drvdata(pdev, chip); - dev_info(dev, "ZX GPIO chip registered\n"); - - return 0; -} - -static const struct of_device_id zx_gpio_match[] = { - { - .compatible = "zte,zx296702-gpio", - }, - { }, -}; - -static struct platform_driver zx_gpio_driver = { - .probe = zx_gpio_probe, - .driver = { - .name = "zx_gpio", - .of_match_table = of_match_ptr(zx_gpio_match), - }, -}; -builtin_platform_driver(zx_gpio_driver) diff --git a/drivers/gpio/gpiolib-of.c b/drivers/gpio/gpiolib-of.c index b4a71119a4b07c24d24f3329678d9940609fb543..baf0153b7bca3f150af5b129453b5f6e284f6d2c 100644 --- a/drivers/gpio/gpiolib-of.c +++ b/drivers/gpio/gpiolib-of.c @@ -1039,3 +1039,14 @@ void of_gpiochip_remove(struct gpio_chip *chip) { of_node_put(chip->of_node); } + +void of_gpio_dev_init(struct gpio_chip *gc, struct gpio_device *gdev) +{ + /* If the gpiochip has an assigned OF node this takes precedence */ + if (gc->of_node) + gdev->dev.of_node = gc->of_node; + else + gc->of_node = gdev->dev.of_node; + if (gdev->dev.of_node) + gdev->dev.fwnode = of_fwnode_handle(gdev->dev.of_node); +} diff --git a/drivers/gpio/gpiolib-of.h b/drivers/gpio/gpiolib-of.h index ed26664f153782fc6670a242b4fb60a6a74b1364..8af2bc899aab251082f11441beb1618183a4b1d4 100644 --- a/drivers/gpio/gpiolib-of.h +++ b/drivers/gpio/gpiolib-of.h @@ -15,6 +15,7 @@ int of_gpiochip_add(struct gpio_chip *gc); void of_gpiochip_remove(struct gpio_chip *gc); int of_gpio_get_count(struct device *dev, const char *con_id); bool of_gpio_need_valid_mask(const struct gpio_chip *gc); +void of_gpio_dev_init(struct gpio_chip *gc, struct gpio_device *gdev); #else static inline struct gpio_desc *of_find_gpio(struct device *dev, const char *con_id, @@ -33,6 +34,10 @@ static inline bool of_gpio_need_valid_mask(const struct gpio_chip *gc) { return false; } +static inline void of_gpio_dev_init(struct gpio_chip *gc, + struct gpio_device *gdev) +{ +} #endif /* CONFIG_OF_GPIO */ extern struct notifier_block gpio_of_notifier; diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c index 97eec8d8dbdc42bb8b3100d1c338c5811f264396..adf55db080d86f1d3c07832972e2a468e53380e3 100644 --- a/drivers/gpio/gpiolib.c +++ b/drivers/gpio/gpiolib.c @@ -56,8 +56,10 @@ static DEFINE_IDA(gpio_ida); static dev_t gpio_devt; #define GPIO_DEV_MAX 256 /* 256 GPIO chip devices supported */ +static int gpio_bus_match(struct device *dev, struct device_driver *drv); static struct bus_type gpio_bus_type = { .name = "gpio", + .match = gpio_bus_match, }; /* @@ -590,13 +592,7 @@ int gpiochip_add_data_with_key(struct gpio_chip *gc, void *data, gdev->dev.of_node = gc->parent->of_node; } -#ifdef CONFIG_OF_GPIO - /* If the gpiochip has an assigned OF node this takes precedence */ - if (gc->of_node) - gdev->dev.of_node = gc->of_node; - else - gc->of_node = gdev->dev.of_node; -#endif + of_gpio_dev_init(gc, gdev); gdev->id = ida_alloc(&gpio_ida, GFP_KERNEL); if (gdev->id < 0) { @@ -3469,6 +3465,10 @@ EXPORT_SYMBOL_GPL(gpiod_add_lookup_table); */ void gpiod_remove_lookup_table(struct gpiod_lookup_table *table) { + /* Nothing to remove */ + if (!table) + return; + mutex_lock(&gpio_lookup_lock); list_del(&table->list); @@ -4211,6 +4211,41 @@ void gpiod_put_array(struct gpio_descs *descs) } EXPORT_SYMBOL_GPL(gpiod_put_array); + +static int gpio_bus_match(struct device *dev, struct device_driver *drv) +{ + /* + * Only match if the fwnode doesn't already have a proper struct device + * created for it. + */ + if (dev->fwnode && dev->fwnode->dev != dev) + return 0; + return 1; +} + +static int gpio_stub_drv_probe(struct device *dev) +{ + /* + * The DT node of some GPIO chips have a "compatible" property, but + * never have a struct device added and probed by a driver to register + * the GPIO chip with gpiolib. In such cases, fw_devlink=on will cause + * the consumers of the GPIO chip to get probe deferred forever because + * they will be waiting for a device associated with the GPIO chip + * firmware node to get added and bound to a driver. + * + * To allow these consumers to probe, we associate the struct + * gpio_device of the GPIO chip with the firmware node and then simply + * bind it to this stub driver. + */ + return 0; +} + +static struct device_driver gpio_stub_drv = { + .name = "gpio_stub_drv", + .bus = &gpio_bus_type, + .probe = gpio_stub_drv_probe, +}; + static int __init gpiolib_dev_init(void) { int ret; @@ -4222,9 +4257,16 @@ static int __init gpiolib_dev_init(void) return ret; } + if (driver_register(&gpio_stub_drv) < 0) { + pr_err("gpiolib: could not register GPIO stub driver\n"); + bus_unregister(&gpio_bus_type); + return ret; + } + ret = alloc_chrdev_region(&gpio_devt, 0, GPIO_DEV_MAX, GPIOCHIP_NAME); if (ret < 0) { pr_err("gpiolib: failed to allocate char dev region\n"); + driver_unregister(&gpio_stub_drv); bus_unregister(&gpio_bus_type); return ret; } diff --git a/drivers/gpu/drm/Kconfig b/drivers/gpu/drm/Kconfig index 0973f408d75fe388bc34e3400c8684ce75fd7bfc..e392a90ca6876cc3dcec88baacb3bc4099bbac6c 100644 --- a/drivers/gpu/drm/Kconfig +++ b/drivers/gpu/drm/Kconfig @@ -15,6 +15,9 @@ menuconfig DRM select I2C_ALGOBIT select DMA_SHARED_BUFFER select SYNC_FILE +# gallium uses SYS_kcmp for os_same_file_description() to de-duplicate +# device and dmabuf fd. Let's make sure that is available for our userspace. + select KCMP help Kernel-level support for the Direct Rendering Infrastructure (DRI) introduced in XFree86 4.0. If you say Y here, you need to select @@ -214,10 +217,6 @@ config DRM_GEM_SHMEM_HELPER help Choose this if you need the GEM shmem helper functions -config DRM_VM - bool - depends on DRM && MMU - config DRM_SCHED tristate depends on DRM @@ -391,7 +390,6 @@ source "drivers/gpu/drm/xlnx/Kconfig" menuconfig DRM_LEGACY bool "Enable legacy drivers (DANGEROUS)" depends on DRM && MMU - select DRM_VM help Enable legacy DRI1 drivers. Those drivers expose unsafe and dangerous APIs to user-space, which can be used to circumvent access diff --git a/drivers/gpu/drm/Makefile b/drivers/gpu/drm/Makefile index fefaff4c832d12e97e36e0fd0b881f4115c44a7a..926adef289db96d7c2d8c7c77f97acf81e6f799b 100644 --- a/drivers/gpu/drm/Makefile +++ b/drivers/gpu/drm/Makefile @@ -5,7 +5,7 @@ drm-y := drm_auth.o drm_cache.o \ drm_file.o drm_gem.o drm_ioctl.o drm_irq.o \ - drm_memory.o drm_drv.o \ + drm_drv.o \ drm_sysfs.o drm_hashtab.o drm_mm.o \ drm_crtc.o drm_fourcc.o drm_modes.o drm_edid.o \ drm_encoder_slave.o \ @@ -20,9 +20,9 @@ drm-y := drm_auth.o drm_cache.o \ drm_client_modeset.o drm_atomic_uapi.o drm_hdcp.o \ drm_managed.o drm_vblank_work.o -drm-$(CONFIG_DRM_LEGACY) += drm_legacy_misc.o drm_bufs.o drm_context.o drm_dma.o drm_scatter.o drm_lock.o +drm-$(CONFIG_DRM_LEGACY) += drm_bufs.o drm_context.o drm_dma.o drm_legacy_misc.o drm_lock.o \ + drm_memory.o drm_scatter.o drm_vm.o drm-$(CONFIG_DRM_LIB_RANDOM) += lib/drm_random.o -drm-$(CONFIG_DRM_VM) += drm_vm.o drm-$(CONFIG_COMPAT) += drm_ioc32.o drm-$(CONFIG_DRM_GEM_CMA_HELPER) += drm_gem_cma_helper.o drm-$(CONFIG_DRM_GEM_SHMEM_HELPER) += drm_gem_shmem_helper.o diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile b/drivers/gpu/drm/amd/amdgpu/Makefile index 6bf6cfaea3f19f88b25b0668c3af722b6e14ca70..13ebb1f71e49753b3ee0579502f6c32498e04e90 100644 --- a/drivers/gpu/drm/amd/amdgpu/Makefile +++ b/drivers/gpu/drm/amd/amdgpu/Makefile @@ -56,7 +56,7 @@ amdgpu-y += amdgpu_device.o amdgpu_kms.o \ amdgpu_gmc.o amdgpu_mmhub.o amdgpu_xgmi.o amdgpu_csa.o amdgpu_ras.o amdgpu_vm_cpu.o \ amdgpu_vm_sdma.o amdgpu_discovery.o amdgpu_ras_eeprom.o amdgpu_nbio.o \ amdgpu_umc.o smu_v11_0_i2c.o amdgpu_fru_eeprom.o amdgpu_rap.o \ - amdgpu_fw_attestation.o + amdgpu_fw_attestation.o amdgpu_securedisplay.o amdgpu-$(CONFIG_PERF_EVENTS) += amdgpu_pmu.o @@ -71,7 +71,7 @@ amdgpu-y += \ vi.o mxgpu_vi.o nbio_v6_1.o soc15.o emu_soc.o mxgpu_ai.o nbio_v7_0.o vega10_reg_init.o \ vega20_reg_init.o nbio_v7_4.o nbio_v2_3.o nv.o navi10_reg_init.o navi14_reg_init.o \ arct_reg_init.o navi12_reg_init.o mxgpu_nv.o sienna_cichlid_reg_init.o vangogh_reg_init.o \ - nbio_v7_2.o dimgrey_cavefish_reg_init.o + nbio_v7_2.o dimgrey_cavefish_reg_init.o hdp_v4_0.o hdp_v5_0.o # add DF block amdgpu-y += \ @@ -97,6 +97,7 @@ amdgpu-y += \ tonga_ih.o \ cz_ih.o \ vega10_ih.o \ + vega20_ih.o \ navi10_ih.o # add PSP block @@ -170,7 +171,8 @@ amdgpu-y += \ # add SMUIO block amdgpu-y += \ smuio_v9_0.o \ - smuio_v11_0.o + smuio_v11_0.o \ + smuio_v11_0_6.o # add amdkfd interfaces amdgpu-y += amdgpu_amdkfd.o diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h index 5993dd0fdd8e7d559fa5e6f730984ad7f3af7a4e..b6879d97c9c9c16fe055fb6c3cfee2f8491ab7fb 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h @@ -55,7 +55,6 @@ #include #include #include -#include #include #include @@ -89,6 +88,7 @@ #include "amdgpu_gfx.h" #include "amdgpu_sdma.h" #include "amdgpu_nbio.h" +#include "amdgpu_hdp.h" #include "amdgpu_dm.h" #include "amdgpu_virt.h" #include "amdgpu_csa.h" @@ -107,6 +107,7 @@ #include "amdgpu_gfxhub.h" #include "amdgpu_df.h" #include "amdgpu_smuio.h" +#include "amdgpu_hdp.h" #define MAX_GPU_INSTANCE 16 @@ -286,7 +287,7 @@ enum amdgpu_kiq_irq { #define MAX_KIQ_REG_WAIT 5000 /* in usecs, 5ms */ #define MAX_KIQ_REG_BAILOUT_INTERVAL 5 /* in msecs, 5ms */ -#define MAX_KIQ_REG_TRY 80 /* 20 -> 80 */ +#define MAX_KIQ_REG_TRY 1000 int amdgpu_device_ip_set_clockgating_state(void *dev, enum amd_ip_block_type block_type, @@ -578,7 +579,8 @@ enum amd_reset_method { AMD_RESET_METHOD_MODE0, AMD_RESET_METHOD_MODE1, AMD_RESET_METHOD_MODE2, - AMD_RESET_METHOD_BACO + AMD_RESET_METHOD_BACO, + AMD_RESET_METHOD_PCI, }; /* @@ -608,7 +610,6 @@ struct amdgpu_asic_funcs { /* invalidate hdp read cache */ void (*invalidate_hdp)(struct amdgpu_device *adev, struct amdgpu_ring *ring); - void (*reset_hdp_ras_error_count)(struct amdgpu_device *adev); /* check if the asic needs a full reset of if soft reset will work */ bool (*need_full_reset)(struct amdgpu_device *adev); /* initialize doorbell layout for specific asic*/ @@ -891,6 +892,7 @@ struct amdgpu_device { /* For pre-DCE11. DCE11 and later are in "struct amdgpu_device->dm" */ struct work_struct hotplug_work; struct amdgpu_irq_src crtc_irq; + struct amdgpu_irq_src vline0_irq; struct amdgpu_irq_src vupdate_irq; struct amdgpu_irq_src pageflip_irq; struct amdgpu_irq_src hpd_irq; @@ -921,6 +923,9 @@ struct amdgpu_device { /* nbio */ struct amdgpu_nbio nbio; + /* hdp */ + struct amdgpu_hdp hdp; + /* smuio */ struct amdgpu_smuio smuio; @@ -1003,6 +1008,12 @@ struct amdgpu_device { bool in_suspend; bool in_hibernate; + /* + * The combination flag in_poweroff_reboot_com used to identify the poweroff + * and reboot opt in the s0i3 system-wide suspend. + */ + bool in_poweroff_reboot_com; + atomic_t in_gpu_reset; enum pp_mp1_state mp1_state; struct rw_semaphore reset_sem; @@ -1202,8 +1213,10 @@ int emu_soc_asic_init(struct amdgpu_device *adev); #define amdgpu_asic_read_bios_from_rom(adev, b, l) (adev)->asic_funcs->read_bios_from_rom((adev), (b), (l)) #define amdgpu_asic_read_register(adev, se, sh, offset, v)((adev)->asic_funcs->read_register((adev), (se), (sh), (offset), (v))) #define amdgpu_asic_get_config_memsize(adev) (adev)->asic_funcs->get_config_memsize((adev)) -#define amdgpu_asic_flush_hdp(adev, r) (adev)->asic_funcs->flush_hdp((adev), (r)) -#define amdgpu_asic_invalidate_hdp(adev, r) (adev)->asic_funcs->invalidate_hdp((adev), (r)) +#define amdgpu_asic_flush_hdp(adev, r) \ + ((adev)->asic_funcs->flush_hdp ? (adev)->asic_funcs->flush_hdp((adev), (r)) : (adev)->hdp.funcs->flush_hdp((adev), (r))) +#define amdgpu_asic_invalidate_hdp(adev, r) \ + ((adev)->asic_funcs->invalidate_hdp ? (adev)->asic_funcs->invalidate_hdp((adev), (r)) : (adev)->hdp.funcs->invalidate_hdp((adev), (r))) #define amdgpu_asic_need_full_reset(adev) (adev)->asic_funcs->need_full_reset((adev)) #define amdgpu_asic_init_doorbell_index(adev) (adev)->asic_funcs->init_doorbell_index((adev)) #define amdgpu_asic_get_pcie_usage(adev, cnt0, cnt1) ((adev)->asic_funcs->get_pcie_usage((adev), (cnt0), (cnt1))) @@ -1222,6 +1235,7 @@ bool amdgpu_device_should_recover_gpu(struct amdgpu_device *adev); int amdgpu_device_gpu_recover(struct amdgpu_device *adev, struct amdgpu_job* job); void amdgpu_device_pci_config_reset(struct amdgpu_device *adev); +int amdgpu_device_pci_reset(struct amdgpu_device *adev); bool amdgpu_device_need_post(struct amdgpu_device *adev); void amdgpu_cs_report_moved_bytes(struct amdgpu_device *adev, u64 num_bytes, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c index db96d69eb45e19e730a67a21af202bccc6c522cb..c5343a5eecbeb6ac11ca5dd31b766abec0080f63 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c @@ -47,12 +47,8 @@ int amdgpu_amdkfd_init(void) amdgpu_amdkfd_total_mem_size = si.totalram - si.totalhigh; amdgpu_amdkfd_total_mem_size *= si.mem_unit; -#ifdef CONFIG_HSA_AMD ret = kgd2kfd_init(); amdgpu_amdkfd_gpuvm_init_mem_limits(); -#else - ret = -ENOENT; -#endif kfd_initialized = !ret; return ret; @@ -696,86 +692,3 @@ bool amdgpu_amdkfd_have_atomics_support(struct kgd_dev *kgd) return adev->have_atomics_support; } - -#ifndef CONFIG_HSA_AMD -bool amdkfd_fence_check_mm(struct dma_fence *f, struct mm_struct *mm) -{ - return false; -} - -void amdgpu_amdkfd_unreserve_memory_limit(struct amdgpu_bo *bo) -{ -} - -int amdgpu_amdkfd_remove_fence_on_pt_pd_bos(struct amdgpu_bo *bo) -{ - return 0; -} - -void amdgpu_amdkfd_gpuvm_destroy_cb(struct amdgpu_device *adev, - struct amdgpu_vm *vm) -{ -} - -struct amdgpu_amdkfd_fence *to_amdgpu_amdkfd_fence(struct dma_fence *f) -{ - return NULL; -} - -int amdgpu_amdkfd_evict_userptr(struct kgd_mem *mem, struct mm_struct *mm) -{ - return 0; -} - -struct kfd_dev *kgd2kfd_probe(struct kgd_dev *kgd, struct pci_dev *pdev, - unsigned int asic_type, bool vf) -{ - return NULL; -} - -bool kgd2kfd_device_init(struct kfd_dev *kfd, - struct drm_device *ddev, - const struct kgd2kfd_shared_resources *gpu_resources) -{ - return false; -} - -void kgd2kfd_device_exit(struct kfd_dev *kfd) -{ -} - -void kgd2kfd_exit(void) -{ -} - -void kgd2kfd_suspend(struct kfd_dev *kfd, bool run_pm) -{ -} - -int kgd2kfd_resume(struct kfd_dev *kfd, bool run_pm) -{ - return 0; -} - -int kgd2kfd_pre_reset(struct kfd_dev *kfd) -{ - return 0; -} - -int kgd2kfd_post_reset(struct kfd_dev *kfd) -{ - return 0; -} - -void kgd2kfd_interrupt(struct kfd_dev *kfd, const void *ih_ring_entry) -{ -} - -void kgd2kfd_set_sram_ecc_flag(struct kfd_dev *kfd) -{ -} - -void kgd2kfd_smi_event_throttle(struct kfd_dev *kfd, uint32_t throttle_bitmask) -{ -} -#endif diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h index ea391ca7f2f1cfb5ded5015b8694b7b6edbc9a5a..a81d9cacf9b8fc31f324972904d1fa7ab49b0b80 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h @@ -94,11 +94,6 @@ enum kgd_engine_type { KGD_ENGINE_MAX }; -struct amdgpu_amdkfd_fence *amdgpu_amdkfd_fence_create(u64 context, - struct mm_struct *mm); -bool amdkfd_fence_check_mm(struct dma_fence *f, struct mm_struct *mm); -struct amdgpu_amdkfd_fence *to_amdgpu_amdkfd_fence(struct dma_fence *f); -int amdgpu_amdkfd_remove_fence_on_pt_pd_bos(struct amdgpu_bo *bo); struct amdkfd_process_info { /* List head of all VMs that belong to a KFD process */ @@ -132,8 +127,6 @@ void amdgpu_amdkfd_interrupt(struct amdgpu_device *adev, void amdgpu_amdkfd_device_probe(struct amdgpu_device *adev); void amdgpu_amdkfd_device_init(struct amdgpu_device *adev); void amdgpu_amdkfd_device_fini(struct amdgpu_device *adev); - -int amdgpu_amdkfd_evict_userptr(struct kgd_mem *mem, struct mm_struct *mm); int amdgpu_amdkfd_submit_ib(struct kgd_dev *kgd, enum kgd_engine_type engine, uint32_t vmid, uint64_t gpu_addr, uint32_t *ib_cmd, uint32_t ib_len); @@ -153,6 +146,38 @@ void amdgpu_amdkfd_gpu_reset(struct kgd_dev *kgd); int amdgpu_queue_mask_bit_to_set_resource_bit(struct amdgpu_device *adev, int queue_bit); +struct amdgpu_amdkfd_fence *amdgpu_amdkfd_fence_create(u64 context, + struct mm_struct *mm); +#if IS_ENABLED(CONFIG_HSA_AMD) +bool amdkfd_fence_check_mm(struct dma_fence *f, struct mm_struct *mm); +struct amdgpu_amdkfd_fence *to_amdgpu_amdkfd_fence(struct dma_fence *f); +int amdgpu_amdkfd_remove_fence_on_pt_pd_bos(struct amdgpu_bo *bo); +int amdgpu_amdkfd_evict_userptr(struct kgd_mem *mem, struct mm_struct *mm); +#else +static inline +bool amdkfd_fence_check_mm(struct dma_fence *f, struct mm_struct *mm) +{ + return false; +} + +static inline +struct amdgpu_amdkfd_fence *to_amdgpu_amdkfd_fence(struct dma_fence *f) +{ + return NULL; +} + +static inline +int amdgpu_amdkfd_remove_fence_on_pt_pd_bos(struct amdgpu_bo *bo) +{ + return 0; +} + +static inline +int amdgpu_amdkfd_evict_userptr(struct kgd_mem *mem, struct mm_struct *mm) +{ + return 0; +} +#endif /* Shared API */ int amdgpu_amdkfd_alloc_gtt_mem(struct kgd_dev *kgd, size_t size, void **mem_obj, uint64_t *gpu_addr, @@ -215,8 +240,6 @@ int amdgpu_amdkfd_gpuvm_acquire_process_vm(struct kgd_dev *kgd, struct file *filp, u32 pasid, void **vm, void **process_info, struct dma_fence **ef); -void amdgpu_amdkfd_gpuvm_destroy_cb(struct amdgpu_device *adev, - struct amdgpu_vm *vm); void amdgpu_amdkfd_gpuvm_destroy_process_vm(struct kgd_dev *kgd, void *vm); void amdgpu_amdkfd_gpuvm_release_process_vm(struct kgd_dev *kgd, void *vm); uint64_t amdgpu_amdkfd_gpuvm_get_process_page_dir(void *vm); @@ -236,23 +259,43 @@ int amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(struct kgd_dev *kgd, struct kgd_mem *mem, void **kptr, uint64_t *size); int amdgpu_amdkfd_gpuvm_restore_process_bos(void *process_info, struct dma_fence **ef); - int amdgpu_amdkfd_gpuvm_get_vm_fault_info(struct kgd_dev *kgd, struct kfd_vm_fault_info *info); - int amdgpu_amdkfd_gpuvm_import_dmabuf(struct kgd_dev *kgd, struct dma_buf *dmabuf, uint64_t va, void *vm, struct kgd_mem **mem, uint64_t *size, uint64_t *mmap_offset); - -void amdgpu_amdkfd_gpuvm_init_mem_limits(void); -void amdgpu_amdkfd_unreserve_memory_limit(struct amdgpu_bo *bo); - int amdgpu_amdkfd_get_tile_config(struct kgd_dev *kgd, struct tile_config *config); +#if IS_ENABLED(CONFIG_HSA_AMD) +void amdgpu_amdkfd_gpuvm_init_mem_limits(void); +void amdgpu_amdkfd_gpuvm_destroy_cb(struct amdgpu_device *adev, + struct amdgpu_vm *vm); +void amdgpu_amdkfd_unreserve_memory_limit(struct amdgpu_bo *bo); +#else +static inline +void amdgpu_amdkfd_gpuvm_init_mem_limits(void) +{ +} +static inline +void amdgpu_amdkfd_gpuvm_destroy_cb(struct amdgpu_device *adev, + struct amdgpu_vm *vm) +{ +} + +static inline +void amdgpu_amdkfd_unreserve_memory_limit(struct amdgpu_bo *bo) +{ +} +#endif /* KGD2KFD callbacks */ +int kgd2kfd_quiesce_mm(struct mm_struct *mm); +int kgd2kfd_resume_mm(struct mm_struct *mm); +int kgd2kfd_schedule_evict_and_restore_process(struct mm_struct *mm, + struct dma_fence *fence); +#if IS_ENABLED(CONFIG_HSA_AMD) int kgd2kfd_init(void); void kgd2kfd_exit(void); struct kfd_dev *kgd2kfd_probe(struct kgd_dev *kgd, struct pci_dev *pdev, @@ -266,11 +309,68 @@ int kgd2kfd_resume(struct kfd_dev *kfd, bool run_pm); int kgd2kfd_pre_reset(struct kfd_dev *kfd); int kgd2kfd_post_reset(struct kfd_dev *kfd); void kgd2kfd_interrupt(struct kfd_dev *kfd, const void *ih_ring_entry); -int kgd2kfd_quiesce_mm(struct mm_struct *mm); -int kgd2kfd_resume_mm(struct mm_struct *mm); -int kgd2kfd_schedule_evict_and_restore_process(struct mm_struct *mm, - struct dma_fence *fence); void kgd2kfd_set_sram_ecc_flag(struct kfd_dev *kfd); void kgd2kfd_smi_event_throttle(struct kfd_dev *kfd, uint32_t throttle_bitmask); +#else +static inline int kgd2kfd_init(void) +{ + return -ENOENT; +} +static inline void kgd2kfd_exit(void) +{ +} + +static inline +struct kfd_dev *kgd2kfd_probe(struct kgd_dev *kgd, struct pci_dev *pdev, + unsigned int asic_type, bool vf) +{ + return NULL; +} + +static inline +bool kgd2kfd_device_init(struct kfd_dev *kfd, struct drm_device *ddev, + const struct kgd2kfd_shared_resources *gpu_resources) +{ + return false; +} + +static inline void kgd2kfd_device_exit(struct kfd_dev *kfd) +{ +} + +static inline void kgd2kfd_suspend(struct kfd_dev *kfd, bool run_pm) +{ +} + +static inline int kgd2kfd_resume(struct kfd_dev *kfd, bool run_pm) +{ + return 0; +} + +static inline int kgd2kfd_pre_reset(struct kfd_dev *kfd) +{ + return 0; +} + +static inline int kgd2kfd_post_reset(struct kfd_dev *kfd) +{ + return 0; +} + +static inline +void kgd2kfd_interrupt(struct kfd_dev *kfd, const void *ih_ring_entry) +{ +} + +static inline +void kgd2kfd_set_sram_ecc_flag(struct kfd_dev *kfd) +{ +} + +static inline +void kgd2kfd_smi_event_throttle(struct kfd_dev *kfd, uint32_t throttle_bitmask) +{ +} +#endif #endif /* AMDGPU_AMDKFD_H_INCLUDED */ diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.c index 4763bab7a4d0dfe309752f1a4e3dafc9922efc58..62aa1a6f64edd65b563a2663282c10b39e72e2b7 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.c @@ -23,7 +23,6 @@ #include "amdgpu_amdkfd.h" #include "gc/gc_10_1_0_offset.h" #include "gc/gc_10_1_0_sh_mask.h" -#include "navi10_enum.h" #include "athub/athub_2_0_0_offset.h" #include "athub/athub_2_0_0_sh_mask.h" #include "oss/osssys_5_0_0_offset.h" diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10_3.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10_3.c index 50016bf9c4279b5d06999ac18c16eb5b0ad66461..fad3b91f74f54972a4aee067bc921c5ec4b5befb 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10_3.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10_3.c @@ -24,7 +24,6 @@ #include "amdgpu_amdkfd.h" #include "gc/gc_10_3_0_offset.h" #include "gc/gc_10_3_0_sh_mask.h" -#include "navi10_enum.h" #include "oss/osssys_5_0_0_offset.h" #include "oss/osssys_5_0_0_sh_mask.h" #include "soc15_common.h" diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c index d1ed4f8df2b76d7dd41286b52426dfbde175f33f..ac0a432a9bf79fc6878e616c1f93f7c05d542705 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c @@ -454,7 +454,7 @@ static int add_bo_to_vm(struct amdgpu_device *adev, struct kgd_mem *mem, struct amdgpu_bo *bo = mem->bo; uint64_t va = mem->va; struct list_head *list_bo_va = &mem->bo_va_list; - unsigned long bo_size = bo->tbo.mem.size; + unsigned long bo_size = bo->tbo.base.size; if (!va) { pr_err("Invalid VA when adding BO to VM\n"); @@ -1277,7 +1277,7 @@ int amdgpu_amdkfd_gpuvm_free_memory_of_gpu( struct kgd_dev *kgd, struct kgd_mem *mem, uint64_t *size) { struct amdkfd_process_info *process_info = mem->process_info; - unsigned long bo_size = mem->bo->tbo.mem.size; + unsigned long bo_size = mem->bo->tbo.base.size; struct kfd_bo_va_list *entry, *tmp; struct bo_vm_reservation_context ctx; struct ttm_validate_buffer *bo_list_entry; @@ -1398,7 +1398,7 @@ int amdgpu_amdkfd_gpuvm_map_memory_to_gpu( mutex_lock(&mem->lock); domain = mem->domain; - bo_size = bo->tbo.mem.size; + bo_size = bo->tbo.base.size; pr_debug("Map VA 0x%llx - 0x%llx to vm %p domain %s\n", mem->va, @@ -1502,7 +1502,7 @@ int amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu( struct amdgpu_device *adev = get_amdgpu_device(kgd); struct amdkfd_process_info *process_info = ((struct amdgpu_vm *)vm)->process_info; - unsigned long bo_size = mem->bo->tbo.mem.size; + unsigned long bo_size = mem->bo->tbo.base.size; struct kfd_bo_va_list *entry; struct bo_vm_reservation_context ctx; int ret; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c index 055f600eeed87f015893e7ac2976b5747f6eb0f4..cfb1a9a044772c828e80b8b80916330033ed61a2 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c @@ -155,7 +155,7 @@ static bool amdgpu_read_bios_from_rom(struct amdgpu_device *adev) u8 header[AMD_VBIOS_SIGNATURE_END+1] = {0}; int len; - if (!adev->asic_funcs->read_bios_from_rom) + if (!adev->asic_funcs || !adev->asic_funcs->read_bios_from_rom) return false; /* validate VBIOS signature */ @@ -348,7 +348,8 @@ static bool amdgpu_read_disabled_bios(struct amdgpu_device *adev) if (adev->flags & AMD_IS_APU) return igp_read_bios_from_vram(adev); else - return amdgpu_asic_read_disabled_bios(adev); + return (!adev->asic_funcs || !adev->asic_funcs->read_disabled_bios) ? + false : amdgpu_asic_read_disabled_bios(adev); } #ifdef CONFIG_ACPI diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c index 594a0108e90fa4e5278220eda33bdcd83311b1ef..3e240b952e7912a36365bdf7d2191e981b08598d 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c @@ -98,8 +98,7 @@ static int amdgpu_cs_bo_handles_chunk(struct amdgpu_cs_parser *p, return 0; error_free: - if (info) - kvfree(info); + kvfree(info); return r; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c index a6667a2ca0db3997f9a37de68d3a6fac19048824..0a25fecf488aba6763798bb50292ae334b994080 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c @@ -35,6 +35,7 @@ #include "amdgpu_dm_debugfs.h" #include "amdgpu_ras.h" #include "amdgpu_rap.h" +#include "amdgpu_securedisplay.h" #include "amdgpu_fw_attestation.h" /** @@ -1427,7 +1428,7 @@ static void amdgpu_ib_preempt_job_recovery(struct drm_gpu_scheduler *sched) struct dma_fence *fence; spin_lock(&sched->job_list_lock); - list_for_each_entry(s_job, &sched->ring_mirror_list, node) { + list_for_each_entry(s_job, &sched->pending_list, list) { fence = sched->ops->run_job(s_job); dma_fence_put(fence); } @@ -1459,10 +1460,10 @@ static void amdgpu_ib_preempt_mark_partial_job(struct amdgpu_ring *ring) no_preempt: spin_lock(&sched->job_list_lock); - list_for_each_entry_safe(s_job, tmp, &sched->ring_mirror_list, node) { + list_for_each_entry_safe(s_job, tmp, &sched->pending_list, list) { if (dma_fence_is_signaled(&s_job->s_fence->finished)) { /* remove job from ring_mirror_list */ - list_del_init(&s_job->node); + list_del_init(&s_job->list); sched->ops->free_job(s_job); continue; } @@ -1669,6 +1670,8 @@ int amdgpu_debugfs_init(struct amdgpu_device *adev) amdgpu_rap_debugfs_init(adev); + amdgpu_securedisplay_debugfs_init(adev); + amdgpu_fw_attestation_debugfs_init(adev); return amdgpu_debugfs_add_files(adev, amdgpu_debugfs_list, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c index cab1ebaf6d6299a365c272b90055cd8e847104fb..6447cd6ca5a8eb54841e2c9999c33a1437b02408 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c @@ -929,6 +929,18 @@ void amdgpu_device_pci_config_reset(struct amdgpu_device *adev) pci_write_config_dword(adev->pdev, 0x7c, AMDGPU_ASIC_RESET_DATA); } +/** + * amdgpu_device_pci_reset - reset the GPU using generic PCI means + * + * @adev: amdgpu_device pointer + * + * Resets the GPU using generic pci reset interfaces (FLR, SBR, etc.). + */ +int amdgpu_device_pci_reset(struct amdgpu_device *adev) +{ + return pci_reset_function(adev->pdev); +} + /* * GPU doorbell aperture helpers function. */ @@ -1105,8 +1117,7 @@ void amdgpu_device_wb_free(struct amdgpu_device *adev, u32 wb) */ int amdgpu_device_resize_fb_bar(struct amdgpu_device *adev) { - u64 space_needed = roundup_pow_of_two(adev->gmc.real_vram_size); - u32 rbar_size = order_base_2(((space_needed >> 20) | 1)) - 1; + int rbar_size = pci_rebar_bytes_to_size(adev->gmc.real_vram_size); struct pci_bus *root; struct resource *res; unsigned i; @@ -1137,6 +1148,10 @@ int amdgpu_device_resize_fb_bar(struct amdgpu_device *adev) if (!res) return 0; + /* Limit the BAR size to what is available */ + rbar_size = min(fls(pci_rebar_get_possible_sizes(adev->pdev, 0)) - 1, + rbar_size); + /* Disable memory decoding while we change the BAR addresses and size */ pci_read_config_word(adev->pdev, PCI_COMMAND, &cmd); pci_write_config_word(adev->pdev, PCI_COMMAND, @@ -1422,24 +1437,22 @@ static void amdgpu_switcheroo_set_state(struct pci_dev *pdev, /* don't suspend or resume card normally */ dev->switch_power_state = DRM_SWITCH_POWER_CHANGING; - pci_set_power_state(dev->pdev, PCI_D0); - amdgpu_device_load_pci_state(dev->pdev); - r = pci_enable_device(dev->pdev); + pci_set_power_state(pdev, PCI_D0); + amdgpu_device_load_pci_state(pdev); + r = pci_enable_device(pdev); if (r) DRM_WARN("pci_enable_device failed (%d)\n", r); amdgpu_device_resume(dev, true); dev->switch_power_state = DRM_SWITCH_POWER_ON; - drm_kms_helper_poll_enable(dev); } else { pr_info("switched off\n"); - drm_kms_helper_poll_disable(dev); dev->switch_power_state = DRM_SWITCH_POWER_CHANGING; amdgpu_device_suspend(dev, true); - amdgpu_device_cache_pci_state(dev->pdev); + amdgpu_device_cache_pci_state(pdev); /* Shut down the device */ - pci_disable_device(dev->pdev); - pci_set_power_state(dev->pdev, PCI_D3cold); + pci_disable_device(pdev); + pci_set_power_state(pdev, PCI_D3cold); dev->switch_power_state = DRM_SWITCH_POWER_OFF; } } @@ -1702,8 +1715,7 @@ static void amdgpu_device_enable_virtual_display(struct amdgpu_device *adev) adev->enable_virtual_display = false; if (amdgpu_virtual_display) { - struct drm_device *ddev = adev_to_drm(adev); - const char *pci_address_name = pci_name(ddev->pdev); + const char *pci_address_name = pci_name(adev->pdev); char *pciaddstr, *pciaddstr_tmp, *pciaddname_tmp, *pciaddname; pciaddstr = kstrdup(amdgpu_virtual_display, GFP_KERNEL); @@ -2666,7 +2678,8 @@ static int amdgpu_device_ip_suspend_phase1(struct amdgpu_device *adev) { int i, r; - if (!amdgpu_acpi_is_s0ix_supported(adev) || amdgpu_in_reset(adev)) { + if (adev->in_poweroff_reboot_com || + !amdgpu_acpi_is_s0ix_supported(adev) || amdgpu_in_reset(adev)) { amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE); amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE); } @@ -3116,7 +3129,10 @@ static int amdgpu_device_get_job_timeout_settings(struct amdgpu_device *adev) */ adev->gfx_timeout = msecs_to_jiffies(10000); adev->sdma_timeout = adev->video_timeout = adev->gfx_timeout; - if (amdgpu_sriov_vf(adev) || amdgpu_passthrough(adev)) + if (amdgpu_sriov_vf(adev)) + adev->compute_timeout = amdgpu_sriov_is_pp_one_vf(adev) ? + msecs_to_jiffies(60000) : msecs_to_jiffies(10000); + else if (amdgpu_passthrough(adev)) adev->compute_timeout = msecs_to_jiffies(60000); else adev->compute_timeout = MAX_SCHEDULE_TIMEOUT; @@ -3396,7 +3412,7 @@ int amdgpu_device_init(struct amdgpu_device *adev, } } - pci_enable_pcie_error_reporting(adev->ddev.pdev); + pci_enable_pcie_error_reporting(adev->pdev); /* Post card if necessary */ if (amdgpu_device_need_post(adev)) { @@ -3719,14 +3735,15 @@ int amdgpu_device_suspend(struct drm_device *dev, bool fbcon) r = amdgpu_device_ip_suspend_phase1(adev); - amdgpu_amdkfd_suspend(adev, !fbcon); + amdgpu_amdkfd_suspend(adev, adev->in_runpm); /* evict vram memory */ amdgpu_bo_evict_vram(adev); amdgpu_fence_driver_suspend(adev); - if (!amdgpu_acpi_is_s0ix_supported(adev) || amdgpu_in_reset(adev)) + if (adev->in_poweroff_reboot_com || + !amdgpu_acpi_is_s0ix_supported(adev) || amdgpu_in_reset(adev)) r = amdgpu_device_ip_suspend_phase2(adev); else amdgpu_gfx_state_change_set(adev, sGpuChangeState_D3Entry); @@ -3803,7 +3820,7 @@ int amdgpu_device_resume(struct drm_device *dev, bool fbcon) } } } - r = amdgpu_amdkfd_resume(adev, !fbcon); + r = amdgpu_amdkfd_resume(adev, adev->in_runpm); if (r) return r; @@ -4154,8 +4171,8 @@ bool amdgpu_device_has_job_running(struct amdgpu_device *adev) continue; spin_lock(&ring->sched.job_list_lock); - job = list_first_entry_or_null(&ring->sched.ring_mirror_list, - struct drm_sched_job, node); + job = list_first_entry_or_null(&ring->sched.pending_list, + struct drm_sched_job, list); spin_unlock(&ring->sched.job_list_lock); if (job) return true; @@ -4205,6 +4222,8 @@ bool amdgpu_device_should_recover_gpu(struct amdgpu_device *adev) case CHIP_NAVI14: case CHIP_NAVI12: case CHIP_SIENNA_CICHLID: + case CHIP_NAVY_FLOUNDER: + case CHIP_DIMGREY_CAVEFISH: break; default: goto disabled; @@ -4454,6 +4473,46 @@ static void amdgpu_device_unlock_adev(struct amdgpu_device *adev) up_write(&adev->reset_sem); } +/* + * to lockup a list of amdgpu devices in a hive safely, if not a hive + * with multiple nodes, it will be similar as amdgpu_device_lock_adev. + * + * unlock won't require roll back. + */ +static int amdgpu_device_lock_hive_adev(struct amdgpu_device *adev, struct amdgpu_hive_info *hive) +{ + struct amdgpu_device *tmp_adev = NULL; + + if (adev->gmc.xgmi.num_physical_nodes > 1) { + if (!hive) { + dev_err(adev->dev, "Hive is NULL while device has multiple xgmi nodes"); + return -ENODEV; + } + list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) { + if (!amdgpu_device_lock_adev(tmp_adev, hive)) + goto roll_back; + } + } else if (!amdgpu_device_lock_adev(adev, hive)) + return -EAGAIN; + + return 0; +roll_back: + if (!list_is_first(&tmp_adev->gmc.xgmi.head, &hive->device_list)) { + /* + * if the lockup iteration break in the middle of a hive, + * it may means there may has a race issue, + * or a hive device locked up independently. + * we may be in trouble and may not, so will try to roll back + * the lock and give out a warnning. + */ + dev_warn(tmp_adev->dev, "Hive lock iteration broke in the middle. Rolling back to unlock"); + list_for_each_entry_continue_reverse(tmp_adev, &hive->device_list, gmc.xgmi.head) { + amdgpu_device_unlock_adev(tmp_adev); + } + } + return -EAGAIN; +} + static void amdgpu_device_resume_display_audio(struct amdgpu_device *adev) { struct pci_dev *p = NULL; @@ -4567,11 +4626,29 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev, DRM_INFO("Bailing on TDR for s_job:%llx, hive: %llx as another already in progress", job ? job->base.id : -1, hive->hive_id); amdgpu_put_xgmi_hive(hive); + if (job) + drm_sched_increase_karma(&job->base); return 0; } mutex_lock(&hive->hive_lock); } + /* + * lock the device before we try to operate the linked list + * if didn't get the device lock, don't touch the linked list since + * others may iterating it. + */ + r = amdgpu_device_lock_hive_adev(adev, hive); + if (r) { + dev_info(adev->dev, "Bailing on TDR for s_job:%llx, as another already in progress", + job ? job->base.id : -1); + + /* even we skipped this reset, still need to set the job to guilty */ + if (job) + drm_sched_increase_karma(&job->base); + goto skip_recovery; + } + /* * Build list of devices to reset. * In case we are in XGMI hive mode, resort the device list @@ -4579,8 +4656,6 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev, */ INIT_LIST_HEAD(&device_list); if (adev->gmc.xgmi.num_physical_nodes > 1) { - if (!hive) - return -ENODEV; if (!list_is_first(&adev->gmc.xgmi.head, &hive->device_list)) list_rotate_to_front(&adev->gmc.xgmi.head, &hive->device_list); device_list_handle = &hive->device_list; @@ -4591,13 +4666,6 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev, /* block all schedulers and reset given job's ring */ list_for_each_entry(tmp_adev, device_list_handle, gmc.xgmi.head) { - if (!amdgpu_device_lock_adev(tmp_adev, hive)) { - dev_info(tmp_adev->dev, "Bailing on TDR for s_job:%llx, as another already in progress", - job ? job->base.id : -1); - r = 0; - goto skip_recovery; - } - /* * Try to put the audio codec into suspend state * before gpu reset started. @@ -4735,7 +4803,7 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev, amdgpu_put_xgmi_hive(hive); } - if (r) + if (r && r != -EAGAIN) dev_info(adev->dev, "GPU reset end with ret = %d\n", r); return r; } @@ -4785,7 +4853,13 @@ static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev) CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 | CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3); } else { - if (speed_cap == PCIE_SPEED_16_0GT) + if (speed_cap == PCIE_SPEED_32_0GT) + adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 | + CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 | + CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3 | + CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN4 | + CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN5); + else if (speed_cap == PCIE_SPEED_16_0GT) adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 | CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 | CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3 | @@ -4805,7 +4879,13 @@ static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev) adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 | CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2); } else { - if (platform_speed_cap == PCIE_SPEED_16_0GT) + if (platform_speed_cap == PCIE_SPEED_32_0GT) + adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 | + CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 | + CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3 | + CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4 | + CAIL_PCIE_LINK_SPEED_SUPPORT_GEN5); + else if (platform_speed_cap == PCIE_SPEED_16_0GT) adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 | CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 | CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3 | @@ -4949,8 +5029,8 @@ pci_ers_result_t amdgpu_pci_error_detected(struct pci_dev *pdev, pci_channel_sta case pci_channel_io_normal: return PCI_ERS_RESULT_CAN_RECOVER; /* Fatal error, prepare for slot reset */ - case pci_channel_io_frozen: - /* + case pci_channel_io_frozen: + /* * Cancel and wait for all TDRs in progress if failing to * set adev->in_gpu_reset in amdgpu_device_lock_adev * @@ -5041,7 +5121,7 @@ pci_ers_result_t amdgpu_pci_slot_reset(struct pci_dev *pdev) goto out; } - adev->in_pci_err_recovery = true; + adev->in_pci_err_recovery = true; r = amdgpu_device_pre_asic_reset(adev, NULL, &need_full_reset); adev->in_pci_err_recovery = false; if (r) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c index e42175e1acf18b758c7ab64077c6f2496f0e4550..47e0b48dc26fd72f0a5e4869a3b0ba677e6db2da 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c @@ -40,6 +40,7 @@ #include #include #include +#include /** * amdgpu_gem_prime_mmap - &drm_driver.gem_prime_mmap implementation @@ -151,9 +152,13 @@ static int amdgpu_dma_buf_attach(struct dma_buf *dmabuf, if (attach->dev->driver == adev->dev->driver) return 0; + r = pm_runtime_get_sync(adev_to_drm(adev)->dev); + if (r < 0) + goto out; + r = amdgpu_bo_reserve(bo, false); if (unlikely(r != 0)) - return r; + goto out; /* * We only create shared fences for internal use, but importers @@ -165,11 +170,15 @@ static int amdgpu_dma_buf_attach(struct dma_buf *dmabuf, */ r = __dma_resv_make_exclusive(bo->tbo.base.resv); if (r) - return r; + goto out; bo->prime_shared_count++; amdgpu_bo_unreserve(bo); return 0; + +out: + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); + return r; } /** @@ -189,6 +198,9 @@ static void amdgpu_dma_buf_detach(struct dma_buf *dmabuf, if (attach->dev->driver != adev->dev->driver && bo->prime_shared_count) bo->prime_shared_count--; + + pm_runtime_mark_last_busy(adev_to_drm(adev)->dev); + pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); } /** @@ -269,7 +281,7 @@ static struct sg_table *amdgpu_dma_buf_map(struct dma_buf_attachment *attach, case TTM_PL_TT: sgt = drm_prime_pages_to_sg(obj->dev, bo->tbo.ttm->pages, - bo->tbo.num_pages); + bo->tbo.ttm->num_pages); if (IS_ERR(sgt)) return sgt; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c index 7169fb5e3d9c47f0c8fd575b11544c31cd83cdb9..4575192d9b087a0e0336ea3618a6a22b0c557a37 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c @@ -132,8 +132,12 @@ uint amdgpu_pg_mask = 0xffffffff; uint amdgpu_sdma_phase_quantum = 32; char *amdgpu_disable_cu = NULL; char *amdgpu_virtual_display = NULL; -/* OverDrive(bit 14) disabled by default*/ -uint amdgpu_pp_feature_mask = 0xffffbfff; + +/* + * OverDrive(bit 14) disabled by default + * GFX DCS(bit 19) disabled by default + */ +uint amdgpu_pp_feature_mask = 0xfff7bfff; uint amdgpu_force_long_training; int amdgpu_job_hang_limit; int amdgpu_lbpw = -1; @@ -789,9 +793,9 @@ module_param_named(tmz, amdgpu_tmz, int, 0444); /** * DOC: reset_method (int) - * GPU reset method (-1 = auto (default), 0 = legacy, 1 = mode0, 2 = mode1, 3 = mode2, 4 = baco) + * GPU reset method (-1 = auto (default), 0 = legacy, 1 = mode0, 2 = mode1, 3 = mode2, 4 = baco, 5 = pci) */ -MODULE_PARM_DESC(reset_method, "GPU reset method (-1 = auto (default), 0 = legacy, 1 = mode0, 2 = mode1, 3 = mode2, 4 = baco/bamaco)"); +MODULE_PARM_DESC(reset_method, "GPU reset method (-1 = auto (default), 0 = legacy, 1 = mode0, 2 = mode1, 3 = mode2, 4 = baco/bamaco, 5 = pci)"); module_param_named(reset_method, amdgpu_reset_method, int, 0444); /** @@ -1094,6 +1098,7 @@ static const struct pci_device_id pciidlist[] = { /* Sienna_Cichlid */ {0x1002, 0x73A0, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_SIENNA_CICHLID}, + {0x1002, 0x73A1, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_SIENNA_CICHLID}, {0x1002, 0x73A2, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_SIENNA_CICHLID}, {0x1002, 0x73A3, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_SIENNA_CICHLID}, {0x1002, 0x73AB, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_SIENNA_CICHLID}, @@ -1206,7 +1211,6 @@ static int amdgpu_pci_probe(struct pci_dev *pdev, if (ret) return ret; - ddev->pdev = pdev; pci_set_drvdata(pdev, ddev); ret = amdgpu_driver_load_kms(adev, ent->driver_data); @@ -1266,7 +1270,9 @@ amdgpu_pci_shutdown(struct pci_dev *pdev) */ if (!amdgpu_passthrough(adev)) adev->mp1_state = PP_MP1_STATE_UNLOAD; + adev->in_poweroff_reboot_com = true; amdgpu_device_ip_suspend(adev); + adev->in_poweroff_reboot_com = false; adev->mp1_state = PP_MP1_STATE_NONE; } @@ -1308,8 +1314,13 @@ static int amdgpu_pmops_thaw(struct device *dev) static int amdgpu_pmops_poweroff(struct device *dev) { struct drm_device *drm_dev = dev_get_drvdata(dev); + struct amdgpu_device *adev = drm_to_adev(drm_dev); + int r; - return amdgpu_device_suspend(drm_dev, true); + adev->in_poweroff_reboot_com = true; + r = amdgpu_device_suspend(drm_dev, true); + adev->in_poweroff_reboot_com = false; + return r; } static int amdgpu_pmops_restore(struct device *dev) @@ -1344,11 +1355,12 @@ static int amdgpu_pmops_runtime_suspend(struct device *dev) adev->in_runpm = true; if (amdgpu_device_supports_atpx(drm_dev)) drm_dev->switch_power_state = DRM_SWITCH_POWER_CHANGING; - drm_kms_helper_poll_disable(drm_dev); ret = amdgpu_device_suspend(drm_dev, false); - if (ret) + if (ret) { + adev->in_runpm = false; return ret; + } if (amdgpu_device_supports_atpx(drm_dev)) { /* Only need to handle PCI state in the driver for ATPX @@ -1401,7 +1413,6 @@ static int amdgpu_pmops_runtime_resume(struct device *dev) amdgpu_device_baco_exit(drm_dev); } ret = amdgpu_device_resume(drm_dev, false); - drm_kms_helper_poll_enable(drm_dev); if (amdgpu_device_supports_atpx(drm_dev)) drm_dev->switch_power_state = DRM_SWITCH_POWER_ON; adev->in_runpm = false; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c index 0bf7d36c6686d4543609b1af3bb3e3480c33239f..51cd49c6f38fd06f58369e3265cafa5367573a3f 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c @@ -271,7 +271,7 @@ static int amdgpufb_create(struct drm_fb_helper *helper, DRM_INFO("fb depth is %d\n", fb->format->depth); DRM_INFO(" pitch is %d\n", fb->pitches[0]); - vga_switcheroo_client_fb_set(adev_to_drm(adev)->pdev, info); + vga_switcheroo_client_fb_set(adev->pdev, info); return 0; out: diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fw_attestation.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fw_attestation.c index 7c6e02e35573c4c0c907178052d6e406654b5d7e..8d1ad294cb026db75c56e7929b486b699cb229bc 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_fw_attestation.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_fw_attestation.c @@ -47,10 +47,9 @@ typedef struct FW_ATT_RECORD uint16_t AttFwIdV2; /* V2 FW ID field */ uint32_t AttFWVersion; /* FW Version */ uint16_t AttFWActiveFunctionID; /* The VF ID (only in VF Attestation Table) */ - uint16_t AttSource; /* FW source indicator */ - uint16_t RecordValid; /* Indicates whether the record is a valid entry */ - uint8_t AttFwTaId; /* Ta ID (only in TA Attestation Table) */ - uint8_t Reserved; + uint8_t AttSource; /* FW source indicator */ + uint8_t RecordValid; /* Indicates whether the record is a valid entry */ + uint32_t AttFwTaId; /* Ta ID (only in TA Attestation Table) */ } FW_ATT_RECORD; static ssize_t amdgpu_fw_attestation_debugfs_read(struct file *f, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c index 174a73eb23f08117528f8a458a68e57d50f89a72..b443907afceadfebab03bec153292bff145d5a0a 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c @@ -619,7 +619,7 @@ int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data, int r = 0; if (args->va_address < AMDGPU_VA_RESERVED_SIZE) { - dev_dbg(&dev->pdev->dev, + dev_dbg(dev->dev, "va_address 0x%LX is in reserved area 0x%LX\n", args->va_address, AMDGPU_VA_RESERVED_SIZE); return -EINVAL; @@ -627,7 +627,7 @@ int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data, if (args->va_address >= AMDGPU_GMC_HOLE_START && args->va_address < AMDGPU_GMC_HOLE_END) { - dev_dbg(&dev->pdev->dev, + dev_dbg(dev->dev, "va_address 0x%LX is in VA hole 0x%LX-0x%LX\n", args->va_address, AMDGPU_GMC_HOLE_START, AMDGPU_GMC_HOLE_END); @@ -639,14 +639,14 @@ int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data, vm_size = adev->vm_manager.max_pfn * AMDGPU_GPU_PAGE_SIZE; vm_size -= AMDGPU_VA_RESERVED_SIZE; if (args->va_address + args->map_size > vm_size) { - dev_dbg(&dev->pdev->dev, + dev_dbg(dev->dev, "va_address 0x%llx is in top reserved area 0x%llx\n", args->va_address + args->map_size, vm_size); return -EINVAL; } if ((args->flags & ~valid_flags) && (args->flags & ~prt_flags)) { - dev_dbg(&dev->pdev->dev, "invalid flags combination 0x%08X\n", + dev_dbg(dev->dev, "invalid flags combination 0x%08X\n", args->flags); return -EINVAL; } @@ -658,7 +658,7 @@ int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data, case AMDGPU_VA_OP_REPLACE: break; default: - dev_dbg(&dev->pdev->dev, "unsupported operation %d\n", + dev_dbg(dev->dev, "unsupported operation %d\n", args->operation); return -EINVAL; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c index cd2c676a2797cc084950f90a0727064c6e076181..8e0a6c62322ec9d62d569d717982580abd58901f 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c @@ -193,15 +193,16 @@ static bool amdgpu_gfx_is_multipipe_capable(struct amdgpu_device *adev) } bool amdgpu_gfx_is_high_priority_compute_queue(struct amdgpu_device *adev, - int pipe, int queue) + struct amdgpu_ring *ring) { - bool multipipe_policy = amdgpu_gfx_is_multipipe_capable(adev); - int cond; - /* Policy: alternate between normal and high priority */ - cond = multipipe_policy ? pipe : queue; - - return ((cond % 2) != 0); + /* Policy: use 1st queue as high priority compute queue if we + * have more than one compute queue. + */ + if (adev->gfx.num_compute_rings > 1 && + ring == &adev->gfx.compute_ring[0]) + return true; + return false; } void amdgpu_gfx_compute_queue_acquire(struct amdgpu_device *adev) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h index 6b5a8f4642cc9abe4d717016bca2962b357fcd46..72dbcd2bc6a630e5ff69d6a335e99fad1f95e5bf 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h @@ -380,7 +380,7 @@ void amdgpu_queue_mask_bit_to_mec_queue(struct amdgpu_device *adev, int bit, bool amdgpu_gfx_is_mec_queue_enabled(struct amdgpu_device *adev, int mec, int pipe, int queue); bool amdgpu_gfx_is_high_priority_compute_queue(struct amdgpu_device *adev, - int pipe, int queue); + struct amdgpu_ring *ring); int amdgpu_gfx_me_queue_to_bit(struct amdgpu_device *adev, int me, int pipe, int queue); void amdgpu_gfx_bit_to_me_queue(struct amdgpu_device *adev, int bit, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c index 6e679db5e46f0e8ceb1b5b26d6d88c3fb48f673d..fe1a39ffda724f5490791da76dff6c90e8646d57 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c @@ -120,7 +120,7 @@ uint64_t amdgpu_gmc_agp_addr(struct ttm_buffer_object *bo) { struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); - if (bo->num_pages != 1 || bo->ttm->caching == ttm_cached) + if (bo->ttm->num_pages != 1 || bo->ttm->caching == ttm_cached) return AMDGPU_BO_INVALID_OFFSET; if (bo->ttm->dma_address[0] + PAGE_SIZE >= adev->gmc.agp_size) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_hdp.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_hdp.h new file mode 100644 index 0000000000000000000000000000000000000000..43caf9f8cc110a8a561ba6e5a8cbdaf287be7778 --- /dev/null +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_hdp.h @@ -0,0 +1,40 @@ +/* + * Copyright 2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ +#ifndef __AMDGPU_HDP_H__ +#define __AMDGPU_HDP_H__ + +struct amdgpu_hdp_funcs { + void (*flush_hdp)(struct amdgpu_device *adev, struct amdgpu_ring *ring); + void (*invalidate_hdp)(struct amdgpu_device *adev, + struct amdgpu_ring *ring); + void (*reset_ras_error_count)(struct amdgpu_device *adev); + void (*update_clock_gating)(struct amdgpu_device *adev, bool enable); + void (*get_clock_gating_state)(struct amdgpu_device *adev, u32 *flags); + void (*init_registers)(struct amdgpu_device *adev); +}; + +struct amdgpu_hdp { + const struct amdgpu_hdp_funcs *funcs; +}; + +#endif /* __AMDGPU_HDP_H__ */ diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_i2c.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_i2c.c index 47cad23a6b9e2b6b81223a55df741e17da7cfbca..bca4dddd5a15b46dabf7946236892d143781d413 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_i2c.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_i2c.c @@ -176,7 +176,7 @@ struct amdgpu_i2c_chan *amdgpu_i2c_create(struct drm_device *dev, i2c->rec = *rec; i2c->adapter.owner = THIS_MODULE; i2c->adapter.class = I2C_CLASS_DDC; - i2c->adapter.dev.parent = &dev->pdev->dev; + i2c->adapter.dev.parent = dev->dev; i2c->dev = dev; i2c_set_adapdata(&i2c->adapter, i2c); mutex_init(&i2c->mutex); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c index 024d0a563a652be59bc054b9572e55af2e7c76c1..7645223ea0ef2173a96f8750aea0a57aaac90bd8 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c @@ -195,6 +195,10 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs, if ((ib->flags & AMDGPU_IB_FLAG_EMIT_MEM_SYNC) && ring->funcs->emit_mem_sync) ring->funcs->emit_mem_sync(ring); + if (ring->funcs->emit_wave_limit && + ring->hw_prio == AMDGPU_GFX_PIPE_PRIO_HIGH) + ring->funcs->emit_wave_limit(ring, true); + if (ring->funcs->insert_start) ring->funcs->insert_start(ring); @@ -295,6 +299,11 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs, ring->current_ctx = fence_ctx; if (vm && ring->funcs->emit_switch_buffer) amdgpu_ring_emit_switch_buffer(ring); + + if (ring->funcs->emit_wave_limit && + ring->hw_prio == AMDGPU_GFX_PIPE_PRIO_HIGH) + ring->funcs->emit_wave_limit(ring, false); + amdgpu_ring_commit(ring); return 0; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c index dcd9b4a8e20bef0932376e0e9acaa4cbba3bbdab..dc852af4f3b76f6a33bfa0784addd7f80f7c443d 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c @@ -205,3 +205,48 @@ int amdgpu_ih_process(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih) return IRQ_HANDLED; } +/** + * amdgpu_ih_decode_iv_helper - decode an interrupt vector + * + * @adev: amdgpu_device pointer + * @ih: ih ring to process + * @entry: IV entry + * + * Decodes the interrupt vector at the current rptr + * position and also advance the position for for Vega10 + * and later GPUs. + */ +void amdgpu_ih_decode_iv_helper(struct amdgpu_device *adev, + struct amdgpu_ih_ring *ih, + struct amdgpu_iv_entry *entry) +{ + /* wptr/rptr are in bytes! */ + u32 ring_index = ih->rptr >> 2; + uint32_t dw[8]; + + dw[0] = le32_to_cpu(ih->ring[ring_index + 0]); + dw[1] = le32_to_cpu(ih->ring[ring_index + 1]); + dw[2] = le32_to_cpu(ih->ring[ring_index + 2]); + dw[3] = le32_to_cpu(ih->ring[ring_index + 3]); + dw[4] = le32_to_cpu(ih->ring[ring_index + 4]); + dw[5] = le32_to_cpu(ih->ring[ring_index + 5]); + dw[6] = le32_to_cpu(ih->ring[ring_index + 6]); + dw[7] = le32_to_cpu(ih->ring[ring_index + 7]); + + entry->client_id = dw[0] & 0xff; + entry->src_id = (dw[0] >> 8) & 0xff; + entry->ring_id = (dw[0] >> 16) & 0xff; + entry->vmid = (dw[0] >> 24) & 0xf; + entry->vmid_src = (dw[0] >> 31); + entry->timestamp = dw[1] | ((u64)(dw[2] & 0xffff) << 32); + entry->timestamp_src = dw[2] >> 31; + entry->pasid = dw[3] & 0xffff; + entry->pasid_src = dw[3] >> 31; + entry->src_data[0] = dw[4]; + entry->src_data[1] = dw[5]; + entry->src_data[2] = dw[6]; + entry->src_data[3] = dw[7]; + + /* wptr/rptr are in bytes! */ + ih->rptr += 32; +} diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.h index 3c9cfe7eecff4f2dd22b53b51886ddf9b1b2a0bc..6ed4a85fc7c31e7916472257dc243a6d2c09afc3 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.h @@ -30,6 +30,18 @@ struct amdgpu_device; struct amdgpu_iv_entry; +struct amdgpu_ih_regs { + uint32_t ih_rb_base; + uint32_t ih_rb_base_hi; + uint32_t ih_rb_cntl; + uint32_t ih_rb_wptr; + uint32_t ih_rb_rptr; + uint32_t ih_doorbell_rptr; + uint32_t ih_rb_wptr_addr_lo; + uint32_t ih_rb_wptr_addr_hi; + uint32_t psp_reg_id; +}; + /* * R6xx+ IH ring */ @@ -53,6 +65,7 @@ struct amdgpu_ih_ring { bool enabled; unsigned rptr; atomic_t lock; + struct amdgpu_ih_regs ih_regs; }; /* provided by the ih block */ @@ -75,5 +88,7 @@ void amdgpu_ih_ring_fini(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih); void amdgpu_ih_ring_write(struct amdgpu_ih_ring *ih, const uint32_t *iv, unsigned int num_dw); int amdgpu_ih_process(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih); - +void amdgpu_ih_decode_iv_helper(struct amdgpu_device *adev, + struct amdgpu_ih_ring *ih, + struct amdgpu_iv_entry *entry); #endif diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c index bea57e8e793f704e7da2cdd22fcf0c0be94cb85d..afbbec82a289c8300cd823b0e41ed1dc221fb166 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c @@ -444,7 +444,8 @@ void amdgpu_irq_dispatch(struct amdgpu_device *adev, } else if (src_id >= AMDGPU_MAX_IRQ_SRC_ID) { DRM_DEBUG("Invalid src_id in IV: %d\n", src_id); - } else if (adev->irq.virq[src_id]) { + } else if ((client_id == AMDGPU_IRQ_CLIENTID_LEGACY) && + adev->irq.virq[src_id]) { generic_handle_irq(irq_find_mapping(adev->irq.domain, src_id)); } else if (!adev->irq.client[client_id].sources) { diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c index dcfe8a3b03ffb9efdbeeba5a560bbc37cd777320..ff48101bab551373f7ef54682af1cac4ad728491 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c @@ -271,7 +271,7 @@ void amdgpu_job_stop_all_jobs_on_sched(struct drm_gpu_scheduler *sched) } /* Signal all jobs already scheduled to HW */ - list_for_each_entry(s_job, &sched->ring_mirror_list, node) { + list_for_each_entry(s_job, &sched->pending_list, list) { struct drm_sched_fence *s_fence = s_job->s_fence; dma_fence_set_error(&s_fence->finished, -EHWPOISON); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c index b16b32797624af9a38bb57094c19e41845a18765..3c37cf1ae8b7f13891e442b9a1fb8d4e6f2d1004 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c @@ -142,7 +142,7 @@ int amdgpu_driver_load_kms(struct amdgpu_device *adev, unsigned long flags) (amdgpu_is_atpx_hybrid() || amdgpu_has_atpx_dgpu_power_cntl()) && ((flags & AMD_IS_APU) == 0) && - !pci_is_thunderbolt_attached(dev->pdev)) + !pci_is_thunderbolt_attached(to_pci_dev(dev->dev))) flags |= AMD_IS_PX; parent = pci_upstream_bridge(adev->pdev); @@ -156,7 +156,7 @@ int amdgpu_driver_load_kms(struct amdgpu_device *adev, unsigned long flags) */ r = amdgpu_device_init(adev, flags); if (r) { - dev_err(&dev->pdev->dev, "Fatal error during GPU init\n"); + dev_err(dev->dev, "Fatal error during GPU init\n"); goto out; } @@ -199,7 +199,7 @@ int amdgpu_driver_load_kms(struct amdgpu_device *adev, unsigned long flags) acpi_status = amdgpu_acpi_init(adev); if (acpi_status) - dev_dbg(&dev->pdev->dev, "Error during ACPI methods call\n"); + dev_dbg(dev->dev, "Error during ACPI methods call\n"); if (adev->runpm) { /* only need to skip on ATPX */ @@ -735,10 +735,10 @@ int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) if (!dev_info) return -ENOMEM; - dev_info->device_id = dev->pdev->device; + dev_info->device_id = adev->pdev->device; dev_info->chip_rev = adev->rev_id; dev_info->external_rev = adev->external_rev_id; - dev_info->pci_rev = dev->pdev->revision; + dev_info->pci_rev = adev->pdev->revision; dev_info->family = adev->family; dev_info->num_shader_engines = adev->gfx.config.max_shader_engines; dev_info->num_shader_arrays_per_engine = adev->gfx.config.max_sh_per_se; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_nbio.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_nbio.h index e62cc0e1a5ad072f5079db6611af8af3b2128d93..7c11bce4514bd0c7f47d255f151603848a7f2832 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_nbio.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_nbio.h @@ -57,7 +57,6 @@ struct amdgpu_nbio_funcs { u32 (*get_pcie_port_data_offset)(struct amdgpu_device *adev); u32 (*get_rev_id)(struct amdgpu_device *adev); void (*mc_access_enable)(struct amdgpu_device *adev, bool enable); - void (*hdp_flush)(struct amdgpu_device *adev, struct amdgpu_ring *ring); u32 (*get_memsize)(struct amdgpu_device *adev); void (*sdma_doorbell_range)(struct amdgpu_device *adev, int instance, bool use_doorbell, int doorbell_index, int doorbell_size); @@ -89,6 +88,7 @@ struct amdgpu_nbio_funcs { int (*ras_late_init)(struct amdgpu_device *adev); void (*enable_aspm)(struct amdgpu_device *adev, bool enable); + void (*program_aspm)(struct amdgpu_device *adev); }; struct amdgpu_nbio { diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c index b4c8e5d5c763cd5416b712e60c3a7ec2a10ed44a..4b29b820544281cc3f1ab3c717866e6c0ea66b00 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c @@ -787,7 +787,7 @@ int amdgpu_bo_kmap(struct amdgpu_bo *bo, void **ptr) if (r < 0) return r; - r = ttm_bo_kmap(&bo->tbo, 0, bo->tbo.num_pages, &bo->kmap); + r = ttm_bo_kmap(&bo->tbo, 0, bo->tbo.mem.num_pages, &bo->kmap); if (r) return r; @@ -911,10 +911,16 @@ int amdgpu_bo_pin_restricted(struct amdgpu_bo *bo, u32 domain, if (bo->tbo.pin_count) { uint32_t mem_type = bo->tbo.mem.mem_type; + uint32_t mem_flags = bo->tbo.mem.placement; if (!(domain & amdgpu_mem_type_to_domain(mem_type))) return -EINVAL; + if ((mem_type == TTM_PL_VRAM) && + (bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS) && + !(mem_flags & TTM_PL_FLAG_CONTIGUOUS)) + return -EINVAL; + ttm_bo_pin(&bo->tbo); if (max_offset != 0) { @@ -930,7 +936,6 @@ int amdgpu_bo_pin_restricted(struct amdgpu_bo *bo, u32 domain, if (bo->tbo.base.import_attach) dma_buf_pin(bo->tbo.base.import_attach); - bo->flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS; /* force to pin into visible video ram */ if (!(bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)) bo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; @@ -983,6 +988,7 @@ int amdgpu_bo_pin_restricted(struct amdgpu_bo *bo, u32 domain, */ int amdgpu_bo_pin(struct amdgpu_bo *bo, u32 domain) { + bo->flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS; return amdgpu_bo_pin_restricted(bo, domain, 0, 0); } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h index 79120ec413967132dc7addb8642f4e21ee791ff4..9ac37569823ff4c7bc9ef22093c4189b6d969bcb 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h @@ -174,12 +174,12 @@ static inline void amdgpu_bo_unreserve(struct amdgpu_bo *bo) static inline unsigned long amdgpu_bo_size(struct amdgpu_bo *bo) { - return bo->tbo.num_pages << PAGE_SHIFT; + return bo->tbo.base.size; } static inline unsigned amdgpu_bo_ngpu_pages(struct amdgpu_bo *bo) { - return (bo->tbo.num_pages << PAGE_SHIFT) / AMDGPU_GPU_PAGE_SIZE; + return bo->tbo.base.size / AMDGPU_GPU_PAGE_SIZE; } static inline unsigned amdgpu_bo_gpu_page_alignment(struct amdgpu_bo *bo) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c index 347fec66942485414a400a4ac1ae2cb795278943..839917eb7bc3c56995f2a5d9a2b80841f461dbb2 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c @@ -36,6 +36,7 @@ #include "psp_v12_0.h" #include "amdgpu_ras.h" +#include "amdgpu_securedisplay.h" static int psp_sysfs_init(struct amdgpu_device *adev); static void psp_sysfs_fini(struct amdgpu_device *adev); @@ -249,7 +250,7 @@ psp_cmd_submit_buf(struct psp_context *psp, { int ret; int index; - int timeout = 2000; + int timeout = 20000; bool ras_intr = false; bool skip_unsupport = false; @@ -282,7 +283,7 @@ psp_cmd_submit_buf(struct psp_context *psp, ras_intr = amdgpu_ras_intr_triggered(); if (ras_intr) break; - msleep(1); + usleep_range(10, 100); amdgpu_asic_invalidate_hdp(psp->adev, NULL); } @@ -1652,6 +1653,175 @@ int psp_rap_invoke(struct psp_context *psp, uint32_t ta_cmd_id) } // RAP end +/* securedisplay start */ +static int psp_securedisplay_init_shared_buf(struct psp_context *psp) +{ + int ret; + + /* + * Allocate 16k memory aligned to 4k from Frame Buffer (local + * physical) for sa ta <-> Driver + */ + ret = amdgpu_bo_create_kernel(psp->adev, PSP_SECUREDISPLAY_SHARED_MEM_SIZE, + PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM, + &psp->securedisplay_context.securedisplay_shared_bo, + &psp->securedisplay_context.securedisplay_shared_mc_addr, + &psp->securedisplay_context.securedisplay_shared_buf); + + return ret; +} + +static int psp_securedisplay_load(struct psp_context *psp) +{ + int ret; + struct psp_gfx_cmd_resp *cmd; + + cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL); + if (!cmd) + return -ENOMEM; + + memset(psp->fw_pri_buf, 0, PSP_1_MEG); + memcpy(psp->fw_pri_buf, psp->ta_securedisplay_start_addr, psp->ta_securedisplay_ucode_size); + + psp_prep_ta_load_cmd_buf(cmd, + psp->fw_pri_mc_addr, + psp->ta_securedisplay_ucode_size, + psp->securedisplay_context.securedisplay_shared_mc_addr, + PSP_SECUREDISPLAY_SHARED_MEM_SIZE); + + ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr); + + if (ret) + goto failed; + + psp->securedisplay_context.securedisplay_initialized = true; + psp->securedisplay_context.session_id = cmd->resp.session_id; + mutex_init(&psp->securedisplay_context.mutex); + +failed: + kfree(cmd); + return ret; +} + +static int psp_securedisplay_unload(struct psp_context *psp) +{ + int ret; + struct psp_gfx_cmd_resp *cmd; + + cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL); + if (!cmd) + return -ENOMEM; + + psp_prep_ta_unload_cmd_buf(cmd, psp->securedisplay_context.session_id); + + ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr); + + kfree(cmd); + + return ret; +} + +static int psp_securedisplay_initialize(struct psp_context *psp) +{ + int ret; + struct securedisplay_cmd *securedisplay_cmd; + + /* + * TODO: bypass the initialize in sriov for now + */ + if (amdgpu_sriov_vf(psp->adev)) + return 0; + + if (!psp->adev->psp.ta_securedisplay_ucode_size || + !psp->adev->psp.ta_securedisplay_start_addr) { + dev_info(psp->adev->dev, "SECUREDISPLAY: securedisplay ta ucode is not available\n"); + return 0; + } + + if (!psp->securedisplay_context.securedisplay_initialized) { + ret = psp_securedisplay_init_shared_buf(psp); + if (ret) + return ret; + } + + ret = psp_securedisplay_load(psp); + if (ret) + return ret; + + psp_prep_securedisplay_cmd_buf(psp, &securedisplay_cmd, + TA_SECUREDISPLAY_COMMAND__QUERY_TA); + + ret = psp_securedisplay_invoke(psp, TA_SECUREDISPLAY_COMMAND__QUERY_TA); + if (ret) { + psp_securedisplay_unload(psp); + + amdgpu_bo_free_kernel(&psp->securedisplay_context.securedisplay_shared_bo, + &psp->securedisplay_context.securedisplay_shared_mc_addr, + &psp->securedisplay_context.securedisplay_shared_buf); + + psp->securedisplay_context.securedisplay_initialized = false; + + dev_err(psp->adev->dev, "SECUREDISPLAY TA initialize fail.\n"); + return -EINVAL; + } + + if (securedisplay_cmd->status != TA_SECUREDISPLAY_STATUS__SUCCESS) { + psp_securedisplay_parse_resp_status(psp, securedisplay_cmd->status); + dev_err(psp->adev->dev, "SECUREDISPLAY: query securedisplay TA failed. ret 0x%x\n", + securedisplay_cmd->securedisplay_out_message.query_ta.query_cmd_ret); + } + + return 0; +} + +static int psp_securedisplay_terminate(struct psp_context *psp) +{ + int ret; + + /* + * TODO:bypass the terminate in sriov for now + */ + if (amdgpu_sriov_vf(psp->adev)) + return 0; + + if (!psp->securedisplay_context.securedisplay_initialized) + return 0; + + ret = psp_securedisplay_unload(psp); + if (ret) + return ret; + + psp->securedisplay_context.securedisplay_initialized = false; + + /* free securedisplay shared memory */ + amdgpu_bo_free_kernel(&psp->securedisplay_context.securedisplay_shared_bo, + &psp->securedisplay_context.securedisplay_shared_mc_addr, + &psp->securedisplay_context.securedisplay_shared_buf); + + return ret; +} + +int psp_securedisplay_invoke(struct psp_context *psp, uint32_t ta_cmd_id) +{ + int ret; + + if (!psp->securedisplay_context.securedisplay_initialized) + return -EINVAL; + + if (ta_cmd_id != TA_SECUREDISPLAY_COMMAND__QUERY_TA && + ta_cmd_id != TA_SECUREDISPLAY_COMMAND__SEND_ROI_CRC) + return -EINVAL; + + mutex_lock(&psp->securedisplay_context.mutex); + + ret = psp_ta_invoke(psp, ta_cmd_id, psp->securedisplay_context.session_id); + + mutex_unlock(&psp->securedisplay_context.mutex); + + return ret; +} +/* SECUREDISPLAY end */ + static int psp_hw_start(struct psp_context *psp) { struct amdgpu_device *adev = psp->adev; @@ -2126,6 +2296,11 @@ static int psp_load_fw(struct amdgpu_device *adev) if (ret) dev_err(psp->adev->dev, "RAP: Failed to initialize RAP\n"); + + ret = psp_securedisplay_initialize(psp); + if (ret) + dev_err(psp->adev->dev, + "SECUREDISPLAY: Failed to initialize SECUREDISPLAY\n"); } return 0; @@ -2176,6 +2351,7 @@ static int psp_hw_fini(void *handle) if (psp->adev->psp.ta_fw) { psp_ras_terminate(psp); + psp_securedisplay_terminate(psp); psp_rap_terminate(psp); psp_dtm_terminate(psp); psp_hdcp_terminate(psp); @@ -2240,6 +2416,11 @@ static int psp_suspend(void *handle) DRM_ERROR("Failed to terminate rap ta\n"); return ret; } + ret = psp_securedisplay_terminate(psp); + if (ret) { + DRM_ERROR("Failed to terminate securedisplay ta\n"); + return ret; + } } ret = psp_asd_unload(psp); @@ -2323,6 +2504,11 @@ static int psp_resume(void *handle) if (ret) dev_err(psp->adev->dev, "RAP: Failed to initialize RAP\n"); + + ret = psp_securedisplay_initialize(psp); + if (ret) + dev_err(psp->adev->dev, + "SECUREDISPLAY: Failed to initialize SECUREDISPLAY\n"); } mutex_unlock(&adev->firmware.mutex); @@ -2629,6 +2815,11 @@ static int parse_ta_bin_descriptor(struct psp_context *psp, psp->ta_rap_ucode_size = le32_to_cpu(desc->size_bytes); psp->ta_rap_start_addr = ucode_start_addr; break; + case TA_FW_TYPE_PSP_SECUREDISPLAY: + psp->ta_securedisplay_ucode_version = le32_to_cpu(desc->fw_version); + psp->ta_securedisplay_ucode_size = le32_to_cpu(desc->size_bytes); + psp->ta_securedisplay_start_addr = ucode_start_addr; + break; default: dev_warn(psp->adev->dev, "Unsupported TA type: %d\n", desc->fw_type); break; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h index da250bc1ac578b9db61abe3caa8f0c6d21c60fa9..cb50ba445f8c86e46c720259f5457ffc5f32e272 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h @@ -30,6 +30,7 @@ #include "ta_xgmi_if.h" #include "ta_ras_if.h" #include "ta_rap_if.h" +#include "ta_secureDisplay_if.h" #define PSP_FENCE_BUFFER_SIZE 0x1000 #define PSP_CMD_BUFFER_SIZE 0x1000 @@ -40,6 +41,7 @@ #define PSP_HDCP_SHARED_MEM_SIZE 0x4000 #define PSP_DTM_SHARED_MEM_SIZE 0x4000 #define PSP_RAP_SHARED_MEM_SIZE 0x4000 +#define PSP_SECUREDISPLAY_SHARED_MEM_SIZE 0x4000 #define PSP_SHARED_MEM_SIZE 0x4000 #define PSP_FW_NAME_LEN 0x24 @@ -171,6 +173,15 @@ struct psp_rap_context { struct mutex mutex; }; +struct psp_securedisplay_context { + bool securedisplay_initialized; + uint32_t session_id; + struct amdgpu_bo *securedisplay_shared_bo; + uint64_t securedisplay_shared_mc_addr; + void *securedisplay_shared_buf; + struct mutex mutex; +}; + #define MEM_TRAIN_SYSTEM_SIGNATURE 0x54534942 #define GDDR6_MEM_TRAINING_DATA_SIZE_IN_BYTES 0x1000 #define GDDR6_MEM_TRAINING_OFFSET 0x8000 @@ -298,12 +309,17 @@ struct psp_context uint32_t ta_rap_ucode_size; uint8_t *ta_rap_start_addr; + uint32_t ta_securedisplay_ucode_version; + uint32_t ta_securedisplay_ucode_size; + uint8_t *ta_securedisplay_start_addr; + struct psp_asd_context asd_context; struct psp_xgmi_context xgmi_context; struct psp_ras_context ras; struct psp_hdcp_context hdcp_context; struct psp_dtm_context dtm_context; struct psp_rap_context rap_context; + struct psp_securedisplay_context securedisplay_context; struct mutex mutex; struct psp_memory_training_context mem_train_ctx; }; @@ -380,6 +396,7 @@ int psp_ras_trigger_error(struct psp_context *psp, int psp_hdcp_invoke(struct psp_context *psp, uint32_t ta_cmd_id); int psp_dtm_invoke(struct psp_context *psp, uint32_t ta_cmd_id); int psp_rap_invoke(struct psp_context *psp, uint32_t ta_cmd_id); +int psp_securedisplay_invoke(struct psp_context *psp, uint32_t ta_cmd_id); int psp_rlc_autoload_start(struct psp_context *psp); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c index 82e952696d24f4730936948f526d2211205cc003..1fb2a91ad30ad797e06d4ae1b26b91b289f30bd5 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c @@ -846,7 +846,7 @@ static int amdgpu_ras_error_inject_xgmi(struct amdgpu_device *adev, if (amdgpu_dpm_allow_xgmi_power_down(adev, true)) dev_warn(adev->dev, "Failed to allow XGMI power down"); - if (amdgpu_dpm_set_df_cstate(adev, DF_CSTATE_DISALLOW)) + if (amdgpu_dpm_set_df_cstate(adev, DF_CSTATE_ALLOW)) dev_warn(adev->dev, "Failed to allow df cstate"); return ret; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c index 1a612f51ecd9e58c078411b08a0f3d1e11e599ae..b644c78475fd8de1e53677bccf3333d4db104fa0 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c @@ -166,7 +166,7 @@ int amdgpu_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring, unsigned int max_dw, struct amdgpu_irq_src *irq_src, unsigned int irq_type, unsigned int hw_prio) { - int r, i; + int r; int sched_hw_submission = amdgpu_sched_hw_submission; u32 *num_sched; u32 hw_ip; @@ -258,8 +258,7 @@ int amdgpu_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring, } ring->max_dw = max_dw; - ring->priority = DRM_SCHED_PRIORITY_NORMAL; - mutex_init(&ring->priority_mutex); + ring->hw_prio = hw_prio; if (!ring->no_scheduler) { hw_ip = ring->funcs->type; @@ -268,9 +267,6 @@ int amdgpu_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring, &ring->sched; } - for (i = DRM_SCHED_PRIORITY_MIN; i < DRM_SCHED_PRIORITY_COUNT; ++i) - atomic_set(&ring->num_jobs[i], 0); - return 0; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h index 7112137689db002b7b73ad600b08578f39b27a57..56acec1075acd7defabce37d0bf6862af174d7f6 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h @@ -197,6 +197,7 @@ struct amdgpu_ring_funcs { void (*soft_recovery)(struct amdgpu_ring *ring, unsigned vmid); int (*preempt_ib)(struct amdgpu_ring *ring); void (*emit_mem_sync)(struct amdgpu_ring *ring); + void (*emit_wave_limit)(struct amdgpu_ring *ring, bool enable); }; struct amdgpu_ring { @@ -242,11 +243,7 @@ struct amdgpu_ring { struct dma_fence *vmid_wait; bool has_compute_vm_bug; bool no_scheduler; - - atomic_t num_jobs[DRM_SCHED_PRIORITY_COUNT]; - struct mutex priority_mutex; - /* protected by priority_mutex */ - int priority; + int hw_prio; #if defined(CONFIG_DEBUG_FS) struct dentry *ent; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_securedisplay.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_securedisplay.c new file mode 100644 index 0000000000000000000000000000000000000000..834440ab9ff74740820f11d35a04074947b1273a --- /dev/null +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_securedisplay.c @@ -0,0 +1,176 @@ +/* + * Copyright 2021 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * + */ +#include +#include + +#include "amdgpu.h" +#include "amdgpu_securedisplay.h" + +/** + * DOC: AMDGPU SECUREDISPLAY debugfs test interface + * + * how to use? + * echo opcode > /dri/xxx/securedisplay_test + * eg. echo 1 > /dri/xxx/securedisplay_test + * eg. echo 2 phy_id > /dri/xxx/securedisplay_test + * + * opcode: + * 1：Query whether TA is responding used only for validation pupose + * 2: Send region of Interest and CRC value to I2C. (uint32)phy_id is + * send to determine which DIO scratch register should be used to get + * ROI and receive i2c_buf as the output. + * + * You can refer more detail from header file ta_securedisplay_if.h + * + */ + +void psp_securedisplay_parse_resp_status(struct psp_context *psp, + enum ta_securedisplay_status status) +{ + switch (status) { + case TA_SECUREDISPLAY_STATUS__SUCCESS: + break; + case TA_SECUREDISPLAY_STATUS__GENERIC_FAILURE: + dev_err(psp->adev->dev, "Secure display: Generic Failure."); + break; + case TA_SECUREDISPLAY_STATUS__INVALID_PARAMETER: + dev_err(psp->adev->dev, "Secure display: Invalid Parameter."); + break; + case TA_SECUREDISPLAY_STATUS__NULL_POINTER: + dev_err(psp->adev->dev, "Secure display: Null Pointer."); + break; + case TA_SECUREDISPLAY_STATUS__I2C_WRITE_ERROR: + dev_err(psp->adev->dev, "Secure display: Failed to write to I2C."); + break; + case TA_SECUREDISPLAY_STATUS__READ_DIO_SCRATCH_ERROR: + dev_err(psp->adev->dev, "Secure display: Failed to Read DIO Scratch Register."); + break; + case TA_SECUREDISPLAY_STATUS__READ_CRC_ERROR: + dev_err(psp->adev->dev, "Secure display: Failed to Read CRC"); + break; + default: + dev_err(psp->adev->dev, "Secure display: Failed to parse status: %d\n", status); + } +} + +void psp_prep_securedisplay_cmd_buf(struct psp_context *psp, struct securedisplay_cmd **cmd, + enum ta_securedisplay_command command_id) +{ + *cmd = (struct securedisplay_cmd *)psp->securedisplay_context.securedisplay_shared_buf; + memset(*cmd, 0, sizeof(struct securedisplay_cmd)); + (*cmd)->status = TA_SECUREDISPLAY_STATUS__GENERIC_FAILURE; + (*cmd)->cmd_id = command_id; +} + +static ssize_t amdgpu_securedisplay_debugfs_write(struct file *f, const char __user *buf, + size_t size, loff_t *pos) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)file_inode(f)->i_private; + struct psp_context *psp = &adev->psp; + struct securedisplay_cmd *securedisplay_cmd; + struct drm_device *dev = adev_to_drm(adev); + uint32_t phy_id; + uint32_t op; + int i; + char str[64]; + char i2c_output[256]; + int ret; + + if (*pos || size > sizeof(str) - 1) + return -EINVAL; + + memset(str, 0, sizeof(str)); + ret = copy_from_user(str, buf, size); + if (ret) + return -EFAULT; + + ret = pm_runtime_get_sync(dev->dev); + if (ret < 0) { + pm_runtime_put_autosuspend(dev->dev); + return ret; + } + + if (size < 3) + sscanf(str, "%u ", &op); + else + sscanf(str, "%u %u", &op, &phy_id); + + switch (op) { + case 1: + psp_prep_securedisplay_cmd_buf(psp, &securedisplay_cmd, + TA_SECUREDISPLAY_COMMAND__QUERY_TA); + ret = psp_securedisplay_invoke(psp, TA_SECUREDISPLAY_COMMAND__QUERY_TA); + if (!ret) { + if (securedisplay_cmd->status == TA_SECUREDISPLAY_STATUS__SUCCESS) + dev_info(adev->dev, "SECUREDISPLAY: query securedisplay TA ret is 0x%X\n", + securedisplay_cmd->securedisplay_out_message.query_ta.query_cmd_ret); + else + psp_securedisplay_parse_resp_status(psp, securedisplay_cmd->status); + } + break; + case 2: + psp_prep_securedisplay_cmd_buf(psp, &securedisplay_cmd, + TA_SECUREDISPLAY_COMMAND__SEND_ROI_CRC); + securedisplay_cmd->securedisplay_in_message.send_roi_crc.phy_id = phy_id; + ret = psp_securedisplay_invoke(psp, TA_SECUREDISPLAY_COMMAND__SEND_ROI_CRC); + if (!ret) { + if (securedisplay_cmd->status == TA_SECUREDISPLAY_STATUS__SUCCESS) { + memset(i2c_output, 0, sizeof(i2c_output)); + for (i = 0; i < TA_SECUREDISPLAY_I2C_BUFFER_SIZE; i++) + sprintf(i2c_output, "%s 0x%X", i2c_output, + securedisplay_cmd->securedisplay_out_message.send_roi_crc.i2c_buf[i]); + dev_info(adev->dev, "SECUREDISPLAY: I2C buffer out put is :%s\n", i2c_output); + } else { + psp_securedisplay_parse_resp_status(psp, securedisplay_cmd->status); + } + } + break; + default: + dev_err(adev->dev, "Invalid input: %s\n", str); + } + + pm_runtime_mark_last_busy(dev->dev); + pm_runtime_put_autosuspend(dev->dev); + + return size; +} + +static const struct file_operations amdgpu_securedisplay_debugfs_ops = { + .owner = THIS_MODULE, + .read = NULL, + .write = amdgpu_securedisplay_debugfs_write, + .llseek = default_llseek +}; + +void amdgpu_securedisplay_debugfs_init(struct amdgpu_device *adev) +{ +#if defined(CONFIG_DEBUG_FS) + + if (!adev->psp.securedisplay_context.securedisplay_initialized) + return; + + debugfs_create_file("securedisplay_test", S_IWUSR, adev_to_drm(adev)->primary->debugfs_root, + adev, &amdgpu_securedisplay_debugfs_ops); +#endif +} diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_securedisplay.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_securedisplay.h new file mode 100644 index 0000000000000000000000000000000000000000..fe98574748f440238eab697c8b9a9190ef02a1f0 --- /dev/null +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_securedisplay.h @@ -0,0 +1,36 @@ +/* + * Copyright 2021 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * + */ +#ifndef _AMDGPU_SECUREDISPLAY_H +#define _AMDGPU_SECUREDISPLAY_H + +#include "amdgpu.h" +#include "ta_secureDisplay_if.h" + +void amdgpu_securedisplay_debugfs_init(struct amdgpu_device *adev); +void psp_securedisplay_parse_resp_status(struct psp_context *psp, + enum ta_securedisplay_status status); +void psp_prep_securedisplay_cmd_buf(struct psp_context *psp, struct securedisplay_cmd **cmd, + enum ta_securedisplay_command command_id); + +#endif diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h index 6752d8b131188471d683efde37263283001100e9..792d20261846514a5bab28d0c701c216bd86e1d0 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h @@ -21,7 +21,7 @@ * */ -#if !defined(_AMDGPU_TRACE_H) || defined(TRACE_HEADER_MULTI_READ) +#if !defined(_AMDGPU_TRACE_H_) || defined(TRACE_HEADER_MULTI_READ) #define _AMDGPU_TRACE_H_ #include @@ -127,7 +127,7 @@ TRACE_EVENT(amdgpu_bo_create, TP_fast_assign( __entry->bo = bo; - __entry->pages = bo->tbo.num_pages; + __entry->pages = bo->tbo.mem.num_pages; __entry->type = bo->tbo.mem.mem_type; __entry->prefer = bo->preferred_domains; __entry->allow = bo->allowed_domains; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c index 4d8f19ab10144ae66e2764c449b58662c7075e2d..9fd2157b133accc9e71894fcdc9515c764270948 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c @@ -46,7 +46,6 @@ #include #include #include -#include #include #include @@ -637,7 +636,7 @@ static int amdgpu_bo_move(struct ttm_buffer_object *bo, bool evict, out: /* update statistics */ - atomic64_add((u64)bo->num_pages << PAGE_SHIFT, &adev->num_bytes_moved); + atomic64_add(bo->base.size, &adev->num_bytes_moved); amdgpu_bo_move_notify(bo, evict, new_mem); return 0; } @@ -918,8 +917,8 @@ static int amdgpu_ttm_tt_pin_userptr(struct ttm_bo_device *bdev, goto release_sg; /* convert SG to linear array of pages and dma addresses */ - drm_prime_sg_to_page_addr_arrays(ttm->sg, ttm->pages, - gtt->ttm.dma_address, ttm->num_pages); + drm_prime_sg_to_dma_addr_array(ttm->sg, gtt->ttm.dma_address, + ttm->num_pages); return 0; @@ -1265,9 +1264,8 @@ static int amdgpu_ttm_tt_populate(struct ttm_bo_device *bdev, ttm->sg = sgt; } - drm_prime_sg_to_page_addr_arrays(ttm->sg, ttm->pages, - gtt->ttm.dma_address, - ttm->num_pages); + drm_prime_sg_to_dma_addr_array(ttm->sg, gtt->ttm.dma_address, + ttm->num_pages); return 0; } @@ -2124,7 +2122,7 @@ int amdgpu_fill_buffer(struct amdgpu_bo *bo, return r; } - num_pages = bo->tbo.num_pages; + num_pages = bo->tbo.mem.num_pages; mm_node = bo->tbo.mem.mm_node; num_loops = 0; while (num_pages) { @@ -2154,7 +2152,7 @@ int amdgpu_fill_buffer(struct amdgpu_bo *bo, } } - num_pages = bo->tbo.num_pages; + num_pages = bo->tbo.mem.num_pages; mm_node = bo->tbo.mem.mm_node; while (num_pages) { diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.h index 0e43b46d3ab50d50db20dfa224a5a43b3550feb9..46449e70348bea565a144fdb0d152b2bd6b8dda5 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.h @@ -122,6 +122,9 @@ struct ta_firmware_header_v1_0 { uint32_t ta_dtm_ucode_version; uint32_t ta_dtm_offset_bytes; uint32_t ta_dtm_size_bytes; + uint32_t ta_securedisplay_ucode_version; + uint32_t ta_securedisplay_offset_bytes; + uint32_t ta_securedisplay_size_bytes; }; enum ta_fw_type { @@ -132,6 +135,7 @@ enum ta_fw_type { TA_FW_TYPE_PSP_HDCP, TA_FW_TYPE_PSP_DTM, TA_FW_TYPE_PSP_RAP, + TA_FW_TYPE_PSP_SECUREDISPLAY, }; struct ta_fw_bin_desc { diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c index 8b989670ed663032b6af9abd57acd24829cf38d5..e2ed4689118a9980baef80b677efaf22614a7787 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c @@ -1170,7 +1170,7 @@ int amdgpu_uvd_get_create_msg(struct amdgpu_ring *ring, uint32_t handle, int r, i; r = amdgpu_bo_create_reserved(adev, 1024, PAGE_SIZE, - AMDGPU_GEM_DOMAIN_VRAM, + AMDGPU_GEM_DOMAIN_GTT, &bo, NULL, (void **)&msg); if (r) return r; @@ -1202,7 +1202,7 @@ int amdgpu_uvd_get_destroy_msg(struct amdgpu_ring *ring, uint32_t handle, int r, i; r = amdgpu_bo_create_reserved(adev, 1024, PAGE_SIZE, - AMDGPU_GEM_DOMAIN_VRAM, + AMDGPU_GEM_DOMAIN_GTT, &bo, NULL, (void **)&msg); if (r) return r; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c index 0d5284b936e487818c85224264b4632edd7902ad..ea6a62f67e3809cc5041bbaa9bb8f889ec3e7bbf 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c @@ -1160,6 +1160,6 @@ int amdgpu_vce_ring_test_ib(struct amdgpu_ring *ring, long timeout) error: dma_fence_put(fence); amdgpu_bo_unreserve(bo); - amdgpu_bo_unref(&bo); + amdgpu_bo_free_kernel(&bo, NULL, NULL); return r; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c index 4a77c7424dfc7d986e35989bd25bc5d6e02c1b24..99b82f3c26177c7d5fecc7ee93ed875d3f9f0be6 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c @@ -496,6 +496,7 @@ static int amdgpu_vcn_dec_send_msg(struct amdgpu_ring *ring, struct amdgpu_job *job; struct amdgpu_ib *ib; uint64_t addr; + void *msg = NULL; int i, r; r = amdgpu_job_alloc_with_ib(adev, 64, @@ -505,6 +506,7 @@ static int amdgpu_vcn_dec_send_msg(struct amdgpu_ring *ring, ib = &job->ibs[0]; addr = amdgpu_bo_gpu_offset(bo); + msg = amdgpu_bo_kptr(bo); ib->ptr[0] = PACKET0(adev->vcn.internal.data0, 0); ib->ptr[1] = addr; ib->ptr[2] = PACKET0(adev->vcn.internal.data1, 0); @@ -523,7 +525,7 @@ static int amdgpu_vcn_dec_send_msg(struct amdgpu_ring *ring, amdgpu_bo_fence(bo, f, false); amdgpu_bo_unreserve(bo); - amdgpu_bo_unref(&bo); + amdgpu_bo_free_kernel(&bo, NULL, (void **)&msg); if (fence) *fence = dma_fence_get(f); @@ -536,7 +538,7 @@ static int amdgpu_vcn_dec_send_msg(struct amdgpu_ring *ring, err: amdgpu_bo_unreserve(bo); - amdgpu_bo_unref(&bo); + amdgpu_bo_free_kernel(&bo, NULL, (void **)&msg); return r; } @@ -890,6 +892,7 @@ int amdgpu_vcn_enc_ring_test_ib(struct amdgpu_ring *ring, long timeout) error: dma_fence_put(fence); amdgpu_bo_unreserve(bo); - amdgpu_bo_unref(&bo); + amdgpu_bo_free_kernel(&bo, NULL, NULL); + return r; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c index 2d51b7694d1fdf5806e0479efa39aaf34e74de88..5da04d45b63795a0305178719dcdbe2025a95cef 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c @@ -560,10 +560,14 @@ static int amdgpu_virt_write_vf2pf_data(struct amdgpu_device *adev) static void amdgpu_virt_update_vf2pf_work_item(struct work_struct *work) { struct amdgpu_device *adev = container_of(work, struct amdgpu_device, virt.vf2pf_work.work); + int ret; - amdgpu_virt_read_pf2vf_data(adev); + ret = amdgpu_virt_read_pf2vf_data(adev); + if (ret) + goto out; amdgpu_virt_write_vf2pf_data(adev); +out: schedule_delayed_work(&(adev->virt.vf2pf_work), adev->virt.vf2pf_update_interval_ms); } @@ -571,8 +575,8 @@ void amdgpu_virt_fini_data_exchange(struct amdgpu_device *adev) { if (adev->virt.vf2pf_update_interval_ms != 0) { DRM_INFO("clean up the vf2pf work item\n"); - flush_delayed_work(&adev->virt.vf2pf_work); cancel_delayed_work_sync(&adev->virt.vf2pf_work); + adev->virt.vf2pf_update_interval_ms = 0; } } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c index 0768c868698365785a3ebc36fdcf92c773d2a4f3..ad91c0c3c42357cd3499df51286a77e86c861cf0 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c @@ -653,9 +653,11 @@ void amdgpu_vm_move_to_lru_tail(struct amdgpu_device *adev, if (!bo->parent) continue; - ttm_bo_move_to_lru_tail(&bo->tbo, &vm->lru_bulk_move); + ttm_bo_move_to_lru_tail(&bo->tbo, &bo->tbo.mem, + &vm->lru_bulk_move); if (bo->shadow) ttm_bo_move_to_lru_tail(&bo->shadow->tbo, + &bo->shadow->tbo.mem, &vm->lru_bulk_move); } spin_unlock(&ttm_bo_glob.lru_lock); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c index d2de2a720a3d86fcdca9feaed5d562fc8141238e..c89b66bb70e2e7c03de3723c304121907029e8d3 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c @@ -473,6 +473,9 @@ static int amdgpu_vram_mgr_new(struct ttm_resource_manager *man, for (i = 0; pages_left >= pages_per_node; ++i) { unsigned long pages = rounddown_pow_of_two(pages_left); + /* Limit maximum size to 2GB due to SG table limitations */ + pages = min(pages, (2UL << (30 - PAGE_SHIFT))); + r = drm_mm_insert_node_in_range(mm, &nodes[i], pages, pages_per_node, 0, place->fpfn, lpfn, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c index 541ef6be390f09d0bfe55872acb9ed39b8ef6284..659b385b27b5a04ca02b9e2a1f7b450ab1a41168 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c @@ -324,7 +324,7 @@ static void amdgpu_xgmi_sysfs_rem_dev_info(struct amdgpu_device *adev, struct amdgpu_hive_info *amdgpu_get_xgmi_hive(struct amdgpu_device *adev) { - struct amdgpu_hive_info *hive = NULL, *tmp = NULL; + struct amdgpu_hive_info *hive = NULL; int ret; if (!adev->gmc.xgmi.hive_id) @@ -337,11 +337,9 @@ struct amdgpu_hive_info *amdgpu_get_xgmi_hive(struct amdgpu_device *adev) mutex_lock(&xgmi_mutex); - if (!list_empty(&xgmi_hive_list)) { - list_for_each_entry_safe(hive, tmp, &xgmi_hive_list, node) { - if (hive->hive_id == adev->gmc.xgmi.hive_id) - goto pro_end; - } + list_for_each_entry(hive, &xgmi_hive_list, node) { + if (hive->hive_id == adev->gmc.xgmi.hive_id) + goto pro_end; } hive = kzalloc(sizeof(*hive), GFP_KERNEL); diff --git a/drivers/gpu/drm/amd/amdgpu/athub_v2_0.c b/drivers/gpu/drm/amd/amdgpu/athub_v2_0.c index 921a69abda55526aab2c40036a9e945547d82f4f..5b90efd6f6d04721fdf828801cabe5b56792a278 100644 --- a/drivers/gpu/drm/amd/amdgpu/athub_v2_0.c +++ b/drivers/gpu/drm/amd/amdgpu/athub_v2_0.c @@ -27,7 +27,6 @@ #include "athub/athub_2_0_0_offset.h" #include "athub/athub_2_0_0_sh_mask.h" #include "athub/athub_2_0_0_default.h" -#include "navi10_enum.h" #include "soc15_common.h" diff --git a/drivers/gpu/drm/amd/amdgpu/athub_v2_1.c b/drivers/gpu/drm/amd/amdgpu/athub_v2_1.c index 66c183ddd43e58ea136377a6f2731aeb7087bf7f..7b1b18350bf92aa7b9baf4330747f8ff072c5413 100644 --- a/drivers/gpu/drm/amd/amdgpu/athub_v2_1.c +++ b/drivers/gpu/drm/amd/amdgpu/athub_v2_1.c @@ -26,7 +26,6 @@ #include "athub/athub_2_1_0_offset.h" #include "athub/athub_2_1_0_sh_mask.h" -#include "navi10_enum.h" #include "soc15_common.h" diff --git a/drivers/gpu/drm/amd/amdgpu/cik.c b/drivers/gpu/drm/amd/amdgpu/cik.c index 13737b317f7c22e5784c9d09d96d200ef06dd200..4d6832cc7fb066511a21d4cd37ad7225cd268beb 100644 --- a/drivers/gpu/drm/amd/amdgpu/cik.c +++ b/drivers/gpu/drm/amd/amdgpu/cik.c @@ -1251,13 +1251,22 @@ static void kv_restore_regs_for_reset(struct amdgpu_device *adev, WREG32(mmGMCON_RENG_EXECUTE, save->gmcon_reng_execute); } -static int cik_gpu_pci_config_reset(struct amdgpu_device *adev) +/** + * cik_asic_pci_config_reset - soft reset GPU + * + * @adev: amdgpu_device pointer + * + * Use PCI Config method to reset the GPU. + * + * Returns 0 for success. + */ +static int cik_asic_pci_config_reset(struct amdgpu_device *adev) { struct kv_reset_save_regs kv_save = { 0 }; u32 i; int r = -EINVAL; - dev_info(adev->dev, "GPU pci config reset\n"); + amdgpu_atombios_scratch_regs_engine_hung(adev, true); if (adev->flags & AMD_IS_APU) kv_save_regs_for_reset(adev, &kv_save); @@ -1285,26 +1294,6 @@ static int cik_gpu_pci_config_reset(struct amdgpu_device *adev) if (adev->flags & AMD_IS_APU) kv_restore_regs_for_reset(adev, &kv_save); - return r; -} - -/** - * cik_asic_pci_config_reset - soft reset GPU - * - * @adev: amdgpu_device pointer - * - * Use PCI Config method to reset the GPU. - * - * Returns 0 for success. - */ -static int cik_asic_pci_config_reset(struct amdgpu_device *adev) -{ - int r; - - amdgpu_atombios_scratch_regs_engine_hung(adev, true); - - r = cik_gpu_pci_config_reset(adev); - amdgpu_atombios_scratch_regs_engine_hung(adev, false); return r; diff --git a/drivers/gpu/drm/amd/amdgpu/cz_ih.c b/drivers/gpu/drm/amd/amdgpu/cz_ih.c index da37f8a900afb737ee1d55b1e62fd76445483542..307c01301c87a7f600e98bf0c927376028abb863 100644 --- a/drivers/gpu/drm/amd/amdgpu/cz_ih.c +++ b/drivers/gpu/drm/amd/amdgpu/cz_ih.c @@ -194,19 +194,30 @@ static u32 cz_ih_get_wptr(struct amdgpu_device *adev, wptr = le32_to_cpu(*ih->wptr_cpu); - if (REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW)) { - wptr = REG_SET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW, 0); - /* When a ring buffer overflow happen start parsing interrupt - * from the last not overwritten vector (wptr + 16). Hopefully - * this should allow us to catchup. - */ - dev_warn(adev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n", - wptr, ih->rptr, (wptr + 16) & ih->ptr_mask); - ih->rptr = (wptr + 16) & ih->ptr_mask; - tmp = RREG32(mmIH_RB_CNTL); - tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 1); - WREG32(mmIH_RB_CNTL, tmp); - } + if (!REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW)) + goto out; + + /* Double check that the overflow wasn't already cleared. */ + wptr = RREG32(mmIH_RB_WPTR); + + if (!REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW)) + goto out; + + wptr = REG_SET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW, 0); + + /* When a ring buffer overflow happen start parsing interrupt + * from the last not overwritten vector (wptr + 16). Hopefully + * this should allow us to catchup. + */ + dev_warn(adev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n", + wptr, ih->rptr, (wptr + 16) & ih->ptr_mask); + ih->rptr = (wptr + 16) & ih->ptr_mask; + tmp = RREG32(mmIH_RB_CNTL); + tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 1); + WREG32(mmIH_RB_CNTL, tmp); + + +out: return (wptr & ih->ptr_mask); } diff --git a/drivers/gpu/drm/amd/amdgpu/dce_virtual.c b/drivers/gpu/drm/amd/amdgpu/dce_virtual.c index ffcc64ec64733f4f99147f6667fb8e55af0eea34..9810af712cc0d1d01164b1255a0db093fa33e6f9 100644 --- a/drivers/gpu/drm/amd/amdgpu/dce_virtual.c +++ b/drivers/gpu/drm/amd/amdgpu/dce_virtual.c @@ -294,7 +294,7 @@ static int dce_virtual_get_modes(struct drm_connector *connector) static const struct mode_size { int w; int h; - } common_modes[21] = { + } common_modes[] = { { 640, 480}, { 720, 480}, { 800, 600}, @@ -312,13 +312,14 @@ static int dce_virtual_get_modes(struct drm_connector *connector) {1600, 1200}, {1920, 1080}, {1920, 1200}, + {2560, 1440}, {4096, 3112}, {3656, 2664}, {3840, 2160}, {4096, 2160}, }; - for (i = 0; i < 21; i++) { + for (i = 0; i < ARRAY_SIZE(common_modes); i++) { mode = drm_cvt_mode(dev, common_modes[i].w, common_modes[i].h, 60, false, false, false); drm_mode_probed_add(connector, mode); } diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c index d86b42a36560183b4d1ef0bde1a199800bec4285..45d1172b7bff93e8ef38e87859467c5944e93e0c 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c @@ -38,7 +38,6 @@ #include "smuio/smuio_11_0_0_offset.h" #include "smuio/smuio_11_0_0_sh_mask.h" #include "navi10_enum.h" -#include "hdp/hdp_5_0_0_offset.h" #include "ivsrcid/gfx/irqsrcs_gfx_10_1.h" #include "soc15.h" @@ -71,6 +70,11 @@ #define GB_ADDR_CONFIG__NUM_PKRS__SHIFT 0x8 #define GB_ADDR_CONFIG__NUM_PKRS_MASK 0x00000700L +#define mmCGTS_TCC_DISABLE_gc_10_3 0x5006 +#define mmCGTS_TCC_DISABLE_gc_10_3_BASE_IDX 1 +#define mmCGTS_USER_TCC_DISABLE_gc_10_3 0x5007 +#define mmCGTS_USER_TCC_DISABLE_gc_10_3_BASE_IDX 1 + #define mmCP_MEC_CNTL_Sienna_Cichlid 0x0f55 #define mmCP_MEC_CNTL_Sienna_Cichlid_BASE_IDX 0 #define mmRLC_SAFE_MODE_Sienna_Cichlid 0x4ca0 @@ -99,10 +103,6 @@ #define mmGCR_GENERAL_CNTL_Sienna_Cichlid 0x1580 #define mmGCR_GENERAL_CNTL_Sienna_Cichlid_BASE_IDX 0 -#define mmCGTS_TCC_DISABLE_Vangogh 0x5006 -#define mmCGTS_TCC_DISABLE_Vangogh_BASE_IDX 1 -#define mmCGTS_USER_TCC_DISABLE_Vangogh 0x5007 -#define mmCGTS_USER_TCC_DISABLE_Vangogh_BASE_IDX 1 #define mmGOLDEN_TSC_COUNT_UPPER_Vangogh 0x0025 #define mmGOLDEN_TSC_COUNT_UPPER_Vangogh_BASE_IDX 1 #define mmGOLDEN_TSC_COUNT_LOWER_Vangogh 0x0026 @@ -125,6 +125,7 @@ #define mmSPI_CONFIG_CNTL_Vangogh_BASE_IDX 1 #define mmGCR_GENERAL_CNTL_Vangogh 0x1580 #define mmGCR_GENERAL_CNTL_Vangogh_BASE_IDX 0 +#define RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG_MASK_Vangogh 0x0000FFFFL #define mmCP_HYP_PFP_UCODE_ADDR 0x5814 #define mmCP_HYP_PFP_UCODE_ADDR_BASE_IDX 1 @@ -3782,9 +3783,6 @@ static void gfx_v10_0_check_gfxoff_flag(struct amdgpu_device *adev) if (!gfx_v10_0_navi10_gfxoff_should_enable(adev)) adev->pm.pp_feature &= ~PP_GFXOFF_MASK; break; - case CHIP_VANGOGH: - adev->pm.pp_feature &= ~PP_GFXOFF_MASK; - break; default: break; } @@ -4494,8 +4492,7 @@ static int gfx_v10_0_compute_ring_init(struct amdgpu_device *adev, int ring_id, irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP + ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec) + ring->pipe; - hw_prio = amdgpu_gfx_is_high_priority_compute_queue(adev, ring->pipe, - ring->queue) ? + hw_prio = amdgpu_gfx_is_high_priority_compute_queue(adev, ring) ? AMDGPU_GFX_PIPE_PRIO_HIGH : AMDGPU_GFX_PIPE_PRIO_NORMAL; /* type-2 packets are deprecated on MEC, use type-3 instead */ r = amdgpu_ring_init(adev, ring, 1024, @@ -4942,15 +4939,12 @@ static void gfx_v10_0_get_tcc_info(struct amdgpu_device *adev) /* TCCs are global (not instanced). */ uint32_t tcc_disable; - switch (adev->asic_type) { - case CHIP_VANGOGH: - tcc_disable = RREG32_SOC15(GC, 0, mmCGTS_TCC_DISABLE_Vangogh) | - RREG32_SOC15(GC, 0, mmCGTS_USER_TCC_DISABLE_Vangogh); - break; - default: + if (adev->asic_type >= CHIP_SIENNA_CICHLID) { + tcc_disable = RREG32_SOC15(GC, 0, mmCGTS_TCC_DISABLE_gc_10_3) | + RREG32_SOC15(GC, 0, mmCGTS_USER_TCC_DISABLE_gc_10_3); + } else { tcc_disable = RREG32_SOC15(GC, 0, mmCGTS_TCC_DISABLE) | - RREG32_SOC15(GC, 0, mmCGTS_USER_TCC_DISABLE); - break; + RREG32_SOC15(GC, 0, mmCGTS_USER_TCC_DISABLE); } adev->gfx.config.tcc_disabled_mask = @@ -5715,7 +5709,7 @@ static int gfx_v10_0_cp_gfx_load_pfp_microcode(struct amdgpu_device *adev) } if (amdgpu_emu_mode == 1) - adev->nbio.funcs->hdp_flush(adev, NULL); + adev->hdp.funcs->flush_hdp(adev, NULL); tmp = RREG32_SOC15(GC, 0, mmCP_PFP_IC_BASE_CNTL); tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, VMID, 0); @@ -5793,7 +5787,7 @@ static int gfx_v10_0_cp_gfx_load_ce_microcode(struct amdgpu_device *adev) } if (amdgpu_emu_mode == 1) - adev->nbio.funcs->hdp_flush(adev, NULL); + adev->hdp.funcs->flush_hdp(adev, NULL); tmp = RREG32_SOC15(GC, 0, mmCP_CE_IC_BASE_CNTL); tmp = REG_SET_FIELD(tmp, CP_CE_IC_BASE_CNTL, VMID, 0); @@ -5870,7 +5864,7 @@ static int gfx_v10_0_cp_gfx_load_me_microcode(struct amdgpu_device *adev) } if (amdgpu_emu_mode == 1) - adev->nbio.funcs->hdp_flush(adev, NULL); + adev->hdp.funcs->flush_hdp(adev, NULL); tmp = RREG32_SOC15(GC, 0, mmCP_ME_IC_BASE_CNTL); tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, VMID, 0); @@ -6239,7 +6233,7 @@ static int gfx_v10_0_cp_compute_load_microcode(struct amdgpu_device *adev) } if (amdgpu_emu_mode == 1) - adev->nbio.funcs->hdp_flush(adev, NULL); + adev->hdp.funcs->flush_hdp(adev, NULL); tmp = RREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_CNTL); tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0); @@ -6547,8 +6541,7 @@ static void gfx_v10_0_compute_mqd_set_priority(struct amdgpu_ring *ring, struct struct amdgpu_device *adev = ring->adev; if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) { - if (amdgpu_gfx_is_high_priority_compute_queue(adev, ring->pipe, - ring->queue)) { + if (amdgpu_gfx_is_high_priority_compute_queue(adev, ring)) { mqd->cp_hqd_pipe_priority = AMDGPU_GFX_PIPE_PRIO_HIGH; mqd->cp_hqd_queue_priority = AMDGPU_GFX_QUEUE_PRIORITY_MAXIMUM; @@ -7847,6 +7840,20 @@ static void gfx_v10_cntl_power_gating(struct amdgpu_device *adev, bool enable) data &= ~RLC_PG_CNTL__GFX_POWER_GATING_ENABLE_MASK; WREG32_SOC15(GC, 0, mmRLC_PG_CNTL, data); + + /* + * CGPG enablement required and the register to program the hysteresis value + * RLC_PG_DELAY_3.CGCG_ACTIVE_BEFORE_CGPG to the desired CGPG hysteresis value + * in refclk count. Note that RLC FW is modified to take 16 bits from + * RLC_PG_DELAY_3[15:0] as the hysteresis instead of just 8 bits. + * + * The recommendation from RLC team is setting RLC_PG_DELAY_3 to 200us(0x4E20) + * as part of CGPG enablement starting point. + */ + if (enable && (adev->pg_flags & AMD_PG_SUPPORT_GFX_PG) && adev->asic_type == CHIP_VANGOGH) { + data = 0x4E20 & RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG_MASK_Vangogh; + WREG32_SOC15(GC, 0, mmRLC_PG_DELAY_3, data); + } } static void gfx_v10_cntl_pg(struct amdgpu_device *adev, bool enable) @@ -7908,6 +7915,7 @@ static int gfx_v10_0_set_powergating_state(void *handle, break; case CHIP_VANGOGH: gfx_v10_cntl_pg(adev, enable); + amdgpu_gfx_off_ctrl(adev, enable); break; default: break; diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c index 37639214cbbbd5edeca3e97a93360c4e4ab32020..84d2eaa381013781ec62d71a2fa63c48a5a63830 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c @@ -29,6 +29,7 @@ #include "amdgpu.h" #include "amdgpu_gfx.h" +#include "amdgpu_ring.h" #include "vi.h" #include "vi_structs.h" #include "vid.h" @@ -1923,8 +1924,7 @@ static int gfx_v8_0_compute_ring_init(struct amdgpu_device *adev, int ring_id, + ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec) + ring->pipe; - hw_prio = amdgpu_gfx_is_high_priority_compute_queue(adev, ring->pipe, - ring->queue) ? + hw_prio = amdgpu_gfx_is_high_priority_compute_queue(adev, ring) ? AMDGPU_GFX_PIPE_PRIO_HIGH : AMDGPU_RING_PRIO_DEFAULT; /* type-2 packets are deprecated on MEC, use type-3 instead */ r = amdgpu_ring_init(adev, ring, 1024, @@ -4442,8 +4442,7 @@ static void gfx_v8_0_mqd_set_priority(struct amdgpu_ring *ring, struct vi_mqd *m struct amdgpu_device *adev = ring->adev; if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) { - if (amdgpu_gfx_is_high_priority_compute_queue(adev, ring->pipe, - ring->queue)) { + if (amdgpu_gfx_is_high_priority_compute_queue(adev, ring)) { mqd->cp_hqd_pipe_priority = AMDGPU_GFX_PIPE_PRIO_HIGH; mqd->cp_hqd_queue_priority = AMDGPU_GFX_QUEUE_PRIORITY_MAXIMUM; @@ -6847,6 +6846,66 @@ static void gfx_v8_0_emit_mem_sync_compute(struct amdgpu_ring *ring) amdgpu_ring_write(ring, 0x0000000A); /* poll interval */ } + +/* mmSPI_WCL_PIPE_PERCENT_CS[0-7]_DEFAULT values are same */ +#define mmSPI_WCL_PIPE_PERCENT_CS_DEFAULT 0x0000007f +static void gfx_v8_0_emit_wave_limit_cs(struct amdgpu_ring *ring, + uint32_t pipe, bool enable) +{ + uint32_t val; + uint32_t wcl_cs_reg; + + val = enable ? 0x1 : mmSPI_WCL_PIPE_PERCENT_CS_DEFAULT; + + switch (pipe) { + case 0: + wcl_cs_reg = mmSPI_WCL_PIPE_PERCENT_CS0; + break; + case 1: + wcl_cs_reg = mmSPI_WCL_PIPE_PERCENT_CS1; + break; + case 2: + wcl_cs_reg = mmSPI_WCL_PIPE_PERCENT_CS2; + break; + case 3: + wcl_cs_reg = mmSPI_WCL_PIPE_PERCENT_CS3; + break; + default: + DRM_DEBUG("invalid pipe %d\n", pipe); + return; + } + + amdgpu_ring_emit_wreg(ring, wcl_cs_reg, val); + +} + +#define mmSPI_WCL_PIPE_PERCENT_GFX_DEFAULT 0x07ffffff +static void gfx_v8_0_emit_wave_limit(struct amdgpu_ring *ring, bool enable) +{ + struct amdgpu_device *adev = ring->adev; + uint32_t val; + int i; + + /* mmSPI_WCL_PIPE_PERCENT_GFX is 7 bit multiplier register to limit + * number of gfx waves. Setting 5 bit will make sure gfx only gets + * around 25% of gpu resources. + */ + val = enable ? 0x1f : mmSPI_WCL_PIPE_PERCENT_GFX_DEFAULT; + amdgpu_ring_emit_wreg(ring, mmSPI_WCL_PIPE_PERCENT_GFX, val); + + /* Restrict waves for normal/low priority compute queues as well + * to get best QoS for high priority compute jobs. + * + * amdgpu controls only 1st ME(0-3 CS pipes). + */ + for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) { + if (i != ring->pipe) + gfx_v8_0_emit_wave_limit_cs(ring, i, enable); + + } + +} + static const struct amd_ip_funcs gfx_v8_0_ip_funcs = { .name = "gfx_v8_0", .early_init = gfx_v8_0_early_init, @@ -6930,7 +6989,9 @@ static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_compute = { 7 + /* gfx_v8_0_ring_emit_pipeline_sync */ VI_FLUSH_GPU_TLB_NUM_WREG * 5 + 7 + /* gfx_v8_0_ring_emit_vm_flush */ 7 + 7 + 7 + /* gfx_v8_0_ring_emit_fence_compute x3 for user fence, vm fence */ - 7, /* gfx_v8_0_emit_mem_sync_compute */ + 7 + /* gfx_v8_0_emit_mem_sync_compute */ + 5 + /* gfx_v8_0_emit_wave_limit for updating mmSPI_WCL_PIPE_PERCENT_GFX register */ + 15, /* for updating 3 mmSPI_WCL_PIPE_PERCENT_CS registers */ .emit_ib_size = 7, /* gfx_v8_0_ring_emit_ib_compute */ .emit_ib = gfx_v8_0_ring_emit_ib_compute, .emit_fence = gfx_v8_0_ring_emit_fence_compute, @@ -6944,6 +7005,7 @@ static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_compute = { .pad_ib = amdgpu_ring_generic_pad_ib, .emit_wreg = gfx_v8_0_ring_emit_wreg, .emit_mem_sync = gfx_v8_0_emit_mem_sync_compute, + .emit_wave_limit = gfx_v8_0_emit_wave_limit, }; static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_kiq = { diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c index 5f4805e4d04ac840ededc9ca72d248cc514e95d6..65db88bb6cbcd6a3d77c64524cbeddba933e7662 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c @@ -38,7 +38,6 @@ #include "gc/gc_9_0_sh_mask.h" #include "vega10_enum.h" -#include "hdp/hdp_4_0_offset.h" #include "soc15_common.h" #include "clearstate_gfx9.h" @@ -53,6 +52,7 @@ #include "asic_reg/pwr/pwr_10_0_offset.h" #include "asic_reg/pwr/pwr_10_0_sh_mask.h" +#include "asic_reg/gc/gc_9_0_default.h" #define GFX9_NUM_GFX_RINGS 1 #define GFX9_MEC_HPD_SIZE 4096 @@ -2228,8 +2228,7 @@ static int gfx_v9_0_compute_ring_init(struct amdgpu_device *adev, int ring_id, irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP + ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec) + ring->pipe; - hw_prio = amdgpu_gfx_is_high_priority_compute_queue(adev, ring->pipe, - ring->queue) ? + hw_prio = amdgpu_gfx_is_high_priority_compute_queue(adev, ring) ? AMDGPU_GFX_PIPE_PRIO_HIGH : AMDGPU_GFX_PIPE_PRIO_NORMAL; /* type-2 packets are deprecated on MEC, use type-3 instead */ return amdgpu_ring_init(adev, ring, 1024, @@ -3391,9 +3390,7 @@ static void gfx_v9_0_mqd_set_priority(struct amdgpu_ring *ring, struct v9_mqd *m struct amdgpu_device *adev = ring->adev; if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) { - if (amdgpu_gfx_is_high_priority_compute_queue(adev, - ring->pipe, - ring->queue)) { + if (amdgpu_gfx_is_high_priority_compute_queue(adev, ring)) { mqd->cp_hqd_pipe_priority = AMDGPU_GFX_PIPE_PRIO_HIGH; mqd->cp_hqd_queue_priority = AMDGPU_GFX_QUEUE_PRIORITY_MAXIMUM; @@ -6671,6 +6668,65 @@ static void gfx_v9_0_emit_mem_sync(struct amdgpu_ring *ring) amdgpu_ring_write(ring, 0x0000000A); /* POLL_INTERVAL */ } +static void gfx_v9_0_emit_wave_limit_cs(struct amdgpu_ring *ring, + uint32_t pipe, bool enable) +{ + struct amdgpu_device *adev = ring->adev; + uint32_t val; + uint32_t wcl_cs_reg; + + /* mmSPI_WCL_PIPE_PERCENT_CS[0-7]_DEFAULT values are same */ + val = enable ? 0x1 : mmSPI_WCL_PIPE_PERCENT_CS0_DEFAULT; + + switch (pipe) { + case 0: + wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS0); + break; + case 1: + wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS1); + break; + case 2: + wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS2); + break; + case 3: + wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS3); + break; + default: + DRM_DEBUG("invalid pipe %d\n", pipe); + return; + } + + amdgpu_ring_emit_wreg(ring, wcl_cs_reg, val); + +} +static void gfx_v9_0_emit_wave_limit(struct amdgpu_ring *ring, bool enable) +{ + struct amdgpu_device *adev = ring->adev; + uint32_t val; + int i; + + + /* mmSPI_WCL_PIPE_PERCENT_GFX is 7 bit multiplier register to limit + * number of gfx waves. Setting 5 bit will make sure gfx only gets + * around 25% of gpu resources. + */ + val = enable ? 0x1f : mmSPI_WCL_PIPE_PERCENT_GFX_DEFAULT; + amdgpu_ring_emit_wreg(ring, + SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_GFX), + val); + + /* Restrict waves for normal/low priority compute queues as well + * to get best QoS for high priority compute jobs. + * + * amdgpu controls only 1st ME(0-3 CS pipes). + */ + for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) { + if (i != ring->pipe) + gfx_v9_0_emit_wave_limit_cs(ring, i, enable); + + } +} + static const struct amd_ip_funcs gfx_v9_0_ip_funcs = { .name = "gfx_v9_0", .early_init = gfx_v9_0_early_init, @@ -6760,7 +6816,9 @@ static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_compute = { SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 + 2 + /* gfx_v9_0_ring_emit_vm_flush */ 8 + 8 + 8 + /* gfx_v9_0_ring_emit_fence x3 for user fence, vm fence */ - 7, /* gfx_v9_0_emit_mem_sync */ + 7 + /* gfx_v9_0_emit_mem_sync */ + 5 + /* gfx_v9_0_emit_wave_limit for updating mmSPI_WCL_PIPE_PERCENT_GFX register */ + 15, /* for updating 3 mmSPI_WCL_PIPE_PERCENT_CS registers */ .emit_ib_size = 7, /* gfx_v9_0_ring_emit_ib_compute */ .emit_ib = gfx_v9_0_ring_emit_ib_compute, .emit_fence = gfx_v9_0_ring_emit_fence, @@ -6776,6 +6834,7 @@ static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_compute = { .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait, .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait, .emit_mem_sync = gfx_v9_0_emit_mem_sync, + .emit_wave_limit = gfx_v9_0_emit_wave_limit, }; static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_kiq = { diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c index 5648c48be77f0481cf1f78afd85285d963e0dfec..3b7c6c31fce1f17590a11b4b03b0e59db224e37c 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c @@ -27,8 +27,6 @@ #include "gmc_v10_0.h" #include "umc_v8_7.h" -#include "hdp/hdp_5_0_0_offset.h" -#include "hdp/hdp_5_0_0_sh_mask.h" #include "athub/athub_2_0_0_sh_mask.h" #include "athub/athub_2_0_0_offset.h" #include "dcn/dcn_2_0_0_offset.h" @@ -312,7 +310,7 @@ static void gmc_v10_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid, int r; /* flush hdp cache */ - adev->nbio.funcs->hdp_flush(adev, NULL); + adev->hdp.funcs->flush_hdp(adev, NULL); /* For SRIOV run time, driver shouldn't access the register through MMIO * Directly use kiq to do the vm invalidation instead @@ -995,7 +993,6 @@ static int gmc_v10_0_gart_enable(struct amdgpu_device *adev) { int r; bool value; - u32 tmp; if (adev->gart.bo == NULL) { dev_err(adev->dev, "No VRAM object for PCIE GART.\n"); @@ -1014,15 +1011,10 @@ static int gmc_v10_0_gart_enable(struct amdgpu_device *adev) if (r) return r; - tmp = RREG32_SOC15(HDP, 0, mmHDP_MISC_CNTL); - tmp |= HDP_MISC_CNTL__FLUSH_INVALIDATE_CACHE_MASK; - WREG32_SOC15(HDP, 0, mmHDP_MISC_CNTL, tmp); - - tmp = RREG32_SOC15(HDP, 0, mmHDP_HOST_PATH_CNTL); - WREG32_SOC15(HDP, 0, mmHDP_HOST_PATH_CNTL, tmp); + adev->hdp.funcs->init_registers(adev); /* Flush HDP after it is initialized */ - adev->nbio.funcs->hdp_flush(adev, NULL); + adev->hdp.funcs->flush_hdp(adev, NULL); value = (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS) ? false : true; diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c index e22268f9dba7315a64e0abe52d68f06806e3ef7b..3686e777c76cb232cfbfbafc66680bb1a073db34 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c @@ -31,8 +31,6 @@ #include "amdgpu_atomfirmware.h" #include "amdgpu_gem.h" -#include "hdp/hdp_4_0_offset.h" -#include "hdp/hdp_4_0_sh_mask.h" #include "gc/gc_9_0_sh_mask.h" #include "dce/dce_12_0_offset.h" #include "dce/dce_12_0_sh_mask.h" @@ -241,60 +239,44 @@ static const char *mmhub_client_ids_vega20[][2] = { }; static const char *mmhub_client_ids_arcturus[][2] = { + [0][0] = "DBGU1", + [1][0] = "XDP", [2][0] = "MP1", - [3][0] = "MP0", - [10][0] = "UTCL2", - [13][0] = "OSS", [14][0] = "HDP", - [15][0] = "SDMA0", - [32+15][0] = "SDMA1", - [64+15][0] = "SDMA2", - [96+15][0] = "SDMA3", - [128+15][0] = "SDMA4", - [160+11][0] = "JPEG", - [160+12][0] = "VCN", - [160+13][0] = "VCNU", - [160+15][0] = "SDMA5", - [192+10][0] = "UTCL2", - [192+11][0] = "JPEG1", - [192+12][0] = "VCN1", - [192+13][0] = "VCN1U", - [192+15][0] = "SDMA6", - [224+15][0] = "SDMA7", + [171][0] = "JPEG", + [172][0] = "VCN", + [173][0] = "VCNU", + [203][0] = "JPEG1", + [204][0] = "VCN1", + [205][0] = "VCN1U", + [256][0] = "SDMA0", + [257][0] = "SDMA1", + [258][0] = "SDMA2", + [259][0] = "SDMA3", + [260][0] = "SDMA4", + [261][0] = "SDMA5", + [262][0] = "SDMA6", + [263][0] = "SDMA7", + [384][0] = "OSS", [0][1] = "DBGU1", [1][1] = "XDP", [2][1] = "MP1", - [3][1] = "MP0", - [13][1] = "OSS", [14][1] = "HDP", - [15][1] = "SDMA0", - [32+15][1] = "SDMA1", - [64+15][1] = "SDMA2", - [96+15][1] = "SDMA3", - [128+15][1] = "SDMA4", - [160+11][1] = "JPEG", - [160+12][1] = "VCN", - [160+13][1] = "VCNU", - [160+15][1] = "SDMA5", - [192+11][1] = "JPEG1", - [192+12][1] = "VCN1", - [192+13][1] = "VCN1U", - [192+15][1] = "SDMA6", - [224+15][1] = "SDMA7", -}; - -static const u32 golden_settings_vega10_hdp[] = -{ - 0xf64, 0x0fffffff, 0x00000000, - 0xf65, 0x0fffffff, 0x00000000, - 0xf66, 0x0fffffff, 0x00000000, - 0xf67, 0x0fffffff, 0x00000000, - 0xf68, 0x0fffffff, 0x00000000, - 0xf6a, 0x0fffffff, 0x00000000, - 0xf6b, 0x0fffffff, 0x00000000, - 0xf6c, 0x0fffffff, 0x00000000, - 0xf6d, 0x0fffffff, 0x00000000, - 0xf6e, 0x0fffffff, 0x00000000, + [171][1] = "JPEG", + [172][1] = "VCN", + [173][1] = "VCNU", + [203][1] = "JPEG1", + [204][1] = "VCN1", + [205][1] = "VCN1U", + [256][1] = "SDMA0", + [257][1] = "SDMA1", + [258][1] = "SDMA2", + [259][1] = "SDMA3", + [260][1] = "SDMA4", + [261][1] = "SDMA5", + [262][1] = "SDMA6", + [263][1] = "SDMA7", + [384][1] = "OSS", }; static const struct soc15_reg_golden golden_settings_mmhub_1_0_0[] = @@ -1571,7 +1553,6 @@ static int gmc_v9_0_hw_init(void *handle) struct amdgpu_device *adev = (struct amdgpu_device *)handle; bool value; int r, i; - u32 tmp; /* The sequence of these two function calls matters.*/ gmc_v9_0_init_golden_registers(adev); @@ -1583,31 +1564,13 @@ static int gmc_v9_0_hw_init(void *handle) WREG32_FIELD15(DCE, 0, VGA_RENDER_CONTROL, VGA_VSTATUS_CNTL, 0); } - amdgpu_device_program_register_sequence(adev, - golden_settings_vega10_hdp, - ARRAY_SIZE(golden_settings_vega10_hdp)); - if (adev->mmhub.funcs->update_power_gating) adev->mmhub.funcs->update_power_gating(adev, true); - switch (adev->asic_type) { - case CHIP_ARCTURUS: - WREG32_FIELD15(HDP, 0, HDP_MMHUB_CNTL, HDP_MMHUB_GCC, 1); - break; - default: - break; - } - - WREG32_FIELD15(HDP, 0, HDP_MISC_CNTL, FLUSH_INVALIDATE_CACHE, 1); - - tmp = RREG32_SOC15(HDP, 0, mmHDP_HOST_PATH_CNTL); - WREG32_SOC15(HDP, 0, mmHDP_HOST_PATH_CNTL, tmp); - - WREG32_SOC15(HDP, 0, mmHDP_NONSURFACE_BASE, (adev->gmc.vram_start >> 8)); - WREG32_SOC15(HDP, 0, mmHDP_NONSURFACE_BASE_HI, (adev->gmc.vram_start >> 40)); + adev->hdp.funcs->init_registers(adev); /* After HDP is initialized, flush HDP.*/ - adev->nbio.funcs->hdp_flush(adev, NULL); + adev->hdp.funcs->flush_hdp(adev, NULL); if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS) value = false; diff --git a/drivers/gpu/drm/amd/amdgpu/hdp_v4_0.c b/drivers/gpu/drm/amd/amdgpu/hdp_v4_0.c new file mode 100644 index 0000000000000000000000000000000000000000..e46621fed5b953952e3ccddbae27fd55aac67fff --- /dev/null +++ b/drivers/gpu/drm/amd/amdgpu/hdp_v4_0.c @@ -0,0 +1,137 @@ +/* + * Copyright 2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ +#include "amdgpu.h" +#include "amdgpu_atombios.h" +#include "hdp_v4_0.h" +#include "amdgpu_ras.h" + +#include "hdp/hdp_4_0_offset.h" +#include "hdp/hdp_4_0_sh_mask.h" +#include + +/* for Vega20 register name change */ +#define mmHDP_MEM_POWER_CTRL 0x00d4 +#define HDP_MEM_POWER_CTRL__IPH_MEM_POWER_CTRL_EN_MASK 0x00000001L +#define HDP_MEM_POWER_CTRL__IPH_MEM_POWER_LS_EN_MASK 0x00000002L +#define HDP_MEM_POWER_CTRL__RC_MEM_POWER_CTRL_EN_MASK 0x00010000L +#define HDP_MEM_POWER_CTRL__RC_MEM_POWER_LS_EN_MASK 0x00020000L +#define mmHDP_MEM_POWER_CTRL_BASE_IDX 0 + +static void hdp_v4_0_flush_hdp(struct amdgpu_device *adev, + struct amdgpu_ring *ring) +{ + if (!ring || !ring->funcs->emit_wreg) + WREG32_NO_KIQ((adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_MEM_FLUSH_CNTL) >> 2, 0); + else + amdgpu_ring_emit_wreg(ring, (adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_MEM_FLUSH_CNTL) >> 2, 0); +} + +static void hdp_v4_0_invalidate_hdp(struct amdgpu_device *adev, + struct amdgpu_ring *ring) +{ + if (!ring || !ring->funcs->emit_wreg) + WREG32_SOC15_NO_KIQ(HDP, 0, mmHDP_READ_CACHE_INVALIDATE, 1); + else + amdgpu_ring_emit_wreg(ring, SOC15_REG_OFFSET( + HDP, 0, mmHDP_READ_CACHE_INVALIDATE), 1); +} + +static void hdp_v4_0_reset_ras_error_count(struct amdgpu_device *adev) +{ + if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__HDP)) + return; + /*read back hdp ras counter to reset it to 0 */ + RREG32_SOC15(HDP, 0, mmHDP_EDC_CNT); +} + +static void hdp_v4_0_update_clock_gating(struct amdgpu_device *adev, + bool enable) +{ + uint32_t def, data; + + if (adev->asic_type == CHIP_VEGA10 || + adev->asic_type == CHIP_VEGA12 || + adev->asic_type == CHIP_RAVEN) { + def = data = RREG32(SOC15_REG_OFFSET(HDP, 0, mmHDP_MEM_POWER_LS)); + + if (enable && (adev->cg_flags & AMD_CG_SUPPORT_HDP_LS)) + data |= HDP_MEM_POWER_LS__LS_ENABLE_MASK; + else + data &= ~HDP_MEM_POWER_LS__LS_ENABLE_MASK; + + if (def != data) + WREG32(SOC15_REG_OFFSET(HDP, 0, mmHDP_MEM_POWER_LS), data); + } else { + def = data = RREG32(SOC15_REG_OFFSET(HDP, 0, mmHDP_MEM_POWER_CTRL)); + + if (enable && (adev->cg_flags & AMD_CG_SUPPORT_HDP_LS)) + data |= HDP_MEM_POWER_CTRL__IPH_MEM_POWER_CTRL_EN_MASK | + HDP_MEM_POWER_CTRL__IPH_MEM_POWER_LS_EN_MASK | + HDP_MEM_POWER_CTRL__RC_MEM_POWER_CTRL_EN_MASK | + HDP_MEM_POWER_CTRL__RC_MEM_POWER_LS_EN_MASK; + else + data &= ~(HDP_MEM_POWER_CTRL__IPH_MEM_POWER_CTRL_EN_MASK | + HDP_MEM_POWER_CTRL__IPH_MEM_POWER_LS_EN_MASK | + HDP_MEM_POWER_CTRL__RC_MEM_POWER_CTRL_EN_MASK | + HDP_MEM_POWER_CTRL__RC_MEM_POWER_LS_EN_MASK); + + if (def != data) + WREG32(SOC15_REG_OFFSET(HDP, 0, mmHDP_MEM_POWER_CTRL), data); + } +} + +static void hdp_v4_0_get_clockgating_state(struct amdgpu_device *adev, + u32 *flags) +{ + int data; + + /* AMD_CG_SUPPORT_HDP_LS */ + data = RREG32(SOC15_REG_OFFSET(HDP, 0, mmHDP_MEM_POWER_LS)); + if (data & HDP_MEM_POWER_LS__LS_ENABLE_MASK) + *flags |= AMD_CG_SUPPORT_HDP_LS; +} + +static void hdp_v4_0_init_registers(struct amdgpu_device *adev) +{ + switch (adev->asic_type) { + case CHIP_ARCTURUS: + WREG32_FIELD15(HDP, 0, HDP_MMHUB_CNTL, HDP_MMHUB_GCC, 1); + break; + default: + break; + } + + WREG32_FIELD15(HDP, 0, HDP_MISC_CNTL, FLUSH_INVALIDATE_CACHE, 1); + + WREG32_SOC15(HDP, 0, mmHDP_NONSURFACE_BASE, (adev->gmc.vram_start >> 8)); + WREG32_SOC15(HDP, 0, mmHDP_NONSURFACE_BASE_HI, (adev->gmc.vram_start >> 40)); +} + +const struct amdgpu_hdp_funcs hdp_v4_0_funcs = { + .flush_hdp = hdp_v4_0_flush_hdp, + .invalidate_hdp = hdp_v4_0_invalidate_hdp, + .reset_ras_error_count = hdp_v4_0_reset_ras_error_count, + .update_clock_gating = hdp_v4_0_update_clock_gating, + .get_clock_gating_state = hdp_v4_0_get_clockgating_state, + .init_registers = hdp_v4_0_init_registers, +}; diff --git a/drivers/gpu/drm/amd/amdgpu/hdp_v4_0.h b/drivers/gpu/drm/amd/amdgpu/hdp_v4_0.h new file mode 100644 index 0000000000000000000000000000000000000000..d1e6399e8c46ef9e86cf9e521f11bbdf73474a26 --- /dev/null +++ b/drivers/gpu/drm/amd/amdgpu/hdp_v4_0.h @@ -0,0 +1,31 @@ +/* + * Copyright 2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __HDP_V4_0_H__ +#define __HDP_V4_0_H__ + +#include "soc15_common.h" + +extern const struct amdgpu_hdp_funcs hdp_v4_0_funcs; + +#endif diff --git a/drivers/gpu/drm/amd/amdgpu/hdp_v5_0.c b/drivers/gpu/drm/amd/amdgpu/hdp_v5_0.c new file mode 100644 index 0000000000000000000000000000000000000000..7a15e669b68db0517fc6f19c6aad62cbdb183457 --- /dev/null +++ b/drivers/gpu/drm/amd/amdgpu/hdp_v5_0.c @@ -0,0 +1,212 @@ +/* + * Copyright 2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ +#include "amdgpu.h" +#include "amdgpu_atombios.h" +#include "hdp_v5_0.h" + +#include "hdp/hdp_5_0_0_offset.h" +#include "hdp/hdp_5_0_0_sh_mask.h" +#include + +static void hdp_v5_0_flush_hdp(struct amdgpu_device *adev, + struct amdgpu_ring *ring) +{ + if (!ring || !ring->funcs->emit_wreg) + WREG32_NO_KIQ((adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_MEM_FLUSH_CNTL) >> 2, 0); + else + amdgpu_ring_emit_wreg(ring, (adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_MEM_FLUSH_CNTL) >> 2, 0); +} + +static void hdp_v5_0_invalidate_hdp(struct amdgpu_device *adev, + struct amdgpu_ring *ring) +{ + if (!ring || !ring->funcs->emit_wreg) { + WREG32_SOC15_NO_KIQ(HDP, 0, mmHDP_READ_CACHE_INVALIDATE, 1); + } else { + amdgpu_ring_emit_wreg(ring, SOC15_REG_OFFSET( + HDP, 0, mmHDP_READ_CACHE_INVALIDATE), 1); + } +} + +static void hdp_v5_0_update_mem_power_gating(struct amdgpu_device *adev, + bool enable) +{ + uint32_t hdp_clk_cntl, hdp_clk_cntl1; + uint32_t hdp_mem_pwr_cntl; + + if (!(adev->cg_flags & (AMD_CG_SUPPORT_HDP_LS | + AMD_CG_SUPPORT_HDP_DS | + AMD_CG_SUPPORT_HDP_SD))) + return; + + hdp_clk_cntl = hdp_clk_cntl1 = RREG32_SOC15(HDP, 0, mmHDP_CLK_CNTL); + hdp_mem_pwr_cntl = RREG32_SOC15(HDP, 0, mmHDP_MEM_POWER_CTRL); + + /* Before doing clock/power mode switch, + * forced on IPH & RC clock */ + hdp_clk_cntl = REG_SET_FIELD(hdp_clk_cntl, HDP_CLK_CNTL, + IPH_MEM_CLK_SOFT_OVERRIDE, 1); + hdp_clk_cntl = REG_SET_FIELD(hdp_clk_cntl, HDP_CLK_CNTL, + RC_MEM_CLK_SOFT_OVERRIDE, 1); + WREG32_SOC15(HDP, 0, mmHDP_CLK_CNTL, hdp_clk_cntl); + + /* HDP 5.0 doesn't support dynamic power mode switch, + * disable clock and power gating before any changing */ + hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl, HDP_MEM_POWER_CTRL, + IPH_MEM_POWER_CTRL_EN, 0); + hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl, HDP_MEM_POWER_CTRL, + IPH_MEM_POWER_LS_EN, 0); + hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl, HDP_MEM_POWER_CTRL, + IPH_MEM_POWER_DS_EN, 0); + hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl, HDP_MEM_POWER_CTRL, + IPH_MEM_POWER_SD_EN, 0); + hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl, HDP_MEM_POWER_CTRL, + RC_MEM_POWER_CTRL_EN, 0); + hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl, HDP_MEM_POWER_CTRL, + RC_MEM_POWER_LS_EN, 0); + hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl, HDP_MEM_POWER_CTRL, + RC_MEM_POWER_DS_EN, 0); + hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl, HDP_MEM_POWER_CTRL, + RC_MEM_POWER_SD_EN, 0); + WREG32_SOC15(HDP, 0, mmHDP_MEM_POWER_CTRL, hdp_mem_pwr_cntl); + + /* only one clock gating mode (LS/DS/SD) can be enabled */ + if (adev->cg_flags & AMD_CG_SUPPORT_HDP_LS) { + hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl, + HDP_MEM_POWER_CTRL, + IPH_MEM_POWER_LS_EN, enable); + hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl, + HDP_MEM_POWER_CTRL, + RC_MEM_POWER_LS_EN, enable); + } else if (adev->cg_flags & AMD_CG_SUPPORT_HDP_DS) { + hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl, + HDP_MEM_POWER_CTRL, + IPH_MEM_POWER_DS_EN, enable); + hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl, + HDP_MEM_POWER_CTRL, + RC_MEM_POWER_DS_EN, enable); + } else if (adev->cg_flags & AMD_CG_SUPPORT_HDP_SD) { + hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl, + HDP_MEM_POWER_CTRL, + IPH_MEM_POWER_SD_EN, enable); + /* RC should not use shut down mode, fallback to ds */ + hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl, + HDP_MEM_POWER_CTRL, + RC_MEM_POWER_DS_EN, enable); + } + + /* confirmed that IPH_MEM_POWER_CTRL_EN and RC_MEM_POWER_CTRL_EN have to + * be set for SRAM LS/DS/SD */ + if (adev->cg_flags & (AMD_CG_SUPPORT_HDP_LS | AMD_CG_SUPPORT_HDP_DS | + AMD_CG_SUPPORT_HDP_SD)) { + hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl, HDP_MEM_POWER_CTRL, + IPH_MEM_POWER_CTRL_EN, 1); + hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl, HDP_MEM_POWER_CTRL, + RC_MEM_POWER_CTRL_EN, 1); + } + + WREG32_SOC15(HDP, 0, mmHDP_MEM_POWER_CTRL, hdp_mem_pwr_cntl); + + /* restore IPH & RC clock override after clock/power mode changing */ + WREG32_SOC15(HDP, 0, mmHDP_CLK_CNTL, hdp_clk_cntl1); +} + +static void hdp_v5_0_update_medium_grain_clock_gating(struct amdgpu_device *adev, + bool enable) +{ + uint32_t hdp_clk_cntl; + + if (!(adev->cg_flags & AMD_CG_SUPPORT_HDP_MGCG)) + return; + + hdp_clk_cntl = RREG32_SOC15(HDP, 0, mmHDP_CLK_CNTL); + + if (enable) { + hdp_clk_cntl &= + ~(uint32_t) + (HDP_CLK_CNTL__IPH_MEM_CLK_SOFT_OVERRIDE_MASK | + HDP_CLK_CNTL__RC_MEM_CLK_SOFT_OVERRIDE_MASK | + HDP_CLK_CNTL__DBUS_CLK_SOFT_OVERRIDE_MASK | + HDP_CLK_CNTL__DYN_CLK_SOFT_OVERRIDE_MASK | + HDP_CLK_CNTL__XDP_REG_CLK_SOFT_OVERRIDE_MASK | + HDP_CLK_CNTL__HDP_REG_CLK_SOFT_OVERRIDE_MASK); + } else { + hdp_clk_cntl |= HDP_CLK_CNTL__IPH_MEM_CLK_SOFT_OVERRIDE_MASK | + HDP_CLK_CNTL__RC_MEM_CLK_SOFT_OVERRIDE_MASK | + HDP_CLK_CNTL__DBUS_CLK_SOFT_OVERRIDE_MASK | + HDP_CLK_CNTL__DYN_CLK_SOFT_OVERRIDE_MASK | + HDP_CLK_CNTL__XDP_REG_CLK_SOFT_OVERRIDE_MASK | + HDP_CLK_CNTL__HDP_REG_CLK_SOFT_OVERRIDE_MASK; + } + + WREG32_SOC15(HDP, 0, mmHDP_CLK_CNTL, hdp_clk_cntl); +} + +static void hdp_v5_0_update_clock_gating(struct amdgpu_device *adev, + bool enable) +{ + hdp_v5_0_update_mem_power_gating(adev, enable); + hdp_v5_0_update_medium_grain_clock_gating(adev, enable); +} + +static void hdp_v5_0_get_clockgating_state(struct amdgpu_device *adev, + u32 *flags) +{ + uint32_t tmp; + + /* AMD_CG_SUPPORT_HDP_MGCG */ + tmp = RREG32_SOC15(HDP, 0, mmHDP_CLK_CNTL); + if (!(tmp & (HDP_CLK_CNTL__IPH_MEM_CLK_SOFT_OVERRIDE_MASK | + HDP_CLK_CNTL__RC_MEM_CLK_SOFT_OVERRIDE_MASK | + HDP_CLK_CNTL__DBUS_CLK_SOFT_OVERRIDE_MASK | + HDP_CLK_CNTL__DYN_CLK_SOFT_OVERRIDE_MASK | + HDP_CLK_CNTL__XDP_REG_CLK_SOFT_OVERRIDE_MASK | + HDP_CLK_CNTL__HDP_REG_CLK_SOFT_OVERRIDE_MASK))) + *flags |= AMD_CG_SUPPORT_HDP_MGCG; + + /* AMD_CG_SUPPORT_HDP_LS/DS/SD */ + tmp = RREG32_SOC15(HDP, 0, mmHDP_MEM_POWER_CTRL); + if (tmp & HDP_MEM_POWER_CTRL__IPH_MEM_POWER_LS_EN_MASK) + *flags |= AMD_CG_SUPPORT_HDP_LS; + else if (tmp & HDP_MEM_POWER_CTRL__IPH_MEM_POWER_DS_EN_MASK) + *flags |= AMD_CG_SUPPORT_HDP_DS; + else if (tmp & HDP_MEM_POWER_CTRL__IPH_MEM_POWER_SD_EN_MASK) + *flags |= AMD_CG_SUPPORT_HDP_SD; +} + +static void hdp_v5_0_init_registers(struct amdgpu_device *adev) +{ + u32 tmp; + + tmp = RREG32_SOC15(HDP, 0, mmHDP_MISC_CNTL); + tmp |= HDP_MISC_CNTL__FLUSH_INVALIDATE_CACHE_MASK; + WREG32_SOC15(HDP, 0, mmHDP_MISC_CNTL, tmp); +} + +const struct amdgpu_hdp_funcs hdp_v5_0_funcs = { + .flush_hdp = hdp_v5_0_flush_hdp, + .invalidate_hdp = hdp_v5_0_invalidate_hdp, + .update_clock_gating = hdp_v5_0_update_clock_gating, + .get_clock_gating_state = hdp_v5_0_get_clockgating_state, + .init_registers = hdp_v5_0_init_registers, +}; diff --git a/drivers/gpu/drm/amd/amdgpu/hdp_v5_0.h b/drivers/gpu/drm/amd/amdgpu/hdp_v5_0.h new file mode 100644 index 0000000000000000000000000000000000000000..2d5ec2b419f384a402a69913b2f5252d42ee5b2e --- /dev/null +++ b/drivers/gpu/drm/amd/amdgpu/hdp_v5_0.h @@ -0,0 +1,31 @@ +/* + * Copyright 2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __HDP_V5_0_H__ +#define __HDP_V5_0_H__ + +#include "soc15_common.h" + +extern const struct amdgpu_hdp_funcs hdp_v5_0_funcs; + +#endif diff --git a/drivers/gpu/drm/amd/amdgpu/iceland_ih.c b/drivers/gpu/drm/amd/amdgpu/iceland_ih.c index 37d8b6ca4dab8a76261d6e4cce1cf641bcdd4ed0..cc957471f31ea04b05fa4e12f4181b9680da6208 100644 --- a/drivers/gpu/drm/amd/amdgpu/iceland_ih.c +++ b/drivers/gpu/drm/amd/amdgpu/iceland_ih.c @@ -194,19 +194,29 @@ static u32 iceland_ih_get_wptr(struct amdgpu_device *adev, wptr = le32_to_cpu(*ih->wptr_cpu); - if (REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW)) { - wptr = REG_SET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW, 0); - /* When a ring buffer overflow happen start parsing interrupt - * from the last not overwritten vector (wptr + 16). Hopefully - * this should allow us to catchup. - */ - dev_warn(adev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n", - wptr, ih->rptr, (wptr + 16) & ih->ptr_mask); - ih->rptr = (wptr + 16) & ih->ptr_mask; - tmp = RREG32(mmIH_RB_CNTL); - tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 1); - WREG32(mmIH_RB_CNTL, tmp); - } + if (!REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW)) + goto out; + + /* Double check that the overflow wasn't already cleared. */ + wptr = RREG32(mmIH_RB_WPTR); + + if (!REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW)) + goto out; + + wptr = REG_SET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW, 0); + /* When a ring buffer overflow happen start parsing interrupt + * from the last not overwritten vector (wptr + 16). Hopefully + * this should allow us to catchup. + */ + dev_warn(adev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n", + wptr, ih->rptr, (wptr + 16) & ih->ptr_mask); + ih->rptr = (wptr + 16) & ih->ptr_mask; + tmp = RREG32(mmIH_RB_CNTL); + tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 1); + WREG32(mmIH_RB_CNTL, tmp); + + +out: return (wptr & ih->ptr_mask); } diff --git a/drivers/gpu/drm/amd/amdgpu/mes_v10_1.c b/drivers/gpu/drm/amd/amdgpu/mes_v10_1.c index 985e454463e1e2cc81d7f132c495269e5dcdfc30..7f30629f21a21dd3b90f35089eb5f27ab9f3172f 100644 --- a/drivers/gpu/drm/amd/amdgpu/mes_v10_1.c +++ b/drivers/gpu/drm/amd/amdgpu/mes_v10_1.c @@ -554,7 +554,7 @@ static int mes_v10_1_allocate_eop_buf(struct amdgpu_device *adev) return r; } - memset(eop, 0, adev->mes.eop_gpu_obj->tbo.mem.size); + memset(eop, 0, adev->mes.eop_gpu_obj->tbo.base.size); amdgpu_bo_kunmap(adev->mes.eop_gpu_obj); amdgpu_bo_unreserve(adev->mes.eop_gpu_obj); diff --git a/drivers/gpu/drm/amd/amdgpu/mmhub_v2_3.c b/drivers/gpu/drm/amd/amdgpu/mmhub_v2_3.c index 1961745e89c73df00c0fcc5198b96039d0f481e7..ab9be5ad5a5fd102f20eaf7581007e2b8827f275 100644 --- a/drivers/gpu/drm/amd/amdgpu/mmhub_v2_3.c +++ b/drivers/gpu/drm/amd/amdgpu/mmhub_v2_3.c @@ -531,12 +531,12 @@ mmhub_v2_3_update_medium_grain_light_sleep(struct amdgpu_device *adev, if (enable && (adev->cg_flags & AMD_CG_SUPPORT_MC_LS)) { data &= ~MM_ATC_L2_CGTT_CLK_CTRL__MGLS_OVERRIDE_MASK; - data1 &= !(DAGB0_WR_CGTT_CLK_CTRL__LS_OVERRIDE_MASK | + data1 &= ~(DAGB0_WR_CGTT_CLK_CTRL__LS_OVERRIDE_MASK | DAGB0_WR_CGTT_CLK_CTRL__LS_OVERRIDE_WRITE_MASK | DAGB0_WR_CGTT_CLK_CTRL__LS_OVERRIDE_READ_MASK | DAGB0_WR_CGTT_CLK_CTRL__LS_OVERRIDE_RETURN_MASK | DAGB0_WR_CGTT_CLK_CTRL__LS_OVERRIDE_REGISTER_MASK); - data2 &= !(DAGB0_RD_CGTT_CLK_CTRL__LS_OVERRIDE_MASK | + data2 &= ~(DAGB0_RD_CGTT_CLK_CTRL__LS_OVERRIDE_MASK | DAGB0_RD_CGTT_CLK_CTRL__LS_OVERRIDE_WRITE_MASK | DAGB0_RD_CGTT_CLK_CTRL__LS_OVERRIDE_READ_MASK | DAGB0_RD_CGTT_CLK_CTRL__LS_OVERRIDE_RETURN_MASK | diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c b/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c index 7767ccca526bbbe489d9313233c46f1a59801bc3..3ee481557fc9741b480284c8330e48723e0b0d5b 100644 --- a/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c +++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c @@ -255,6 +255,7 @@ static void xgpu_ai_mailbox_flr_work(struct work_struct *work) if (!down_read_trylock(&adev->reset_sem)) return; + amdgpu_virt_fini_data_exchange(adev); atomic_set(&adev->in_gpu_reset, 1); do { diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c b/drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c index dd5c1e6ce0098716024897b57b2c3d828a32f818..48e588d3c4098c84dcd5900426c235463eb962f9 100644 --- a/drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c +++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c @@ -276,6 +276,7 @@ static void xgpu_nv_mailbox_flr_work(struct work_struct *work) if (!down_read_trylock(&adev->reset_sem)) return; + amdgpu_virt_fini_data_exchange(adev); atomic_set(&adev->in_gpu_reset, 1); do { diff --git a/drivers/gpu/drm/amd/amdgpu/navi10_ih.c b/drivers/gpu/drm/amd/amdgpu/navi10_ih.c index 7ba229e43799f214ade4bb778a08961f635d37cf..f4e4040bbd25505e5df76bf0687bae008b57bfef 100644 --- a/drivers/gpu/drm/amd/amdgpu/navi10_ih.c +++ b/drivers/gpu/drm/amd/amdgpu/navi10_ih.c @@ -39,6 +39,53 @@ static void navi10_ih_set_interrupt_funcs(struct amdgpu_device *adev); +/** + * navi10_ih_init_register_offset - Initialize register offset for ih rings + * + * @adev: amdgpu_device pointer + * + * Initialize register offset ih rings (NAVI10). + */ +static void navi10_ih_init_register_offset(struct amdgpu_device *adev) +{ + struct amdgpu_ih_regs *ih_regs; + + if (adev->irq.ih.ring_size) { + ih_regs = &adev->irq.ih.ih_regs; + ih_regs->ih_rb_base = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE); + ih_regs->ih_rb_base_hi = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE_HI); + ih_regs->ih_rb_cntl = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL); + ih_regs->ih_rb_wptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR); + ih_regs->ih_rb_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_RPTR); + ih_regs->ih_doorbell_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_DOORBELL_RPTR); + ih_regs->ih_rb_wptr_addr_lo = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR_ADDR_LO); + ih_regs->ih_rb_wptr_addr_hi = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR_ADDR_HI); + ih_regs->psp_reg_id = PSP_REG_IH_RB_CNTL; + } + + if (adev->irq.ih1.ring_size) { + ih_regs = &adev->irq.ih1.ih_regs; + ih_regs->ih_rb_base = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE_RING1); + ih_regs->ih_rb_base_hi = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE_HI_RING1); + ih_regs->ih_rb_cntl = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL_RING1); + ih_regs->ih_rb_wptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR_RING1); + ih_regs->ih_rb_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_RPTR_RING1); + ih_regs->ih_doorbell_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_DOORBELL_RPTR_RING1); + ih_regs->psp_reg_id = PSP_REG_IH_RB_CNTL_RING1; + } + + if (adev->irq.ih2.ring_size) { + ih_regs = &adev->irq.ih2.ih_regs; + ih_regs->ih_rb_base = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE_RING2); + ih_regs->ih_rb_base_hi = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE_HI_RING2); + ih_regs->ih_rb_cntl = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL_RING2); + ih_regs->ih_rb_wptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR_RING2); + ih_regs->ih_rb_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_RPTR_RING2); + ih_regs->ih_doorbell_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_DOORBELL_RPTR_RING2); + ih_regs->psp_reg_id = PSP_REG_IH_RB_CNTL_RING2; + } +} + /** * force_update_wptr_for_self_int - Force update the wptr for self interrupt * @@ -82,133 +129,66 @@ force_update_wptr_for_self_int(struct amdgpu_device *adev, } /** - * navi10_ih_enable_interrupts - Enable the interrupt ring buffer + * navi10_ih_toggle_ring_interrupts - toggle the interrupt ring buffer * * @adev: amdgpu_device pointer + * @ih: amdgpu_ih_ring pointet + * @enable: true - enable the interrupts, false - disable the interrupts * - * Enable the interrupt ring buffer (NAVI10). + * Toggle the interrupt ring buffer (NAVI10) */ -static void navi10_ih_enable_interrupts(struct amdgpu_device *adev) +static int navi10_ih_toggle_ring_interrupts(struct amdgpu_device *adev, + struct amdgpu_ih_ring *ih, + bool enable) { - u32 ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL); - - ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_ENABLE, 1); - ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, ENABLE_INTR, 1); - if (amdgpu_sriov_vf(adev) && adev->asic_type < CHIP_NAVI10) { - if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL, ih_rb_cntl)) { - DRM_ERROR("PSP program IH_RB_CNTL failed!\n"); - return; - } - } else { - WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL, ih_rb_cntl); - } + struct amdgpu_ih_regs *ih_regs; + uint32_t tmp; - adev->irq.ih.enabled = true; + ih_regs = &ih->ih_regs; - if (adev->irq.ih1.ring_size) { - ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1); - ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL_RING1, - RB_ENABLE, 1); - if (amdgpu_sriov_vf(adev) && adev->asic_type < CHIP_NAVI10) { - if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL_RING1, - ih_rb_cntl)) { - DRM_ERROR("program IH_RB_CNTL_RING1 failed!\n"); - return; - } - } else { - WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1, ih_rb_cntl); - } - adev->irq.ih1.enabled = true; - } + tmp = RREG32(ih_regs->ih_rb_cntl); + tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, RB_ENABLE, (enable ? 1 : 0)); + /* enable_intr field is only valid in ring0 */ + if (ih == &adev->irq.ih) + tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, ENABLE_INTR, (enable ? 1 : 0)); + WREG32(ih_regs->ih_rb_cntl, tmp); - if (adev->irq.ih2.ring_size) { - ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING2); - ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL_RING2, - RB_ENABLE, 1); - if (amdgpu_sriov_vf(adev) && adev->asic_type < CHIP_NAVI10) { - if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL_RING2, - ih_rb_cntl)) { - DRM_ERROR("program IH_RB_CNTL_RING2 failed!\n"); - return; - } - } else { - WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING2, ih_rb_cntl); - } - adev->irq.ih2.enabled = true; + if (enable) { + ih->enabled = true; + } else { + /* set rptr, wptr to 0 */ + WREG32(ih_regs->ih_rb_rptr, 0); + WREG32(ih_regs->ih_rb_wptr, 0); + ih->enabled = false; + ih->rptr = 0; } - if (adev->irq.ih_soft.ring_size) - adev->irq.ih_soft.enabled = true; + return 0; } /** - * navi10_ih_disable_interrupts - Disable the interrupt ring buffer + * navi10_ih_toggle_interrupts - Toggle all the available interrupt ring buffers * * @adev: amdgpu_device pointer + * @enable: enable or disable interrupt ring buffers * - * Disable the interrupt ring buffer (NAVI10). + * Toggle all the available interrupt ring buffers (NAVI10). */ -static void navi10_ih_disable_interrupts(struct amdgpu_device *adev) +static int navi10_ih_toggle_interrupts(struct amdgpu_device *adev, bool enable) { - u32 ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL); - - ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_ENABLE, 0); - ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, ENABLE_INTR, 0); - if (amdgpu_sriov_vf(adev) && adev->asic_type < CHIP_NAVI10) { - if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL, ih_rb_cntl)) { - DRM_ERROR("PSP program IH_RB_CNTL failed!\n"); - return; - } - } else { - WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL, ih_rb_cntl); - } - - /* set rptr, wptr to 0 */ - WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR, 0); - WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR, 0); - adev->irq.ih.enabled = false; - adev->irq.ih.rptr = 0; - - if (adev->irq.ih1.ring_size) { - ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1); - ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL_RING1, - RB_ENABLE, 0); - if (amdgpu_sriov_vf(adev) && adev->asic_type < CHIP_NAVI10) { - if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL_RING1, - ih_rb_cntl)) { - DRM_ERROR("program IH_RB_CNTL_RING1 failed!\n"); - return; - } - } else { - WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1, ih_rb_cntl); - } - /* set rptr, wptr to 0 */ - WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR_RING1, 0); - WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_RING1, 0); - adev->irq.ih1.enabled = false; - adev->irq.ih1.rptr = 0; - } + struct amdgpu_ih_ring *ih[] = {&adev->irq.ih, &adev->irq.ih1, &adev->irq.ih2}; + int i; + int r; - if (adev->irq.ih2.ring_size) { - ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING2); - ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL_RING2, - RB_ENABLE, 0); - if (amdgpu_sriov_vf(adev) && adev->asic_type < CHIP_NAVI10) { - if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL_RING2, - ih_rb_cntl)) { - DRM_ERROR("program IH_RB_CNTL_RING2 failed!\n"); - return; - } - } else { - WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING2, ih_rb_cntl); + for (i = 0; i < ARRAY_SIZE(ih); i++) { + if (ih[i]->ring_size) { + r = navi10_ih_toggle_ring_interrupts(adev, ih[i], enable); + if (r) + return r; } - /* set rptr, wptr to 0 */ - WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR_RING2, 0); - WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_RING2, 0); - adev->irq.ih2.enabled = false; - adev->irq.ih2.rptr = 0; } + return 0; } static uint32_t navi10_ih_rb_cntl(struct amdgpu_ih_ring *ih, uint32_t ih_rb_cntl) @@ -253,22 +233,49 @@ static uint32_t navi10_ih_doorbell_rptr(struct amdgpu_ih_ring *ih) return ih_doorbell_rtpr; } -static void navi10_ih_reroute_ih(struct amdgpu_device *adev) +/** + * navi10_ih_enable_ring - enable an ih ring buffer + * + * @adev: amdgpu_device pointer + * @ih: amdgpu_ih_ring pointer + * + * Enable an ih ring buffer (NAVI10) + */ +static int navi10_ih_enable_ring(struct amdgpu_device *adev, + struct amdgpu_ih_ring *ih) { + struct amdgpu_ih_regs *ih_regs; uint32_t tmp; - /* Reroute to IH ring 1 for VMC */ - WREG32_SOC15(OSSSYS, 0, mmIH_CLIENT_CFG_INDEX, 0x12); - tmp = RREG32_SOC15(OSSSYS, 0, mmIH_CLIENT_CFG_DATA); - tmp = REG_SET_FIELD(tmp, IH_CLIENT_CFG_DATA, CLIENT_TYPE, 1); - tmp = REG_SET_FIELD(tmp, IH_CLIENT_CFG_DATA, RING_ID, 1); - WREG32_SOC15(OSSSYS, 0, mmIH_CLIENT_CFG_DATA, tmp); - - /* Reroute IH ring 1 for UMC */ - WREG32_SOC15(OSSSYS, 0, mmIH_CLIENT_CFG_INDEX, 0x1B); - tmp = RREG32_SOC15(OSSSYS, 0, mmIH_CLIENT_CFG_DATA); - tmp = REG_SET_FIELD(tmp, IH_CLIENT_CFG_DATA, RING_ID, 1); - WREG32_SOC15(OSSSYS, 0, mmIH_CLIENT_CFG_DATA, tmp); + ih_regs = &ih->ih_regs; + + /* Ring Buffer base. [39:8] of 40-bit address of the beginning of the ring buffer*/ + WREG32(ih_regs->ih_rb_base, ih->gpu_addr >> 8); + WREG32(ih_regs->ih_rb_base_hi, (ih->gpu_addr >> 40) & 0xff); + + tmp = RREG32(ih_regs->ih_rb_cntl); + tmp = navi10_ih_rb_cntl(ih, tmp); + if (ih == &adev->irq.ih) + tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, RPTR_REARM, !!adev->irq.msi_enabled); + if (ih == &adev->irq.ih1) { + tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, WPTR_OVERFLOW_ENABLE, 0); + tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, RB_FULL_DRAIN_ENABLE, 1); + } + WREG32(ih_regs->ih_rb_cntl, tmp); + + if (ih == &adev->irq.ih) { + /* set the ih ring 0 writeback address whether it's enabled or not */ + WREG32(ih_regs->ih_rb_wptr_addr_lo, lower_32_bits(ih->wptr_addr)); + WREG32(ih_regs->ih_rb_wptr_addr_hi, upper_32_bits(ih->wptr_addr) & 0xFFFF); + } + + /* set rptr, wptr to 0 */ + WREG32(ih_regs->ih_rb_wptr, 0); + WREG32(ih_regs->ih_rb_rptr, 0); + + WREG32(ih_regs->ih_doorbell_rptr, navi10_ih_doorbell_rptr(ih)); + + return 0; } /** @@ -284,36 +291,21 @@ static void navi10_ih_reroute_ih(struct amdgpu_device *adev) */ static int navi10_ih_irq_init(struct amdgpu_device *adev) { - struct amdgpu_ih_ring *ih = &adev->irq.ih; - u32 ih_rb_cntl, ih_chicken; + struct amdgpu_ih_ring *ih[] = {&adev->irq.ih, &adev->irq.ih1, &adev->irq.ih2}; + u32 ih_chicken; u32 tmp; + int ret; + int i; /* disable irqs */ - navi10_ih_disable_interrupts(adev); + ret = navi10_ih_toggle_interrupts(adev, false); + if (ret) + return ret; adev->nbio.funcs->ih_control(adev); - /* Ring Buffer base. [39:8] of 40-bit address of the beginning of the ring buffer*/ - WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE, ih->gpu_addr >> 8); - WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE_HI, (ih->gpu_addr >> 40) & 0xff); - - ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL); - ih_rb_cntl = navi10_ih_rb_cntl(ih, ih_rb_cntl); - ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RPTR_REARM, - !!adev->irq.msi_enabled); - if (amdgpu_sriov_vf(adev) && adev->asic_type < CHIP_NAVI10) { - if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL, ih_rb_cntl)) { - DRM_ERROR("PSP program IH_RB_CNTL failed!\n"); - return -ETIMEDOUT; - } - } else { - WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL, ih_rb_cntl); - } - if (adev->irq.ih1.ring_size) - navi10_ih_reroute_ih(adev); - if (unlikely(adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT)) { - if (ih->use_bus_addr) { + if (ih[0]->use_bus_addr) { switch (adev->asic_type) { case CHIP_SIENNA_CICHLID: case CHIP_NAVY_FLOUNDER: @@ -334,77 +326,17 @@ static int navi10_ih_irq_init(struct amdgpu_device *adev) } } - /* set the writeback address whether it's enabled or not */ - WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_ADDR_LO, - lower_32_bits(ih->wptr_addr)); - WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_ADDR_HI, - upper_32_bits(ih->wptr_addr) & 0xFFFF); - - /* set rptr, wptr to 0 */ - WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR, 0); - WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR, 0); - - WREG32_SOC15(OSSSYS, 0, mmIH_DOORBELL_RPTR, - navi10_ih_doorbell_rptr(ih)); - - adev->nbio.funcs->ih_doorbell_range(adev, ih->use_doorbell, - ih->doorbell_index); - - ih = &adev->irq.ih1; - if (ih->ring_size) { - WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE_RING1, ih->gpu_addr >> 8); - WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE_HI_RING1, - (ih->gpu_addr >> 40) & 0xff); - - ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1); - ih_rb_cntl = navi10_ih_rb_cntl(ih, ih_rb_cntl); - ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, - WPTR_OVERFLOW_ENABLE, 0); - ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, - RB_FULL_DRAIN_ENABLE, 1); - if (amdgpu_sriov_vf(adev) && adev->asic_type < CHIP_NAVI10) { - if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL_RING1, - ih_rb_cntl)) { - DRM_ERROR("program IH_RB_CNTL_RING1 failed!\n"); - return -ETIMEDOUT; - } - } else { - WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1, ih_rb_cntl); + for (i = 0; i < ARRAY_SIZE(ih); i++) { + if (ih[i]->ring_size) { + ret = navi10_ih_enable_ring(adev, ih[i]); + if (ret) + return ret; } - /* set rptr, wptr to 0 */ - WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_RING1, 0); - WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR_RING1, 0); - - WREG32_SOC15(OSSSYS, 0, mmIH_DOORBELL_RPTR_RING1, - navi10_ih_doorbell_rptr(ih)); - } - - ih = &adev->irq.ih2; - if (ih->ring_size) { - WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE_RING2, ih->gpu_addr >> 8); - WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE_HI_RING2, - (ih->gpu_addr >> 40) & 0xff); - - ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING2); - ih_rb_cntl = navi10_ih_rb_cntl(ih, ih_rb_cntl); - - if (amdgpu_sriov_vf(adev) && adev->asic_type < CHIP_NAVI10) { - if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL_RING2, - ih_rb_cntl)) { - DRM_ERROR("program IH_RB_CNTL_RING2 failed!\n"); - return -ETIMEDOUT; - } - } else { - WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING2, ih_rb_cntl); - } - /* set rptr, wptr to 0 */ - WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_RING2, 0); - WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR_RING2, 0); - - WREG32_SOC15(OSSSYS, 0, mmIH_DOORBELL_RPTR_RING2, - navi10_ih_doorbell_rptr(ih)); } + /* update doorbell range for ih ring 0*/ + adev->nbio.funcs->ih_doorbell_range(adev, ih[0]->use_doorbell, + ih[0]->doorbell_index); tmp = RREG32_SOC15(OSSSYS, 0, mmIH_STORM_CLIENT_LIST_CNTL); tmp = REG_SET_FIELD(tmp, IH_STORM_CLIENT_LIST_CNTL, @@ -418,10 +350,15 @@ static int navi10_ih_irq_init(struct amdgpu_device *adev) pci_set_master(adev->pdev); /* enable interrupts */ - navi10_ih_enable_interrupts(adev); + ret = navi10_ih_toggle_interrupts(adev, true); + if (ret) + return ret; /* enable wptr force update for self int */ force_update_wptr_for_self_int(adev, 0, 8, true); + if (adev->irq.ih_soft.ring_size) + adev->irq.ih_soft.enabled = true; + return 0; } @@ -435,7 +372,7 @@ static int navi10_ih_irq_init(struct amdgpu_device *adev) static void navi10_ih_irq_disable(struct amdgpu_device *adev) { force_update_wptr_for_self_int(adev, 0, 8, false); - navi10_ih_disable_interrupts(adev); + navi10_ih_toggle_interrupts(adev, false); /* Wait and acknowledge irq */ mdelay(1); @@ -455,23 +392,16 @@ static void navi10_ih_irq_disable(struct amdgpu_device *adev) static u32 navi10_ih_get_wptr(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih) { - u32 wptr, reg, tmp; + u32 wptr, tmp; + struct amdgpu_ih_regs *ih_regs; wptr = le32_to_cpu(*ih->wptr_cpu); + ih_regs = &ih->ih_regs; if (!REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW)) goto out; - if (ih == &adev->irq.ih) - reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR); - else if (ih == &adev->irq.ih1) - reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR_RING1); - else if (ih == &adev->irq.ih2) - reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR_RING2); - else - BUG(); - - wptr = RREG32_NO_KIQ(reg); + wptr = RREG32_NO_KIQ(ih_regs->ih_rb_wptr); if (!REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW)) goto out; wptr = REG_SET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW, 0); @@ -486,67 +416,13 @@ static u32 navi10_ih_get_wptr(struct amdgpu_device *adev, wptr, ih->rptr, tmp); ih->rptr = tmp; - if (ih == &adev->irq.ih) - reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL); - else if (ih == &adev->irq.ih1) - reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL_RING1); - else if (ih == &adev->irq.ih2) - reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL_RING2); - else - BUG(); - - tmp = RREG32_NO_KIQ(reg); + tmp = RREG32_NO_KIQ(ih_regs->ih_rb_cntl); tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 1); - WREG32_NO_KIQ(reg, tmp); + WREG32_NO_KIQ(ih_regs->ih_rb_cntl, tmp); out: return (wptr & ih->ptr_mask); } -/** - * navi10_ih_decode_iv - decode an interrupt vector - * - * @adev: amdgpu_device pointer - * @ih: IH ring buffer to decode - * @entry: IV entry to place decoded information into - * - * Decodes the interrupt vector at the current rptr - * position and also advance the position. - */ -static void navi10_ih_decode_iv(struct amdgpu_device *adev, - struct amdgpu_ih_ring *ih, - struct amdgpu_iv_entry *entry) -{ - /* wptr/rptr are in bytes! */ - u32 ring_index = ih->rptr >> 2; - uint32_t dw[8]; - - dw[0] = le32_to_cpu(ih->ring[ring_index + 0]); - dw[1] = le32_to_cpu(ih->ring[ring_index + 1]); - dw[2] = le32_to_cpu(ih->ring[ring_index + 2]); - dw[3] = le32_to_cpu(ih->ring[ring_index + 3]); - dw[4] = le32_to_cpu(ih->ring[ring_index + 4]); - dw[5] = le32_to_cpu(ih->ring[ring_index + 5]); - dw[6] = le32_to_cpu(ih->ring[ring_index + 6]); - dw[7] = le32_to_cpu(ih->ring[ring_index + 7]); - - entry->client_id = dw[0] & 0xff; - entry->src_id = (dw[0] >> 8) & 0xff; - entry->ring_id = (dw[0] >> 16) & 0xff; - entry->vmid = (dw[0] >> 24) & 0xf; - entry->vmid_src = (dw[0] >> 31); - entry->timestamp = dw[1] | ((u64)(dw[2] & 0xffff) << 32); - entry->timestamp_src = dw[2] >> 31; - entry->pasid = dw[3] & 0xffff; - entry->pasid_src = dw[3] >> 31; - entry->src_data[0] = dw[4]; - entry->src_data[1] = dw[5]; - entry->src_data[2] = dw[6]; - entry->src_data[3] = dw[7]; - - /* wptr/rptr are in bytes! */ - ih->rptr += 32; -} - /** * navi10_ih_irq_rearm - rearm IRQ if lost * @@ -557,22 +433,15 @@ static void navi10_ih_decode_iv(struct amdgpu_device *adev, static void navi10_ih_irq_rearm(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih) { - uint32_t reg_rptr = 0; uint32_t v = 0; uint32_t i = 0; + struct amdgpu_ih_regs *ih_regs; - if (ih == &adev->irq.ih) - reg_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_RPTR); - else if (ih == &adev->irq.ih1) - reg_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_RPTR_RING1); - else if (ih == &adev->irq.ih2) - reg_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_RPTR_RING2); - else - return; + ih_regs = &ih->ih_regs; /* Rearm IRQ / re-write doorbell if doorbell write is lost */ for (i = 0; i < MAX_REARM_RETRY; i++) { - v = RREG32_NO_KIQ(reg_rptr); + v = RREG32_NO_KIQ(ih_regs->ih_rb_rptr); if ((v < ih->ring_size) && (v != ih->rptr)) WDOORBELL32(ih->doorbell_index, ih->rptr); else @@ -591,6 +460,8 @@ static void navi10_ih_irq_rearm(struct amdgpu_device *adev, static void navi10_ih_set_rptr(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih) { + struct amdgpu_ih_regs *ih_regs; + if (ih->use_doorbell) { /* XXX check if swapping is necessary on BE */ *ih->rptr_cpu = ih->rptr; @@ -598,12 +469,9 @@ static void navi10_ih_set_rptr(struct amdgpu_device *adev, if (amdgpu_sriov_vf(adev)) navi10_ih_irq_rearm(adev, ih); - } else if (ih == &adev->irq.ih) { - WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR, ih->rptr); - } else if (ih == &adev->irq.ih1) { - WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR_RING1, ih->rptr); - } else if (ih == &adev->irq.ih2) { - WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR_RING2, ih->rptr); + } else { + ih_regs = &ih->ih_regs; + WREG32(ih_regs->ih_rb_rptr, ih->rptr); } } @@ -685,23 +553,8 @@ static int navi10_ih_sw_init(void *handle) adev->irq.ih1.ring_size = 0; adev->irq.ih2.ring_size = 0; - if (adev->asic_type < CHIP_NAVI10) { - r = amdgpu_ih_ring_init(adev, &adev->irq.ih1, PAGE_SIZE, true); - if (r) - return r; - - adev->irq.ih1.use_doorbell = true; - adev->irq.ih1.doorbell_index = - (adev->doorbell_index.ih + 1) << 1; - - r = amdgpu_ih_ring_init(adev, &adev->irq.ih2, PAGE_SIZE, true); - if (r) - return r; - - adev->irq.ih2.use_doorbell = true; - adev->irq.ih2.doorbell_index = - (adev->doorbell_index.ih + 2) << 1; - } + /* initialize ih control registers offset */ + navi10_ih_init_register_offset(adev); r = amdgpu_ih_ring_init(adev, &adev->irq.ih_soft, PAGE_SIZE, true); if (r) @@ -717,6 +570,7 @@ static int navi10_ih_sw_fini(void *handle) struct amdgpu_device *adev = (struct amdgpu_device *)handle; amdgpu_irq_fini(adev); + amdgpu_ih_ring_fini(adev, &adev->irq.ih_soft); amdgpu_ih_ring_fini(adev, &adev->irq.ih2); amdgpu_ih_ring_fini(adev, &adev->irq.ih1); amdgpu_ih_ring_fini(adev, &adev->irq.ih); @@ -848,7 +702,7 @@ static const struct amd_ip_funcs navi10_ih_ip_funcs = { static const struct amdgpu_ih_funcs navi10_ih_funcs = { .get_wptr = navi10_ih_get_wptr, - .decode_iv = navi10_ih_decode_iv, + .decode_iv = amdgpu_ih_decode_iv_helper, .set_rptr = navi10_ih_set_rptr }; diff --git a/drivers/gpu/drm/amd/amdgpu/nbio_v2_3.c b/drivers/gpu/drm/amd/amdgpu/nbio_v2_3.c index b5c3db16c2b01a74b25bfdccc557d2612fb01ca0..05ddec7ba7e2e3d7702be889616ff5bcfeea5c6b 100644 --- a/drivers/gpu/drm/amd/amdgpu/nbio_v2_3.c +++ b/drivers/gpu/drm/amd/amdgpu/nbio_v2_3.c @@ -34,6 +34,14 @@ #define smnCPM_CONTROL 0x11180460 #define smnPCIE_CNTL2 0x11180070 #define smnPCIE_LC_CNTL 0x11140280 +#define smnPCIE_LC_CNTL3 0x111402d4 +#define smnPCIE_LC_CNTL6 0x111402ec +#define smnPCIE_LC_CNTL7 0x111402f0 +#define smnBIF_CFG_DEV0_EPF0_DEVICE_CNTL2 0x1014008c +#define smnRCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL 0x10123538 +#define smnBIF_CFG_DEV0_EPF0_PCIE_LTR_CAP 0x10140324 +#define smnPSWUSP0_PCIE_LC_CNTL2 0x111402c4 +#define smnNBIF_MGCG_CTRL_LCLK 0x1013a21c #define mmBIF_SDMA2_DOORBELL_RANGE 0x01d6 #define mmBIF_SDMA2_DOORBELL_RANGE_BASE_IDX 2 @@ -80,15 +88,6 @@ static void nbio_v2_3_mc_access_enable(struct amdgpu_device *adev, bool enable) WREG32_SOC15(NBIO, 0, mmBIF_FB_EN, 0); } -static void nbio_v2_3_hdp_flush(struct amdgpu_device *adev, - struct amdgpu_ring *ring) -{ - if (!ring || !ring->funcs->emit_wreg) - WREG32_NO_KIQ((adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_MEM_FLUSH_CNTL) >> 2, 0); - else - amdgpu_ring_emit_wreg(ring, (adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_MEM_FLUSH_CNTL) >> 2, 0); -} - static u32 nbio_v2_3_get_memsize(struct amdgpu_device *adev) { return RREG32_SOC15(NBIO, 0, mmRCC_DEV0_EPF0_RCC_CONFIG_MEMSIZE); @@ -359,6 +358,111 @@ static void nbio_v2_3_enable_aspm(struct amdgpu_device *adev, WREG32_PCIE(smnPCIE_LC_CNTL, data); } +static void nbio_v2_3_program_ltr(struct amdgpu_device *adev) +{ + uint32_t def, data; + + WREG32_PCIE(smnRCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL, 0x75EB); + + def = data = RREG32_SOC15(NBIO, 0, mmRCC_BIF_STRAP2); + data &= ~RCC_BIF_STRAP2__STRAP_LTR_IN_ASPML1_DIS_MASK; + if (def != data) + WREG32_SOC15(NBIO, 0, mmRCC_BIF_STRAP2, data); + + def = data = RREG32_PCIE(smnRCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL); + data &= ~EP_PCIE_TX_LTR_CNTL__LTR_PRIV_MSG_DIS_IN_PM_NON_D0_MASK; + if (def != data) + WREG32_PCIE(smnRCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL, data); + + def = data = RREG32_PCIE(smnBIF_CFG_DEV0_EPF0_DEVICE_CNTL2); + data |= BIF_CFG_DEV0_EPF0_DEVICE_CNTL2__LTR_EN_MASK; + if (def != data) + WREG32_PCIE(smnBIF_CFG_DEV0_EPF0_DEVICE_CNTL2, data); +} + +static void nbio_v2_3_program_aspm(struct amdgpu_device *adev) +{ + uint32_t def, data; + + def = data = RREG32_PCIE(smnPCIE_LC_CNTL); + data &= ~PCIE_LC_CNTL__LC_L1_INACTIVITY_MASK; + data &= ~PCIE_LC_CNTL__LC_L0S_INACTIVITY_MASK; + data |= PCIE_LC_CNTL__LC_PMI_TO_L1_DIS_MASK; + if (def != data) + WREG32_PCIE(smnPCIE_LC_CNTL, data); + + def = data = RREG32_PCIE(smnPCIE_LC_CNTL7); + data |= PCIE_LC_CNTL7__LC_NBIF_ASPM_INPUT_EN_MASK; + if (def != data) + WREG32_PCIE(smnPCIE_LC_CNTL7, data); + + def = data = RREG32_PCIE(smnNBIF_MGCG_CTRL_LCLK); + data |= NBIF_MGCG_CTRL_LCLK__NBIF_MGCG_REG_DIS_LCLK_MASK; + if (def != data) + WREG32_PCIE(smnNBIF_MGCG_CTRL_LCLK, data); + + def = data = RREG32_PCIE(smnPCIE_LC_CNTL3); + data |= PCIE_LC_CNTL3__LC_DSC_DONT_ENTER_L23_AFTER_PME_ACK_MASK; + if (def != data) + WREG32_PCIE(smnPCIE_LC_CNTL3, data); + + def = data = RREG32_SOC15(NBIO, 0, mmRCC_BIF_STRAP3); + data &= ~RCC_BIF_STRAP3__STRAP_VLINK_ASPM_IDLE_TIMER_MASK; + data &= ~RCC_BIF_STRAP3__STRAP_VLINK_PM_L1_ENTRY_TIMER_MASK; + if (def != data) + WREG32_SOC15(NBIO, 0, mmRCC_BIF_STRAP3, data); + + def = data = RREG32_SOC15(NBIO, 0, mmRCC_BIF_STRAP5); + data &= ~RCC_BIF_STRAP5__STRAP_VLINK_LDN_ENTRY_TIMER_MASK; + if (def != data) + WREG32_SOC15(NBIO, 0, mmRCC_BIF_STRAP5, data); + + def = data = RREG32_PCIE(smnBIF_CFG_DEV0_EPF0_DEVICE_CNTL2); + data &= ~BIF_CFG_DEV0_EPF0_DEVICE_CNTL2__LTR_EN_MASK; + if (def != data) + WREG32_PCIE(smnBIF_CFG_DEV0_EPF0_DEVICE_CNTL2, data); + + WREG32_PCIE(smnBIF_CFG_DEV0_EPF0_PCIE_LTR_CAP, 0x10011001); + + def = data = RREG32_PCIE(smnPSWUSP0_PCIE_LC_CNTL2); + data |= PSWUSP0_PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L1_MASK | + PSWUSP0_PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L23_MASK; + data &= ~PSWUSP0_PCIE_LC_CNTL2__LC_RCV_L0_TO_RCV_L0S_DIS_MASK; + if (def != data) + WREG32_PCIE(smnPSWUSP0_PCIE_LC_CNTL2, data); + + def = data = RREG32_PCIE(smnPCIE_LC_CNTL6); + data |= PCIE_LC_CNTL6__LC_L1_POWERDOWN_MASK | + PCIE_LC_CNTL6__LC_RX_L0S_STANDBY_EN_MASK; + if (def != data) + WREG32_PCIE(smnPCIE_LC_CNTL6, data); + + nbio_v2_3_program_ltr(adev); + + def = data = RREG32_SOC15(NBIO, 0, mmRCC_BIF_STRAP3); + data |= 0x5DE0 << RCC_BIF_STRAP3__STRAP_VLINK_ASPM_IDLE_TIMER__SHIFT; + data |= 0x0010 << RCC_BIF_STRAP3__STRAP_VLINK_PM_L1_ENTRY_TIMER__SHIFT; + if (def != data) + WREG32_SOC15(NBIO, 0, mmRCC_BIF_STRAP3, data); + + def = data = RREG32_SOC15(NBIO, 0, mmRCC_BIF_STRAP5); + data |= 0x0010 << RCC_BIF_STRAP5__STRAP_VLINK_LDN_ENTRY_TIMER__SHIFT; + if (def != data) + WREG32_SOC15(NBIO, 0, mmRCC_BIF_STRAP5, data); + + def = data = RREG32_PCIE(smnPCIE_LC_CNTL); + data &= ~PCIE_LC_CNTL__LC_L0S_INACTIVITY_MASK; + data |= 0x9 << PCIE_LC_CNTL__LC_L1_INACTIVITY__SHIFT; + data |= 0x1 << PCIE_LC_CNTL__LC_PMI_TO_L1_DIS__SHIFT; + if (def != data) + WREG32_PCIE(smnPCIE_LC_CNTL, data); + + def = data = RREG32_PCIE(smnPCIE_LC_CNTL3); + data &= ~PCIE_LC_CNTL3__LC_DSC_DONT_ENTER_L23_AFTER_PME_ACK_MASK; + if (def != data) + WREG32_PCIE(smnPCIE_LC_CNTL3, data); +} + const struct amdgpu_nbio_funcs nbio_v2_3_funcs = { .get_hdp_flush_req_offset = nbio_v2_3_get_hdp_flush_req_offset, .get_hdp_flush_done_offset = nbio_v2_3_get_hdp_flush_done_offset, @@ -366,7 +470,6 @@ const struct amdgpu_nbio_funcs nbio_v2_3_funcs = { .get_pcie_data_offset = nbio_v2_3_get_pcie_data_offset, .get_rev_id = nbio_v2_3_get_rev_id, .mc_access_enable = nbio_v2_3_mc_access_enable, - .hdp_flush = nbio_v2_3_hdp_flush, .get_memsize = nbio_v2_3_get_memsize, .sdma_doorbell_range = nbio_v2_3_sdma_doorbell_range, .vcn_doorbell_range = nbio_v2_3_vcn_doorbell_range, @@ -380,4 +483,5 @@ const struct amdgpu_nbio_funcs nbio_v2_3_funcs = { .init_registers = nbio_v2_3_init_registers, .remap_hdp_registers = nbio_v2_3_remap_hdp_registers, .enable_aspm = nbio_v2_3_enable_aspm, + .program_aspm = nbio_v2_3_program_aspm, }; diff --git a/drivers/gpu/drm/amd/amdgpu/nbio_v6_1.c b/drivers/gpu/drm/amd/amdgpu/nbio_v6_1.c index d2f1fe55d3888038f1e6567bb62bd38a639da801..83ea063388fddda6ab3d3090fc040d95d429d1e7 100644 --- a/drivers/gpu/drm/amd/amdgpu/nbio_v6_1.c +++ b/drivers/gpu/drm/amd/amdgpu/nbio_v6_1.c @@ -29,6 +29,15 @@ #include "nbio/nbio_6_1_sh_mask.h" #include "nbio/nbio_6_1_smn.h" #include "vega10_enum.h" +#include + +static void nbio_v6_1_remap_hdp_registers(struct amdgpu_device *adev) +{ + WREG32_SOC15(NBIO, 0, mmREMAP_HDP_MEM_FLUSH_CNTL, + adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_MEM_FLUSH_CNTL); + WREG32_SOC15(NBIO, 0, mmREMAP_HDP_REG_FLUSH_CNTL, + adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_REG_FLUSH_CNTL); +} static u32 nbio_v6_1_get_rev_id(struct amdgpu_device *adev) { @@ -50,18 +59,6 @@ static void nbio_v6_1_mc_access_enable(struct amdgpu_device *adev, bool enable) WREG32_SOC15(NBIO, 0, mmBIF_FB_EN, 0); } -static void nbio_v6_1_hdp_flush(struct amdgpu_device *adev, - struct amdgpu_ring *ring) -{ - if (!ring || !ring->funcs->emit_wreg) - WREG32_SOC15_NO_KIQ(NBIO, 0, - mmBIF_BX_PF0_HDP_MEM_COHERENCY_FLUSH_CNTL, - 0); - else - amdgpu_ring_emit_wreg(ring, SOC15_REG_OFFSET( - NBIO, 0, mmBIF_BX_PF0_HDP_MEM_COHERENCY_FLUSH_CNTL), 0); -} - static u32 nbio_v6_1_get_memsize(struct amdgpu_device *adev) { return RREG32_SOC15(NBIO, 0, mmRCC_PF_0_0_RCC_CONFIG_MEMSIZE); @@ -266,7 +263,6 @@ const struct amdgpu_nbio_funcs nbio_v6_1_funcs = { .get_pcie_data_offset = nbio_v6_1_get_pcie_data_offset, .get_rev_id = nbio_v6_1_get_rev_id, .mc_access_enable = nbio_v6_1_mc_access_enable, - .hdp_flush = nbio_v6_1_hdp_flush, .get_memsize = nbio_v6_1_get_memsize, .sdma_doorbell_range = nbio_v6_1_sdma_doorbell_range, .enable_doorbell_aperture = nbio_v6_1_enable_doorbell_aperture, @@ -277,4 +273,5 @@ const struct amdgpu_nbio_funcs nbio_v6_1_funcs = { .get_clockgating_state = nbio_v6_1_get_clockgating_state, .ih_control = nbio_v6_1_ih_control, .init_registers = nbio_v6_1_init_registers, + .remap_hdp_registers = nbio_v6_1_remap_hdp_registers, }; diff --git a/drivers/gpu/drm/amd/amdgpu/nbio_v7_0.c b/drivers/gpu/drm/amd/amdgpu/nbio_v7_0.c index ae685813c419db8ad0e4abf60f265a59feb07296..3c00666a13e16b45213a98da1aa8151628640d5c 100644 --- a/drivers/gpu/drm/amd/amdgpu/nbio_v7_0.c +++ b/drivers/gpu/drm/amd/amdgpu/nbio_v7_0.c @@ -60,15 +60,6 @@ static void nbio_v7_0_mc_access_enable(struct amdgpu_device *adev, bool enable) WREG32_SOC15(NBIO, 0, mmBIF_FB_EN, 0); } -static void nbio_v7_0_hdp_flush(struct amdgpu_device *adev, - struct amdgpu_ring *ring) -{ - if (!ring || !ring->funcs->emit_wreg) - WREG32_NO_KIQ((adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_MEM_FLUSH_CNTL) >> 2, 0); - else - amdgpu_ring_emit_wreg(ring, (adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_MEM_FLUSH_CNTL) >> 2, 0); -} - static u32 nbio_v7_0_get_memsize(struct amdgpu_device *adev) { return RREG32_SOC15(NBIO, 0, mmRCC_CONFIG_MEMSIZE); @@ -292,7 +283,6 @@ const struct amdgpu_nbio_funcs nbio_v7_0_funcs = { .get_pcie_data_offset = nbio_v7_0_get_pcie_data_offset, .get_rev_id = nbio_v7_0_get_rev_id, .mc_access_enable = nbio_v7_0_mc_access_enable, - .hdp_flush = nbio_v7_0_hdp_flush, .get_memsize = nbio_v7_0_get_memsize, .sdma_doorbell_range = nbio_v7_0_sdma_doorbell_range, .vcn_doorbell_range = nbio_v7_0_vcn_doorbell_range, diff --git a/drivers/gpu/drm/amd/amdgpu/nbio_v7_2.c b/drivers/gpu/drm/amd/amdgpu/nbio_v7_2.c index aa36022670f9dd2cb4bc3b6f75c22b02fc983e1a..598ce0e9362772a7eb0f26057be9feaadf424215 100644 --- a/drivers/gpu/drm/amd/amdgpu/nbio_v7_2.c +++ b/drivers/gpu/drm/amd/amdgpu/nbio_v7_2.c @@ -56,15 +56,6 @@ static void nbio_v7_2_mc_access_enable(struct amdgpu_device *adev, bool enable) WREG32_SOC15(NBIO, 0, regBIF_BX0_BIF_FB_EN, 0); } -static void nbio_v7_2_hdp_flush(struct amdgpu_device *adev, - struct amdgpu_ring *ring) -{ - if (!ring || !ring->funcs->emit_wreg) - WREG32_NO_KIQ((adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_MEM_FLUSH_CNTL) >> 2, 0); - else - amdgpu_ring_emit_wreg(ring, (adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_MEM_FLUSH_CNTL) >> 2, 0); -} - static u32 nbio_v7_2_get_memsize(struct amdgpu_device *adev) { return RREG32_SOC15(NBIO, 0, regRCC_DEV0_EPF0_0_RCC_CONFIG_MEMSIZE); @@ -325,7 +316,6 @@ const struct amdgpu_nbio_funcs nbio_v7_2_funcs = { .get_pcie_port_data_offset = nbio_v7_2_get_pcie_port_data_offset, .get_rev_id = nbio_v7_2_get_rev_id, .mc_access_enable = nbio_v7_2_mc_access_enable, - .hdp_flush = nbio_v7_2_hdp_flush, .get_memsize = nbio_v7_2_get_memsize, .sdma_doorbell_range = nbio_v7_2_sdma_doorbell_range, .vcn_doorbell_range = nbio_v7_2_vcn_doorbell_range, diff --git a/drivers/gpu/drm/amd/amdgpu/nbio_v7_4.c b/drivers/gpu/drm/amd/amdgpu/nbio_v7_4.c index eadc9526d33fe6418c2c0606a6e90e9a194c9f5b..4bc1d1434065f097db3523f6a47767e120206cc2 100644 --- a/drivers/gpu/drm/amd/amdgpu/nbio_v7_4.c +++ b/drivers/gpu/drm/amd/amdgpu/nbio_v7_4.c @@ -82,15 +82,6 @@ static void nbio_v7_4_mc_access_enable(struct amdgpu_device *adev, bool enable) WREG32_SOC15(NBIO, 0, mmBIF_FB_EN, 0); } -static void nbio_v7_4_hdp_flush(struct amdgpu_device *adev, - struct amdgpu_ring *ring) -{ - if (!ring || !ring->funcs->emit_wreg) - WREG32_NO_KIQ((adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_MEM_FLUSH_CNTL) >> 2, 0); - else - amdgpu_ring_emit_wreg(ring, (adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_MEM_FLUSH_CNTL) >> 2, 0); -} - static u32 nbio_v7_4_get_memsize(struct amdgpu_device *adev) { return RREG32_SOC15(NBIO, 0, mmRCC_CONFIG_MEMSIZE); @@ -541,7 +532,6 @@ const struct amdgpu_nbio_funcs nbio_v7_4_funcs = { .get_pcie_data_offset = nbio_v7_4_get_pcie_data_offset, .get_rev_id = nbio_v7_4_get_rev_id, .mc_access_enable = nbio_v7_4_mc_access_enable, - .hdp_flush = nbio_v7_4_hdp_flush, .get_memsize = nbio_v7_4_get_memsize, .sdma_doorbell_range = nbio_v7_4_sdma_doorbell_range, .vcn_doorbell_range = nbio_v7_4_vcn_doorbell_range, diff --git a/drivers/gpu/drm/amd/amdgpu/nv.c b/drivers/gpu/drm/amd/amdgpu/nv.c index 6bee3677394ac011131369778f10a4dcd56668c1..160fa5f598053b28f7d5afafd0c3e0d97e27cb1b 100644 --- a/drivers/gpu/drm/amd/amdgpu/nv.c +++ b/drivers/gpu/drm/amd/amdgpu/nv.c @@ -38,9 +38,6 @@ #include "gc/gc_10_1_0_offset.h" #include "gc/gc_10_1_0_sh_mask.h" -#include "hdp/hdp_5_0_0_offset.h" -#include "hdp/hdp_5_0_0_sh_mask.h" -#include "smuio/smuio_11_0_0_offset.h" #include "mp/mp_11_0_offset.h" #include "soc15.h" @@ -50,6 +47,7 @@ #include "mmhub_v2_0.h" #include "nbio_v2_3.h" #include "nbio_v7_2.h" +#include "hdp_v5_0.h" #include "nv.h" #include "navi10_ih.h" #include "gfx_v10_0.h" @@ -62,6 +60,8 @@ #include "dce_virtual.h" #include "mes_v10_1.h" #include "mxgpu_nv.h" +#include "smuio_v11_0.h" +#include "smuio_v11_0_6.h" static const struct amd_ip_funcs nv_common_ip_funcs; @@ -203,6 +203,7 @@ static bool nv_read_bios_from_rom(struct amdgpu_device *adev, { u32 *dw_ptr; u32 i, length_dw; + u32 rom_index_offset, rom_data_offset; if (bios == NULL) return false; @@ -215,11 +216,16 @@ static bool nv_read_bios_from_rom(struct amdgpu_device *adev, dw_ptr = (u32 *)bios; length_dw = ALIGN(length_bytes, 4) / 4; + rom_index_offset = + adev->smuio.funcs->get_rom_index_offset(adev); + rom_data_offset = + adev->smuio.funcs->get_rom_data_offset(adev); + /* set rom index to 0 */ - WREG32(SOC15_REG_OFFSET(SMUIO, 0, mmROM_INDEX), 0); + WREG32(rom_index_offset, 0); /* read out the rom data */ for (i = 0; i < length_dw; i++) - dw_ptr[i] = RREG32(SOC15_REG_OFFSET(SMUIO, 0, mmROM_DATA)); + dw_ptr[i] = RREG32(rom_data_offset); return true; } @@ -336,6 +342,38 @@ static int nv_asic_mode1_reset(struct amdgpu_device *adev) return ret; } +static int nv_asic_mode2_reset(struct amdgpu_device *adev) +{ + u32 i; + int ret = 0; + + amdgpu_atombios_scratch_regs_engine_hung(adev, true); + + /* disable BM */ + pci_clear_master(adev->pdev); + + amdgpu_device_cache_pci_state(adev->pdev); + + ret = amdgpu_dpm_mode2_reset(adev); + if (ret) + dev_err(adev->dev, "GPU mode2 reset failed\n"); + + amdgpu_device_load_pci_state(adev->pdev); + + /* wait for asic to come out of reset */ + for (i = 0; i < adev->usec_timeout; i++) { + u32 memsize = adev->nbio.funcs->get_memsize(adev); + + if (memsize != 0xffffffff) + break; + udelay(1); + } + + amdgpu_atombios_scratch_regs_engine_hung(adev, false); + + return ret; +} + static bool nv_asic_supports_baco(struct amdgpu_device *adev) { struct smu_context *smu = &adev->smu; @@ -352,7 +390,9 @@ nv_asic_reset_method(struct amdgpu_device *adev) struct smu_context *smu = &adev->smu; if (amdgpu_reset_method == AMD_RESET_METHOD_MODE1 || - amdgpu_reset_method == AMD_RESET_METHOD_BACO) + amdgpu_reset_method == AMD_RESET_METHOD_MODE2 || + amdgpu_reset_method == AMD_RESET_METHOD_BACO || + amdgpu_reset_method == AMD_RESET_METHOD_PCI) return amdgpu_reset_method; if (amdgpu_reset_method != -1) @@ -360,6 +400,8 @@ nv_asic_reset_method(struct amdgpu_device *adev) amdgpu_reset_method); switch (adev->asic_type) { + case CHIP_VANGOGH: + return AMD_RESET_METHOD_MODE2; case CHIP_SIENNA_CICHLID: case CHIP_NAVY_FLOUNDER: case CHIP_DIMGREY_CAVEFISH: @@ -377,7 +419,16 @@ static int nv_asic_reset(struct amdgpu_device *adev) int ret = 0; struct smu_context *smu = &adev->smu; - if (nv_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) { + /* skip reset on vangogh for now */ + if (adev->asic_type == CHIP_VANGOGH) + return 0; + + switch (nv_asic_reset_method(adev)) { + case AMD_RESET_METHOD_PCI: + dev_info(adev->dev, "PCI reset\n"); + ret = amdgpu_device_pci_reset(adev); + break; + case AMD_RESET_METHOD_BACO: dev_info(adev->dev, "BACO reset\n"); ret = smu_baco_enter(smu); @@ -386,9 +437,15 @@ static int nv_asic_reset(struct amdgpu_device *adev) ret = smu_baco_exit(smu); if (ret) return ret; - } else { + break; + case AMD_RESET_METHOD_MODE2: + dev_info(adev->dev, "MODE2 reset\n"); + ret = nv_asic_mode2_reset(adev); + break; + default: dev_info(adev->dev, "MODE1 reset\n"); ret = nv_asic_mode1_reset(adev); + break; } return ret; @@ -423,11 +480,14 @@ static void nv_pcie_gen3_enable(struct amdgpu_device *adev) static void nv_program_aspm(struct amdgpu_device *adev) { - - if (amdgpu_aspm == 0) + if (amdgpu_aspm != 1) return; - /* todo */ + if ((adev->asic_type >= CHIP_SIENNA_CICHLID) && + !(adev->flags & AMD_IS_APU) && + (adev->nbio.funcs->program_aspm)) + adev->nbio.funcs->program_aspm(adev); + } static void nv_enable_doorbell_aperture(struct amdgpu_device *adev, @@ -514,6 +574,12 @@ int nv_set_ip_blocks(struct amdgpu_device *adev) adev->nbio.funcs = &nbio_v2_3_funcs; adev->nbio.hdp_flush_reg = &nbio_v2_3_hdp_flush_reg; } + adev->hdp.funcs = &hdp_v5_0_funcs; + + if (adev->asic_type >= CHIP_SIENNA_CICHLID) + adev->smuio.funcs = &smuio_v11_0_6_funcs; + else + adev->smuio.funcs = &smuio_v11_0_funcs; if (adev->asic_type == CHIP_SIENNA_CICHLID) adev->gmc.xgmi.supported = true; @@ -669,22 +735,6 @@ static uint32_t nv_get_rev_id(struct amdgpu_device *adev) return adev->nbio.funcs->get_rev_id(adev); } -static void nv_flush_hdp(struct amdgpu_device *adev, struct amdgpu_ring *ring) -{ - adev->nbio.funcs->hdp_flush(adev, ring); -} - -static void nv_invalidate_hdp(struct amdgpu_device *adev, - struct amdgpu_ring *ring) -{ - if (!ring || !ring->funcs->emit_wreg) { - WREG32_SOC15_NO_KIQ(HDP, 0, mmHDP_READ_CACHE_INVALIDATE, 1); - } else { - amdgpu_ring_emit_wreg(ring, SOC15_REG_OFFSET( - HDP, 0, mmHDP_READ_CACHE_INVALIDATE), 1); - } -} - static bool nv_need_full_reset(struct amdgpu_device *adev) { return true; @@ -768,10 +818,10 @@ static int nv_update_umd_stable_pstate(struct amdgpu_device *adev, * The ASPM function is not fully enabled and verified on * Navi yet. Temporarily skip this until ASPM enabled. */ -#if 0 - if (adev->nbio.funcs->enable_aspm) + if ((adev->asic_type >= CHIP_SIENNA_CICHLID) && + !(adev->flags & AMD_IS_APU) && + (adev->nbio.funcs->enable_aspm)) adev->nbio.funcs->enable_aspm(adev, !enter); -#endif return 0; } @@ -788,8 +838,6 @@ static const struct amdgpu_asic_funcs nv_asic_funcs = .set_uvd_clocks = &nv_set_uvd_clocks, .set_vce_clocks = &nv_set_vce_clocks, .get_config_memsize = &nv_get_config_memsize, - .flush_hdp = &nv_flush_hdp, - .invalidate_hdp = &nv_invalidate_hdp, .init_doorbell_index = &nv_init_doorbell_index, .need_full_reset = &nv_need_full_reset, .need_reset_on_init = &nv_need_reset_on_init, @@ -1080,120 +1128,6 @@ static int nv_common_soft_reset(void *handle) return 0; } -static void nv_update_hdp_mem_power_gating(struct amdgpu_device *adev, - bool enable) -{ - uint32_t hdp_clk_cntl, hdp_clk_cntl1; - uint32_t hdp_mem_pwr_cntl; - - if (!(adev->cg_flags & (AMD_CG_SUPPORT_HDP_LS | - AMD_CG_SUPPORT_HDP_DS | - AMD_CG_SUPPORT_HDP_SD))) - return; - - hdp_clk_cntl = hdp_clk_cntl1 = RREG32_SOC15(HDP, 0, mmHDP_CLK_CNTL); - hdp_mem_pwr_cntl = RREG32_SOC15(HDP, 0, mmHDP_MEM_POWER_CTRL); - - /* Before doing clock/power mode switch, - * forced on IPH & RC clock */ - hdp_clk_cntl = REG_SET_FIELD(hdp_clk_cntl, HDP_CLK_CNTL, - IPH_MEM_CLK_SOFT_OVERRIDE, 1); - hdp_clk_cntl = REG_SET_FIELD(hdp_clk_cntl, HDP_CLK_CNTL, - RC_MEM_CLK_SOFT_OVERRIDE, 1); - WREG32_SOC15(HDP, 0, mmHDP_CLK_CNTL, hdp_clk_cntl); - - /* HDP 5.0 doesn't support dynamic power mode switch, - * disable clock and power gating before any changing */ - hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl, HDP_MEM_POWER_CTRL, - IPH_MEM_POWER_CTRL_EN, 0); - hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl, HDP_MEM_POWER_CTRL, - IPH_MEM_POWER_LS_EN, 0); - hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl, HDP_MEM_POWER_CTRL, - IPH_MEM_POWER_DS_EN, 0); - hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl, HDP_MEM_POWER_CTRL, - IPH_MEM_POWER_SD_EN, 0); - hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl, HDP_MEM_POWER_CTRL, - RC_MEM_POWER_CTRL_EN, 0); - hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl, HDP_MEM_POWER_CTRL, - RC_MEM_POWER_LS_EN, 0); - hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl, HDP_MEM_POWER_CTRL, - RC_MEM_POWER_DS_EN, 0); - hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl, HDP_MEM_POWER_CTRL, - RC_MEM_POWER_SD_EN, 0); - WREG32_SOC15(HDP, 0, mmHDP_MEM_POWER_CTRL, hdp_mem_pwr_cntl); - - /* only one clock gating mode (LS/DS/SD) can be enabled */ - if (adev->cg_flags & AMD_CG_SUPPORT_HDP_LS) { - hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl, - HDP_MEM_POWER_CTRL, - IPH_MEM_POWER_LS_EN, enable); - hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl, - HDP_MEM_POWER_CTRL, - RC_MEM_POWER_LS_EN, enable); - } else if (adev->cg_flags & AMD_CG_SUPPORT_HDP_DS) { - hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl, - HDP_MEM_POWER_CTRL, - IPH_MEM_POWER_DS_EN, enable); - hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl, - HDP_MEM_POWER_CTRL, - RC_MEM_POWER_DS_EN, enable); - } else if (adev->cg_flags & AMD_CG_SUPPORT_HDP_SD) { - hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl, - HDP_MEM_POWER_CTRL, - IPH_MEM_POWER_SD_EN, enable); - /* RC should not use shut down mode, fallback to ds */ - hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl, - HDP_MEM_POWER_CTRL, - RC_MEM_POWER_DS_EN, enable); - } - - /* confirmed that IPH_MEM_POWER_CTRL_EN and RC_MEM_POWER_CTRL_EN have to - * be set for SRAM LS/DS/SD */ - if (adev->cg_flags & (AMD_CG_SUPPORT_HDP_LS | AMD_CG_SUPPORT_HDP_DS | - AMD_CG_SUPPORT_HDP_SD)) { - hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl, HDP_MEM_POWER_CTRL, - IPH_MEM_POWER_CTRL_EN, 1); - hdp_mem_pwr_cntl = REG_SET_FIELD(hdp_mem_pwr_cntl, HDP_MEM_POWER_CTRL, - RC_MEM_POWER_CTRL_EN, 1); - } - - WREG32_SOC15(HDP, 0, mmHDP_MEM_POWER_CTRL, hdp_mem_pwr_cntl); - - /* restore IPH & RC clock override after clock/power mode changing */ - WREG32_SOC15(HDP, 0, mmHDP_CLK_CNTL, hdp_clk_cntl1); -} - -static void nv_update_hdp_clock_gating(struct amdgpu_device *adev, - bool enable) -{ - uint32_t hdp_clk_cntl; - - if (!(adev->cg_flags & AMD_CG_SUPPORT_HDP_MGCG)) - return; - - hdp_clk_cntl = RREG32_SOC15(HDP, 0, mmHDP_CLK_CNTL); - - if (enable) { - hdp_clk_cntl &= - ~(uint32_t) - (HDP_CLK_CNTL__IPH_MEM_CLK_SOFT_OVERRIDE_MASK | - HDP_CLK_CNTL__RC_MEM_CLK_SOFT_OVERRIDE_MASK | - HDP_CLK_CNTL__DBUS_CLK_SOFT_OVERRIDE_MASK | - HDP_CLK_CNTL__DYN_CLK_SOFT_OVERRIDE_MASK | - HDP_CLK_CNTL__XDP_REG_CLK_SOFT_OVERRIDE_MASK | - HDP_CLK_CNTL__HDP_REG_CLK_SOFT_OVERRIDE_MASK); - } else { - hdp_clk_cntl |= HDP_CLK_CNTL__IPH_MEM_CLK_SOFT_OVERRIDE_MASK | - HDP_CLK_CNTL__RC_MEM_CLK_SOFT_OVERRIDE_MASK | - HDP_CLK_CNTL__DBUS_CLK_SOFT_OVERRIDE_MASK | - HDP_CLK_CNTL__DYN_CLK_SOFT_OVERRIDE_MASK | - HDP_CLK_CNTL__XDP_REG_CLK_SOFT_OVERRIDE_MASK | - HDP_CLK_CNTL__HDP_REG_CLK_SOFT_OVERRIDE_MASK; - } - - WREG32_SOC15(HDP, 0, mmHDP_CLK_CNTL, hdp_clk_cntl); -} - static int nv_common_set_clockgating_state(void *handle, enum amd_clockgating_state state) { @@ -1213,9 +1147,9 @@ static int nv_common_set_clockgating_state(void *handle, state == AMD_CG_STATE_GATE); adev->nbio.funcs->update_medium_grain_light_sleep(adev, state == AMD_CG_STATE_GATE); - nv_update_hdp_mem_power_gating(adev, - state == AMD_CG_STATE_GATE); - nv_update_hdp_clock_gating(adev, + adev->hdp.funcs->update_clock_gating(adev, + state == AMD_CG_STATE_GATE); + adev->smuio.funcs->update_rom_clock_gating(adev, state == AMD_CG_STATE_GATE); break; default: @@ -1234,31 +1168,15 @@ static int nv_common_set_powergating_state(void *handle, static void nv_common_get_clockgating_state(void *handle, u32 *flags) { struct amdgpu_device *adev = (struct amdgpu_device *)handle; - uint32_t tmp; if (amdgpu_sriov_vf(adev)) *flags = 0; adev->nbio.funcs->get_clockgating_state(adev, flags); - /* AMD_CG_SUPPORT_HDP_MGCG */ - tmp = RREG32_SOC15(HDP, 0, mmHDP_CLK_CNTL); - if (!(tmp & (HDP_CLK_CNTL__IPH_MEM_CLK_SOFT_OVERRIDE_MASK | - HDP_CLK_CNTL__RC_MEM_CLK_SOFT_OVERRIDE_MASK | - HDP_CLK_CNTL__DBUS_CLK_SOFT_OVERRIDE_MASK | - HDP_CLK_CNTL__DYN_CLK_SOFT_OVERRIDE_MASK | - HDP_CLK_CNTL__XDP_REG_CLK_SOFT_OVERRIDE_MASK | - HDP_CLK_CNTL__HDP_REG_CLK_SOFT_OVERRIDE_MASK))) - *flags |= AMD_CG_SUPPORT_HDP_MGCG; - - /* AMD_CG_SUPPORT_HDP_LS/DS/SD */ - tmp = RREG32_SOC15(HDP, 0, mmHDP_MEM_POWER_CTRL); - if (tmp & HDP_MEM_POWER_CTRL__IPH_MEM_POWER_LS_EN_MASK) - *flags |= AMD_CG_SUPPORT_HDP_LS; - else if (tmp & HDP_MEM_POWER_CTRL__IPH_MEM_POWER_DS_EN_MASK) - *flags |= AMD_CG_SUPPORT_HDP_DS; - else if (tmp & HDP_MEM_POWER_CTRL__IPH_MEM_POWER_SD_EN_MASK) - *flags |= AMD_CG_SUPPORT_HDP_SD; + adev->hdp.funcs->get_clock_gating_state(adev, flags); + + adev->smuio.funcs->get_clock_gating_state(adev, flags); return; } diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v10_0.c b/drivers/gpu/drm/amd/amdgpu/psp_v10_0.c index d7f92634eba271e40e5c1e219dfbb28a563a0bbb..4b1cc5e9ee926472d52ca02956312562309abff1 100644 --- a/drivers/gpu/drm/amd/amdgpu/psp_v10_0.c +++ b/drivers/gpu/drm/amd/amdgpu/psp_v10_0.c @@ -92,8 +92,6 @@ static int psp_v10_0_init_microcode(struct psp_context *psp) (uint8_t *)ta_hdr + le32_to_cpu(ta_hdr->header.ucode_array_offset_bytes); - adev->psp.ta_fw_version = le32_to_cpu(ta_hdr->header.ucode_version); - adev->psp.ta_dtm_ucode_version = le32_to_cpu(ta_hdr->ta_dtm_ucode_version); adev->psp.ta_dtm_ucode_size = @@ -101,6 +99,16 @@ static int psp_v10_0_init_microcode(struct psp_context *psp) adev->psp.ta_dtm_start_addr = (uint8_t *)adev->psp.ta_hdcp_start_addr + le32_to_cpu(ta_hdr->ta_dtm_offset_bytes); + + adev->psp.ta_securedisplay_ucode_version = + le32_to_cpu(ta_hdr->ta_securedisplay_ucode_version); + adev->psp.ta_securedisplay_ucode_size = + le32_to_cpu(ta_hdr->ta_securedisplay_size_bytes); + adev->psp.ta_securedisplay_start_addr = + (uint8_t *)adev->psp.ta_hdcp_start_addr + + le32_to_cpu(ta_hdr->ta_securedisplay_offset_bytes); + + adev->psp.ta_fw_version = le32_to_cpu(ta_hdr->header.ucode_version); } return 0; diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v11_0.c b/drivers/gpu/drm/amd/amdgpu/psp_v11_0.c index bd4248c93c49ff5a59d2f255a1a543524805bd66..c325d6f53a71eed8932bb40c8d42878e4a262982 100644 --- a/drivers/gpu/drm/amd/amdgpu/psp_v11_0.c +++ b/drivers/gpu/drm/amd/amdgpu/psp_v11_0.c @@ -392,37 +392,6 @@ static int psp_v11_0_bootloader_load_sos(struct psp_context *psp) return ret; } -static void psp_v11_0_reroute_ih(struct psp_context *psp) -{ - struct amdgpu_device *adev = psp->adev; - uint32_t tmp; - - /* Change IH ring for VMC */ - tmp = REG_SET_FIELD(0, IH_CLIENT_CFG_DATA, CREDIT_RETURN_ADDR, 0x1244b); - tmp = REG_SET_FIELD(tmp, IH_CLIENT_CFG_DATA, CLIENT_TYPE, 1); - tmp = REG_SET_FIELD(tmp, IH_CLIENT_CFG_DATA, RING_ID, 1); - - WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_69, 3); - WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_70, tmp); - WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_64, GFX_CTRL_CMD_ID_GBR_IH_SET); - - mdelay(20); - psp_wait_for(psp, SOC15_REG_OFFSET(MP0, 0, mmMP0_SMN_C2PMSG_64), - 0x80000000, 0x8000FFFF, false); - - /* Change IH ring for UMC */ - tmp = REG_SET_FIELD(0, IH_CLIENT_CFG_DATA, CREDIT_RETURN_ADDR, 0x1216b); - tmp = REG_SET_FIELD(tmp, IH_CLIENT_CFG_DATA, RING_ID, 1); - - WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_69, 4); - WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_70, tmp); - WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_64, GFX_CTRL_CMD_ID_GBR_IH_SET); - - mdelay(20); - psp_wait_for(psp, SOC15_REG_OFFSET(MP0, 0, mmMP0_SMN_C2PMSG_64), - 0x80000000, 0x8000FFFF, false); -} - static int psp_v11_0_ring_init(struct psp_context *psp, enum psp_ring_type ring_type) { @@ -430,11 +399,6 @@ static int psp_v11_0_ring_init(struct psp_context *psp, struct psp_ring *ring; struct amdgpu_device *adev = psp->adev; - if ((!amdgpu_sriov_vf(adev)) && - !(adev->asic_type >= CHIP_SIENNA_CICHLID && - adev->asic_type <= CHIP_DIMGREY_CAVEFISH)) - psp_v11_0_reroute_ih(psp); - ring = &psp->km_ring; ring->ring_type = ring_type; @@ -726,7 +690,7 @@ static int psp_v11_0_memory_training(struct psp_context *psp, uint32_t ops) } memcpy_toio(adev->mman.aper_base_kaddr, buf, sz); - adev->nbio.funcs->hdp_flush(adev, NULL); + adev->hdp.funcs->flush_hdp(adev, NULL); vfree(buf); } diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c index ce56e93c68867afddc126f14e40963a67bb6b921..c8c22c1d1e65546613a433a8f0853d0da6c6c455 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c @@ -46,7 +46,6 @@ #include "sdma6/sdma6_4_2_2_sh_mask.h" #include "sdma7/sdma7_4_2_2_offset.h" #include "sdma7/sdma7_4_2_2_sh_mask.h" -#include "hdp/hdp_4_0_offset.h" #include "sdma0/sdma0_4_1_default.h" #include "soc15_common.h" diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c index b208b81005bb72aa1c251afa2c5160d94af42c84..d345e324837ddf2152757e7119cf563882770ed6 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c @@ -32,7 +32,6 @@ #include "gc/gc_10_1_0_offset.h" #include "gc/gc_10_1_0_sh_mask.h" -#include "hdp/hdp_5_0_0_offset.h" #include "ivsrcid/sdma0/irqsrcs_sdma0_5_0.h" #include "ivsrcid/sdma1/irqsrcs_sdma1_5_0.h" diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c b/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c index f1ba36a094daf0964d1d3d8ea5d1780710713c93..690a5090475a57d93e57eca61f181899dd9a1966 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c @@ -119,15 +119,7 @@ static int sdma_v5_2_init_inst_ctx(struct amdgpu_sdma_instance *sdma_inst) static void sdma_v5_2_destroy_inst_ctx(struct amdgpu_device *adev) { - int i; - - for (i = 0; i < adev->sdma.num_instances; i++) { - release_firmware(adev->sdma.instance[i].fw); - adev->sdma.instance[i].fw = NULL; - - if (adev->asic_type == CHIP_SIENNA_CICHLID) - break; - } + release_firmware(adev->sdma.instance[0].fw); memset((void *)adev->sdma.instance, 0, sizeof(struct amdgpu_sdma_instance) * AMDGPU_MAX_SDMA_INSTANCES); @@ -185,23 +177,10 @@ static int sdma_v5_2_init_microcode(struct amdgpu_device *adev) if (err) goto out; - for (i = 1; i < adev->sdma.num_instances; i++) { - if (adev->asic_type >= CHIP_SIENNA_CICHLID && - adev->asic_type <= CHIP_DIMGREY_CAVEFISH) { - memcpy((void *)&adev->sdma.instance[i], - (void *)&adev->sdma.instance[0], - sizeof(struct amdgpu_sdma_instance)); - } else { - snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_sdma%d.bin", chip_name, i); - err = request_firmware(&adev->sdma.instance[i].fw, fw_name, adev->dev); - if (err) - goto out; - - err = sdma_v5_2_init_inst_ctx(&adev->sdma.instance[i]); - if (err) - goto out; - } - } + for (i = 1; i < adev->sdma.num_instances; i++) + memcpy((void *)&adev->sdma.instance[i], + (void *)&adev->sdma.instance[0], + sizeof(struct amdgpu_sdma_instance)); DRM_DEBUG("psp_load == '%s'\n", adev->firmware.load_type == AMDGPU_FW_LOAD_PSP ? "true" : "false"); diff --git a/drivers/gpu/drm/amd/amdgpu/si.c b/drivers/gpu/drm/amd/amdgpu/si.c index 3cf0589bfea5e4ecf5bbf44b2701d0773a908ceb..6b5cf7882a12d03aa133ec30a5c1b0177162c6e1 100644 --- a/drivers/gpu/drm/amd/amdgpu/si.c +++ b/drivers/gpu/drm/amd/amdgpu/si.c @@ -1270,7 +1270,7 @@ static int si_gpu_pci_config_reset(struct amdgpu_device *adev) u32 i; int r = -EINVAL; - dev_info(adev->dev, "GPU pci config reset\n"); + amdgpu_atombios_scratch_regs_engine_hung(adev, true); /* set mclk/sclk to bypass */ si_set_clk_bypass_mode(adev); @@ -1294,20 +1294,6 @@ static int si_gpu_pci_config_reset(struct amdgpu_device *adev) } udelay(1); } - - return r; -} - -static int si_asic_reset(struct amdgpu_device *adev) -{ - int r; - - dev_info(adev->dev, "PCI CONFIG reset\n"); - - amdgpu_atombios_scratch_regs_engine_hung(adev, true); - - r = si_gpu_pci_config_reset(adev); - amdgpu_atombios_scratch_regs_engine_hung(adev, false); return r; @@ -1321,14 +1307,34 @@ static bool si_asic_supports_baco(struct amdgpu_device *adev) static enum amd_reset_method si_asic_reset_method(struct amdgpu_device *adev) { - if (amdgpu_reset_method != AMD_RESET_METHOD_LEGACY && - amdgpu_reset_method != -1) + if (amdgpu_reset_method == AMD_RESET_METHOD_PCI) + return amdgpu_reset_method; + else if (amdgpu_reset_method != AMD_RESET_METHOD_LEGACY && + amdgpu_reset_method != -1) dev_warn(adev->dev, "Specified reset method:%d isn't supported, using AUTO instead.\n", - amdgpu_reset_method); + amdgpu_reset_method); return AMD_RESET_METHOD_LEGACY; } +static int si_asic_reset(struct amdgpu_device *adev) +{ + int r; + + switch (si_asic_reset_method(adev)) { + case AMD_RESET_METHOD_PCI: + dev_info(adev->dev, "PCI reset\n"); + r = amdgpu_device_pci_reset(adev); + break; + default: + dev_info(adev->dev, "PCI CONFIG reset\n"); + r = si_gpu_pci_config_reset(adev); + break; + } + + return r; +} + static u32 si_get_config_memsize(struct amdgpu_device *adev) { return RREG32(mmCONFIG_MEMSIZE); diff --git a/drivers/gpu/drm/amd/amdgpu/smuio_v11_0_6.c b/drivers/gpu/drm/amd/amdgpu/smuio_v11_0_6.c new file mode 100644 index 0000000000000000000000000000000000000000..3a18dbb55c32815b4ace96a3417f87fa15e7529b --- /dev/null +++ b/drivers/gpu/drm/amd/amdgpu/smuio_v11_0_6.c @@ -0,0 +1,77 @@ +/* + * Copyright 2021 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ +#include "amdgpu.h" +#include "smuio_v11_0_6.h" +#include "smuio/smuio_11_0_6_offset.h" +#include "smuio/smuio_11_0_6_sh_mask.h" + +static u32 smuio_v11_0_6_get_rom_index_offset(struct amdgpu_device *adev) +{ + return SOC15_REG_OFFSET(SMUIO, 0, mmROM_INDEX); +} + +static u32 smuio_v11_0_6_get_rom_data_offset(struct amdgpu_device *adev) +{ + return SOC15_REG_OFFSET(SMUIO, 0, mmROM_DATA); +} + +static void smuio_v11_0_6_update_rom_clock_gating(struct amdgpu_device *adev, bool enable) +{ + u32 def, data; + + /* enable/disable ROM CG is not supported on APU */ + if (adev->flags & AMD_IS_APU) + return; + + def = data = RREG32_SOC15(SMUIO, 0, mmCGTT_ROM_CLK_CTRL0); + + if (enable && (adev->cg_flags & AMD_CG_SUPPORT_ROM_MGCG)) + data &= ~(CGTT_ROM_CLK_CTRL0__SOFT_OVERRIDE0_MASK | + CGTT_ROM_CLK_CTRL0__SOFT_OVERRIDE1_MASK); + else + data |= CGTT_ROM_CLK_CTRL0__SOFT_OVERRIDE0_MASK | + CGTT_ROM_CLK_CTRL0__SOFT_OVERRIDE1_MASK; + + if (def != data) + WREG32_SOC15(SMUIO, 0, mmCGTT_ROM_CLK_CTRL0, data); +} + +static void smuio_v11_0_6_get_clock_gating_state(struct amdgpu_device *adev, u32 *flags) +{ + u32 data; + + /* CGTT_ROM_CLK_CTRL0 is not available for APU */ + if (adev->flags & AMD_IS_APU) + return; + + data = RREG32_SOC15(SMUIO, 0, mmCGTT_ROM_CLK_CTRL0); + if (!(data & CGTT_ROM_CLK_CTRL0__SOFT_OVERRIDE0_MASK)) + *flags |= AMD_CG_SUPPORT_ROM_MGCG; +} + +const struct amdgpu_smuio_funcs smuio_v11_0_6_funcs = { + .get_rom_index_offset = smuio_v11_0_6_get_rom_index_offset, + .get_rom_data_offset = smuio_v11_0_6_get_rom_data_offset, + .update_rom_clock_gating = smuio_v11_0_6_update_rom_clock_gating, + .get_clock_gating_state = smuio_v11_0_6_get_clock_gating_state, +}; diff --git a/drivers/gpu/drm/amd/amdgpu/smuio_v11_0_6.h b/drivers/gpu/drm/amd/amdgpu/smuio_v11_0_6.h new file mode 100644 index 0000000000000000000000000000000000000000..3c3f4ab0bc9b8e9a1969b403e43d37ce669a7ab5 --- /dev/null +++ b/drivers/gpu/drm/amd/amdgpu/smuio_v11_0_6.h @@ -0,0 +1,30 @@ +/* + * Copyright 2021 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ +#ifndef __SMUIO_V11_0_6_H__ +#define __SMUIO_V11_0_6_H__ + +#include "soc15_common.h" + +extern const struct amdgpu_smuio_funcs smuio_v11_0_6_funcs; + +#endif /* __SMUIO_V11_0_6_H__ */ diff --git a/drivers/gpu/drm/amd/amdgpu/soc15.c b/drivers/gpu/drm/amd/amdgpu/soc15.c index 0b3516c4eefb33881c235ba44d8338c0cef2515e..1221aa6b40a9f53d8b7c18d88727ad9815cebfd8 100644 --- a/drivers/gpu/drm/amd/amdgpu/soc15.c +++ b/drivers/gpu/drm/amd/amdgpu/soc15.c @@ -40,8 +40,6 @@ #include "gc/gc_9_0_sh_mask.h" #include "sdma0/sdma0_4_0_offset.h" #include "sdma1/sdma1_4_0_offset.h" -#include "hdp/hdp_4_0_offset.h" -#include "hdp/hdp_4_0_sh_mask.h" #include "nbio/nbio_7_0_default.h" #include "nbio/nbio_7_0_offset.h" #include "nbio/nbio_7_0_sh_mask.h" @@ -59,7 +57,9 @@ #include "nbio_v6_1.h" #include "nbio_v7_0.h" #include "nbio_v7_4.h" +#include "hdp_v4_0.h" #include "vega10_ih.h" +#include "vega20_ih.h" #include "navi10_ih.h" #include "sdma_v4_0.h" #include "uvd_v7_0.h" @@ -83,14 +83,6 @@ #define mmMP0_MISC_LIGHT_SLEEP_CTRL 0x01ba #define mmMP0_MISC_LIGHT_SLEEP_CTRL_BASE_IDX 0 -/* for Vega20 register name change */ -#define mmHDP_MEM_POWER_CTRL 0x00d4 -#define HDP_MEM_POWER_CTRL__IPH_MEM_POWER_CTRL_EN_MASK 0x00000001L -#define HDP_MEM_POWER_CTRL__IPH_MEM_POWER_LS_EN_MASK 0x00000002L -#define HDP_MEM_POWER_CTRL__RC_MEM_POWER_CTRL_EN_MASK 0x00010000L -#define HDP_MEM_POWER_CTRL__RC_MEM_POWER_LS_EN_MASK 0x00020000L -#define mmHDP_MEM_POWER_CTRL_BASE_IDX 0 - /* * Indirect registers accessor */ @@ -241,6 +233,8 @@ static u32 soc15_get_xclk(struct amdgpu_device *adev) { u32 reference_clock = adev->clock.spll.reference_freq; + if (adev->asic_type == CHIP_RENOIR) + return 10000; if (adev->asic_type == CHIP_RAVEN) return reference_clock / 4; @@ -487,7 +481,8 @@ soc15_asic_reset_method(struct amdgpu_device *adev) if (amdgpu_reset_method == AMD_RESET_METHOD_MODE1 || amdgpu_reset_method == AMD_RESET_METHOD_MODE2 || - amdgpu_reset_method == AMD_RESET_METHOD_BACO) + amdgpu_reset_method == AMD_RESET_METHOD_BACO || + amdgpu_reset_method == AMD_RESET_METHOD_PCI) return amdgpu_reset_method; if (amdgpu_reset_method != -1) @@ -532,15 +527,18 @@ static int soc15_asic_reset(struct amdgpu_device *adev) return 0; switch (soc15_asic_reset_method(adev)) { - case AMD_RESET_METHOD_BACO: - dev_info(adev->dev, "BACO reset\n"); - return soc15_asic_baco_reset(adev); - case AMD_RESET_METHOD_MODE2: - dev_info(adev->dev, "MODE2 reset\n"); - return amdgpu_dpm_mode2_reset(adev); - default: - dev_info(adev->dev, "MODE1 reset\n"); - return soc15_asic_mode1_reset(adev); + case AMD_RESET_METHOD_PCI: + dev_info(adev->dev, "PCI reset\n"); + return amdgpu_device_pci_reset(adev); + case AMD_RESET_METHOD_BACO: + dev_info(adev->dev, "BACO reset\n"); + return soc15_asic_baco_reset(adev); + case AMD_RESET_METHOD_MODE2: + dev_info(adev->dev, "MODE2 reset\n"); + return amdgpu_dpm_mode2_reset(adev); + default: + dev_info(adev->dev, "MODE1 reset\n"); + return soc15_asic_mode1_reset(adev); } } @@ -699,6 +697,7 @@ int soc15_set_ip_blocks(struct amdgpu_device *adev) adev->nbio.funcs = &nbio_v6_1_funcs; adev->nbio.hdp_flush_reg = &nbio_v6_1_hdp_flush_reg; } + adev->hdp.funcs = &hdp_v4_0_funcs; if (adev->asic_type == CHIP_VEGA20 || adev->asic_type == CHIP_ARCTURUS) adev->df.funcs = &df_v3_6_funcs; @@ -729,12 +728,12 @@ int soc15_set_ip_blocks(struct amdgpu_device *adev) amdgpu_device_ip_block_add(adev, &psp_v3_1_ip_block); } if (adev->asic_type == CHIP_VEGA20) - amdgpu_device_ip_block_add(adev, &navi10_ih_ip_block); + amdgpu_device_ip_block_add(adev, &vega20_ih_ip_block); else amdgpu_device_ip_block_add(adev, &vega10_ih_ip_block); } else { if (adev->asic_type == CHIP_VEGA20) - amdgpu_device_ip_block_add(adev, &navi10_ih_ip_block); + amdgpu_device_ip_block_add(adev, &vega20_ih_ip_block); else amdgpu_device_ip_block_add(adev, &vega10_ih_ip_block); if (likely(adev->firmware.load_type == AMDGPU_FW_LOAD_PSP)) { @@ -787,9 +786,9 @@ int soc15_set_ip_blocks(struct amdgpu_device *adev) if (amdgpu_sriov_vf(adev)) { if (likely(adev->firmware.load_type == AMDGPU_FW_LOAD_PSP)) amdgpu_device_ip_block_add(adev, &psp_v11_0_ip_block); - amdgpu_device_ip_block_add(adev, &navi10_ih_ip_block); + amdgpu_device_ip_block_add(adev, &vega20_ih_ip_block); } else { - amdgpu_device_ip_block_add(adev, &navi10_ih_ip_block); + amdgpu_device_ip_block_add(adev, &vega20_ih_ip_block); if (likely(adev->firmware.load_type == AMDGPU_FW_LOAD_PSP)) amdgpu_device_ip_block_add(adev, &psp_v11_0_ip_block); } @@ -834,35 +833,12 @@ int soc15_set_ip_blocks(struct amdgpu_device *adev) return 0; } -static void soc15_flush_hdp(struct amdgpu_device *adev, struct amdgpu_ring *ring) -{ - adev->nbio.funcs->hdp_flush(adev, ring); -} - -static void soc15_invalidate_hdp(struct amdgpu_device *adev, - struct amdgpu_ring *ring) -{ - if (!ring || !ring->funcs->emit_wreg) - WREG32_SOC15_NO_KIQ(HDP, 0, mmHDP_READ_CACHE_INVALIDATE, 1); - else - amdgpu_ring_emit_wreg(ring, SOC15_REG_OFFSET( - HDP, 0, mmHDP_READ_CACHE_INVALIDATE), 1); -} - static bool soc15_need_full_reset(struct amdgpu_device *adev) { /* change this when we implement soft reset */ return true; } -static void vega20_reset_hdp_ras_error_count(struct amdgpu_device *adev) -{ - if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__HDP)) - return; - /*read back hdp ras counter to reset it to 0 */ - RREG32_SOC15(HDP, 0, mmHDP_EDC_CNT); -} - static void soc15_get_pcie_usage(struct amdgpu_device *adev, uint64_t *count0, uint64_t *count1) { @@ -1011,8 +987,6 @@ static const struct amdgpu_asic_funcs soc15_asic_funcs = .set_uvd_clocks = &soc15_set_uvd_clocks, .set_vce_clocks = &soc15_set_vce_clocks, .get_config_memsize = &soc15_get_config_memsize, - .flush_hdp = &soc15_flush_hdp, - .invalidate_hdp = &soc15_invalidate_hdp, .need_full_reset = &soc15_need_full_reset, .init_doorbell_index = &vega10_doorbell_index_init, .get_pcie_usage = &soc15_get_pcie_usage, @@ -1034,9 +1008,6 @@ static const struct amdgpu_asic_funcs vega20_asic_funcs = .set_uvd_clocks = &soc15_set_uvd_clocks, .set_vce_clocks = &soc15_set_vce_clocks, .get_config_memsize = &soc15_get_config_memsize, - .flush_hdp = &soc15_flush_hdp, - .invalidate_hdp = &soc15_invalidate_hdp, - .reset_hdp_ras_error_count = &vega20_reset_hdp_ras_error_count, .need_full_reset = &soc15_need_full_reset, .init_doorbell_index = &vega20_doorbell_index_init, .get_pcie_usage = &vega20_get_pcie_usage, @@ -1294,9 +1265,8 @@ static int soc15_common_late_init(void *handle) if (amdgpu_sriov_vf(adev)) xgpu_ai_mailbox_get_irq(adev); - if (adev->asic_funcs && - adev->asic_funcs->reset_hdp_ras_error_count) - adev->asic_funcs->reset_hdp_ras_error_count(adev); + if (adev->hdp.funcs->reset_ras_error_count) + adev->hdp.funcs->reset_ras_error_count(adev); if (adev->nbio.funcs->ras_late_init) r = adev->nbio.funcs->ras_late_init(adev); @@ -1422,41 +1392,6 @@ static int soc15_common_soft_reset(void *handle) return 0; } -static void soc15_update_hdp_light_sleep(struct amdgpu_device *adev, bool enable) -{ - uint32_t def, data; - - if (adev->asic_type == CHIP_VEGA20 || - adev->asic_type == CHIP_ARCTURUS || - adev->asic_type == CHIP_RENOIR) { - def = data = RREG32(SOC15_REG_OFFSET(HDP, 0, mmHDP_MEM_POWER_CTRL)); - - if (enable && (adev->cg_flags & AMD_CG_SUPPORT_HDP_LS)) - data |= HDP_MEM_POWER_CTRL__IPH_MEM_POWER_CTRL_EN_MASK | - HDP_MEM_POWER_CTRL__IPH_MEM_POWER_LS_EN_MASK | - HDP_MEM_POWER_CTRL__RC_MEM_POWER_CTRL_EN_MASK | - HDP_MEM_POWER_CTRL__RC_MEM_POWER_LS_EN_MASK; - else - data &= ~(HDP_MEM_POWER_CTRL__IPH_MEM_POWER_CTRL_EN_MASK | - HDP_MEM_POWER_CTRL__IPH_MEM_POWER_LS_EN_MASK | - HDP_MEM_POWER_CTRL__RC_MEM_POWER_CTRL_EN_MASK | - HDP_MEM_POWER_CTRL__RC_MEM_POWER_LS_EN_MASK); - - if (def != data) - WREG32(SOC15_REG_OFFSET(HDP, 0, mmHDP_MEM_POWER_CTRL), data); - } else { - def = data = RREG32(SOC15_REG_OFFSET(HDP, 0, mmHDP_MEM_POWER_LS)); - - if (enable && (adev->cg_flags & AMD_CG_SUPPORT_HDP_LS)) - data |= HDP_MEM_POWER_LS__LS_ENABLE_MASK; - else - data &= ~HDP_MEM_POWER_LS__LS_ENABLE_MASK; - - if (def != data) - WREG32(SOC15_REG_OFFSET(HDP, 0, mmHDP_MEM_POWER_LS), data); - } -} - static void soc15_update_drm_clock_gating(struct amdgpu_device *adev, bool enable) { uint32_t def, data; @@ -1517,7 +1452,7 @@ static int soc15_common_set_clockgating_state(void *handle, state == AMD_CG_STATE_GATE); adev->nbio.funcs->update_medium_grain_light_sleep(adev, state == AMD_CG_STATE_GATE); - soc15_update_hdp_light_sleep(adev, + adev->hdp.funcs->update_clock_gating(adev, state == AMD_CG_STATE_GATE); soc15_update_drm_clock_gating(adev, state == AMD_CG_STATE_GATE); @@ -1534,7 +1469,7 @@ static int soc15_common_set_clockgating_state(void *handle, state == AMD_CG_STATE_GATE); adev->nbio.funcs->update_medium_grain_light_sleep(adev, state == AMD_CG_STATE_GATE); - soc15_update_hdp_light_sleep(adev, + adev->hdp.funcs->update_clock_gating(adev, state == AMD_CG_STATE_GATE); soc15_update_drm_clock_gating(adev, state == AMD_CG_STATE_GATE); @@ -1542,7 +1477,7 @@ static int soc15_common_set_clockgating_state(void *handle, state == AMD_CG_STATE_GATE); break; case CHIP_ARCTURUS: - soc15_update_hdp_light_sleep(adev, + adev->hdp.funcs->update_clock_gating(adev, state == AMD_CG_STATE_GATE); break; default: @@ -1561,10 +1496,7 @@ static void soc15_common_get_clockgating_state(void *handle, u32 *flags) adev->nbio.funcs->get_clockgating_state(adev, flags); - /* AMD_CG_SUPPORT_HDP_LS */ - data = RREG32(SOC15_REG_OFFSET(HDP, 0, mmHDP_MEM_POWER_LS)); - if (data & HDP_MEM_POWER_LS__LS_ENABLE_MASK) - *flags |= AMD_CG_SUPPORT_HDP_LS; + adev->hdp.funcs->get_clock_gating_state(adev, flags); /* AMD_CG_SUPPORT_DRM_MGCG */ data = RREG32(SOC15_REG_OFFSET(MP0, 0, mmMP0_MISC_CGTT_CTRL0)); diff --git a/drivers/gpu/drm/amd/amdgpu/ta_secureDisplay_if.h b/drivers/gpu/drm/amd/amdgpu/ta_secureDisplay_if.h new file mode 100644 index 0000000000000000000000000000000000000000..5039375bb1d42b8dc27d261d93652df85ca86287 --- /dev/null +++ b/drivers/gpu/drm/amd/amdgpu/ta_secureDisplay_if.h @@ -0,0 +1,154 @@ +/* + * Copyright 2019 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef _TA_SECUREDISPLAY_IF_H +#define _TA_SECUREDISPLAY_IF_H + +/** Secure Display related enumerations */ +/**********************************************************/ + +/** @enum ta_securedisplay_command + * Secure Display Command ID + */ +enum ta_securedisplay_command { + /* Query whether TA is responding used only for validation purpose */ + TA_SECUREDISPLAY_COMMAND__QUERY_TA = 1, + /* Send region of Interest and CRC value to I2C */ + TA_SECUREDISPLAY_COMMAND__SEND_ROI_CRC = 2, + /* Maximum Command ID */ + TA_SECUREDISPLAY_COMMAND__MAX_ID = 0x7FFFFFFF, +}; + +/** @enum ta_securedisplay_status + * Secure Display status returns in shared buffer status + */ +enum ta_securedisplay_status { + TA_SECUREDISPLAY_STATUS__SUCCESS = 0x00, /* Success */ + TA_SECUREDISPLAY_STATUS__GENERIC_FAILURE = 0x01, /* Generic Failure */ + TA_SECUREDISPLAY_STATUS__INVALID_PARAMETER = 0x02, /* Invalid Parameter */ + TA_SECUREDISPLAY_STATUS__NULL_POINTER = 0x03, /* Null Pointer*/ + TA_SECUREDISPLAY_STATUS__I2C_WRITE_ERROR = 0x04, /* Fail to Write to I2C */ + TA_SECUREDISPLAY_STATUS__READ_DIO_SCRATCH_ERROR = 0x05, /*Fail Read DIO Scratch Register*/ + TA_SECUREDISPLAY_STATUS__READ_CRC_ERROR = 0x06, /* Fail to Read CRC*/ + + TA_SECUREDISPLAY_STATUS__MAX = 0x7FFFFFFF,/* Maximum Value for status*/ +}; + +/** @enum ta_securedisplay_max_phy + * Physical ID number to use for reading corresponding DIO Scratch register for ROI + */ +enum ta_securedisplay_max_phy { + TA_SECUREDISPLAY_PHY0 = 0, + TA_SECUREDISPLAY_PHY1 = 1, + TA_SECUREDISPLAY_PHY2 = 2, + TA_SECUREDISPLAY_PHY3 = 3, + TA_SECUREDISPLAY_MAX_PHY = 4, +}; + +/** @enum ta_securedisplay_ta_query_cmd_ret + * A predefined specific reteurn value which is 0xAB only used to validate + * communication to Secure Display TA is functional. + * This value is used to validate whether TA is responding successfully + */ +enum ta_securedisplay_ta_query_cmd_ret { + /* This is a value to validate if TA is loaded successfully */ + TA_SECUREDISPLAY_QUERY_CMD_RET = 0xAB, +}; + +/** @enum ta_securedisplay_buffer_size + * I2C Buffer size which contains 8 bytes of ROI (X start, X end, Y start, Y end) + * and 6 bytes of CRC( R,G,B) and 1 byte for physical ID + */ +enum ta_securedisplay_buffer_size { + /* 15 bytes = 8 byte (ROI) + 6 byte(CRC) + 1 byte(phy_id) */ + TA_SECUREDISPLAY_I2C_BUFFER_SIZE = 15, +}; + +/** Input/output structures for Secure Display commands */ +/**********************************************************/ +/** + * Input structures + */ + +/** @struct ta_securedisplay_send_roi_crc_input + * Physical ID to determine which DIO scratch register should be used to get ROI + */ +struct ta_securedisplay_send_roi_crc_input { + uint32_t phy_id; /* Physical ID */ +}; + +/** @union ta_securedisplay_cmd_input + * Input buffer + */ +union ta_securedisplay_cmd_input { + /* send ROI and CRC input buffer format */ + struct ta_securedisplay_send_roi_crc_input send_roi_crc; + uint32_t reserved[4]; +}; + +/** + * Output structures + */ + +/** @struct ta_securedisplay_query_ta_output + * Output buffer format for query TA whether TA is responding used only for validation purpose + */ +struct ta_securedisplay_query_ta_output { + /* return value from TA when it is queried for validation purpose only */ + uint32_t query_cmd_ret; +}; + +/** @struct ta_securedisplay_send_roi_crc_output + * Output buffer format for send ROI CRC command which will pass I2c buffer created inside TA + * and used to write to I2C used only for validation purpose + */ +struct ta_securedisplay_send_roi_crc_output { + uint8_t i2c_buf[TA_SECUREDISPLAY_I2C_BUFFER_SIZE]; /* I2C buffer */ + uint8_t reserved; +}; + +/** @union ta_securedisplay_cmd_output + * Output buffer + */ +union ta_securedisplay_cmd_output { + /* Query TA output buffer format used only for validation purpose*/ + struct ta_securedisplay_query_ta_output query_ta; + /* Send ROI CRC output buffer format used only for validation purpose */ + struct ta_securedisplay_send_roi_crc_output send_roi_crc; + uint32_t reserved[4]; +}; + +/** @struct securedisplay_cmd + * Secure Display Command which is shared buffer memory + */ +struct securedisplay_cmd { + uint32_t cmd_id; /* +0 Bytes Command ID */ + enum ta_securedisplay_status status; /* +4 Bytes Status of Secure Display TA */ + uint32_t reserved[2]; /* +8 Bytes Reserved */ + union ta_securedisplay_cmd_input securedisplay_in_message; /* +16 Bytes Input Buffer */ + union ta_securedisplay_cmd_output securedisplay_out_message;/* +32 Bytes Output Buffer */ + /**@note Total 48 Bytes */ +}; + +#endif //_TA_SECUREDISPLAY_IF_H + diff --git a/drivers/gpu/drm/amd/amdgpu/tonga_ih.c b/drivers/gpu/drm/amd/amdgpu/tonga_ih.c index ce3319993b4bd4553025a34a92cb41e7280b483c..249fcbee7871cc6525858a8b8284fde598bfb719 100644 --- a/drivers/gpu/drm/amd/amdgpu/tonga_ih.c +++ b/drivers/gpu/drm/amd/amdgpu/tonga_ih.c @@ -196,19 +196,30 @@ static u32 tonga_ih_get_wptr(struct amdgpu_device *adev, wptr = le32_to_cpu(*ih->wptr_cpu); - if (REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW)) { - wptr = REG_SET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW, 0); - /* When a ring buffer overflow happen start parsing interrupt - * from the last not overwritten vector (wptr + 16). Hopefully - * this should allow us to catchup. - */ - dev_warn(adev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n", - wptr, ih->rptr, (wptr + 16) & ih->ptr_mask); - ih->rptr = (wptr + 16) & ih->ptr_mask; - tmp = RREG32(mmIH_RB_CNTL); - tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 1); - WREG32(mmIH_RB_CNTL, tmp); - } + if (!REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW)) + goto out; + + /* Double check that the overflow wasn't already cleared. */ + wptr = RREG32(mmIH_RB_WPTR); + + if (!REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW)) + goto out; + + wptr = REG_SET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW, 0); + + /* When a ring buffer overflow happen start parsing interrupt + * from the last not overwritten vector (wptr + 16). Hopefully + * this should allow us to catchup. + */ + + dev_warn(adev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n", + wptr, ih->rptr, (wptr + 16) & ih->ptr_mask); + ih->rptr = (wptr + 16) & ih->ptr_mask; + tmp = RREG32(mmIH_RB_CNTL); + tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 1); + WREG32(mmIH_RB_CNTL, tmp); + +out: return (wptr & ih->ptr_mask); } diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c b/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c index 312ecf6d24a042d4bbd05d1e0962e158f9d6e5f2..7cd67cb2ac5f114e945b16041e79fbef6c5ab465 100644 --- a/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c +++ b/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c @@ -36,7 +36,6 @@ #include "vce/vce_4_0_default.h" #include "vce/vce_4_0_sh_mask.h" #include "nbif/nbif_6_1_offset.h" -#include "hdp/hdp_4_0_offset.h" #include "mmhub/mmhub_1_0_offset.h" #include "mmhub/mmhub_1_0_sh_mask.h" #include "ivsrcid/uvd/irqsrcs_uvd_7_0.h" diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c b/drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c index c734e31a9e653cbdfbf05ba3e6903ef6e1ff2ab6..6117931fa8d798275219aceea41eee0ed15db652 100644 --- a/drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c +++ b/drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c @@ -32,7 +32,6 @@ #include "vcn/vcn_1_0_offset.h" #include "vcn/vcn_1_0_sh_mask.h" -#include "hdp/hdp_4_0_offset.h" #include "mmhub/mmhub_9_1_offset.h" #include "mmhub/mmhub_9_1_sh_mask.h" diff --git a/drivers/gpu/drm/amd/amdgpu/vega10_ih.c b/drivers/gpu/drm/amd/amdgpu/vega10_ih.c index e5ae31eb744ed95a155a9ac5d30abb0ad9e6cbbf..88626d83e07bebd5eeec19fba53e4b3d0dfdb6ec 100644 --- a/drivers/gpu/drm/amd/amdgpu/vega10_ih.c +++ b/drivers/gpu/drm/amd/amdgpu/vega10_ih.c @@ -38,132 +38,120 @@ static void vega10_ih_set_interrupt_funcs(struct amdgpu_device *adev); /** - * vega10_ih_enable_interrupts - Enable the interrupt ring buffer + * vega10_ih_init_register_offset - Initialize register offset for ih rings * * @adev: amdgpu_device pointer * - * Enable the interrupt ring buffer (VEGA10). + * Initialize register offset ih rings (VEGA10). */ -static void vega10_ih_enable_interrupts(struct amdgpu_device *adev) +static void vega10_ih_init_register_offset(struct amdgpu_device *adev) { - u32 ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL); - - ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_ENABLE, 1); - ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, ENABLE_INTR, 1); - if (amdgpu_sriov_vf(adev)) { - if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL, ih_rb_cntl)) { - DRM_ERROR("PSP program IH_RB_CNTL failed!\n"); - return; - } - } else { - WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL, ih_rb_cntl); + struct amdgpu_ih_regs *ih_regs; + + if (adev->irq.ih.ring_size) { + ih_regs = &adev->irq.ih.ih_regs; + ih_regs->ih_rb_base = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE); + ih_regs->ih_rb_base_hi = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE_HI); + ih_regs->ih_rb_cntl = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL); + ih_regs->ih_rb_wptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR); + ih_regs->ih_rb_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_RPTR); + ih_regs->ih_doorbell_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_DOORBELL_RPTR); + ih_regs->ih_rb_wptr_addr_lo = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR_ADDR_LO); + ih_regs->ih_rb_wptr_addr_hi = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR_ADDR_HI); + ih_regs->psp_reg_id = PSP_REG_IH_RB_CNTL; } - adev->irq.ih.enabled = true; if (adev->irq.ih1.ring_size) { - ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1); - ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL_RING1, - RB_ENABLE, 1); - if (amdgpu_sriov_vf(adev)) { - if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL_RING1, - ih_rb_cntl)) { - DRM_ERROR("program IH_RB_CNTL_RING1 failed!\n"); - return; - } - } else { - WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1, ih_rb_cntl); - } - adev->irq.ih1.enabled = true; + ih_regs = &adev->irq.ih1.ih_regs; + ih_regs->ih_rb_base = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE_RING1); + ih_regs->ih_rb_base_hi = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE_HI_RING1); + ih_regs->ih_rb_cntl = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL_RING1); + ih_regs->ih_rb_wptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR_RING1); + ih_regs->ih_rb_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_RPTR_RING1); + ih_regs->ih_doorbell_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_DOORBELL_RPTR_RING1); + ih_regs->psp_reg_id = PSP_REG_IH_RB_CNTL_RING1; } if (adev->irq.ih2.ring_size) { - ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING2); - ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL_RING2, - RB_ENABLE, 1); - if (amdgpu_sriov_vf(adev)) { - if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL_RING2, - ih_rb_cntl)) { - DRM_ERROR("program IH_RB_CNTL_RING2 failed!\n"); - return; - } - } else { - WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING2, ih_rb_cntl); - } - adev->irq.ih2.enabled = true; + ih_regs = &adev->irq.ih2.ih_regs; + ih_regs->ih_rb_base = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE_RING2); + ih_regs->ih_rb_base_hi = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE_HI_RING2); + ih_regs->ih_rb_cntl = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL_RING2); + ih_regs->ih_rb_wptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR_RING2); + ih_regs->ih_rb_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_RPTR_RING2); + ih_regs->ih_doorbell_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_DOORBELL_RPTR_RING2); + ih_regs->psp_reg_id = PSP_REG_IH_RB_CNTL_RING2; } - - if (adev->irq.ih_soft.ring_size) - adev->irq.ih_soft.enabled = true; } /** - * vega10_ih_disable_interrupts - Disable the interrupt ring buffer + * vega10_ih_toggle_ring_interrupts - toggle the interrupt ring buffer * * @adev: amdgpu_device pointer + * @ih: amdgpu_ih_ring pointet + * @enable: true - enable the interrupts, false - disable the interrupts * - * Disable the interrupt ring buffer (VEGA10). + * Toggle the interrupt ring buffer (VEGA10) */ -static void vega10_ih_disable_interrupts(struct amdgpu_device *adev) +static int vega10_ih_toggle_ring_interrupts(struct amdgpu_device *adev, + struct amdgpu_ih_ring *ih, + bool enable) { - u32 ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL); + struct amdgpu_ih_regs *ih_regs; + uint32_t tmp; - ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_ENABLE, 0); - ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, ENABLE_INTR, 0); + ih_regs = &ih->ih_regs; + + tmp = RREG32(ih_regs->ih_rb_cntl); + tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, RB_ENABLE, (enable ? 1 : 0)); + /* enable_intr field is only valid in ring0 */ + if (ih == &adev->irq.ih) + tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, ENABLE_INTR, (enable ? 1 : 0)); if (amdgpu_sriov_vf(adev)) { - if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL, ih_rb_cntl)) { - DRM_ERROR("PSP program IH_RB_CNTL failed!\n"); - return; + if (psp_reg_program(&adev->psp, ih_regs->psp_reg_id, tmp)) { + dev_err(adev->dev, "PSP program IH_RB_CNTL failed!\n"); + return -ETIMEDOUT; } } else { - WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL, ih_rb_cntl); + WREG32(ih_regs->ih_rb_cntl, tmp); } - /* set rptr, wptr to 0 */ - WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR, 0); - WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR, 0); - adev->irq.ih.enabled = false; - adev->irq.ih.rptr = 0; - - if (adev->irq.ih1.ring_size) { - ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1); - ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL_RING1, - RB_ENABLE, 0); - if (amdgpu_sriov_vf(adev)) { - if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL_RING1, - ih_rb_cntl)) { - DRM_ERROR("program IH_RB_CNTL_RING1 failed!\n"); - return; - } - } else { - WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1, ih_rb_cntl); - } + if (enable) { + ih->enabled = true; + } else { /* set rptr, wptr to 0 */ - WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR_RING1, 0); - WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_RING1, 0); - adev->irq.ih1.enabled = false; - adev->irq.ih1.rptr = 0; + WREG32(ih_regs->ih_rb_rptr, 0); + WREG32(ih_regs->ih_rb_wptr, 0); + ih->enabled = false; + ih->rptr = 0; } - if (adev->irq.ih2.ring_size) { - ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING2); - ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL_RING2, - RB_ENABLE, 0); - if (amdgpu_sriov_vf(adev)) { - if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL_RING2, - ih_rb_cntl)) { - DRM_ERROR("program IH_RB_CNTL_RING2 failed!\n"); - return; - } - } else { - WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING2, ih_rb_cntl); - } + return 0; +} - /* set rptr, wptr to 0 */ - WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR_RING2, 0); - WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_RING2, 0); - adev->irq.ih2.enabled = false; - adev->irq.ih2.rptr = 0; +/** + * vega10_ih_toggle_interrupts - Toggle all the available interrupt ring buffers + * + * @adev: amdgpu_device pointer + * @enable: enable or disable interrupt ring buffers + * + * Toggle all the available interrupt ring buffers (VEGA10). + */ +static int vega10_ih_toggle_interrupts(struct amdgpu_device *adev, bool enable) +{ + struct amdgpu_ih_ring *ih[] = {&adev->irq.ih, &adev->irq.ih1, &adev->irq.ih2}; + int i; + int r; + + for (i = 0; i < ARRAY_SIZE(ih); i++) { + if (ih[i]->ring_size) { + r = vega10_ih_toggle_ring_interrupts(adev, ih[i], enable); + if (r) + return r; + } } + + return 0; } static uint32_t vega10_ih_rb_cntl(struct amdgpu_ih_ring *ih, uint32_t ih_rb_cntl) @@ -208,6 +196,58 @@ static uint32_t vega10_ih_doorbell_rptr(struct amdgpu_ih_ring *ih) return ih_doorbell_rtpr; } +/** + * vega10_ih_enable_ring - enable an ih ring buffer + * + * @adev: amdgpu_device pointer + * @ih: amdgpu_ih_ring pointer + * + * Enable an ih ring buffer (VEGA10) + */ +static int vega10_ih_enable_ring(struct amdgpu_device *adev, + struct amdgpu_ih_ring *ih) +{ + struct amdgpu_ih_regs *ih_regs; + uint32_t tmp; + + ih_regs = &ih->ih_regs; + + /* Ring Buffer base. [39:8] of 40-bit address of the beginning of the ring buffer*/ + WREG32(ih_regs->ih_rb_base, ih->gpu_addr >> 8); + WREG32(ih_regs->ih_rb_base_hi, (ih->gpu_addr >> 40) & 0xff); + + tmp = RREG32(ih_regs->ih_rb_cntl); + tmp = vega10_ih_rb_cntl(ih, tmp); + if (ih == &adev->irq.ih) + tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, RPTR_REARM, !!adev->irq.msi_enabled); + if (ih == &adev->irq.ih1) { + tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, WPTR_OVERFLOW_ENABLE, 0); + tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, RB_FULL_DRAIN_ENABLE, 1); + } + if (amdgpu_sriov_vf(adev)) { + if (psp_reg_program(&adev->psp, ih_regs->psp_reg_id, tmp)) { + dev_err(adev->dev, "PSP program IH_RB_CNTL failed!\n"); + return -ETIMEDOUT; + } + } else { + WREG32(ih_regs->ih_rb_cntl, tmp); + } + + if (ih == &adev->irq.ih) { + /* set the ih ring 0 writeback address whether it's enabled or not */ + WREG32(ih_regs->ih_rb_wptr_addr_lo, lower_32_bits(ih->wptr_addr)); + WREG32(ih_regs->ih_rb_wptr_addr_hi, upper_32_bits(ih->wptr_addr) & 0xFFFF); + } + + /* set rptr, wptr to 0 */ + WREG32(ih_regs->ih_rb_wptr, 0); + WREG32(ih_regs->ih_rb_rptr, 0); + + WREG32(ih_regs->ih_doorbell_rptr, vega10_ih_doorbell_rptr(ih)); + + return 0; +} + /** * vega10_ih_irq_init - init and enable the interrupt ring * @@ -221,116 +261,34 @@ static uint32_t vega10_ih_doorbell_rptr(struct amdgpu_ih_ring *ih) */ static int vega10_ih_irq_init(struct amdgpu_device *adev) { - struct amdgpu_ih_ring *ih; - u32 ih_rb_cntl, ih_chicken; - int ret = 0; + struct amdgpu_ih_ring *ih[] = {&adev->irq.ih, &adev->irq.ih1, &adev->irq.ih2}; + u32 ih_chicken; + int ret; + int i; u32 tmp; /* disable irqs */ - vega10_ih_disable_interrupts(adev); + ret = vega10_ih_toggle_interrupts(adev, false); + if (ret) + return ret; adev->nbio.funcs->ih_control(adev); - ih = &adev->irq.ih; - /* Ring Buffer base. [39:8] of 40-bit address of the beginning of the ring buffer*/ - WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE, ih->gpu_addr >> 8); - WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE_HI, (ih->gpu_addr >> 40) & 0xff); - - ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL); - ih_rb_cntl = vega10_ih_rb_cntl(ih, ih_rb_cntl); - ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RPTR_REARM, - !!adev->irq.msi_enabled); - if (amdgpu_sriov_vf(adev)) { - if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL, ih_rb_cntl)) { - DRM_ERROR("PSP program IH_RB_CNTL failed!\n"); - return -ETIMEDOUT; - } - } else { - WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL, ih_rb_cntl); - } - - if ((adev->asic_type == CHIP_ARCTURUS && - adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) || - adev->asic_type == CHIP_RENOIR) { + if (adev->asic_type == CHIP_RENOIR) { ih_chicken = RREG32_SOC15(OSSSYS, 0, mmIH_CHICKEN); if (adev->irq.ih.use_bus_addr) { ih_chicken = REG_SET_FIELD(ih_chicken, IH_CHICKEN, MC_SPACE_GPA_ENABLE, 1); - } else { - ih_chicken = REG_SET_FIELD(ih_chicken, IH_CHICKEN, - MC_SPACE_FBPA_ENABLE, 1); } WREG32_SOC15(OSSSYS, 0, mmIH_CHICKEN, ih_chicken); } - /* set the writeback address whether it's enabled or not */ - WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_ADDR_LO, - lower_32_bits(ih->wptr_addr)); - WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_ADDR_HI, - upper_32_bits(ih->wptr_addr) & 0xFFFF); - - /* set rptr, wptr to 0 */ - WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR, 0); - WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR, 0); - - WREG32_SOC15(OSSSYS, 0, mmIH_DOORBELL_RPTR, - vega10_ih_doorbell_rptr(ih)); - - ih = &adev->irq.ih1; - if (ih->ring_size) { - WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE_RING1, ih->gpu_addr >> 8); - WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE_HI_RING1, - (ih->gpu_addr >> 40) & 0xff); - - ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1); - ih_rb_cntl = vega10_ih_rb_cntl(ih, ih_rb_cntl); - ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, - WPTR_OVERFLOW_ENABLE, 0); - ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, - RB_FULL_DRAIN_ENABLE, 1); - if (amdgpu_sriov_vf(adev)) { - if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL_RING1, - ih_rb_cntl)) { - DRM_ERROR("program IH_RB_CNTL_RING1 failed!\n"); - return -ETIMEDOUT; - } - } else { - WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1, ih_rb_cntl); + for (i = 0; i < ARRAY_SIZE(ih); i++) { + if (ih[i]->ring_size) { + ret = vega10_ih_enable_ring(adev, ih[i]); + if (ret) + return ret; } - - /* set rptr, wptr to 0 */ - WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_RING1, 0); - WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR_RING1, 0); - - WREG32_SOC15(OSSSYS, 0, mmIH_DOORBELL_RPTR_RING1, - vega10_ih_doorbell_rptr(ih)); - } - - ih = &adev->irq.ih2; - if (ih->ring_size) { - WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE_RING2, ih->gpu_addr >> 8); - WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE_HI_RING2, - (ih->gpu_addr >> 40) & 0xff); - - ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING2); - ih_rb_cntl = vega10_ih_rb_cntl(ih, ih_rb_cntl); - - if (amdgpu_sriov_vf(adev)) { - if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL_RING2, - ih_rb_cntl)) { - DRM_ERROR("program IH_RB_CNTL_RING2 failed!\n"); - return -ETIMEDOUT; - } - } else { - WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING2, ih_rb_cntl); - } - - /* set rptr, wptr to 0 */ - WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_RING2, 0); - WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR_RING2, 0); - - WREG32_SOC15(OSSSYS, 0, mmIH_DOORBELL_RPTR_RING2, - vega10_ih_doorbell_rptr(ih)); } tmp = RREG32_SOC15(OSSSYS, 0, mmIH_STORM_CLIENT_LIST_CNTL); @@ -345,9 +303,14 @@ static int vega10_ih_irq_init(struct amdgpu_device *adev) pci_set_master(adev->pdev); /* enable interrupts */ - vega10_ih_enable_interrupts(adev); + ret = vega10_ih_toggle_interrupts(adev, true); + if (ret) + return ret; - return ret; + if (adev->irq.ih_soft.ring_size) + adev->irq.ih_soft.enabled = true; + + return 0; } /** @@ -359,7 +322,7 @@ static int vega10_ih_irq_init(struct amdgpu_device *adev) */ static void vega10_ih_irq_disable(struct amdgpu_device *adev) { - vega10_ih_disable_interrupts(adev); + vega10_ih_toggle_interrupts(adev, false); /* Wait and acknowledge irq */ mdelay(1); @@ -379,25 +342,17 @@ static void vega10_ih_irq_disable(struct amdgpu_device *adev) static u32 vega10_ih_get_wptr(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih) { - u32 wptr, reg, tmp; + u32 wptr, tmp; + struct amdgpu_ih_regs *ih_regs; wptr = le32_to_cpu(*ih->wptr_cpu); + ih_regs = &ih->ih_regs; if (!REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW)) goto out; /* Double check that the overflow wasn't already cleared. */ - - if (ih == &adev->irq.ih) - reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR); - else if (ih == &adev->irq.ih1) - reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR_RING1); - else if (ih == &adev->irq.ih2) - reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR_RING2); - else - BUG(); - - wptr = RREG32_NO_KIQ(reg); + wptr = RREG32_NO_KIQ(ih_regs->ih_rb_wptr); if (!REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW)) goto out; @@ -413,68 +368,14 @@ static u32 vega10_ih_get_wptr(struct amdgpu_device *adev, wptr, ih->rptr, tmp); ih->rptr = tmp; - if (ih == &adev->irq.ih) - reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL); - else if (ih == &adev->irq.ih1) - reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL_RING1); - else if (ih == &adev->irq.ih2) - reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL_RING2); - else - BUG(); - - tmp = RREG32_NO_KIQ(reg); + tmp = RREG32_NO_KIQ(ih_regs->ih_rb_cntl); tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 1); - WREG32_NO_KIQ(reg, tmp); + WREG32_NO_KIQ(ih_regs->ih_rb_cntl, tmp); out: return (wptr & ih->ptr_mask); } -/** - * vega10_ih_decode_iv - decode an interrupt vector - * - * @adev: amdgpu_device pointer - * @ih: IH ring buffer to decode - * @entry: IV entry to place decoded information into - * - * Decodes the interrupt vector at the current rptr - * position and also advance the position. - */ -static void vega10_ih_decode_iv(struct amdgpu_device *adev, - struct amdgpu_ih_ring *ih, - struct amdgpu_iv_entry *entry) -{ - /* wptr/rptr are in bytes! */ - u32 ring_index = ih->rptr >> 2; - uint32_t dw[8]; - - dw[0] = le32_to_cpu(ih->ring[ring_index + 0]); - dw[1] = le32_to_cpu(ih->ring[ring_index + 1]); - dw[2] = le32_to_cpu(ih->ring[ring_index + 2]); - dw[3] = le32_to_cpu(ih->ring[ring_index + 3]); - dw[4] = le32_to_cpu(ih->ring[ring_index + 4]); - dw[5] = le32_to_cpu(ih->ring[ring_index + 5]); - dw[6] = le32_to_cpu(ih->ring[ring_index + 6]); - dw[7] = le32_to_cpu(ih->ring[ring_index + 7]); - - entry->client_id = dw[0] & 0xff; - entry->src_id = (dw[0] >> 8) & 0xff; - entry->ring_id = (dw[0] >> 16) & 0xff; - entry->vmid = (dw[0] >> 24) & 0xf; - entry->vmid_src = (dw[0] >> 31); - entry->timestamp = dw[1] | ((u64)(dw[2] & 0xffff) << 32); - entry->timestamp_src = dw[2] >> 31; - entry->pasid = dw[3] & 0xffff; - entry->pasid_src = dw[3] >> 31; - entry->src_data[0] = dw[4]; - entry->src_data[1] = dw[5]; - entry->src_data[2] = dw[6]; - entry->src_data[3] = dw[7]; - - /* wptr/rptr are in bytes! */ - ih->rptr += 32; -} - /** * vega10_ih_irq_rearm - rearm IRQ if lost * @@ -485,22 +386,14 @@ static void vega10_ih_decode_iv(struct amdgpu_device *adev, static void vega10_ih_irq_rearm(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih) { - uint32_t reg_rptr = 0; uint32_t v = 0; uint32_t i = 0; + struct amdgpu_ih_regs *ih_regs; - if (ih == &adev->irq.ih) - reg_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_RPTR); - else if (ih == &adev->irq.ih1) - reg_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_RPTR_RING1); - else if (ih == &adev->irq.ih2) - reg_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_RPTR_RING2); - else - return; - + ih_regs = &ih->ih_regs; /* Rearm IRQ / re-wwrite doorbell if doorbell write is lost */ for (i = 0; i < MAX_REARM_RETRY; i++) { - v = RREG32_NO_KIQ(reg_rptr); + v = RREG32_NO_KIQ(ih_regs->ih_rb_rptr); if ((v < ih->ring_size) && (v != ih->rptr)) WDOORBELL32(ih->doorbell_index, ih->rptr); else @@ -519,6 +412,8 @@ static void vega10_ih_irq_rearm(struct amdgpu_device *adev, static void vega10_ih_set_rptr(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih) { + struct amdgpu_ih_regs *ih_regs; + if (ih->use_doorbell) { /* XXX check if swapping is necessary on BE */ *ih->rptr_cpu = ih->rptr; @@ -526,12 +421,9 @@ static void vega10_ih_set_rptr(struct amdgpu_device *adev, if (amdgpu_sriov_vf(adev)) vega10_ih_irq_rearm(adev, ih); - } else if (ih == &adev->irq.ih) { - WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR, ih->rptr); - } else if (ih == &adev->irq.ih1) { - WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR_RING1, ih->rptr); - } else if (ih == &adev->irq.ih2) { - WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR_RING2, ih->rptr); + } else { + ih_regs = &ih->ih_regs; + WREG32(ih_regs->ih_rb_rptr, ih->rptr); } } @@ -600,19 +492,23 @@ static int vega10_ih_sw_init(void *handle) adev->irq.ih.use_doorbell = true; adev->irq.ih.doorbell_index = adev->doorbell_index.ih << 1; - r = amdgpu_ih_ring_init(adev, &adev->irq.ih1, PAGE_SIZE, true); - if (r) - return r; + if (!(adev->flags & AMD_IS_APU)) { + r = amdgpu_ih_ring_init(adev, &adev->irq.ih1, PAGE_SIZE, true); + if (r) + return r; - adev->irq.ih1.use_doorbell = true; - adev->irq.ih1.doorbell_index = (adev->doorbell_index.ih + 1) << 1; + adev->irq.ih1.use_doorbell = true; + adev->irq.ih1.doorbell_index = (adev->doorbell_index.ih + 1) << 1; - r = amdgpu_ih_ring_init(adev, &adev->irq.ih2, PAGE_SIZE, true); - if (r) - return r; + r = amdgpu_ih_ring_init(adev, &adev->irq.ih2, PAGE_SIZE, true); + if (r) + return r; - adev->irq.ih2.use_doorbell = true; - adev->irq.ih2.doorbell_index = (adev->doorbell_index.ih + 2) << 1; + adev->irq.ih2.use_doorbell = true; + adev->irq.ih2.doorbell_index = (adev->doorbell_index.ih + 2) << 1; + } + /* initialize ih control registers offset */ + vega10_ih_init_register_offset(adev); r = amdgpu_ih_ring_init(adev, &adev->irq.ih_soft, PAGE_SIZE, true); if (r) @@ -628,6 +524,7 @@ static int vega10_ih_sw_fini(void *handle) struct amdgpu_device *adev = (struct amdgpu_device *)handle; amdgpu_irq_fini(adev); + amdgpu_ih_ring_fini(adev, &adev->irq.ih_soft); amdgpu_ih_ring_fini(adev, &adev->irq.ih2); amdgpu_ih_ring_fini(adev, &adev->irq.ih1); amdgpu_ih_ring_fini(adev, &adev->irq.ih); @@ -698,15 +595,11 @@ static void vega10_ih_update_clockgating_state(struct amdgpu_device *adev, def = data = RREG32_SOC15(OSSSYS, 0, mmIH_CLK_CTRL); field_val = enable ? 0 : 1; /** - * Vega10 does not have IH_RETRY_INT_CAM_MEM_CLK_SOFT_OVERRIDE - * and IH_BUFFER_MEM_CLK_SOFT_OVERRIDE field. + * Vega10/12 and RAVEN don't have IH_BUFFER_MEM_CLK_SOFT_OVERRIDE field. */ - if (adev->asic_type > CHIP_VEGA10) { - data = REG_SET_FIELD(data, IH_CLK_CTRL, - IH_RETRY_INT_CAM_MEM_CLK_SOFT_OVERRIDE, field_val); + if (adev->asic_type == CHIP_RENOIR) data = REG_SET_FIELD(data, IH_CLK_CTRL, IH_BUFFER_MEM_CLK_SOFT_OVERRIDE, field_val); - } data = REG_SET_FIELD(data, IH_CLK_CTRL, DBUS_MUX_CLK_SOFT_OVERRIDE, field_val); @@ -759,7 +652,7 @@ const struct amd_ip_funcs vega10_ih_ip_funcs = { static const struct amdgpu_ih_funcs vega10_ih_funcs = { .get_wptr = vega10_ih_get_wptr, - .decode_iv = vega10_ih_decode_iv, + .decode_iv = amdgpu_ih_decode_iv_helper, .set_rptr = vega10_ih_set_rptr }; diff --git a/drivers/gpu/drm/amd/amdgpu/vega20_ih.c b/drivers/gpu/drm/amd/amdgpu/vega20_ih.c new file mode 100644 index 0000000000000000000000000000000000000000..5a3c867d58811437f0a06ceaa398d2e552ca12fd --- /dev/null +++ b/drivers/gpu/drm/amd/amdgpu/vega20_ih.c @@ -0,0 +1,703 @@ +/* + * Copyright 2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include + +#include "amdgpu.h" +#include "amdgpu_ih.h" +#include "soc15.h" + +#include "oss/osssys_4_2_0_offset.h" +#include "oss/osssys_4_2_0_sh_mask.h" + +#include "soc15_common.h" +#include "vega20_ih.h" + +#define MAX_REARM_RETRY 10 + +static void vega20_ih_set_interrupt_funcs(struct amdgpu_device *adev); + +/** + * vega20_ih_init_register_offset - Initialize register offset for ih rings + * + * @adev: amdgpu_device pointer + * + * Initialize register offset ih rings (VEGA20). + */ +static void vega20_ih_init_register_offset(struct amdgpu_device *adev) +{ + struct amdgpu_ih_regs *ih_regs; + + if (adev->irq.ih.ring_size) { + ih_regs = &adev->irq.ih.ih_regs; + ih_regs->ih_rb_base = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE); + ih_regs->ih_rb_base_hi = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE_HI); + ih_regs->ih_rb_cntl = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL); + ih_regs->ih_rb_wptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR); + ih_regs->ih_rb_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_RPTR); + ih_regs->ih_doorbell_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_DOORBELL_RPTR); + ih_regs->ih_rb_wptr_addr_lo = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR_ADDR_LO); + ih_regs->ih_rb_wptr_addr_hi = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR_ADDR_HI); + ih_regs->psp_reg_id = PSP_REG_IH_RB_CNTL; + } + + if (adev->irq.ih1.ring_size) { + ih_regs = &adev->irq.ih1.ih_regs; + ih_regs->ih_rb_base = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE_RING1); + ih_regs->ih_rb_base_hi = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE_HI_RING1); + ih_regs->ih_rb_cntl = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL_RING1); + ih_regs->ih_rb_wptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR_RING1); + ih_regs->ih_rb_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_RPTR_RING1); + ih_regs->ih_doorbell_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_DOORBELL_RPTR_RING1); + ih_regs->psp_reg_id = PSP_REG_IH_RB_CNTL_RING1; + } + + if (adev->irq.ih2.ring_size) { + ih_regs = &adev->irq.ih2.ih_regs; + ih_regs->ih_rb_base = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE_RING2); + ih_regs->ih_rb_base_hi = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE_HI_RING2); + ih_regs->ih_rb_cntl = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL_RING2); + ih_regs->ih_rb_wptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR_RING2); + ih_regs->ih_rb_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_RPTR_RING2); + ih_regs->ih_doorbell_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_DOORBELL_RPTR_RING2); + ih_regs->psp_reg_id = PSP_REG_IH_RB_CNTL_RING2; + } +} + +/** + * vega20_ih_toggle_ring_interrupts - toggle the interrupt ring buffer + * + * @adev: amdgpu_device pointer + * @ih: amdgpu_ih_ring pointer + * @enable: true - enable the interrupts, false - disable the interrupts + * + * Toggle the interrupt ring buffer (VEGA20) + */ +static int vega20_ih_toggle_ring_interrupts(struct amdgpu_device *adev, + struct amdgpu_ih_ring *ih, + bool enable) +{ + struct amdgpu_ih_regs *ih_regs; + uint32_t tmp; + + ih_regs = &ih->ih_regs; + + tmp = RREG32(ih_regs->ih_rb_cntl); + tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, RB_ENABLE, (enable ? 1 : 0)); + /* enable_intr field is only valid in ring0 */ + if (ih == &adev->irq.ih) + tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, ENABLE_INTR, (enable ? 1 : 0)); + if (amdgpu_sriov_vf(adev)) { + if (psp_reg_program(&adev->psp, ih_regs->psp_reg_id, tmp)) { + dev_err(adev->dev, "PSP program IH_RB_CNTL failed!\n"); + return -ETIMEDOUT; + } + } else { + WREG32(ih_regs->ih_rb_cntl, tmp); + } + + if (enable) { + ih->enabled = true; + } else { + /* set rptr, wptr to 0 */ + WREG32(ih_regs->ih_rb_rptr, 0); + WREG32(ih_regs->ih_rb_wptr, 0); + ih->enabled = false; + ih->rptr = 0; + } + + return 0; +} + +/** + * vega20_ih_toggle_interrupts - Toggle all the available interrupt ring buffers + * + * @adev: amdgpu_device pointer + * @enable: enable or disable interrupt ring buffers + * + * Toggle all the available interrupt ring buffers (VEGA20). + */ +static int vega20_ih_toggle_interrupts(struct amdgpu_device *adev, bool enable) +{ + struct amdgpu_ih_ring *ih[] = {&adev->irq.ih, &adev->irq.ih1, &adev->irq.ih2}; + int i; + int r; + + for (i = 0; i < ARRAY_SIZE(ih); i++) { + if (ih[i]->ring_size) { + r = vega20_ih_toggle_ring_interrupts(adev, ih[i], enable); + if (r) + return r; + } + } + + return 0; +} + +static uint32_t vega20_ih_rb_cntl(struct amdgpu_ih_ring *ih, uint32_t ih_rb_cntl) +{ + int rb_bufsz = order_base_2(ih->ring_size / 4); + + ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, + MC_SPACE, ih->use_bus_addr ? 1 : 4); + ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, + WPTR_OVERFLOW_CLEAR, 1); + ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, + WPTR_OVERFLOW_ENABLE, 1); + ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_SIZE, rb_bufsz); + /* Ring Buffer write pointer writeback. If enabled, IH_RB_WPTR register + * value is written to memory + */ + ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, + WPTR_WRITEBACK_ENABLE, 1); + ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_SNOOP, 1); + ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_RO, 0); + ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_VMID, 0); + + return ih_rb_cntl; +} + +static uint32_t vega20_ih_doorbell_rptr(struct amdgpu_ih_ring *ih) +{ + u32 ih_doorbell_rtpr = 0; + + if (ih->use_doorbell) { + ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr, + IH_DOORBELL_RPTR, OFFSET, + ih->doorbell_index); + ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr, + IH_DOORBELL_RPTR, + ENABLE, 1); + } else { + ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr, + IH_DOORBELL_RPTR, + ENABLE, 0); + } + return ih_doorbell_rtpr; +} + +/** + * vega20_ih_enable_ring - enable an ih ring buffer + * + * @adev: amdgpu_device pointer + * @ih: amdgpu_ih_ring pointer + * + * Enable an ih ring buffer (VEGA20) + */ +static int vega20_ih_enable_ring(struct amdgpu_device *adev, + struct amdgpu_ih_ring *ih) +{ + struct amdgpu_ih_regs *ih_regs; + uint32_t tmp; + + ih_regs = &ih->ih_regs; + + /* Ring Buffer base. [39:8] of 40-bit address of the beginning of the ring buffer*/ + WREG32(ih_regs->ih_rb_base, ih->gpu_addr >> 8); + WREG32(ih_regs->ih_rb_base_hi, (ih->gpu_addr >> 40) & 0xff); + + tmp = RREG32(ih_regs->ih_rb_cntl); + tmp = vega20_ih_rb_cntl(ih, tmp); + if (ih == &adev->irq.ih) + tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, RPTR_REARM, !!adev->irq.msi_enabled); + if (ih == &adev->irq.ih1) { + tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, WPTR_OVERFLOW_ENABLE, 0); + tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, RB_FULL_DRAIN_ENABLE, 1); + } + if (amdgpu_sriov_vf(adev)) { + if (psp_reg_program(&adev->psp, ih_regs->psp_reg_id, tmp)) { + dev_err(adev->dev, "PSP program IH_RB_CNTL failed!\n"); + return -ETIMEDOUT; + } + } else { + WREG32(ih_regs->ih_rb_cntl, tmp); + } + + if (ih == &adev->irq.ih) { + /* set the ih ring 0 writeback address whether it's enabled or not */ + WREG32(ih_regs->ih_rb_wptr_addr_lo, lower_32_bits(ih->wptr_addr)); + WREG32(ih_regs->ih_rb_wptr_addr_hi, upper_32_bits(ih->wptr_addr) & 0xFFFF); + } + + /* set rptr, wptr to 0 */ + WREG32(ih_regs->ih_rb_wptr, 0); + WREG32(ih_regs->ih_rb_rptr, 0); + + WREG32(ih_regs->ih_doorbell_rptr, vega20_ih_doorbell_rptr(ih)); + + return 0; +} + +/** + * vega20_ih_reroute_ih - reroute VMC/UTCL2 ih to an ih ring + * + * @adev: amdgpu_device pointer + * + * Reroute VMC and UMC interrupts on primary ih ring to + * ih ring 1 so they won't lose when bunches of page faults + * interrupts overwhelms the interrupt handler(VEGA20) + */ +static void vega20_ih_reroute_ih(struct amdgpu_device *adev) +{ + uint32_t tmp; + + /* vega20 ih reroute will go through psp + * this function is only used for arcturus + */ + if (adev->asic_type == CHIP_ARCTURUS) { + /* Reroute to IH ring 1 for VMC */ + WREG32_SOC15(OSSSYS, 0, mmIH_CLIENT_CFG_INDEX, 0x12); + tmp = RREG32_SOC15(OSSSYS, 0, mmIH_CLIENT_CFG_DATA); + tmp = REG_SET_FIELD(tmp, IH_CLIENT_CFG_DATA, CLIENT_TYPE, 1); + tmp = REG_SET_FIELD(tmp, IH_CLIENT_CFG_DATA, RING_ID, 1); + WREG32_SOC15(OSSSYS, 0, mmIH_CLIENT_CFG_DATA, tmp); + + /* Reroute IH ring 1 for UTCL2 */ + WREG32_SOC15(OSSSYS, 0, mmIH_CLIENT_CFG_INDEX, 0x1B); + tmp = RREG32_SOC15(OSSSYS, 0, mmIH_CLIENT_CFG_DATA); + tmp = REG_SET_FIELD(tmp, IH_CLIENT_CFG_DATA, RING_ID, 1); + WREG32_SOC15(OSSSYS, 0, mmIH_CLIENT_CFG_DATA, tmp); + } +} + +/** + * vega20_ih_irq_init - init and enable the interrupt ring + * + * @adev: amdgpu_device pointer + * + * Allocate a ring buffer for the interrupt controller, + * enable the RLC, disable interrupts, enable the IH + * ring buffer and enable it (VI). + * Called at device load and reume. + * Returns 0 for success, errors for failure. + */ +static int vega20_ih_irq_init(struct amdgpu_device *adev) +{ + struct amdgpu_ih_ring *ih[] = {&adev->irq.ih, &adev->irq.ih1, &adev->irq.ih2}; + u32 ih_chicken; + int ret; + int i; + u32 tmp; + + /* disable irqs */ + ret = vega20_ih_toggle_interrupts(adev, false); + if (ret) + return ret; + + adev->nbio.funcs->ih_control(adev); + + if (adev->asic_type == CHIP_ARCTURUS && + adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) { + ih_chicken = RREG32_SOC15(OSSSYS, 0, mmIH_CHICKEN); + if (adev->irq.ih.use_bus_addr) { + ih_chicken = REG_SET_FIELD(ih_chicken, IH_CHICKEN, + MC_SPACE_GPA_ENABLE, 1); + } + WREG32_SOC15(OSSSYS, 0, mmIH_CHICKEN, ih_chicken); + } + + for (i = 0; i < ARRAY_SIZE(ih); i++) { + if (ih[i]->ring_size) { + if (i == 1) + vega20_ih_reroute_ih(adev); + ret = vega20_ih_enable_ring(adev, ih[i]); + if (ret) + return ret; + } + } + + tmp = RREG32_SOC15(OSSSYS, 0, mmIH_STORM_CLIENT_LIST_CNTL); + tmp = REG_SET_FIELD(tmp, IH_STORM_CLIENT_LIST_CNTL, + CLIENT18_IS_STORM_CLIENT, 1); + WREG32_SOC15(OSSSYS, 0, mmIH_STORM_CLIENT_LIST_CNTL, tmp); + + tmp = RREG32_SOC15(OSSSYS, 0, mmIH_INT_FLOOD_CNTL); + tmp = REG_SET_FIELD(tmp, IH_INT_FLOOD_CNTL, FLOOD_CNTL_ENABLE, 1); + WREG32_SOC15(OSSSYS, 0, mmIH_INT_FLOOD_CNTL, tmp); + + pci_set_master(adev->pdev); + + /* enable interrupts */ + ret = vega20_ih_toggle_interrupts(adev, true); + if (ret) + return ret; + + if (adev->irq.ih_soft.ring_size) + adev->irq.ih_soft.enabled = true; + + return 0; +} + +/** + * vega20_ih_irq_disable - disable interrupts + * + * @adev: amdgpu_device pointer + * + * Disable interrupts on the hw (VEGA20). + */ +static void vega20_ih_irq_disable(struct amdgpu_device *adev) +{ + vega20_ih_toggle_interrupts(adev, false); + + /* Wait and acknowledge irq */ + mdelay(1); +} + +/** + * vega20_ih_get_wptr - get the IH ring buffer wptr + * + * @adev: amdgpu_device pointer + * @ih: amdgpu_ih_ring pointer + * + * Get the IH ring buffer wptr from either the register + * or the writeback memory buffer (VEGA20). Also check for + * ring buffer overflow and deal with it. + * Returns the value of the wptr. + */ +static u32 vega20_ih_get_wptr(struct amdgpu_device *adev, + struct amdgpu_ih_ring *ih) +{ + u32 wptr, tmp; + struct amdgpu_ih_regs *ih_regs; + + wptr = le32_to_cpu(*ih->wptr_cpu); + ih_regs = &ih->ih_regs; + + if (!REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW)) + goto out; + + /* Double check that the overflow wasn't already cleared. */ + wptr = RREG32_NO_KIQ(ih_regs->ih_rb_wptr); + if (!REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW)) + goto out; + + wptr = REG_SET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW, 0); + + /* When a ring buffer overflow happen start parsing interrupt + * from the last not overwritten vector (wptr + 32). Hopefully + * this should allow us to catchup. + */ + tmp = (wptr + 32) & ih->ptr_mask; + dev_warn(adev->dev, "IH ring buffer overflow " + "(0x%08X, 0x%08X, 0x%08X)\n", + wptr, ih->rptr, tmp); + ih->rptr = tmp; + + tmp = RREG32_NO_KIQ(ih_regs->ih_rb_cntl); + tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 1); + WREG32_NO_KIQ(ih_regs->ih_rb_cntl, tmp); + +out: + return (wptr & ih->ptr_mask); +} + +/** + * vega20_ih_irq_rearm - rearm IRQ if lost + * + * @adev: amdgpu_device pointer + * @ih: amdgpu_ih_ring pointer + * + */ +static void vega20_ih_irq_rearm(struct amdgpu_device *adev, + struct amdgpu_ih_ring *ih) +{ + uint32_t v = 0; + uint32_t i = 0; + struct amdgpu_ih_regs *ih_regs; + + ih_regs = &ih->ih_regs; + + /* Rearm IRQ / re-wwrite doorbell if doorbell write is lost */ + for (i = 0; i < MAX_REARM_RETRY; i++) { + v = RREG32_NO_KIQ(ih_regs->ih_rb_rptr); + if ((v < ih->ring_size) && (v != ih->rptr)) + WDOORBELL32(ih->doorbell_index, ih->rptr); + else + break; + } +} + +/** + * vega20_ih_set_rptr - set the IH ring buffer rptr + * + * @adev: amdgpu_device pointer + * @ih: amdgpu_ih_ring pointer + * + * Set the IH ring buffer rptr. + */ +static void vega20_ih_set_rptr(struct amdgpu_device *adev, + struct amdgpu_ih_ring *ih) +{ + struct amdgpu_ih_regs *ih_regs; + + if (ih->use_doorbell) { + /* XXX check if swapping is necessary on BE */ + *ih->rptr_cpu = ih->rptr; + WDOORBELL32(ih->doorbell_index, ih->rptr); + + if (amdgpu_sriov_vf(adev)) + vega20_ih_irq_rearm(adev, ih); + } else { + ih_regs = &ih->ih_regs; + WREG32(ih_regs->ih_rb_rptr, ih->rptr); + } +} + +/** + * vega20_ih_self_irq - dispatch work for ring 1 and 2 + * + * @adev: amdgpu_device pointer + * @source: irq source + * @entry: IV with WPTR update + * + * Update the WPTR from the IV and schedule work to handle the entries. + */ +static int vega20_ih_self_irq(struct amdgpu_device *adev, + struct amdgpu_irq_src *source, + struct amdgpu_iv_entry *entry) +{ + uint32_t wptr = cpu_to_le32(entry->src_data[0]); + + switch (entry->ring_id) { + case 1: + *adev->irq.ih1.wptr_cpu = wptr; + schedule_work(&adev->irq.ih1_work); + break; + case 2: + *adev->irq.ih2.wptr_cpu = wptr; + schedule_work(&adev->irq.ih2_work); + break; + default: break; + } + return 0; +} + +static const struct amdgpu_irq_src_funcs vega20_ih_self_irq_funcs = { + .process = vega20_ih_self_irq, +}; + +static void vega20_ih_set_self_irq_funcs(struct amdgpu_device *adev) +{ + adev->irq.self_irq.num_types = 0; + adev->irq.self_irq.funcs = &vega20_ih_self_irq_funcs; +} + +static int vega20_ih_early_init(void *handle) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)handle; + + vega20_ih_set_interrupt_funcs(adev); + vega20_ih_set_self_irq_funcs(adev); + return 0; +} + +static int vega20_ih_sw_init(void *handle) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)handle; + int r; + + r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_IH, 0, + &adev->irq.self_irq); + if (r) + return r; + + r = amdgpu_ih_ring_init(adev, &adev->irq.ih, 256 * 1024, true); + if (r) + return r; + + adev->irq.ih.use_doorbell = true; + adev->irq.ih.doorbell_index = adev->doorbell_index.ih << 1; + + r = amdgpu_ih_ring_init(adev, &adev->irq.ih1, PAGE_SIZE, true); + if (r) + return r; + + adev->irq.ih1.use_doorbell = true; + adev->irq.ih1.doorbell_index = (adev->doorbell_index.ih + 1) << 1; + + r = amdgpu_ih_ring_init(adev, &adev->irq.ih2, PAGE_SIZE, true); + if (r) + return r; + + adev->irq.ih2.use_doorbell = true; + adev->irq.ih2.doorbell_index = (adev->doorbell_index.ih + 2) << 1; + + /* initialize ih control registers offset */ + vega20_ih_init_register_offset(adev); + + r = amdgpu_ih_ring_init(adev, &adev->irq.ih_soft, PAGE_SIZE, true); + if (r) + return r; + + r = amdgpu_irq_init(adev); + + return r; +} + +static int vega20_ih_sw_fini(void *handle) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)handle; + + amdgpu_irq_fini(adev); + amdgpu_ih_ring_fini(adev, &adev->irq.ih_soft); + amdgpu_ih_ring_fini(adev, &adev->irq.ih2); + amdgpu_ih_ring_fini(adev, &adev->irq.ih1); + amdgpu_ih_ring_fini(adev, &adev->irq.ih); + + return 0; +} + +static int vega20_ih_hw_init(void *handle) +{ + int r; + struct amdgpu_device *adev = (struct amdgpu_device *)handle; + + r = vega20_ih_irq_init(adev); + if (r) + return r; + + return 0; +} + +static int vega20_ih_hw_fini(void *handle) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)handle; + + vega20_ih_irq_disable(adev); + + return 0; +} + +static int vega20_ih_suspend(void *handle) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)handle; + + return vega20_ih_hw_fini(adev); +} + +static int vega20_ih_resume(void *handle) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)handle; + + return vega20_ih_hw_init(adev); +} + +static bool vega20_ih_is_idle(void *handle) +{ + /* todo */ + return true; +} + +static int vega20_ih_wait_for_idle(void *handle) +{ + /* todo */ + return -ETIMEDOUT; +} + +static int vega20_ih_soft_reset(void *handle) +{ + /* todo */ + + return 0; +} + +static void vega20_ih_update_clockgating_state(struct amdgpu_device *adev, + bool enable) +{ + uint32_t data, def, field_val; + + if (adev->cg_flags & AMD_CG_SUPPORT_IH_CG) { + def = data = RREG32_SOC15(OSSSYS, 0, mmIH_CLK_CTRL); + field_val = enable ? 0 : 1; + data = REG_SET_FIELD(data, IH_CLK_CTRL, + IH_RETRY_INT_CAM_MEM_CLK_SOFT_OVERRIDE, field_val); + data = REG_SET_FIELD(data, IH_CLK_CTRL, + IH_BUFFER_MEM_CLK_SOFT_OVERRIDE, field_val); + data = REG_SET_FIELD(data, IH_CLK_CTRL, + DBUS_MUX_CLK_SOFT_OVERRIDE, field_val); + data = REG_SET_FIELD(data, IH_CLK_CTRL, + OSSSYS_SHARE_CLK_SOFT_OVERRIDE, field_val); + data = REG_SET_FIELD(data, IH_CLK_CTRL, + LIMIT_SMN_CLK_SOFT_OVERRIDE, field_val); + data = REG_SET_FIELD(data, IH_CLK_CTRL, + DYN_CLK_SOFT_OVERRIDE, field_val); + data = REG_SET_FIELD(data, IH_CLK_CTRL, + REG_CLK_SOFT_OVERRIDE, field_val); + if (def != data) + WREG32_SOC15(OSSSYS, 0, mmIH_CLK_CTRL, data); + } +} + +static int vega20_ih_set_clockgating_state(void *handle, + enum amd_clockgating_state state) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)handle; + + vega20_ih_update_clockgating_state(adev, + state == AMD_CG_STATE_GATE); + return 0; + +} + +static int vega20_ih_set_powergating_state(void *handle, + enum amd_powergating_state state) +{ + return 0; +} + +const struct amd_ip_funcs vega20_ih_ip_funcs = { + .name = "vega20_ih", + .early_init = vega20_ih_early_init, + .late_init = NULL, + .sw_init = vega20_ih_sw_init, + .sw_fini = vega20_ih_sw_fini, + .hw_init = vega20_ih_hw_init, + .hw_fini = vega20_ih_hw_fini, + .suspend = vega20_ih_suspend, + .resume = vega20_ih_resume, + .is_idle = vega20_ih_is_idle, + .wait_for_idle = vega20_ih_wait_for_idle, + .soft_reset = vega20_ih_soft_reset, + .set_clockgating_state = vega20_ih_set_clockgating_state, + .set_powergating_state = vega20_ih_set_powergating_state, +}; + +static const struct amdgpu_ih_funcs vega20_ih_funcs = { + .get_wptr = vega20_ih_get_wptr, + .decode_iv = amdgpu_ih_decode_iv_helper, + .set_rptr = vega20_ih_set_rptr +}; + +static void vega20_ih_set_interrupt_funcs(struct amdgpu_device *adev) +{ + adev->irq.ih_funcs = &vega20_ih_funcs; +} + +const struct amdgpu_ip_block_version vega20_ih_ip_block = +{ + .type = AMD_IP_BLOCK_TYPE_IH, + .major = 4, + .minor = 2, + .rev = 0, + .funcs = &vega20_ih_ip_funcs, +}; diff --git a/drivers/gpu/drm/amd/amdgpu/vega20_ih.h b/drivers/gpu/drm/amd/amdgpu/vega20_ih.h new file mode 100644 index 0000000000000000000000000000000000000000..7af6d8758ee38cb3a6e1a1a0a8db62ab60df8f22 --- /dev/null +++ b/drivers/gpu/drm/amd/amdgpu/vega20_ih.h @@ -0,0 +1,30 @@ +/* + * Copyright 2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef __VEGA20_IH_H__ +#define __VEGA20_IH_H__ + +extern const struct amd_ip_funcs vega20_ih_ip_funcs; +extern const struct amdgpu_ip_block_version vega20_ih_ip_block; + +#endif diff --git a/drivers/gpu/drm/amd/amdgpu/vi.c b/drivers/gpu/drm/amd/amdgpu/vi.c index d56b474b3a219f26767a55c6866a0dbd6cc76599..eafb76aebd004e3b1717d2499a271d5377152001 100644 --- a/drivers/gpu/drm/amd/amdgpu/vi.c +++ b/drivers/gpu/drm/amd/amdgpu/vi.c @@ -642,11 +642,21 @@ static int vi_read_register(struct amdgpu_device *adev, u32 se_num, return -EINVAL; } -static int vi_gpu_pci_config_reset(struct amdgpu_device *adev) +/** + * vi_asic_pci_config_reset - soft reset GPU + * + * @adev: amdgpu_device pointer + * + * Use PCI Config method to reset the GPU. + * + * Returns 0 for success. + */ +static int vi_asic_pci_config_reset(struct amdgpu_device *adev) { u32 i; + int r = -EINVAL; - dev_info(adev->dev, "GPU pci config reset\n"); + amdgpu_atombios_scratch_regs_engine_hung(adev, true); /* disable BM */ pci_clear_master(adev->pdev); @@ -661,29 +671,11 @@ static int vi_gpu_pci_config_reset(struct amdgpu_device *adev) /* enable BM */ pci_set_master(adev->pdev); adev->has_hw_reset = true; - return 0; + r = 0; + break; } udelay(1); } - return -EINVAL; -} - -/** - * vi_asic_pci_config_reset - soft reset GPU - * - * @adev: amdgpu_device pointer - * - * Use PCI Config method to reset the GPU. - * - * Returns 0 for success. - */ -static int vi_asic_pci_config_reset(struct amdgpu_device *adev) -{ - int r; - - amdgpu_atombios_scratch_regs_engine_hung(adev, true); - - r = vi_gpu_pci_config_reset(adev); amdgpu_atombios_scratch_regs_engine_hung(adev, false); diff --git a/drivers/gpu/drm/amd/amdkfd/Kconfig b/drivers/gpu/drm/amd/amdkfd/Kconfig index e8fb10c41f16710e0383411e03e2b9539d526888..f02c938f75dad1b712a1f8a8818ef889bf2fc7c7 100644 --- a/drivers/gpu/drm/amd/amdkfd/Kconfig +++ b/drivers/gpu/drm/amd/amdkfd/Kconfig @@ -7,6 +7,8 @@ config HSA_AMD bool "HSA kernel driver for AMD GPU devices" depends on DRM_AMDGPU && (X86_64 || ARM64 || PPC64) imply AMD_IOMMU_V2 if X86_64 + select HMM_MIRROR select MMU_NOTIFIER + select DRM_AMDGPU_USERPTR help Enable this if you want to use HSA features on AMD GPU devices. diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h index 16262e5d93f5c33a0af6041cd1f7a471e9d361bf..7351dd195274e9b15d18be51e0f85378f63b2c3e 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h @@ -243,11 +243,11 @@ get_sh_mem_bases_nybble_64(struct kfd_process_device *pdd) static inline void dqm_lock(struct device_queue_manager *dqm) { mutex_lock(&dqm->lock_hidden); - dqm->saved_flags = memalloc_nofs_save(); + dqm->saved_flags = memalloc_noreclaim_save(); } static inline void dqm_unlock(struct device_queue_manager *dqm) { - memalloc_nofs_restore(dqm->saved_flags); + memalloc_noreclaim_restore(dqm->saved_flags); mutex_unlock(&dqm->lock_hidden); } diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c b/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c index 241bd6ff79f4add030995f015c895d25237412ce..74a460be077bf8c138e9748d10ff7022f44c7564 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c @@ -44,6 +44,25 @@ static bool event_interrupt_isr_v9(struct kfd_dev *dev, client_id = SOC15_CLIENT_ID_FROM_IH_ENTRY(ih_ring_entry); pasid = SOC15_PASID_FROM_IH_ENTRY(ih_ring_entry); + /* Only handle clients we care about */ + if (client_id != SOC15_IH_CLIENTID_GRBM_CP && + client_id != SOC15_IH_CLIENTID_SDMA0 && + client_id != SOC15_IH_CLIENTID_SDMA1 && + client_id != SOC15_IH_CLIENTID_SDMA2 && + client_id != SOC15_IH_CLIENTID_SDMA3 && + client_id != SOC15_IH_CLIENTID_SDMA4 && + client_id != SOC15_IH_CLIENTID_SDMA5 && + client_id != SOC15_IH_CLIENTID_SDMA6 && + client_id != SOC15_IH_CLIENTID_SDMA7 && + client_id != SOC15_IH_CLIENTID_VMC && + client_id != SOC15_IH_CLIENTID_VMC1 && + client_id != SOC15_IH_CLIENTID_UTCL2 && + client_id != SOC15_IH_CLIENTID_SE0SH && + client_id != SOC15_IH_CLIENTID_SE1SH && + client_id != SOC15_IH_CLIENTID_SE2SH && + client_id != SOC15_IH_CLIENTID_SE3SH) + return false; + /* This is a known issue for gfx9. Under non HWS, pasid is not set * in the interrupt payload, so we need to find out the pasid on our * own. @@ -96,17 +115,30 @@ static void event_interrupt_wq_v9(struct kfd_dev *dev, vmid = SOC15_VMID_FROM_IH_ENTRY(ih_ring_entry); context_id = SOC15_CONTEXT_ID0_FROM_IH_ENTRY(ih_ring_entry); - if (source_id == SOC15_INTSRC_CP_END_OF_PIPE) - kfd_signal_event_interrupt(pasid, context_id, 32); - else if (source_id == SOC15_INTSRC_SDMA_TRAP) - kfd_signal_event_interrupt(pasid, context_id & 0xfffffff, 28); - else if (source_id == SOC15_INTSRC_SQ_INTERRUPT_MSG) - kfd_signal_event_interrupt(pasid, context_id & 0xffffff, 24); - else if (source_id == SOC15_INTSRC_CP_BAD_OPCODE) - kfd_signal_hw_exception_event(pasid); - else if (client_id == SOC15_IH_CLIENTID_VMC || - client_id == SOC15_IH_CLIENTID_VMC1 || - client_id == SOC15_IH_CLIENTID_UTCL2) { + if (client_id == SOC15_IH_CLIENTID_GRBM_CP || + client_id == SOC15_IH_CLIENTID_SE0SH || + client_id == SOC15_IH_CLIENTID_SE1SH || + client_id == SOC15_IH_CLIENTID_SE2SH || + client_id == SOC15_IH_CLIENTID_SE3SH) { + if (source_id == SOC15_INTSRC_CP_END_OF_PIPE) + kfd_signal_event_interrupt(pasid, context_id, 32); + else if (source_id == SOC15_INTSRC_SQ_INTERRUPT_MSG) + kfd_signal_event_interrupt(pasid, context_id & 0xffffff, 24); + else if (source_id == SOC15_INTSRC_CP_BAD_OPCODE) + kfd_signal_hw_exception_event(pasid); + } else if (client_id == SOC15_IH_CLIENTID_SDMA0 || + client_id == SOC15_IH_CLIENTID_SDMA1 || + client_id == SOC15_IH_CLIENTID_SDMA2 || + client_id == SOC15_IH_CLIENTID_SDMA3 || + client_id == SOC15_IH_CLIENTID_SDMA4 || + client_id == SOC15_IH_CLIENTID_SDMA5 || + client_id == SOC15_IH_CLIENTID_SDMA6 || + client_id == SOC15_IH_CLIENTID_SDMA7) { + if (source_id == SOC15_INTSRC_SDMA_TRAP) + kfd_signal_event_interrupt(pasid, context_id & 0xfffffff, 28); + } else if (client_id == SOC15_IH_CLIENTID_VMC || + client_id == SOC15_IH_CLIENTID_VMC1 || + client_id == SOC15_IH_CLIENTID_UTCL2) { struct kfd_vm_fault_info info = {0}; uint16_t ring_id = SOC15_RING_ID_FROM_IH_ENTRY(ih_ring_entry); diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c index a3fc2387381956169043c19c2de341833c80a757..0be72789ccbc066a49f1b0589b21024628a73930 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c @@ -497,8 +497,6 @@ static ssize_t node_show(struct kobject *kobj, struct attribute *attr, dev->node_props.num_sdma_queues_per_engine); sysfs_show_32bit_prop(buffer, offs, "num_cp_queues", dev->node_props.num_cp_queues); - sysfs_show_64bit_prop(buffer, offs, "unique_id", - dev->node_props.unique_id); if (dev->gpu) { log_max_watch_addr = @@ -529,6 +527,9 @@ static ssize_t node_show(struct kobject *kobj, struct attribute *attr, dev->node_props.capability); sysfs_show_32bit_prop(buffer, offs, "sdma_fw_version", dev->gpu->sdma_fw_version); + sysfs_show_64bit_prop(buffer, offs, "unique_id", + amdgpu_amdkfd_get_unique_id(dev->gpu->kgd)); + } return sysfs_show_32bit_prop(buffer, offs, "max_engine_clk_ccompute", @@ -1340,7 +1341,6 @@ int kfd_topology_add_device(struct kfd_dev *gpu) dev->gpu->dqm->sched_policy != KFD_SCHED_POLICY_NO_HWS) ? amdgpu_amdkfd_get_num_gws(dev->gpu->kgd) : 0; dev->node_props.num_cp_queues = get_cp_queues_num(dev->gpu->dqm); - dev->node_props.unique_id = amdgpu_amdkfd_get_unique_id(dev->gpu->kgd); kfd_fill_mem_clk_max_info(dev); kfd_fill_iolink_non_crat_info(dev); diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_topology.h b/drivers/gpu/drm/amd/amdkfd/kfd_topology.h index 326d9b26b7aa7fbbab553671ee1a0d348739d8bf..416fd910e12e1b055d579d366f3e7f31f500d1dd 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_topology.h +++ b/drivers/gpu/drm/amd/amdkfd/kfd_topology.h @@ -57,7 +57,6 @@ struct kfd_node_properties { uint64_t hive_id; - uint64_t unique_id; uint32_t cpu_cores_count; uint32_t simd_count; uint32_t mem_banks_count; diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c index 961abf1cf040cc0372d0d9c74d456684f81cb07d..3e1fd1e7d09f1d4e3c017d5607bef69c19948a58 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c @@ -60,7 +60,6 @@ #include #include -#include #include #include #include @@ -938,7 +937,49 @@ static void mmhub_read_system_context(struct amdgpu_device *adev, struct dc_phy_ } #endif +#if defined(CONFIG_DRM_AMD_DC_DCN) +static void event_mall_stutter(struct work_struct *work) +{ + + struct vblank_workqueue *vblank_work = container_of(work, struct vblank_workqueue, mall_work); + struct amdgpu_display_manager *dm = vblank_work->dm; + + mutex_lock(&dm->dc_lock); + + if (vblank_work->enable) + dm->active_vblank_irq_count++; + else + dm->active_vblank_irq_count--; + + + dc_allow_idle_optimizations( + dm->dc, dm->active_vblank_irq_count == 0 ? true : false); + + DRM_DEBUG_DRIVER("Allow idle optimizations (MALL): %d\n", dm->active_vblank_irq_count == 0); + + + mutex_unlock(&dm->dc_lock); +} + +static struct vblank_workqueue *vblank_create_workqueue(struct amdgpu_device *adev, struct dc *dc) +{ + int max_caps = dc->caps.max_links; + struct vblank_workqueue *vblank_work; + int i = 0; + + vblank_work = kcalloc(max_caps, sizeof(*vblank_work), GFP_KERNEL); + if (ZERO_OR_NULL_PTR(vblank_work)) { + kfree(vblank_work); + return NULL; + } + + for (i = 0; i < max_caps; i++) + INIT_WORK(&vblank_work[i].mall_work, event_mall_stutter); + + return vblank_work; +} +#endif static int amdgpu_dm_init(struct amdgpu_device *adev) { struct dc_init_data init_data; @@ -958,6 +999,9 @@ static int amdgpu_dm_init(struct amdgpu_device *adev) mutex_init(&adev->dm.dc_lock); mutex_init(&adev->dm.audio_lock); +#if defined(CONFIG_DRM_AMD_DC_DCN) + spin_lock_init(&adev->dm.vblank_lock); +#endif if(amdgpu_dm_irq_init(adev)) { DRM_ERROR("amdgpu: failed to initialize DM IRQ support.\n"); @@ -1016,8 +1060,6 @@ static int amdgpu_dm_init(struct amdgpu_device *adev) init_data.flags.power_down_display_on_boot = true; - init_data.soc_bounding_box = adev->dm.soc_bounding_box; - /* Display Core create. */ adev->dm.dc = dc_create(&init_data); @@ -1074,6 +1116,17 @@ static int amdgpu_dm_init(struct amdgpu_device *adev) amdgpu_dm_init_color_mod(); +#if defined(CONFIG_DRM_AMD_DC_DCN) + if (adev->dm.dc->caps.max_links > 0) { + adev->dm.vblank_workqueue = vblank_create_workqueue(adev, adev->dm.dc); + + if (!adev->dm.vblank_workqueue) + DRM_ERROR("amdgpu: failed to initialize vblank_workqueue.\n"); + else + DRM_DEBUG_DRIVER("amdgpu: vblank_workqueue init done %p.\n", adev->dm.vblank_workqueue); + } +#endif + #ifdef CONFIG_DRM_AMD_DC_HDCP if (adev->dm.dc->caps.max_links > 0 && adev->asic_type >= CHIP_RAVEN) { adev->dm.hdcp_workqueue = hdcp_create_workqueue(adev, &init_params.cp_psp, adev->dm.dc); @@ -1131,7 +1184,7 @@ static void amdgpu_dm_fini(struct amdgpu_device *adev) #ifdef CONFIG_DRM_AMD_DC_HDCP if (adev->dm.hdcp_workqueue) { - hdcp_destroy(adev->dm.hdcp_workqueue); + hdcp_destroy(&adev->dev->kobj, adev->dm.hdcp_workqueue); adev->dm.hdcp_workqueue = NULL; } @@ -1778,6 +1831,11 @@ static int dm_suspend(void *handle) if (amdgpu_in_reset(adev)) { mutex_lock(&dm->dc_lock); + +#if defined(CONFIG_DRM_AMD_DC_DCN) + dc_allow_idle_optimizations(adev->dm.dc, false); +#endif + dm->cached_dc_state = dc_copy_state(dm->dc->current_state); dm_gpureset_toggle_interrupts(adev, dm->cached_dc_state, false); @@ -1934,7 +1992,7 @@ static void dm_gpureset_commit_state(struct dc_state *dc_state, dc_commit_updates_for_stream( dm->dc, bundle->surface_updates, dc_state->stream_status->plane_count, - dc_state->streams[k], &bundle->stream_update); + dc_state->streams[k], &bundle->stream_update, dc_state); } cleanup: @@ -1965,7 +2023,8 @@ static void dm_set_dpms_off(struct dc_link *link) stream_update.stream = stream_state; dc_commit_updates_for_stream(stream_state->ctx->dc, NULL, 0, - stream_state, &stream_update); + stream_state, &stream_update, + stream_state->ctx->dc->current_state); mutex_unlock(&adev->dm.dc_lock); } @@ -3719,10 +3778,53 @@ static const struct drm_encoder_funcs amdgpu_dm_encoder_funcs = { }; +static void get_min_max_dc_plane_scaling(struct drm_device *dev, + struct drm_framebuffer *fb, + int *min_downscale, int *max_upscale) +{ + struct amdgpu_device *adev = drm_to_adev(dev); + struct dc *dc = adev->dm.dc; + /* Caps for all supported planes are the same on DCE and DCN 1 - 3 */ + struct dc_plane_cap *plane_cap = &dc->caps.planes[0]; + + switch (fb->format->format) { + case DRM_FORMAT_P010: + case DRM_FORMAT_NV12: + case DRM_FORMAT_NV21: + *max_upscale = plane_cap->max_upscale_factor.nv12; + *min_downscale = plane_cap->max_downscale_factor.nv12; + break; + + case DRM_FORMAT_XRGB16161616F: + case DRM_FORMAT_ARGB16161616F: + case DRM_FORMAT_XBGR16161616F: + case DRM_FORMAT_ABGR16161616F: + *max_upscale = plane_cap->max_upscale_factor.fp16; + *min_downscale = plane_cap->max_downscale_factor.fp16; + break; + + default: + *max_upscale = plane_cap->max_upscale_factor.argb8888; + *min_downscale = plane_cap->max_downscale_factor.argb8888; + break; + } + + /* + * A factor of 1 in the plane_cap means to not allow scaling, ie. use a + * scaling factor of 1.0 == 1000 units. + */ + if (*max_upscale == 1) + *max_upscale = 1000; + + if (*min_downscale == 1) + *min_downscale = 1000; +} + + static int fill_dc_scaling_info(const struct drm_plane_state *state, struct dc_scaling_info *scaling_info) { - int scale_w, scale_h; + int scale_w, scale_h, min_downscale, max_upscale; memset(scaling_info, 0, sizeof(*scaling_info)); @@ -3754,17 +3856,25 @@ static int fill_dc_scaling_info(const struct drm_plane_state *state, /* DRM doesn't specify clipping on destination output. */ scaling_info->clip_rect = scaling_info->dst_rect; - /* TODO: Validate scaling per-format with DC plane caps */ + /* Validate scaling per-format with DC plane caps */ + if (state->plane && state->plane->dev && state->fb) { + get_min_max_dc_plane_scaling(state->plane->dev, state->fb, + &min_downscale, &max_upscale); + } else { + min_downscale = 250; + max_upscale = 16000; + } + scale_w = scaling_info->dst_rect.width * 1000 / scaling_info->src_rect.width; - if (scale_w < 250 || scale_w > 16000) + if (scale_w < min_downscale || scale_w > max_upscale) return -EINVAL; scale_h = scaling_info->dst_rect.height * 1000 / scaling_info->src_rect.height; - if (scale_h < 250 || scale_h > 16000) + if (scale_h < min_downscale || scale_h > max_upscale) return -EINVAL; /* @@ -5321,6 +5431,10 @@ static inline int dm_set_vblank(struct drm_crtc *crtc, bool enable) struct amdgpu_crtc *acrtc = to_amdgpu_crtc(crtc); struct amdgpu_device *adev = drm_to_adev(crtc->dev); struct dm_crtc_state *acrtc_state = to_dm_crtc_state(crtc->state); +#if defined(CONFIG_DRM_AMD_DC_DCN) + struct amdgpu_display_manager *dm = &adev->dm; + unsigned long flags; +#endif int rc = 0; if (enable) { @@ -5336,7 +5450,23 @@ static inline int dm_set_vblank(struct drm_crtc *crtc, bool enable) return rc; irq_source = IRQ_TYPE_VBLANK + acrtc->otg_inst; - return dc_interrupt_set(adev->dm.dc, irq_source, enable) ? 0 : -EBUSY; + + if (!dc_interrupt_set(adev->dm.dc, irq_source, enable)) + return -EBUSY; + + if (amdgpu_in_reset(adev)) + return 0; + +#if defined(CONFIG_DRM_AMD_DC_DCN) + spin_lock_irqsave(&dm->vblank_lock, flags); + dm->vblank_workqueue->dm = dm; + dm->vblank_workqueue->otg_inst = acrtc->otg_inst; + dm->vblank_workqueue->enable = enable; + spin_unlock_irqrestore(&dm->vblank_lock, flags); + schedule_work(&dm->vblank_workqueue->mall_work); +#endif + + return 0; } static int dm_enable_vblank(struct drm_crtc *crtc) @@ -5353,7 +5483,6 @@ static void dm_disable_vblank(struct drm_crtc *crtc) static const struct drm_crtc_funcs amdgpu_dm_crtc_funcs = { .reset = dm_crtc_reset_state, .destroy = amdgpu_dm_crtc_destroy, - .gamma_set = drm_atomic_helper_legacy_gamma_set, .set_config = drm_atomic_helper_set_config, .page_flip = drm_atomic_helper_page_flip, .atomic_duplicate_state = dm_crtc_duplicate_state, @@ -6328,12 +6457,51 @@ static void dm_plane_helper_cleanup_fb(struct drm_plane *plane, static int dm_plane_helper_check_state(struct drm_plane_state *state, struct drm_crtc_state *new_crtc_state) { - int max_downscale = 0; - int max_upscale = INT_MAX; + struct drm_framebuffer *fb = state->fb; + int min_downscale, max_upscale; + int min_scale = 0; + int max_scale = INT_MAX; + + /* Plane enabled? Validate viewport and get scaling factors from plane caps. */ + if (fb && state->crtc) { + /* Validate viewport to cover the case when only the position changes */ + if (state->plane->type != DRM_PLANE_TYPE_CURSOR) { + int viewport_width = state->crtc_w; + int viewport_height = state->crtc_h; + + if (state->crtc_x < 0) + viewport_width += state->crtc_x; + else if (state->crtc_x + state->crtc_w > new_crtc_state->mode.crtc_hdisplay) + viewport_width = new_crtc_state->mode.crtc_hdisplay - state->crtc_x; + + if (state->crtc_y < 0) + viewport_height += state->crtc_y; + else if (state->crtc_y + state->crtc_h > new_crtc_state->mode.crtc_vdisplay) + viewport_height = new_crtc_state->mode.crtc_vdisplay - state->crtc_y; + + /* If completely outside of screen, viewport_width and/or viewport_height will be negative, + * which is still OK to satisfy the condition below, thereby also covering these cases + * (when plane is completely outside of screen). + * x2 for width is because of pipe-split. + */ + if (viewport_width < MIN_VIEWPORT_SIZE*2 || viewport_height < MIN_VIEWPORT_SIZE) + return -EINVAL; + } + + /* Get min/max allowed scaling factors from plane caps. */ + get_min_max_dc_plane_scaling(state->crtc->dev, fb, + &min_downscale, &max_upscale); + /* + * Convert to drm convention: 16.16 fixed point, instead of dc's + * 1.0 == 1000. Also drm scaling is src/dst instead of dc's + * dst/src, so min_scale = 1.0 / max_upscale, etc. + */ + min_scale = (1000 << 16) / max_upscale; + max_scale = (1000 << 16) / min_downscale; + } - /* TODO: These should be checked against DC plane caps */ return drm_atomic_helper_check_plane_state( - state, new_crtc_state, max_downscale, max_upscale, true, true); + state, new_crtc_state, min_scale, max_scale, true, true); } static int dm_plane_atomic_check(struct drm_plane *plane, @@ -7548,7 +7716,7 @@ static void amdgpu_dm_commit_planes(struct drm_atomic_state *state, struct drm_crtc *pcrtc, bool wait_for_vblank) { - int i; + uint32_t i; uint64_t timestamp_ns; struct drm_plane *plane; struct drm_plane_state *old_plane_state, *new_plane_state; @@ -7589,7 +7757,7 @@ static void amdgpu_dm_commit_planes(struct drm_atomic_state *state, amdgpu_dm_commit_cursors(state); /* update planes when needed */ - for_each_oldnew_plane_in_state_reverse(state, plane, old_plane_state, new_plane_state, i) { + for_each_oldnew_plane_in_state(state, plane, old_plane_state, new_plane_state, i) { struct drm_crtc *crtc = new_plane_state->crtc; struct drm_crtc_state *new_crtc_state; struct drm_framebuffer *fb = new_plane_state->fb; @@ -7812,7 +7980,8 @@ static void amdgpu_dm_commit_planes(struct drm_atomic_state *state, bundle->surface_updates, planes_count, acrtc_state->stream, - &bundle->stream_update); + &bundle->stream_update, + dc_state); /** * Enable or disable the interrupts on the backend. @@ -8148,13 +8317,13 @@ static void amdgpu_dm_atomic_commit_tail(struct drm_atomic_state *state) struct dm_connector_state *dm_new_con_state = to_dm_connector_state(new_con_state); struct dm_connector_state *dm_old_con_state = to_dm_connector_state(old_con_state); struct amdgpu_crtc *acrtc = to_amdgpu_crtc(dm_new_con_state->base.crtc); - struct dc_surface_update surface_updates[MAX_SURFACES]; + struct dc_surface_update dummy_updates[MAX_SURFACES]; struct dc_stream_update stream_update; struct dc_info_packet hdr_packet; struct dc_stream_status *status = NULL; bool abm_changed, hdr_changed, scaling_changed; - memset(&surface_updates, 0, sizeof(surface_updates)); + memset(&dummy_updates, 0, sizeof(dummy_updates)); memset(&stream_update, 0, sizeof(stream_update)); if (acrtc) { @@ -8211,15 +8380,16 @@ static void amdgpu_dm_atomic_commit_tail(struct drm_atomic_state *state) * To fix this, DC should permit updating only stream properties. */ for (j = 0; j < status->plane_count; j++) - surface_updates[j].surface = status->plane_states[j]; + dummy_updates[j].surface = status->plane_states[0]; mutex_lock(&dm->dc_lock); dc_commit_updates_for_stream(dm->dc, - surface_updates, + dummy_updates, status->plane_count, dm_new_crtc_state->stream, - &stream_update); + &stream_update, + dc_state); mutex_unlock(&dm->dc_lock); } @@ -9588,6 +9758,10 @@ void amdgpu_dm_update_freesync_caps(struct drm_connector *connector, amdgpu_dm_connector->max_vfreq = range->max_vfreq; amdgpu_dm_connector->pixel_clock_mhz = range->pixel_clock_mhz * 10; + + connector->display_info.monitor_range.min_vfreq = range->min_vfreq; + connector->display_info.monitor_range.max_vfreq = range->max_vfreq; + break; } diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h index 1182dafcef0228fd8818cfd82e597066a7abc3c0..8bfe901cf237426cbcff64e857a8ed33396f855f 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h @@ -58,10 +58,10 @@ /* Forward declarations */ struct amdgpu_device; struct drm_device; -struct amdgpu_dm_irq_handler_data; struct dc; struct amdgpu_bo; struct dmub_srv; +struct dc_plane_state; struct common_irq_params { struct amdgpu_device *adev; @@ -92,6 +92,20 @@ struct dm_compressor_info { uint64_t gpu_addr; }; +/** + * struct vblank_workqueue - Works to be executed in a separate thread during vblank + * @mall_work: work for mall stutter + * @dm: amdgpu display manager device + * @otg_inst: otg instance of which vblank is being set + * @enable: true if enable vblank + */ +struct vblank_workqueue { + struct work_struct mall_work; + struct amdgpu_display_manager *dm; + int otg_inst; + bool enable; +}; + /** * struct amdgpu_dm_backlight_caps - Information about backlight * @@ -243,6 +257,15 @@ struct amdgpu_display_manager { */ struct mutex audio_lock; + /** + * @vblank_work_lock: + * + * Guards access to deferred vblank work state. + */ +#if defined(CONFIG_DRM_AMD_DC_DCN) + spinlock_t vblank_lock; +#endif + /** * @audio_component: * @@ -321,6 +344,10 @@ struct amdgpu_display_manager { struct hdcp_workqueue *hdcp_workqueue; #endif +#if defined(CONFIG_DRM_AMD_DC_DCN) + struct vblank_workqueue *vblank_workqueue; +#endif + struct drm_atomic_state *cached_state; struct dc_state *cached_dc_state; @@ -336,6 +363,13 @@ struct amdgpu_display_manager { */ const struct gpu_info_soc_bounding_box_v1_0 *soc_bounding_box; + /** + * @active_vblank_irq_count: + * + * number of currently active vblank irqs + */ + uint32_t active_vblank_irq_count; + /** * @mst_encoders: * @@ -412,11 +446,6 @@ struct amdgpu_dm_connector { extern const struct amdgpu_ip_block_version dm_ip_block; -struct amdgpu_framebuffer; -struct amdgpu_display_manager; -struct dc_validation_set; -struct dc_plane_state; - struct dm_plane_state { struct drm_plane_state base; struct dc_plane_state *dc_state; diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c index 11459fb09a372a5b165aa62084344480c1638998..360952129b6d5a2b6ffe6bbc3a3004a4c1e20946 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c @@ -691,7 +691,7 @@ static ssize_t dp_phy_test_pattern_debugfs_write(struct file *f, const char __us return size; } -/** +/* * Returns the DMCUB tracebuffer contents. * Example usage: cat /sys/kernel/debug/dri/0/amdgpu_dm_dmub_tracebuffer */ @@ -735,7 +735,7 @@ static int dmub_tracebuffer_show(struct seq_file *m, void *data) return 0; } -/** +/* * Returns the DMCUB firmware state contents. * Example usage: cat /sys/kernel/debug/dri/0/amdgpu_dm_dmub_fw_state */ @@ -1063,7 +1063,7 @@ static int dp_dsc_fec_support_show(struct seq_file *m, void *data) * echo 0 > /sys/kernel/debug/dri/0/DP-X/trigger_hotplug * */ -static ssize_t dp_trigger_hotplug(struct file *f, const char __user *buf, +static ssize_t trigger_hotplug(struct file *f, const char __user *buf, size_t size, loff_t *pos) { struct amdgpu_dm_connector *aconnector = file_inode(f)->i_private; @@ -2214,9 +2214,9 @@ static const struct file_operations dp_dsc_slice_bpg_offset_debugfs_fops = { .llseek = default_llseek }; -static const struct file_operations dp_trigger_hotplug_debugfs_fops = { +static const struct file_operations trigger_hotplug_debugfs_fops = { .owner = THIS_MODULE, - .write = dp_trigger_hotplug, + .write = trigger_hotplug, .llseek = default_llseek }; @@ -2270,7 +2270,6 @@ static const struct { const struct file_operations *fops; } dp_debugfs_entries[] = { {"link_settings", &dp_link_settings_debugfs_fops}, - {"trigger_hotplug", &dp_trigger_hotplug_debugfs_fops}, {"phy_settings", &dp_phy_settings_debugfs_fop}, {"test_pattern", &dp_phy_test_pattern_fops}, #ifdef CONFIG_DRM_AMD_DC_HDCP @@ -2367,6 +2366,9 @@ void connector_debugfs_init(struct amdgpu_dm_connector *connector) debugfs_create_file("output_bpc", 0644, dir, connector, &output_bpc_fops); + debugfs_create_file("trigger_hotplug", 0644, dir, connector, + &trigger_hotplug_debugfs_fops); + connector->debugfs_dpcd_address = 0; connector->debugfs_dpcd_size = 0; diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_hdcp.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_hdcp.c index c2cd184f0bbd4b9f3baacda407a64e0bd22a87e7..0cdbfcd475ec90bf3c4d036ccdf512422fb2c116 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_hdcp.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_hdcp.c @@ -376,7 +376,7 @@ static void event_cpirq(struct work_struct *work) } -void hdcp_destroy(struct hdcp_workqueue *hdcp_work) +void hdcp_destroy(struct kobject *kobj, struct hdcp_workqueue *hdcp_work) { int i = 0; @@ -385,6 +385,7 @@ void hdcp_destroy(struct hdcp_workqueue *hdcp_work) cancel_delayed_work_sync(&hdcp_work[i].watchdog_timer_dwork); } + sysfs_remove_bin_file(kobj, &hdcp_work[0].attr); kfree(hdcp_work->srm); kfree(hdcp_work->srm_temp); kfree(hdcp_work); @@ -449,11 +450,12 @@ static void update_config(void *handle, struct cp_psp_stream_config *config) link->mode = mod_hdcp_signal_type_to_operation_mode(aconnector->dc_sink->sink_signal); display->controller = CONTROLLER_ID_D0 + config->otg_inst; - display->dig_fe = config->stream_enc_inst; - link->dig_be = config->link_enc_inst; + display->dig_fe = config->dig_fe; + link->dig_be = config->dig_be; link->ddc_line = aconnector->dc_link->ddc_hw_inst + 1; link->dp.rev = aconnector->dc_link->dpcd_caps.dpcd_rev.raw; - link->dp.mst_supported = config->mst_supported; + link->dp.assr_enabled = config->assr_enabled; + link->dp.mst_enabled = config->mst_enabled; display->adjust.disable = 1; link->adjust.auth_delay = 3; link->adjust.hdcp1.disable = 0; diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_hdcp.h b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_hdcp.h index 5159b3a5e5b03b3e2c688ffab93735eac9cf2f10..09294ff122fead72611bf7cf909ee5504f311e3c 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_hdcp.h +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_hdcp.h @@ -69,7 +69,7 @@ void hdcp_update_display(struct hdcp_workqueue *hdcp_work, void hdcp_reset_display(struct hdcp_workqueue *work, unsigned int link_index); void hdcp_handle_cpirq(struct hdcp_workqueue *work, unsigned int link_index); -void hdcp_destroy(struct hdcp_workqueue *work); +void hdcp_destroy(struct kobject *kobj, struct hdcp_workqueue *work); struct hdcp_workqueue *hdcp_create_workqueue(struct amdgpu_device *adev, struct cp_psp *cp_psp, struct dc *dc); diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c index f6f487e9fe2d3ffbbeb9ceaaad53516bc76170c5..5159399f82391cebe695579d5f20bfd00c448c98 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c @@ -25,7 +25,6 @@ #include #include -#include #include #include @@ -527,11 +526,11 @@ bool dm_helpers_submit_i2c( bool dm_helpers_dp_write_dsc_enable( struct dc_context *ctx, const struct dc_stream_state *stream, - bool enable -) + bool enable) { uint8_t enable_dsc = enable ? 1 : 0; struct amdgpu_dm_connector *aconnector; + uint8_t ret; if (!stream) return false; @@ -542,13 +541,13 @@ bool dm_helpers_dp_write_dsc_enable( if (!aconnector->dsc_aux) return false; - return (drm_dp_dpcd_write(aconnector->dsc_aux, DP_DSC_ENABLE, &enable_dsc, 1) >= 0); + ret = drm_dp_dpcd_write(aconnector->dsc_aux, DP_DSC_ENABLE, &enable_dsc, 1); } if (stream->signal == SIGNAL_TYPE_DISPLAY_PORT) return dm_helpers_dp_write_dpcd(ctx, stream->link, DP_DSC_ENABLE, &enable_dsc, 1); - return false; + return (ret > 0); } bool dm_helpers_is_dp_sink_present(struct dc_link *link) diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.c index 26ed70e5538ae164a533c9afa1315e4c4cf93807..e0000c180ed1f035b37263c8e8d8e14e059fefa9 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.c @@ -662,6 +662,20 @@ static int amdgpu_dm_set_crtc_irq_state(struct amdgpu_device *adev, __func__); } +static int amdgpu_dm_set_vline0_irq_state(struct amdgpu_device *adev, + struct amdgpu_irq_src *source, + unsigned int crtc_id, + enum amdgpu_interrupt_state state) +{ + return dm_irq_state( + adev, + source, + crtc_id, + state, + IRQ_TYPE_VLINE0, + __func__); +} + static int amdgpu_dm_set_vupdate_irq_state(struct amdgpu_device *adev, struct amdgpu_irq_src *source, unsigned int crtc_id, @@ -681,6 +695,11 @@ static const struct amdgpu_irq_src_funcs dm_crtc_irq_funcs = { .process = amdgpu_dm_irq_handler, }; +static const struct amdgpu_irq_src_funcs dm_vline0_irq_funcs = { + .set = amdgpu_dm_set_vline0_irq_state, + .process = amdgpu_dm_irq_handler, +}; + static const struct amdgpu_irq_src_funcs dm_vupdate_irq_funcs = { .set = amdgpu_dm_set_vupdate_irq_state, .process = amdgpu_dm_irq_handler, @@ -702,6 +721,9 @@ void amdgpu_dm_set_irq_funcs(struct amdgpu_device *adev) adev->crtc_irq.num_types = adev->mode_info.num_crtc; adev->crtc_irq.funcs = &dm_crtc_irq_funcs; + adev->vline0_irq.num_types = adev->mode_info.num_crtc; + adev->vline0_irq.funcs = &dm_vline0_irq_funcs; + adev->vupdate_irq.num_types = adev->mode_info.num_crtc; adev->vupdate_irq.funcs = &dm_vupdate_irq_funcs; diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c index f2d8cf34be46e92bbd2c83cf832835389a8936b6..41b09ab22233a438674807a746a31d1fa7570647 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c @@ -23,7 +23,6 @@ * */ -#include #include #include #include diff --git a/drivers/gpu/drm/amd/display/dc/Makefile b/drivers/gpu/drm/amd/display/dc/Makefile index bf8fe0471b8fc0cfca5db88adbe375a6efd502dd..5bf2f2375b40df208e3f41572bffc84134e1ac23 100644 --- a/drivers/gpu/drm/amd/display/dc/Makefile +++ b/drivers/gpu/drm/amd/display/dc/Makefile @@ -69,5 +69,7 @@ AMD_DISPLAY_FILES += $(AMD_DISPLAY_CORE) AMD_DISPLAY_FILES += $(AMD_DM_REG_UPDATE) DC_DMUB += dc_dmub_srv.o +DC_EDID += dc_edid_parser.o AMD_DISPLAY_DMUB = $(addprefix $(AMDDALPATH)/dc/,$(DC_DMUB)) -AMD_DISPLAY_FILES += $(AMD_DISPLAY_DMUB) +AMD_DISPLAY_EDID = $(addprefix $(AMDDALPATH)/dc/,$(DC_EDID)) +AMD_DISPLAY_FILES += $(AMD_DISPLAY_DMUB) $(AMD_DISPLAY_EDID) diff --git a/drivers/gpu/drm/amd/display/dc/basics/conversion.c b/drivers/gpu/drm/amd/display/dc/basics/conversion.c index 24ed03d8cda7484d69659386d818024105800344..6767fab55c260d4869095c3baaf9517c5b831dd3 100644 --- a/drivers/gpu/drm/amd/display/dc/basics/conversion.c +++ b/drivers/gpu/drm/amd/display/dc/basics/conversion.c @@ -73,12 +73,9 @@ uint16_t fixed_point_to_int_frac( return result; } -/** -* convert_float_matrix -* This converts a double into HW register spec defined format S2D13. -* @param : -* @return None -*/ +/* + * convert_float_matrix - This converts a double into HW register spec defined format S2D13. + */ void convert_float_matrix( uint16_t *matrix, struct fixed31_32 *flt, diff --git a/drivers/gpu/drm/amd/display/dc/basics/dc_common.c b/drivers/gpu/drm/amd/display/dc/basics/dc_common.c index b2fc4f8e64825086855b28a11099312f2d06e5f8..ad04ef98e652b1a214d73207245f40adc57fafde 100644 --- a/drivers/gpu/drm/amd/display/dc/basics/dc_common.c +++ b/drivers/gpu/drm/amd/display/dc/basics/dc_common.c @@ -49,20 +49,24 @@ bool is_rgb_cspace(enum dc_color_space output_color_space) } } -bool is_lower_pipe_tree_visible(struct pipe_ctx *pipe_ctx) +bool is_child_pipe_tree_visible(struct pipe_ctx *pipe_ctx) { if (pipe_ctx->plane_state && pipe_ctx->plane_state->visible) return true; - if (pipe_ctx->bottom_pipe && is_lower_pipe_tree_visible(pipe_ctx->bottom_pipe)) + if (pipe_ctx->bottom_pipe && is_child_pipe_tree_visible(pipe_ctx->bottom_pipe)) + return true; + if (pipe_ctx->next_odm_pipe && is_child_pipe_tree_visible(pipe_ctx->next_odm_pipe)) return true; return false; } -bool is_upper_pipe_tree_visible(struct pipe_ctx *pipe_ctx) +bool is_parent_pipe_tree_visible(struct pipe_ctx *pipe_ctx) { if (pipe_ctx->plane_state && pipe_ctx->plane_state->visible) return true; - if (pipe_ctx->top_pipe && is_upper_pipe_tree_visible(pipe_ctx->top_pipe)) + if (pipe_ctx->top_pipe && is_parent_pipe_tree_visible(pipe_ctx->top_pipe)) + return true; + if (pipe_ctx->prev_odm_pipe && is_parent_pipe_tree_visible(pipe_ctx->prev_odm_pipe)) return true; return false; } @@ -71,9 +75,13 @@ bool is_pipe_tree_visible(struct pipe_ctx *pipe_ctx) { if (pipe_ctx->plane_state && pipe_ctx->plane_state->visible) return true; - if (pipe_ctx->top_pipe && is_upper_pipe_tree_visible(pipe_ctx->top_pipe)) + if (pipe_ctx->top_pipe && is_parent_pipe_tree_visible(pipe_ctx->top_pipe)) + return true; + if (pipe_ctx->bottom_pipe && is_child_pipe_tree_visible(pipe_ctx->bottom_pipe)) + return true; + if (pipe_ctx->prev_odm_pipe && is_parent_pipe_tree_visible(pipe_ctx->prev_odm_pipe)) return true; - if (pipe_ctx->bottom_pipe && is_lower_pipe_tree_visible(pipe_ctx->bottom_pipe)) + if (pipe_ctx->next_odm_pipe && is_child_pipe_tree_visible(pipe_ctx->next_odm_pipe)) return true; return false; } diff --git a/drivers/gpu/drm/amd/display/dc/basics/dc_common.h b/drivers/gpu/drm/amd/display/dc/basics/dc_common.h index 7c0cbf47e8cebabb05244a00f7f8b121fd78dc1a..b061497480b8d1ec151d3159571e8c8b905945c0 100644 --- a/drivers/gpu/drm/amd/display/dc/basics/dc_common.h +++ b/drivers/gpu/drm/amd/display/dc/basics/dc_common.h @@ -30,9 +30,9 @@ bool is_rgb_cspace(enum dc_color_space output_color_space); -bool is_lower_pipe_tree_visible(struct pipe_ctx *pipe_ctx); +bool is_child_pipe_tree_visible(struct pipe_ctx *pipe_ctx); -bool is_upper_pipe_tree_visible(struct pipe_ctx *pipe_ctx); +bool is_parent_pipe_tree_visible(struct pipe_ctx *pipe_ctx); bool is_pipe_tree_visible(struct pipe_ctx *pipe_ctx); diff --git a/drivers/gpu/drm/amd/display/dc/bios/bios_parser.c b/drivers/gpu/drm/amd/display/dc/bios/bios_parser.c index 23a373ca94b5ca7c387add3f984c4ba77eb2612d..c67d21a5ee52ff0e2b92da67ce0bbe2d10be850b 100644 --- a/drivers/gpu/drm/amd/display/dc/bios/bios_parser.c +++ b/drivers/gpu/drm/amd/display/dc/bios/bios_parser.c @@ -911,11 +911,11 @@ static enum bp_result get_ss_info_from_tbl( * ver 2.1 can co-exist with SS_Info table. Expect ASIC_InternalSS_Info ver 3.1, * there is only one entry for each signal /ss id. However, there is * no planning of supporting multiple spread Sprectum entry for EverGreen - * @param [in] this - * @param [in] signal, ASSignalType to be converted to info index - * @param [in] index, number of entries that match the converted info index - * @param [out] ss_info, sprectrum information structure, - * @return Bios parser result code + * @dcb: pointer to the DC BIOS + * @signal: ASSignalType to be converted to info index + * @index: number of entries that match the converted info index + * @ss_info: sprectrum information structure, + * return: Bios parser result code */ static enum bp_result bios_parser_get_spread_spectrum_info( struct dc_bios *dcb, @@ -985,10 +985,10 @@ static enum bp_result get_ss_info_from_internal_ss_info_tbl_V2_1( * There can not be more than 1 entry for ASIC_InternalSS_Info Ver 2.1 or * SS_Info. * - * @param this - * @param id, spread sprectrum info index - * @param pSSinfo, sprectrum information structure, - * @return Bios parser result code + * @bp: pointer to the BIOS parser + * @id: spread sprectrum info index + * @ss_info: sprectrum information structure, + * return: BIOS parser result code */ static enum bp_result get_ss_info_from_tbl( struct bios_parser *bp, @@ -1011,9 +1011,10 @@ static enum bp_result get_ss_info_from_tbl( * from the VBIOS * There will not be multiple entry for Ver 2.1 * - * @param id, spread sprectrum info index - * @param pSSinfo, sprectrum information structure, - * @return Bios parser result code + * @bp: pointer to the Bios parser + * @id: spread sprectrum info index + * @info: sprectrum information structure, + * return: Bios parser result code */ static enum bp_result get_ss_info_from_internal_ss_info_tbl_V2_1( struct bios_parser *bp, @@ -1076,9 +1077,10 @@ static enum bp_result get_ss_info_from_internal_ss_info_tbl_V2_1( * of entries that matches the id * for, the SS_Info table, there should not be more than 1 entry match. * - * @param [in] id, spread sprectrum id - * @param [out] pSSinfo, sprectrum information structure, - * @return Bios parser result code + * @bp: pointer to the Bios parser + * @id: spread sprectrum id + * @ss_info: sprectrum information structure, + * return: Bios parser result code */ static enum bp_result get_ss_info_from_ss_info_table( struct bios_parser *bp, @@ -1451,16 +1453,14 @@ static enum bp_result get_embedded_panel_info_v1_3( } /** - * bios_parser_get_encoder_cap_info + * bios_parser_get_encoder_cap_info - get encoder capability + * information of input object id * - * @brief - * Get encoder capability information of input object id - * - * @param object_id, Object id - * @param object_id, encoder cap information structure - * - * @return Bios parser result code + * @dcb: pointer to the DC BIOS + * @object_id: object id + * @info: encoder cap information structure * + * return: Bios parser result code */ static enum bp_result bios_parser_get_encoder_cap_info( struct dc_bios *dcb, @@ -1490,17 +1490,12 @@ static enum bp_result bios_parser_get_encoder_cap_info( } /** - * get_encoder_cap_record - * - * @brief - * Get encoder cap record for the object - * - * @param object, ATOM object + * get_encoder_cap_record - Get encoder cap record for the object * - * @return atom encoder cap record - * - * @note - * search all records to find the ATOM_ENCODER_CAP_RECORD_V2 record + * @bp: pointer to the BIOS parser + * @object: ATOM object + * return: atom encoder cap record + * note: search all records to find the ATOM_ENCODER_CAP_RECORD_V2 record */ static ATOM_ENCODER_CAP_RECORD_V2 *get_encoder_cap_record( struct bios_parser *bp, @@ -1557,8 +1552,9 @@ static uint32_t get_ss_entry_number_from_ss_info_tbl( * Get Number of SpreadSpectrum Entry from the ASIC_InternalSS_Info table from * the VBIOS that match the SSid (to be converted from signal) * - * @param[in] signal, ASSignalType to be converted to SSid - * @return number of SS Entry that match the signal + * @dcb: pointer to the DC BIOS + * @signal: ASSignalType to be converted to SSid + * return: number of SS Entry that match the signal */ static uint32_t bios_parser_get_ss_entry_number( struct dc_bios *dcb, @@ -1608,10 +1604,10 @@ static uint32_t bios_parser_get_ss_entry_number( * get_ss_entry_number_from_ss_info_tbl * Get Number of spread spectrum entry from the SS_Info table from the VBIOS. * - * @note There can only be one entry for each id for SS_Info Table - * - * @param [in] id, spread spectrum id - * @return number of SS Entry that match the id + * @bp: pointer to the BIOS parser + * @id: spread spectrum id + * return: number of SS Entry that match the id + * note: There can only be one entry for each id for SS_Info Table */ static uint32_t get_ss_entry_number_from_ss_info_tbl( struct bios_parser *bp, @@ -1679,8 +1675,9 @@ static uint32_t get_ss_entry_number_from_ss_info_tbl( * There can not be more than 1 entry for ASIC_InternalSS_Info Ver 2.1 or * SS_Info. * - * @param id, spread sprectrum info index - * @return Bios parser result code + * @bp: pointer to the BIOS parser + * @id: spread sprectrum info index + * return: Bios parser result code */ static uint32_t get_ss_entry_number(struct bios_parser *bp, uint32_t id) { @@ -1696,8 +1693,9 @@ static uint32_t get_ss_entry_number(struct bios_parser *bp, uint32_t id) * Ver 2.1 from the VBIOS * There will not be multiple entry for Ver 2.1 * - * @param id, spread sprectrum info index - * @return number of SS Entry that match the id + * @bp: pointer to the BIOS parser + * @id: spread sprectrum info index + * return: number of SS Entry that match the id */ static uint32_t get_ss_entry_number_from_internal_ss_info_tbl_v2_1( struct bios_parser *bp, @@ -1731,8 +1729,9 @@ static uint32_t get_ss_entry_number_from_internal_ss_info_tbl_v2_1( * Get Number of SpreadSpectrum Entry from the ASIC_InternalSS_Info table of * the VBIOS that matches id * - * @param[in] id, spread sprectrum id - * @return number of SS Entry that match the id + * @bp: pointer to the BIOS parser + * @id: spread sprectrum id + * return: number of SS Entry that match the id */ static uint32_t get_ss_entry_number_from_internal_ss_info_tbl_V3_1( struct bios_parser *bp, @@ -1767,10 +1766,11 @@ static uint32_t get_ss_entry_number_from_internal_ss_info_tbl_V3_1( * bios_parser_get_gpio_pin_info * Get GpioPin information of input gpio id * - * @param gpio_id, GPIO ID - * @param info, GpioPin information structure - * @return Bios parser result code - * @note + * @dcb: pointer to the DC BIOS + * @gpio_id: GPIO ID + * @info: GpioPin information structure + * return: Bios parser result code + * note: * to get the GPIO PIN INFO, we need: * 1. get the GPIO_ID from other object table, see GetHPDInfo() * 2. in DATA_TABLE.GPIO_Pin_LUT, search all records, to get the registerA @@ -2197,13 +2197,10 @@ static uint32_t get_support_mask_for_device_id(struct device_id device_id) } /** - * bios_parser_set_scratch_critical_state - * - * @brief - * update critical state bit in VBIOS scratch register - * - * @param - * bool - to set or reset state + * bios_parser_set_scratch_critical_state - update critical state + * bit in VBIOS scratch register + * @dcb: pointer to the DC BIOS + * @state: set or reset state */ static void bios_parser_set_scratch_critical_state( struct dc_bios *dcb, @@ -2222,7 +2219,7 @@ static void bios_parser_set_scratch_critical_state( * bios_parser *bp - [in]BIOS parser handler to get master data table * integrated_info *info - [out] store and output integrated info * - * @return + * return: * enum bp_result - BP_RESULT_OK if information is available, * BP_RESULT_BADBIOSTABLE otherwise. */ @@ -2372,7 +2369,7 @@ static enum bp_result get_integrated_info_v8( * bios_parser *bp - [in]BIOS parser handler to get master data table * integrated_info *info - [out] store and output integrated info * - * @return + * return: * enum bp_result - BP_RESULT_OK if information is available, * BP_RESULT_BADBIOSTABLE otherwise. */ @@ -2509,7 +2506,7 @@ static enum bp_result get_integrated_info_v9( * bios_parser *bp - [in]BIOS parser handler to get master data table * integrated_info *info - [out] store and output integrated info * - * @return + * return: * enum bp_result - BP_RESULT_OK if information is available, * BP_RESULT_BADBIOSTABLE otherwise. */ @@ -2585,7 +2582,7 @@ static struct integrated_info *bios_parser_create_integrated_info( return NULL; } -enum bp_result update_slot_layout_info( +static enum bp_result update_slot_layout_info( struct dc_bios *dcb, unsigned int i, struct slot_layout_info *slot_layout_info, @@ -2689,7 +2686,7 @@ enum bp_result update_slot_layout_info( } -enum bp_result get_bracket_layout_record( +static enum bp_result get_bracket_layout_record( struct dc_bios *dcb, unsigned int bracket_layout_id, struct slot_layout_info *slot_layout_info) diff --git a/drivers/gpu/drm/amd/display/dc/bios/bios_parser2.c b/drivers/gpu/drm/amd/display/dc/bios/bios_parser2.c index 670c26583817817eb5b24d3a80e0806fbe65deee..9f9fda3118d1fbc730983f6c1c0f7cea839af408 100644 --- a/drivers/gpu/drm/amd/display/dc/bios/bios_parser2.c +++ b/drivers/gpu/drm/amd/display/dc/bios/bios_parser2.c @@ -485,10 +485,11 @@ static struct atom_hpd_int_record *get_hpd_record( * bios_parser_get_gpio_pin_info * Get GpioPin information of input gpio id * - * @param gpio_id, GPIO ID - * @param info, GpioPin information structure - * @return Bios parser result code - * @note + * @dcb: pointer to the DC BIOS + * @gpio_id: GPIO ID + * @info: GpioPin information structure + * return: Bios parser result code + * note: * to get the GPIO PIN INFO, we need: * 1. get the GPIO_ID from other object table, see GetHPDInfo() * 2. in DATA_TABLE.GPIO_Pin_LUT, search all records, @@ -801,11 +802,11 @@ static enum bp_result get_ss_info_v4_2( * ver 3.1, * there is only one entry for each signal /ss id. However, there is * no planning of supporting multiple spread Sprectum entry for EverGreen - * @param [in] this - * @param [in] signal, ASSignalType to be converted to info index - * @param [in] index, number of entries that match the converted info index - * @param [out] ss_info, sprectrum information structure, - * @return Bios parser result code + * @dcb: pointer to the DC BIOS + * @signal: ASSignalType to be converted to info index + * @index: number of entries that match the converted info index + * @ss_info: sprectrum information structure, + * return: Bios parser result code */ static enum bp_result bios_parser_get_spread_spectrum_info( struct dc_bios *dcb, @@ -1196,13 +1197,11 @@ static bool bios_parser_is_accelerated_mode( } /** - * bios_parser_set_scratch_critical_state + * bios_parser_set_scratch_critical_state - update critical state bit + * in VBIOS scratch register * - * @brief - * update critical state bit in VBIOS scratch register - * - * @param - * bool - to set or reset state + * @dcb: pointer to the DC BIO + * @state: set or reset state */ static void bios_parser_set_scratch_critical_state( struct dc_bios *dcb, diff --git a/drivers/gpu/drm/amd/display/dc/bios/command_table.c b/drivers/gpu/drm/amd/display/dc/bios/command_table.c index 070459e3e4070dcb31fa5b678673f1e07e386231..afc10b954ffa7b67cb52131e6ac32e3bb0036e1f 100644 --- a/drivers/gpu/drm/amd/display/dc/bios/command_table.c +++ b/drivers/gpu/drm/amd/display/dc/bios/command_table.c @@ -245,6 +245,23 @@ static enum bp_result encoder_control_digx_v3( cntl->enable_dp_audio); params.ucLaneNum = (uint8_t)(cntl->lanes_number); + switch (cntl->color_depth) { + case COLOR_DEPTH_888: + params.ucBitPerColor = PANEL_8BIT_PER_COLOR; + break; + case COLOR_DEPTH_101010: + params.ucBitPerColor = PANEL_10BIT_PER_COLOR; + break; + case COLOR_DEPTH_121212: + params.ucBitPerColor = PANEL_12BIT_PER_COLOR; + break; + case COLOR_DEPTH_161616: + params.ucBitPerColor = PANEL_16BIT_PER_COLOR; + break; + default: + break; + } + if (EXEC_BIOS_CMD_TABLE(DIGxEncoderControl, params)) result = BP_RESULT_OK; @@ -274,6 +291,23 @@ static enum bp_result encoder_control_digx_v4( cntl->enable_dp_audio)); params.ucLaneNum = (uint8_t)(cntl->lanes_number); + switch (cntl->color_depth) { + case COLOR_DEPTH_888: + params.ucBitPerColor = PANEL_8BIT_PER_COLOR; + break; + case COLOR_DEPTH_101010: + params.ucBitPerColor = PANEL_10BIT_PER_COLOR; + break; + case COLOR_DEPTH_121212: + params.ucBitPerColor = PANEL_12BIT_PER_COLOR; + break; + case COLOR_DEPTH_161616: + params.ucBitPerColor = PANEL_16BIT_PER_COLOR; + break; + default: + break; + } + if (EXEC_BIOS_CMD_TABLE(DIGxEncoderControl, params)) result = BP_RESULT_OK; @@ -1057,6 +1091,19 @@ static enum bp_result set_pixel_clock_v5( * driver choose program it itself, i.e. here we program it * to 888 by default. */ + if (bp_params->signal_type == SIGNAL_TYPE_HDMI_TYPE_A) + switch (bp_params->color_depth) { + case TRANSMITTER_COLOR_DEPTH_30: + /* yes this is correct, the atom define is wrong */ + clk.sPCLKInput.ucMiscInfo |= PIXEL_CLOCK_V5_MISC_HDMI_32BPP; + break; + case TRANSMITTER_COLOR_DEPTH_36: + /* yes this is correct, the atom define is wrong */ + clk.sPCLKInput.ucMiscInfo |= PIXEL_CLOCK_V5_MISC_HDMI_30BPP; + break; + default: + break; + } if (EXEC_BIOS_CMD_TABLE(SetPixelClock, clk)) result = BP_RESULT_OK; @@ -1135,6 +1182,20 @@ static enum bp_result set_pixel_clock_v6( * driver choose program it itself, i.e. here we pass required * target rate that includes deep color. */ + if (bp_params->signal_type == SIGNAL_TYPE_HDMI_TYPE_A) + switch (bp_params->color_depth) { + case TRANSMITTER_COLOR_DEPTH_30: + clk.sPCLKInput.ucMiscInfo |= PIXEL_CLOCK_V6_MISC_HDMI_30BPP_V6; + break; + case TRANSMITTER_COLOR_DEPTH_36: + clk.sPCLKInput.ucMiscInfo |= PIXEL_CLOCK_V6_MISC_HDMI_36BPP_V6; + break; + case TRANSMITTER_COLOR_DEPTH_48: + clk.sPCLKInput.ucMiscInfo |= PIXEL_CLOCK_V6_MISC_HDMI_48BPP; + break; + default: + break; + } if (EXEC_BIOS_CMD_TABLE(SetPixelClock, clk)) result = BP_RESULT_OK; diff --git a/drivers/gpu/drm/amd/display/dc/bios/command_table_helper.c b/drivers/gpu/drm/amd/display/dc/bios/command_table_helper.c index 48b4ef03fc8f83f7720c1abe7e19ab3344224801..5b77251e0590924215b5a034964103728f19f459 100644 --- a/drivers/gpu/drm/amd/display/dc/bios/command_table_helper.c +++ b/drivers/gpu/drm/amd/display/dc/bios/command_table_helper.c @@ -114,18 +114,14 @@ bool dal_cmd_table_helper_controller_id_to_atom( } /** -* translate_transmitter_bp_to_atom -* -* @brief -* Translate the Transmitter to the corresponding ATOM BIOS value -* -* @param -* input transmitter -* output digitalTransmitter -* // =00: Digital Transmitter1 ( UNIPHY linkAB ) -* // =01: Digital Transmitter2 ( UNIPHY linkCD ) -* // =02: Digital Transmitter3 ( UNIPHY linkEF ) -*/ + * translate_transmitter_bp_to_atom - Translate the Transmitter to the + * corresponding ATOM BIOS value + * @t: transmitter + * returns: output digitalTransmitter + * // =00: Digital Transmitter1 ( UNIPHY linkAB ) + * // =01: Digital Transmitter2 ( UNIPHY linkCD ) + * // =02: Digital Transmitter3 ( UNIPHY linkEF ) + */ uint8_t dal_cmd_table_helper_transmitter_bp_to_atom( enum transmitter t) { diff --git a/drivers/gpu/drm/amd/display/dc/bios/command_table_helper2.c b/drivers/gpu/drm/amd/display/dc/bios/command_table_helper2.c index 7736c92d55c402881251c41f5f29164c211afe86..455ee2be15a36f57a2d7dff8b22591b3db1a567d 100644 --- a/drivers/gpu/drm/amd/display/dc/bios/command_table_helper2.c +++ b/drivers/gpu/drm/amd/display/dc/bios/command_table_helper2.c @@ -128,18 +128,14 @@ bool dal_cmd_table_helper_controller_id_to_atom2( } /** -* translate_transmitter_bp_to_atom -* -* @brief -* Translate the Transmitter to the corresponding ATOM BIOS value -* -* @param -* input transmitter -* output digitalTransmitter -* // =00: Digital Transmitter1 ( UNIPHY linkAB ) -* // =01: Digital Transmitter2 ( UNIPHY linkCD ) -* // =02: Digital Transmitter3 ( UNIPHY linkEF ) -*/ + * translate_transmitter_bp_to_atom2 - Translate the Transmitter to the + * corresponding ATOM BIOS value + * @t: transmitter + * returns: digitalTransmitter + * // =00: Digital Transmitter1 ( UNIPHY linkAB ) + * // =01: Digital Transmitter2 ( UNIPHY linkCD ) + * // =02: Digital Transmitter3 ( UNIPHY linkEF ) + */ uint8_t dal_cmd_table_helper_transmitter_bp_to_atom2( enum transmitter t) { diff --git a/drivers/gpu/drm/amd/display/dc/calcs/dce_calcs.c b/drivers/gpu/drm/amd/display/dc/calcs/dce_calcs.c index ef41b287cbe2355c313ecf8cc84efa055690b39d..e633f8a51edb6b810cdd549d48d5982634b85bc6 100644 --- a/drivers/gpu/drm/amd/display/dc/calcs/dce_calcs.c +++ b/drivers/gpu/drm/amd/display/dc/calcs/dce_calcs.c @@ -106,7 +106,6 @@ static void calculate_bandwidth( bool lpt_enabled; enum bw_defines sclk_message; enum bw_defines yclk_message; - enum bw_defines v_filter_init_mode[maximum_number_of_surfaces]; enum bw_defines tiling_mode[maximum_number_of_surfaces]; enum bw_defines surface_type[maximum_number_of_surfaces]; enum bw_defines voltage; @@ -792,12 +791,8 @@ static void calculate_bandwidth( data->v_filter_init[i] = bw_add(data->v_filter_init[i], bw_int_to_fixed(1)); } if (data->stereo_mode[i] == bw_def_top_bottom) { - v_filter_init_mode[i] = bw_def_manual; data->v_filter_init[i] = bw_min2(data->v_filter_init[i], bw_int_to_fixed(4)); } - else { - v_filter_init_mode[i] = bw_def_auto; - } if (data->stereo_mode[i] == bw_def_top_bottom) { data->num_lines_at_frame_start = bw_int_to_fixed(1); } @@ -2730,7 +2725,7 @@ void bw_calcs_init(struct bw_calcs_dceip *bw_dceip, } -/** +/* * Compare calculated (required) clocks against the clocks available at * maximum voltage (max Performance Level). */ @@ -3001,13 +2996,12 @@ static bool all_displays_in_sync(const struct pipe_ctx pipe[], return true; } -/** +/* * Return: * true - Display(s) configuration supported. * In this case 'calcs_output' contains data for HW programming * false - Display(s) configuration not supported (not enough bandwidth). */ - bool bw_calcs(struct dc_context *ctx, const struct bw_calcs_dceip *dceip, const struct bw_calcs_vbios *vbios, @@ -3028,7 +3022,7 @@ bool bw_calcs(struct dc_context *ctx, calcs_output->all_displays_in_sync = false; if (data->number_of_displays != 0) { - uint8_t yclk_lvl, sclk_lvl; + uint8_t yclk_lvl; struct bw_fixed high_sclk = vbios->high_sclk; struct bw_fixed mid1_sclk = vbios->mid1_sclk; struct bw_fixed mid2_sclk = vbios->mid2_sclk; @@ -3049,7 +3043,6 @@ bool bw_calcs(struct dc_context *ctx, calculate_bandwidth(dceip, vbios, data); yclk_lvl = data->y_clk_level; - sclk_lvl = data->sclk_level; calcs_output->nbp_state_change_enable = data->nbp_state_change_enable; diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn10/rv1_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn10/rv1_clk_mgr.c index 75b8240ed0595b073af4622a39817e812b81c0c4..e133edc587d318098e5576acbff9f30584f8fe68 100644 --- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn10/rv1_clk_mgr.c +++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn10/rv1_clk_mgr.c @@ -187,17 +187,6 @@ static void ramp_up_dispclk_with_dpp( clk_mgr->base.clks.max_supported_dppclk_khz = new_clocks->max_supported_dppclk_khz; } -static bool is_mpo_enabled(struct dc_state *context) -{ - int i; - - for (i = 0; i < context->stream_count; i++) { - if (context->stream_status[i].plane_count > 1) - return true; - } - return false; -} - static void rv1_update_clocks(struct clk_mgr *clk_mgr_base, struct dc_state *context, bool safe_to_lower) @@ -295,22 +284,9 @@ static void rv1_update_clocks(struct clk_mgr *clk_mgr_base, if (pp_smu->set_hard_min_fclk_by_freq && pp_smu->set_hard_min_dcfclk_by_freq && pp_smu->set_min_deep_sleep_dcfclk) { - // Only increase clocks when display is active and MPO is enabled - if (display_count && is_mpo_enabled(context)) { - pp_smu->set_hard_min_fclk_by_freq(&pp_smu->pp_smu, - ((new_clocks->fclk_khz / 1000) * 101) / 100); - pp_smu->set_hard_min_dcfclk_by_freq(&pp_smu->pp_smu, - ((new_clocks->dcfclk_khz / 1000) * 101) / 100); - pp_smu->set_min_deep_sleep_dcfclk(&pp_smu->pp_smu, - (new_clocks->dcfclk_deep_sleep_khz + 999) / 1000); - } else { - pp_smu->set_hard_min_fclk_by_freq(&pp_smu->pp_smu, - new_clocks->fclk_khz / 1000); - pp_smu->set_hard_min_dcfclk_by_freq(&pp_smu->pp_smu, - new_clocks->dcfclk_khz / 1000); - pp_smu->set_min_deep_sleep_dcfclk(&pp_smu->pp_smu, - (new_clocks->dcfclk_deep_sleep_khz + 999) / 1000); - } + pp_smu->set_hard_min_fclk_by_freq(&pp_smu->pp_smu, new_clocks->fclk_khz / 1000); + pp_smu->set_hard_min_dcfclk_by_freq(&pp_smu->pp_smu, new_clocks->dcfclk_khz / 1000); + pp_smu->set_min_deep_sleep_dcfclk(&pp_smu->pp_smu, (new_clocks->dcfclk_deep_sleep_khz + 999) / 1000); } } } diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn20/dcn20_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn20/dcn20_clk_mgr.c index f2114bc910bf0077e7453c9131c936b555ca0237..ec9dc265cde0c4d2def10eeb8f353074e219bd35 100644 --- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn20/dcn20_clk_mgr.c +++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn20/dcn20_clk_mgr.c @@ -257,8 +257,7 @@ void dcn2_update_clocks(struct clk_mgr *clk_mgr_base, if (update_dppclk || update_dispclk) dcn20_update_clocks_update_dentist(clk_mgr); // always update dtos unless clock is lowered and not safe to lower - if (new_clocks->dppclk_khz >= dc->current_state->bw_ctx.bw.dcn.clk.dppclk_khz) - dcn20_update_clocks_update_dpp_dto(clk_mgr, context, safe_to_lower); + dcn20_update_clocks_update_dpp_dto(clk_mgr, context, safe_to_lower); } } diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn30/dcn30_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn30/dcn30_clk_mgr.c index ab98c259ef69562a4b9b6c096fee5535a6bab022..c7e5a64e06afb0a5735c5b33ce0a90ac2e938bb0 100644 --- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn30/dcn30_clk_mgr.c +++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn30/dcn30_clk_mgr.c @@ -146,15 +146,15 @@ static noinline void dcn3_build_wm_range_table(struct clk_mgr_internal *clk_mgr) clk_mgr->base.bw_params->wm_table.nv_entries[WM_C].pmfw_breakdown.max_uclk = 0xFFFF; /* Set D - MALL - SR enter and exit times adjusted for MALL */ -// clk_mgr->base.bw_params->wm_table.nv_entries[WM_D].valid = true; -// clk_mgr->base.bw_params->wm_table.nv_entries[WM_D].dml_input.pstate_latency_us = pstate_latency_us; -// clk_mgr->base.bw_params->wm_table.nv_entries[WM_D].dml_input.sr_exit_time_us = 2; -// clk_mgr->base.bw_params->wm_table.nv_entries[WM_D].dml_input.sr_enter_plus_exit_time_us = 4; -// clk_mgr->base.bw_params->wm_table.nv_entries[WM_D].pmfw_breakdown.wm_type = WATERMARKS_MALL; -// clk_mgr->base.bw_params->wm_table.nv_entries[WM_D].pmfw_breakdown.min_dcfclk = 0; -// clk_mgr->base.bw_params->wm_table.nv_entries[WM_D].pmfw_breakdown.max_dcfclk = 0xFFFF; -// clk_mgr->base.bw_params->wm_table.nv_entries[WM_D].pmfw_breakdown.min_uclk = min_uclk_mhz; -// clk_mgr->base.bw_params->wm_table.nv_entries[WM_D].pmfw_breakdown.max_uclk = 0xFFFF; + clk_mgr->base.bw_params->wm_table.nv_entries[WM_D].valid = true; + clk_mgr->base.bw_params->wm_table.nv_entries[WM_D].dml_input.pstate_latency_us = pstate_latency_us; + clk_mgr->base.bw_params->wm_table.nv_entries[WM_D].dml_input.sr_exit_time_us = 2; + clk_mgr->base.bw_params->wm_table.nv_entries[WM_D].dml_input.sr_enter_plus_exit_time_us = 4; + clk_mgr->base.bw_params->wm_table.nv_entries[WM_D].pmfw_breakdown.wm_type = WATERMARKS_MALL; + clk_mgr->base.bw_params->wm_table.nv_entries[WM_D].pmfw_breakdown.min_dcfclk = 0; + clk_mgr->base.bw_params->wm_table.nv_entries[WM_D].pmfw_breakdown.max_dcfclk = 0xFFFF; + clk_mgr->base.bw_params->wm_table.nv_entries[WM_D].pmfw_breakdown.min_uclk = min_uclk_mhz; + clk_mgr->base.bw_params->wm_table.nv_entries[WM_D].pmfw_breakdown.max_uclk = 0xFFFF; } void dcn3_init_clocks(struct clk_mgr *clk_mgr_base) diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn301/dcn301_smu.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn301/dcn301_smu.c index cfa8e02cf103c7c751aff3ab728881dfb3085840..68942bbc7472847b7f13fddef0d3d1399441c93a 100644 --- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn301/dcn301_smu.c +++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn301/dcn301_smu.c @@ -103,7 +103,7 @@ int dcn301_smu_send_msg_with_param( /* Trigger the message transaction by writing the message ID */ REG_WRITE(MP1_SMN_C2PMSG_67, msg_id); - result = dcn301_smu_wait_for_response(clk_mgr, 10, 1000); + result = dcn301_smu_wait_for_response(clk_mgr, 10, 200000); ASSERT(result == VBIOSSMC_Result_OK); diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn301/vg_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn301/vg_clk_mgr.c index 991b9c5beaa301ba9311c2ed6c23a62100c6bcd8..aadb801447a7cb0c3cc1eef7d56ececa1546a214 100644 --- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn301/vg_clk_mgr.c +++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn301/vg_clk_mgr.c @@ -75,7 +75,8 @@ int vg_get_active_display_cnt_wa( const struct dc_link *link = dc->links[i]; /* abusing the fact that the dig and phy are coupled to see if the phy is enabled */ - if (link->link_enc->funcs->is_dig_enabled(link->link_enc)) + if (link->link_enc->funcs->is_dig_enabled && + link->link_enc->funcs->is_dig_enabled(link->link_enc)) display_count++; } diff --git a/drivers/gpu/drm/amd/display/dc/core/dc.c b/drivers/gpu/drm/amd/display/dc/core/dc.c index 6cf1a5a2a5ecc0f5dda72f60930325b1df4de9cb..8f8a13c7cf73d6bdc6efca08cc63c143d3be607b 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc.c @@ -175,6 +175,8 @@ static bool create_links( connectors_num = bios->funcs->get_connectors_number(bios); + DC_LOG_DC("BIOS object table - number of connectors: %d", connectors_num); + if (connectors_num > ENUM_ID_COUNT) { dm_error( "DC: Number of connectors %d exceeds maximum of %d!\n", @@ -193,6 +195,8 @@ static bool create_links( struct link_init_data link_init_params = {0}; struct dc_link *link; + DC_LOG_DC("BIOS object table - printing link object info for connector number: %d, link_index: %d", i, dc->link_count); + link_init_params.ctx = dc->ctx; /* next BIOS object table connector */ link_init_params.connector_index = i; @@ -201,30 +205,14 @@ static bool create_links( link = link_create(&link_init_params); if (link) { - bool should_destory_link = false; - - if (link->connector_signal == SIGNAL_TYPE_EDP) { - if (dc->config.edp_not_connected) { - if (!IS_DIAG_DC(dc->ctx->dce_environment)) - should_destory_link = true; - } else { - enum dc_connection_type type; - dc_link_detect_sink(link, &type); - if (type == dc_connection_none) - should_destory_link = true; - } - } - - if (dc->config.force_enum_edp || !should_destory_link) { dc->links[dc->link_count] = link; link->dc = dc; ++dc->link_count; - } else { - link_destroy(&link); - } } } + DC_LOG_DC("BIOS object table - end"); + for (i = 0; i < num_virtual_links; i++) { struct dc_link *link = kzalloc(sizeof(*link), GFP_KERNEL); struct encoder_init_data enc_init = {0}; @@ -284,20 +272,16 @@ static void dc_perf_trace_destroy(struct dc_perf_trace **perf_trace) } /** - ***************************************************************************** - * Function: dc_stream_adjust_vmin_vmax + * dc_stream_adjust_vmin_vmax: * - * @brief - * Looks up the pipe context of dc_stream_state and updates the - * vertical_total_min and vertical_total_max of the DRR, Dynamic Refresh - * Rate, which is a power-saving feature that targets reducing panel - * refresh rate while the screen is static + * Looks up the pipe context of dc_stream_state and updates the + * vertical_total_min and vertical_total_max of the DRR, Dynamic Refresh + * Rate, which is a power-saving feature that targets reducing panel + * refresh rate while the screen is static * - * @param [in] dc: dc reference - * @param [in] stream: Initial dc stream state - * @param [in] adjust: Updated parameters for vertical_total_min and - * vertical_total_max - ***************************************************************************** + * @dc: dc reference + * @stream: Initial dc stream state + * @adjust: Updated parameters for vertical_total_min and vertical_total_max */ bool dc_stream_adjust_vmin_vmax(struct dc *dc, struct dc_stream_state *stream, @@ -355,6 +339,7 @@ bool dc_stream_get_crtc_position(struct dc *dc, * @dc: DC Object * @stream: The stream to configure CRC on. * @enable: Enable CRC if true, disable otherwise. + * @crc_window: CRC window (x/y start/end) information * @continuous: Capture CRC on every frame if true. Otherwise, only capture * once. * @@ -420,7 +405,9 @@ bool dc_stream_configure_crc(struct dc *dc, struct dc_stream_state *stream, * dc_stream_get_crc() - Get CRC values for the given stream. * @dc: DC object * @stream: The DC stream state of the stream to get CRCs from. - * @r_cr, g_y, b_cb: CRC values for the three channels are stored here. + * @r_cr: CRC value for the first of the 3 channels stored here. + * @g_y: CRC value for the second of the 3 channels stored here. + * @b_cb: CRC value for the third of the 3 channels stored here. * * dc_stream_configure_crc needs to be called beforehand to enable CRCs. * Return false if stream is not found, or if CRCs are not enabled. @@ -707,7 +694,6 @@ static bool dc_construct(struct dc *dc, } dc->dcn_ip = dcn_ip; - dc->soc_bounding_box = init_params->soc_bounding_box; #endif if (!dc_construct_ctx(dc, init_params)) { @@ -757,6 +743,10 @@ static bool dc_construct(struct dc *dc, if (!dc->res_pool) goto fail; + /* set i2c speed if not done by the respective dcnxxx__resource.c */ + if (dc->caps.i2c_speed_in_khz_hdcp == 0) + dc->caps.i2c_speed_in_khz_hdcp = dc->caps.i2c_speed_in_khz; + dc->clk_mgr = dc_clk_mgr_create(dc->ctx, dc->res_pool->pp_smu, dc->res_pool->dccg); if (!dc->clk_mgr) goto fail; @@ -764,8 +754,6 @@ static bool dc_construct(struct dc *dc, dc->clk_mgr->force_smu_not_present = init_params->force_smu_not_present; #endif - dc->debug.force_ignore_link_settings = init_params->force_ignore_link_settings; - if (dc->res_pool->funcs->update_bw_bounding_box) dc->res_pool->funcs->update_bw_bounding_box(dc, dc->clk_mgr->bw_params); @@ -803,7 +791,8 @@ static void disable_all_writeback_pipes_for_stream( stream->writeback_info[i].wb_enabled = false; } -void apply_ctx_interdependent_lock(struct dc *dc, struct dc_state *context, struct dc_stream_state *stream, bool lock) +static void apply_ctx_interdependent_lock(struct dc *dc, struct dc_state *context, + struct dc_stream_state *stream, bool lock) { int i = 0; @@ -964,19 +953,15 @@ struct dc *dc_create(const struct dc_init_data *init_params) struct dc *dc = kzalloc(sizeof(*dc), GFP_KERNEL); unsigned int full_pipe_count; - if (NULL == dc) - goto alloc_fail; + if (!dc) + return NULL; if (init_params->dce_environment == DCE_ENV_VIRTUAL_HW) { - if (false == dc_construct_ctx(dc, init_params)) { - dc_destruct(dc); - goto construct_fail; - } + if (!dc_construct_ctx(dc, init_params)) + goto destruct_dc; } else { - if (false == dc_construct(dc, init_params)) { - dc_destruct(dc); - goto construct_fail; - } + if (!dc_construct(dc, init_params)) + goto destruct_dc; full_pipe_count = dc->res_pool->pipe_count; if (dc->res_pool->underlay_pipe_index != NO_UNDERLAY_PIPE) @@ -1007,15 +992,36 @@ struct dc *dc_create(const struct dc_init_data *init_params) return dc; -construct_fail: +destruct_dc: + dc_destruct(dc); kfree(dc); - -alloc_fail: return NULL; } +static void detect_edp_presence(struct dc *dc) +{ + struct dc_link *edp_link = get_edp_link(dc); + bool edp_sink_present = true; + + if (!edp_link) + return; + + if (dc->config.edp_not_connected) { + edp_sink_present = false; + } else { + enum dc_connection_type type; + dc_link_detect_sink(edp_link, &type); + if (type == dc_connection_none) + edp_sink_present = false; + } + + edp_link->edp_sink_present = edp_sink_present; +} + void dc_hardware_init(struct dc *dc) { + + detect_edp_presence(dc); if (dc->ctx->dce_environment != DCE_ENV_VIRTUAL_HW) dc->hwss.init_hw(dc); } @@ -1493,7 +1499,7 @@ bool dc_commit_state(struct dc *dc, struct dc_state *context) enum dc_status result = DC_ERROR_UNEXPECTED; int i; - if (false == context_changed(dc, context)) + if (!context_changed(dc, context)) return DC_OK; DC_LOG_DC("%s: %d streams\n", @@ -1540,7 +1546,7 @@ bool dc_acquire_release_mpc_3dlut( if (found_pipe_idx) { if (acquire && pool->funcs->acquire_post_bldn_3dlut) ret = pool->funcs->acquire_post_bldn_3dlut(res_ctx, pool, mpcc_id, lut, shaper); - else if (acquire == false && pool->funcs->release_post_bldn_3dlut) + else if (!acquire && pool->funcs->release_post_bldn_3dlut) ret = pool->funcs->release_post_bldn_3dlut(res_ctx, pool, lut, shaper); } } @@ -2016,7 +2022,7 @@ static enum surface_update_type check_update_surfaces_for_stream( return overall_type; } -/** +/* * dc_check_update_surfaces_for_stream() - Determine update type (fast, med, or full) * * See :c:type:`enum surface_update_type ` for explanation of update types @@ -2270,6 +2276,9 @@ static void copy_stream_update_to_stream(struct dc *dc, if (update->dither_option) stream->dither_option = *update->dither_option; + + if (update->pending_test_pattern) + stream->test_pattern = *update->pending_test_pattern; /* update current stream with writeback info */ if (update->wb_update) { int i; @@ -2366,6 +2375,15 @@ static void commit_planes_do_stream_update(struct dc *dc, } } + if (stream_update->pending_test_pattern) { + dc_link_dp_set_test_pattern(stream->link, + stream->test_pattern.type, + stream->test_pattern.color_space, + stream->test_pattern.p_link_settings, + stream->test_pattern.p_custom_pattern, + stream->test_pattern.cust_pattern_size); + } + /* Full fe update*/ if (update_type == UPDATE_TYPE_FAST) continue; @@ -2679,7 +2697,8 @@ void dc_commit_updates_for_stream(struct dc *dc, struct dc_surface_update *srf_updates, int surface_count, struct dc_stream_state *stream, - struct dc_stream_update *stream_update) + struct dc_stream_update *stream_update, + struct dc_state *state) { const struct dc_stream_status *stream_status; enum surface_update_type update_type; @@ -2698,12 +2717,6 @@ void dc_commit_updates_for_stream(struct dc *dc, if (update_type >= UPDATE_TYPE_FULL) { - struct dc_plane_state *new_planes[MAX_SURFACES]; - - memset(new_planes, 0, sizeof(new_planes)); - - for (i = 0; i < surface_count; i++) - new_planes[i] = srf_updates[i].surface; /* initialize scratch memory for building context */ context = dc_create_state(dc); @@ -2712,21 +2725,15 @@ void dc_commit_updates_for_stream(struct dc *dc, return; } - dc_resource_state_copy_construct( - dc->current_state, context); + dc_resource_state_copy_construct(state, context); - /*remove old surfaces from context */ - if (!dc_rem_all_planes_for_stream(dc, stream, context)) { - DC_ERROR("Failed to remove streams for new validate context!\n"); - return; - } + for (i = 0; i < dc->res_pool->pipe_count; i++) { + struct pipe_ctx *new_pipe = &context->res_ctx.pipe_ctx[i]; + struct pipe_ctx *old_pipe = &dc->current_state->res_ctx.pipe_ctx[i]; - /* add surface to context */ - if (!dc_add_all_planes_for_stream(dc, stream, new_planes, surface_count, context)) { - DC_ERROR("Failed to add streams for new validate context!\n"); - return; + if (new_pipe->plane_state && new_pipe->plane_state != old_pipe->plane_state) + new_pipe->plane_state->force_full_update = true; } - } @@ -2830,7 +2837,7 @@ enum dc_irq_source dc_interrupt_to_irq_source( return dal_irq_service_to_irq_source(dc->res_pool->irqs, src_id, ext_id); } -/** +/* * dc_interrupt_set() - Enable/disable an AMD hw interrupt source */ bool dc_interrupt_set(struct dc *dc, enum dc_irq_source src, bool enable) @@ -2964,7 +2971,7 @@ static bool link_add_remote_sink_helper(struct dc_link *dc_link, struct dc_sink return true; } -/** +/* * dc_link_add_remote_sink() - Create a sink and attach it to an existing link * * EDID length is in bytes @@ -3027,7 +3034,7 @@ struct dc_sink *dc_link_add_remote_sink( return NULL; } -/** +/* * dc_link_remove_remote_sink() - Remove a remote sink from a dc_link * * Note that this just removes the struct dc_sink - it doesn't @@ -3154,9 +3161,11 @@ void dc_lock_memory_clock_frequency(struct dc *dc) core_link_enable_stream(dc->current_state, &dc->current_state->res_ctx.pipe_ctx[i]); } -bool dc_is_plane_eligible_for_idle_optimizaitons(struct dc *dc, - struct dc_plane_state *plane) +bool dc_is_plane_eligible_for_idle_optimizations(struct dc *dc, struct dc_plane_state *plane, + struct dc_cursor_attributes *cursor_attr) { + if (dc->hwss.does_plane_fit_in_mall && dc->hwss.does_plane_fit_in_mall(dc, plane, cursor_attr)) + return true; return false; } diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_link.c b/drivers/gpu/drm/amd/display/dc/core/dc_link.c index f4a2088ab17928ee2a7c14150f7b4ddab2e0a843..fa5059f71727a16463d990e1ebd54c7fcc480634 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc_link.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc_link.c @@ -203,9 +203,21 @@ static bool program_hpd_filter(const struct dc_link *link) return result; } +bool dc_link_wait_for_t12(struct dc_link *link) +{ + if (link->connector_signal == SIGNAL_TYPE_EDP && link->dc->hwss.edp_wait_for_T12) { + link->dc->hwss.edp_wait_for_T12(link); + + return true; + } + + return false; +} + /** * dc_link_detect_sink() - Determine if there is a sink connected * + * @link: pointer to the dc link * @type: Returned connection type * Does not detect downstream devices, such as MST sinks * or display connected through active dongles @@ -342,7 +354,7 @@ static enum signal_type get_basic_signal_type(struct graphics_object_id encoder, return SIGNAL_TYPE_NONE; } -/** +/* * dc_link_is_dp_sink_present() - Check if there is a native DP * or passive DP-HDMI dongle connected */ @@ -596,8 +608,6 @@ static void query_hdcp_capability(enum signal_type signal, struct dc_link *link) dc_process_hdcp_msg(signal, link, &msg22); if (signal == SIGNAL_TYPE_DISPLAY_PORT || signal == SIGNAL_TYPE_DISPLAY_PORT_MST) { - enum hdcp_message_status status = HDCP_MESSAGE_UNSUPPORTED; - msg14.data = &link->hdcp_caps.bcaps.raw; msg14.length = sizeof(link->hdcp_caps.bcaps.raw); msg14.msg_id = HDCP_MESSAGE_ID_READ_BCAPS; @@ -605,7 +615,7 @@ static void query_hdcp_capability(enum signal_type signal, struct dc_link *link) msg14.link = HDCP_LINK_PRIMARY; msg14.max_retries = 5; - status = dc_process_hdcp_msg(signal, link, &msg14); + dc_process_hdcp_msg(signal, link, &msg14); } } @@ -830,7 +840,7 @@ static bool wait_for_entering_dp_alt_mode(struct dc_link *link) return false; } -/** +/* * dc_link_detect() - Detect if a sink is attached to a given link * * link->local_sink is created or destroyed as needed. @@ -1065,9 +1075,6 @@ static bool dc_link_detect_helper(struct dc_link *link, break; } - if (link->local_sink->edid_caps.panel_patch.disable_fec) - link->ctx->dc->debug.disable_fec = true; - // Check if edid is the same if ((prev_sink) && (edid_status == EDID_THE_SAME || edid_status == EDID_OK)) @@ -1366,13 +1373,17 @@ static bool dc_link_construct(struct dc_link *link, struct dc_context *dc_ctx = init_params->ctx; struct encoder_init_data enc_init_data = { 0 }; struct panel_cntl_init_data panel_cntl_init_data = { 0 }; - struct integrated_info info = {{{ 0 }}}; + struct integrated_info *info; struct dc_bios *bios = init_params->dc->ctx->dc_bios; const struct dc_vbios_funcs *bp_funcs = bios->funcs; struct bp_disp_connector_caps_info disp_connect_caps_info = { 0 }; DC_LOGGER_INIT(dc_ctx->logger); + info = kzalloc(sizeof(*info), GFP_KERNEL); + if (!info) + goto create_fail; + link->irq_source_hpd = DC_IRQ_SOURCE_INVALID; link->irq_source_hpd_rx = DC_IRQ_SOURCE_INVALID; @@ -1390,10 +1401,12 @@ static bool dc_link_construct(struct dc_link *link, link->link_id = bios->funcs->get_connector_id(bios, init_params->connector_index); + DC_LOG_DC("BIOS object table - link_id: %d", link->link_id.id); if (bios->funcs->get_disp_connector_caps_info) { bios->funcs->get_disp_connector_caps_info(bios, link->link_id, &disp_connect_caps_info); link->is_internal_display = disp_connect_caps_info.INTERNAL_DISPLAY; + DC_LOG_DC("BIOS object table - is_internal_display: %d", link->is_internal_display); } if (link->link_id.type != OBJECT_TYPE_CONNECTOR) { @@ -1408,10 +1421,14 @@ static bool dc_link_construct(struct dc_link *link, link->hpd_gpio = get_hpd_gpio(link->ctx->dc_bios, link->link_id, link->ctx->gpio_service); + if (link->hpd_gpio) { dal_gpio_open(link->hpd_gpio, GPIO_MODE_INTERRUPT); dal_gpio_unlock_pin(link->hpd_gpio); link->irq_source_hpd = dal_irq_get_source(link->hpd_gpio); + + DC_LOG_DC("BIOS object table - hpd_gpio id: %d", link->hpd_gpio->id); + DC_LOG_DC("BIOS object table - hpd_gpio en: %d", link->hpd_gpio->en); } switch (link->link_id.id) { @@ -1470,6 +1487,11 @@ static bool dc_link_construct(struct dc_link *link, goto ddc_create_fail; } + if (!link->ddc->ddc_pin) { + DC_ERROR("Failed to get I2C info for connector!\n"); + goto ddc_create_fail; + } + link->ddc_hw_inst = dal_ddc_get_line(dal_ddc_service_get_ddc_pin(link->ddc)); @@ -1508,6 +1530,8 @@ static bool dc_link_construct(struct dc_link *link, goto link_enc_create_fail; } + DC_LOG_DC("BIOS object table - DP_IS_USB_C: %d", link->link_enc->features.flags.bits.DP_IS_USB_C); + link->link_enc_hw_inst = link->link_enc->transmitter; for (i = 0; i < 4; i++) { @@ -1530,16 +1554,20 @@ static bool dc_link_construct(struct dc_link *link, if (link->device_tag.dev_id.device_type == DEVICE_TYPE_LCD && link->connector_signal == SIGNAL_TYPE_RGB) continue; + + DC_LOG_DC("BIOS object table - device_tag.acpi_device: %d", link->device_tag.acpi_device); + DC_LOG_DC("BIOS object table - device_tag.dev_id.device_type: %d", link->device_tag.dev_id.device_type); + DC_LOG_DC("BIOS object table - device_tag.dev_id.enum_id: %d", link->device_tag.dev_id.enum_id); break; } if (bios->integrated_info) - info = *bios->integrated_info; + memcpy(info, bios->integrated_info, sizeof(*info)); /* Look for channel mapping corresponding to connector and device tag */ for (i = 0; i < MAX_NUMBER_OF_EXT_DISPLAY_PATH; i++) { struct external_display_path *path = - &info.ext_disp_conn_info.path[i]; + &info->ext_disp_conn_info.path[i]; if (path->device_connector_id.enum_id == link->link_id.enum_id && path->device_connector_id.id == link->link_id.id && @@ -1548,10 +1576,14 @@ static bool dc_link_construct(struct dc_link *link, path->device_acpi_enum == link->device_tag.acpi_device) { link->ddi_channel_mapping = path->channel_mapping; link->chip_caps = path->caps; + DC_LOG_DC("BIOS object table - ddi_channel_mapping: 0x%04X", link->ddi_channel_mapping.raw); + DC_LOG_DC("BIOS object table - chip_caps: %d", link->chip_caps); } else if (path->device_tag == link->device_tag.dev_id.raw_device_tag) { link->ddi_channel_mapping = path->channel_mapping; link->chip_caps = path->caps; + DC_LOG_DC("BIOS object table - ddi_channel_mapping: 0x%04X", link->ddi_channel_mapping.raw); + DC_LOG_DC("BIOS object table - chip_caps: %d", link->chip_caps); } break; } @@ -1570,6 +1602,7 @@ static bool dc_link_construct(struct dc_link *link, link->psr_settings.psr_version = DC_PSR_VERSION_UNSUPPORTED; + DC_LOG_DC("BIOS object table - %s finished successfully.\n", __func__); return true; device_tag_fail: link->link_enc->funcs->destroy(&link->link_enc); @@ -1586,6 +1619,9 @@ static bool dc_link_construct(struct dc_link *link, link->hpd_gpio = NULL; } + DC_LOG_DC("BIOS object table - %s failed.\n", __func__); + kfree(info); + return false; } @@ -3133,17 +3169,17 @@ static void update_psp_stream_config(struct pipe_ctx *pipe_ctx, bool dpms_off) { struct cp_psp *cp_psp = &pipe_ctx->stream->ctx->cp_psp; if (cp_psp && cp_psp->funcs.update_stream_config) { - struct cp_psp_stream_config config; - - memset(&config, 0, sizeof(config)); + struct cp_psp_stream_config config = {0}; + enum dp_panel_mode panel_mode = + dp_get_panel_mode(pipe_ctx->stream->link); config.otg_inst = (uint8_t) pipe_ctx->stream_res.tg->inst; - /*stream_enc_inst*/ - config.stream_enc_inst = (uint8_t) pipe_ctx->stream_res.stream_enc->stream_enc_inst; - config.link_enc_inst = pipe_ctx->stream->link->link_enc_hw_inst; + config.dig_fe = (uint8_t) pipe_ctx->stream_res.stream_enc->stream_enc_inst; + config.dig_be = pipe_ctx->stream->link->link_enc_hw_inst; config.dpms_off = dpms_off; config.dm_stream_ctx = pipe_ctx->stream->dm_stream_context; - config.mst_supported = (pipe_ctx->stream->signal == + config.assr_enabled = (panel_mode == DP_PANEL_MODE_EDP); + config.mst_enabled = (pipe_ctx->stream->signal == SIGNAL_TYPE_DISPLAY_PORT_MST); cp_psp->funcs.update_stream_config(cp_psp->handle, &config); } @@ -3396,10 +3432,7 @@ void core_link_set_avmute(struct pipe_ctx *pipe_ctx, bool enable) } /** - ***************************************************************************** - * Function: dc_link_enable_hpd_filter - * - * @brief + * dc_link_enable_hpd_filter: * If enable is true, programs HPD filter on associated HPD line using * delay_on_disconnect/delay_on_connect values dependent on * link->connector_signal @@ -3407,9 +3440,8 @@ void core_link_set_avmute(struct pipe_ctx *pipe_ctx, bool enable) * If enable is false, programs HPD filter on associated HPD line with no * delays on connect or disconnect * - * @param [in] link: pointer to the dc link - * @param [in] enable: boolean specifying whether to enable hbd - ***************************************************************************** + * @link: pointer to the dc link + * @enable: boolean specifying whether to enable hbd */ void dc_link_enable_hpd_filter(struct dc_link *link, bool enable) { @@ -3635,7 +3667,7 @@ uint32_t dc_link_bandwidth_kbps( link_bw_kbps *= 8; /* 8 bits per byte*/ link_bw_kbps *= link_setting->lane_count; - if (dc_link_is_fec_supported(link) && !link->dc->debug.disable_fec) { + if (dc_link_should_enable_fec(link)) { /* Account for FEC overhead. * We have to do it based on caps, * and not based on FEC being set ready, @@ -3656,8 +3688,8 @@ uint32_t dc_link_bandwidth_kbps( * but the difference is minimal and is in a safe direction, * which all works well around potential ambiguity of DP 1.4a spec. */ - link_bw_kbps = mul_u64_u32_shr(BIT_ULL(32) * 970LL / 1000, - link_bw_kbps, 32); + long long fec_link_bw_kbps = link_bw_kbps * 970LL; + link_bw_kbps = (uint32_t)(div64_s64(fec_link_bw_kbps, 1000LL)); } return link_bw_kbps; @@ -3687,3 +3719,19 @@ bool dc_link_is_fec_supported(const struct dc_link *link) !IS_FPGA_MAXIMUS_DC(link->ctx->dce_environment)); } +bool dc_link_should_enable_fec(const struct dc_link *link) +{ + bool is_fec_disable = false; + bool ret = false; + + if ((link->connector_signal != SIGNAL_TYPE_DISPLAY_PORT_MST && + link->local_sink && + link->local_sink->edid_caps.panel_patch.disable_fec) || + link->connector_signal == SIGNAL_TYPE_EDP) // Disable FEC for eDP + is_fec_disable = true; + + if (dc_link_is_fec_supported(link) && !link->dc->debug.disable_fec && !is_fec_disable) + ret = true; + + return ret; +} diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_link_ddc.c b/drivers/gpu/drm/amd/display/dc/core/dc_link_ddc.c index c5936e0643606dc29c9d1cf57c1cc3acac22a356..ae6484ab567b7a2a2d2ea348fa835d0ab8a6b818 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc_link_ddc.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc_link_ddc.c @@ -37,12 +37,16 @@ #include "dc_link_ddc.h" #include "dce/dce_aux.h" -/*DP to Dual link DVI converter*/ +#define DC_LOGGER_INIT(logger) + +static const uint8_t DP_VGA_DONGLE_BRANCH_DEV_NAME[] = "DpVga"; +/* DP to Dual link DVI converter */ static const uint8_t DP_DVI_CONVERTER_ID_4[] = "m2DVIa"; static const uint8_t DP_DVI_CONVERTER_ID_5[] = "3393N2"; #define AUX_POWER_UP_WA_DELAY 500 #define I2C_OVER_AUX_DEFER_WA_DELAY 70 +#define DPVGA_DONGLE_AUX_DEFER_WA_DELAY 40 #define I2C_OVER_AUX_DEFER_WA_DELAY_1MS 1 /* CV smart dongle slave address for retrieving supported HDTV modes*/ @@ -194,6 +198,10 @@ static void ddc_service_construct( if (BP_RESULT_OK != dcb->funcs->get_i2c_info(dcb, init_data->id, &i2c_info)) { ddc_service->ddc_pin = NULL; } else { + DC_LOGGER_INIT(ddc_service->ctx->logger); + DC_LOG_DC("BIOS object table - i2c_line: %d", i2c_info.i2c_line); + DC_LOG_DC("BIOS object table - i2c_engine_id: %d", i2c_info.i2c_engine_id); + hw_info.ddc_channel = i2c_info.i2c_line; if (ddc_service->link != NULL) hw_info.hw_supported = i2c_info.i2c_hw_assist; @@ -286,6 +294,15 @@ static uint32_t defer_delay_converter_wa( { struct dc_link *link = ddc->link; + if (link->dpcd_caps.dongle_type == DISPLAY_DONGLE_DP_VGA_CONVERTER && + link->dpcd_caps.branch_dev_id == DP_BRANCH_DEVICE_ID_0080E1 && + !memcmp(link->dpcd_caps.branch_dev_name, + DP_VGA_DONGLE_BRANCH_DEV_NAME, + sizeof(link->dpcd_caps.branch_dev_name))) + + return defer_delay > DPVGA_DONGLE_AUX_DEFER_WA_DELAY ? + defer_delay : DPVGA_DONGLE_AUX_DEFER_WA_DELAY; + if (link->dpcd_caps.branch_dev_id == DP_BRANCH_DEVICE_ID_0080E1 && !memcmp(link->dpcd_caps.branch_dev_name, DP_DVI_CONVERTER_ID_4, diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c b/drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c index 1e4794e2825caf2b23b330255b899ec6c5614a05..c1391bfb7a9bc58de9460bc778e2f4fac0120318 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c @@ -126,9 +126,7 @@ static void dpcd_set_training_pattern( static enum dc_dp_training_pattern decide_cr_training_pattern( const struct dc_link_settings *link_settings) { - enum dc_dp_training_pattern pattern = DP_TRAINING_PATTERN_SEQUENCE_1; - - return pattern; + return DP_TRAINING_PATTERN_SEQUENCE_1; } static enum dc_dp_training_pattern decide_eq_training_pattern(struct dc_link *link, @@ -3710,7 +3708,7 @@ bool detect_dp_sink_caps(struct dc_link *link) /* TODO save sink caps in link->sink */ } -enum dc_link_rate linkRateInKHzToLinkRateMultiplier(uint32_t link_rate_in_khz) +static enum dc_link_rate linkRateInKHzToLinkRateMultiplier(uint32_t link_rate_in_khz) { enum dc_link_rate link_rate; // LinkRate is normally stored as a multiplier of 0.27 Gbps per lane. Do the translation. @@ -4348,7 +4346,7 @@ void dp_set_fec_ready(struct dc_link *link, bool ready) struct link_encoder *link_enc = link->link_enc; uint8_t fec_config = 0; - if (!dc_link_is_fec_supported(link) || link->dc->debug.disable_fec) + if (!dc_link_should_enable_fec(link)) return; if (link_enc->funcs->fec_set_ready && @@ -4383,7 +4381,7 @@ void dp_set_fec_enable(struct dc_link *link, bool enable) { struct link_encoder *link_enc = link->link_enc; - if (!dc_link_is_fec_supported(link) || link->dc->debug.disable_fec) + if (!dc_link_should_enable_fec(link)) return; if (link_enc->funcs->fec_set_enable && @@ -4409,24 +4407,39 @@ void dp_set_fec_enable(struct dc_link *link, bool enable) void dpcd_set_source_specific_data(struct dc_link *link) { if (!link->dc->vendor_signature.is_valid) { - enum dc_status result_write_min_hblank = DC_NOT_SUPPORTED; - struct dpcd_amd_signature amd_signature; - amd_signature.AMD_IEEE_TxSignature_byte1 = 0x0; - amd_signature.AMD_IEEE_TxSignature_byte2 = 0x0; - amd_signature.AMD_IEEE_TxSignature_byte3 = 0x1A; - amd_signature.device_id_byte1 = + enum dc_status __maybe_unused result_write_min_hblank = DC_NOT_SUPPORTED; + struct dpcd_amd_signature amd_signature = {0}; + struct dpcd_amd_device_id amd_device_id = {0}; + + amd_device_id.device_id_byte1 = (uint8_t)(link->ctx->asic_id.chip_id); - amd_signature.device_id_byte2 = + amd_device_id.device_id_byte2 = (uint8_t)(link->ctx->asic_id.chip_id >> 8); - memset(&amd_signature.zero, 0, 4); - amd_signature.dce_version = + amd_device_id.dce_version = (uint8_t)(link->ctx->dce_version); - amd_signature.dal_version_byte1 = 0x0; // needed? where to get? - amd_signature.dal_version_byte2 = 0x0; // needed? where to get? + amd_device_id.dal_version_byte1 = 0x0; // needed? where to get? + amd_device_id.dal_version_byte2 = 0x0; // needed? where to get? - core_link_write_dpcd(link, DP_SOURCE_OUI, + core_link_read_dpcd(link, DP_SOURCE_OUI, + (uint8_t *)(&amd_signature), + sizeof(amd_signature)); + + if (!((amd_signature.AMD_IEEE_TxSignature_byte1 == 0x0) && + (amd_signature.AMD_IEEE_TxSignature_byte2 == 0x0) && + (amd_signature.AMD_IEEE_TxSignature_byte3 == 0x1A))) { + + amd_signature.AMD_IEEE_TxSignature_byte1 = 0x0; + amd_signature.AMD_IEEE_TxSignature_byte2 = 0x0; + amd_signature.AMD_IEEE_TxSignature_byte3 = 0x1A; + + core_link_write_dpcd(link, DP_SOURCE_OUI, (uint8_t *)(&amd_signature), sizeof(amd_signature)); + } + + core_link_write_dpcd(link, DP_SOURCE_OUI+0x03, + (uint8_t *)(&amd_device_id), + sizeof(amd_device_id)); if (link->ctx->dce_version >= DCN_VERSION_2_0 && link->dc->caps.min_horizontal_blanking_period != 0) { diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_resource.c b/drivers/gpu/drm/amd/display/dc/core/dc_resource.c index 07c22556480bec65c5f522b79117e2bf61b23d57..0c26c2ade782c4ea69713e814d186e06ca755e39 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc_resource.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc_resource.c @@ -1117,7 +1117,7 @@ static void calculate_inits_and_adj_vp(struct pipe_ctx *pipe_ctx) * We also need to make sure pipe_ctx->plane_res.scl_data.h_active uses the * original h_border_left value in its calculation. */ -int shift_border_left_to_dst(struct pipe_ctx *pipe_ctx) +static int shift_border_left_to_dst(struct pipe_ctx *pipe_ctx) { int store_h_border_left = pipe_ctx->stream->timing.h_border_left; @@ -1128,8 +1128,8 @@ int shift_border_left_to_dst(struct pipe_ctx *pipe_ctx) return store_h_border_left; } -void restore_border_left_from_dst(struct pipe_ctx *pipe_ctx, - int store_h_border_left) +static void restore_border_left_from_dst(struct pipe_ctx *pipe_ctx, + int store_h_border_left) { pipe_ctx->stream->dst.x -= store_h_border_left; pipe_ctx->stream->timing.h_border_left = store_h_border_left; @@ -1153,8 +1153,8 @@ bool resource_build_scaling_params(struct pipe_ctx *pipe_ctx) calculate_viewport(pipe_ctx); - if (pipe_ctx->plane_res.scl_data.viewport.height < 12 || - pipe_ctx->plane_res.scl_data.viewport.width < 12) { + if (pipe_ctx->plane_res.scl_data.viewport.height < MIN_VIEWPORT_SIZE || + pipe_ctx->plane_res.scl_data.viewport.width < MIN_VIEWPORT_SIZE) { if (store_h_border_left) { restore_border_left_from_dst(pipe_ctx, store_h_border_left); @@ -1697,7 +1697,7 @@ static bool are_stream_backends_same( return true; } -/** +/* * dc_is_stream_unchanged() - Compare two stream states for equivalence. * * Checks if there a difference between the two states @@ -1718,7 +1718,7 @@ bool dc_is_stream_unchanged( return true; } -/** +/* * dc_is_stream_scaling_unchanged() - Compare scaling rectangles of two streams. */ bool dc_is_stream_scaling_unchanged( @@ -1833,7 +1833,7 @@ static struct audio *find_first_free_audio( return 0; } -/** +/* * dc_add_stream_to_ctx() - Add a new dc_stream_state to a dc_state. */ enum dc_status dc_add_stream_to_ctx( @@ -1860,7 +1860,7 @@ enum dc_status dc_add_stream_to_ctx( return res; } -/** +/* * dc_remove_stream_from_ctx() - Remove a stream from a dc_state. */ enum dc_status dc_remove_stream_from_ctx( @@ -2075,6 +2075,20 @@ static int acquire_resource_from_hw_enabled_state( return -1; } +static void mark_seamless_boot_stream( + const struct dc *dc, + struct dc_stream_state *stream) +{ + struct dc_bios *dcb = dc->ctx->dc_bios; + + /* TODO: Check Linux */ + if (dc->config.allow_seamless_boot_optimization && + !dcb->funcs->is_accelerated_mode(dcb)) { + if (dc_validate_seamless_boot_timing(dc, stream->sink, &stream->timing)) + stream->apply_seamless_boot_optimization = true; + } +} + enum dc_status resource_map_pool_resources( const struct dc *dc, struct dc_state *context, @@ -2085,22 +2099,20 @@ enum dc_status resource_map_pool_resources( struct dc_context *dc_ctx = dc->ctx; struct pipe_ctx *pipe_ctx = NULL; int pipe_idx = -1; - struct dc_bios *dcb = dc->ctx->dc_bios; calculate_phy_pix_clks(stream); - /* TODO: Check Linux */ - if (dc->config.allow_seamless_boot_optimization && - !dcb->funcs->is_accelerated_mode(dcb)) { - if (dc_validate_seamless_boot_timing(dc, stream->sink, &stream->timing)) - stream->apply_seamless_boot_optimization = true; - } + mark_seamless_boot_stream(dc, stream); - if (stream->apply_seamless_boot_optimization) + if (stream->apply_seamless_boot_optimization) { pipe_idx = acquire_resource_from_hw_enabled_state( &context->res_ctx, pool, stream); + if (pipe_idx < 0) + /* hw resource was assigned to other stream */ + stream->apply_seamless_boot_optimization = false; + } if (pipe_idx < 0) /* acquire new resources */ diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_stream.c b/drivers/gpu/drm/amd/display/dc/core/dc_stream.c index c103f858375d060f6cdd79c1ab4fc058edbc1978..25fa712a7847464b4f4215d631d6ba8a1e8e4919 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc_stream.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc_stream.c @@ -244,7 +244,7 @@ struct dc_stream_status *dc_stream_get_status( } #ifndef TRIM_FSFT -/** +/* * dc_optimize_timing_for_fsft() - dc to optimize timing */ bool dc_optimize_timing_for_fsft( @@ -260,8 +260,7 @@ bool dc_optimize_timing_for_fsft( } #endif - -/** +/* * dc_stream_set_cursor_attributes() - Update cursor attributes and set cursor surface address */ bool dc_stream_set_cursor_attributes( diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_surface.c b/drivers/gpu/drm/amd/display/dc/core/dc_surface.c index 3d7d27435f15eb099d7061a5e4707a52848c8d47..e6b9c6a718413a6cd2570c879ca11c52cf8775f8 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc_surface.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc_surface.c @@ -115,7 +115,7 @@ struct dc_plane_state *dc_create_plane_state(struct dc *dc) return plane_state; } -/** +/* ***************************************************************************** * Function: dc_plane_get_status * diff --git a/drivers/gpu/drm/amd/display/dc/dc.h b/drivers/gpu/drm/amd/display/dc/dc.h index 3aedadb34548ef2be2f33ae6fed2a6a530047eda..4eee3a55fa3018ad0f65bfe50f1e949fbf4de7ac 100644 --- a/drivers/gpu/drm/amd/display/dc/dc.h +++ b/drivers/gpu/drm/amd/display/dc/dc.h @@ -42,12 +42,13 @@ #include "inc/hw/dmcu.h" #include "dml/display_mode_lib.h" -#define DC_VER "3.2.116" +#define DC_VER "3.2.122" #define MAX_SURFACES 3 #define MAX_PLANES 6 #define MAX_STREAMS 6 #define MAX_SINKS_PER_LINK 4 +#define MIN_VIEWPORT_SIZE 12 /******************************************************************************* * Display Core Interfaces @@ -171,6 +172,9 @@ struct dc_caps { bool dmcub_support; uint32_t num_of_internal_disp; enum dp_protocol_version max_dp_protocol_version; + unsigned int mall_size_per_mem_channel; + unsigned int mall_size_total; + unsigned int cursor_cache_size; struct dc_plane_cap planes[MAX_PLANES]; struct dc_color_caps color; }; @@ -481,7 +485,6 @@ struct dc_debug_options { bool performance_trace; bool az_endpoint_mute_only; bool always_use_regamma; - bool p010_mpo_support; bool recovery_enabled; bool avoid_vbios_exec_table; bool scl_reset_length10; @@ -499,6 +502,9 @@ struct dc_debug_options { bool dmcub_emulation; #if defined(CONFIG_DRM_AMD_DC_DCN) bool disable_idle_power_optimizations; + unsigned int mall_size_override; + unsigned int mall_additional_timer_percent; + bool mall_error_as_fatal; #endif bool dmub_command_table; /* for testing only */ struct dc_bw_validation_profile bw_val_profile; @@ -521,7 +527,6 @@ struct dc_debug_options { bool usbc_combo_phy_reset_wa; bool disable_dsc; bool enable_dram_clock_change_one_display_vactive; - bool force_ignore_link_settings; union mem_low_power_enable_options enable_mem_low_power; }; @@ -633,7 +638,6 @@ struct dc { const char *build_id; struct vm_helper *vm_helper; - const struct gpu_info_soc_bounding_box_v1_0 *soc_bounding_box; }; enum frame_buffer_mode { @@ -671,16 +675,10 @@ struct dc_init_data { struct dc_config flags; uint64_t log_mask; - /** - * gpu_info FW provided soc bounding box struct or 0 if not - * available in FW - */ - const struct gpu_info_soc_bounding_box_v1_0 *soc_bounding_box; struct dpcd_vendor_signature vendor_signature; #if defined(CONFIG_DRM_AMD_DC_DCN) bool force_smu_not_present; #endif - bool force_ignore_link_settings; }; struct dc_callback_init { @@ -1269,8 +1267,8 @@ enum dc_status dc_set_clock(struct dc *dc, enum dc_clock_type clock_type, uint32 void dc_get_clock(struct dc *dc, enum dc_clock_type clock_type, struct dc_clock_config *clock_cfg); #if defined(CONFIG_DRM_AMD_DC_DCN) -bool dc_is_plane_eligible_for_idle_optimizations(struct dc *dc, - struct dc_plane_state *plane); +bool dc_is_plane_eligible_for_idle_optimizations(struct dc *dc, struct dc_plane_state *plane, + struct dc_cursor_attributes *cursor_attr); void dc_allow_idle_optimizations(struct dc *dc, bool allow); diff --git a/drivers/gpu/drm/amd/display/dc/dc_dp_types.h b/drivers/gpu/drm/amd/display/dc/dc_dp_types.h index 80a2191a3115ca55771aa75bd8bdfc88d152630c..cc6fb838420e8c00c7fe20cabd5b88147e55f2ba 100644 --- a/drivers/gpu/drm/amd/display/dc/dc_dp_types.h +++ b/drivers/gpu/drm/amd/display/dc/dc_dp_types.h @@ -451,6 +451,9 @@ struct dpcd_amd_signature { uint8_t AMD_IEEE_TxSignature_byte1; uint8_t AMD_IEEE_TxSignature_byte2; uint8_t AMD_IEEE_TxSignature_byte3; +}; + +struct dpcd_amd_device_id { uint8_t device_id_byte1; uint8_t device_id_byte2; uint8_t zero[4]; diff --git a/drivers/gpu/drm/amd/display/dc/dc_edid_parser.c b/drivers/gpu/drm/amd/display/dc/dc_edid_parser.c new file mode 100644 index 0000000000000000000000000000000000000000..0db5b49e9d5e9055b65e90e61ecee4501b6f4b8a --- /dev/null +++ b/drivers/gpu/drm/amd/display/dc/dc_edid_parser.c @@ -0,0 +1,80 @@ +/* + * Copyright 2021 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: AMD + * + */ + +#include "dce/dce_dmcu.h" +#include "dc_edid_parser.h" + +bool dc_edid_parser_send_cea(struct dc *dc, + int offset, + int total_length, + uint8_t *data, + int length) +{ + struct dmcu *dmcu = dc->res_pool->dmcu; + + if (dmcu && + dmcu->funcs->is_dmcu_initialized(dmcu) && + dmcu->funcs->send_edid_cea) { + return dmcu->funcs->send_edid_cea(dmcu, + offset, + total_length, + data, + length); + } + + return false; +} + +bool dc_edid_parser_recv_cea_ack(struct dc *dc, int *offset) +{ + struct dmcu *dmcu = dc->res_pool->dmcu; + + if (dmcu && + dmcu->funcs->is_dmcu_initialized(dmcu) && + dmcu->funcs->recv_edid_cea_ack) { + return dmcu->funcs->recv_edid_cea_ack(dmcu, offset); + } + + return false; +} + +bool dc_edid_parser_recv_amd_vsdb(struct dc *dc, + int *version, + int *min_frame_rate, + int *max_frame_rate) +{ + struct dmcu *dmcu = dc->res_pool->dmcu; + + if (dmcu && + dmcu->funcs->is_dmcu_initialized(dmcu) && + dmcu->funcs->recv_amd_vsdb) { + return dmcu->funcs->recv_amd_vsdb(dmcu, + version, + min_frame_rate, + max_frame_rate); + } + + return false; +} diff --git a/drivers/gpu/drm/amd/display/dc/dc_edid_parser.h b/drivers/gpu/drm/amd/display/dc/dc_edid_parser.h new file mode 100644 index 0000000000000000000000000000000000000000..da67ec06f0a2f360369c24dc0e58c737aa01baa1 --- /dev/null +++ b/drivers/gpu/drm/amd/display/dc/dc_edid_parser.h @@ -0,0 +1,44 @@ +/* + * Copyright 2021 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: AMD + * + */ + +#ifndef _DC_EDID_PARSER_H_ +#define _DC_EDID_PARSER_H_ + +#include "core_types.h" + +bool dc_edid_parser_send_cea(struct dc *dc, + int offset, + int total_length, + uint8_t *data, + int length); + +bool dc_edid_parser_recv_cea_ack(struct dc *dc, int *offset); + +bool dc_edid_parser_recv_amd_vsdb(struct dc *dc, + int *version, + int *min_frame_rate, + int *max_frame_rate); + +#endif /* _DC_EDID_PARSER_H_ */ diff --git a/drivers/gpu/drm/amd/display/dc/dc_helper.c b/drivers/gpu/drm/amd/display/dc/dc_helper.c index 57edb25fc3812f5c31c821512f0418e09b044f49..a612ba6dc3898526c6ad925a52b96a067cd62382 100644 --- a/drivers/gpu/drm/amd/display/dc/dc_helper.c +++ b/drivers/gpu/drm/amd/display/dc/dc_helper.c @@ -34,6 +34,7 @@ #include "dc.h" #include "dc_dmub_srv.h" +#include "reg_helper.h" static inline void submit_dmub_read_modify_write( struct dc_reg_helper_state *offload, diff --git a/drivers/gpu/drm/amd/display/dc/dc_hw_types.h b/drivers/gpu/drm/amd/display/dc/dc_hw_types.h index 701aa7178a896346809b1e6628544a3755184946..b41e6367b15ef967ad8e89c59dc8232e4df497de 100644 --- a/drivers/gpu/drm/amd/display/dc/dc_hw_types.h +++ b/drivers/gpu/drm/amd/display/dc/dc_hw_types.h @@ -71,6 +71,7 @@ struct dc_plane_address { union { struct{ PHYSICAL_ADDRESS_LOC addr; + PHYSICAL_ADDRESS_LOC cursor_cache_addr; PHYSICAL_ADDRESS_LOC meta_addr; union large_integer dcc_const_color; } grph; diff --git a/drivers/gpu/drm/amd/display/dc/dc_link.h b/drivers/gpu/drm/amd/display/dc/dc_link.h index 6d9a60c9dcc070e1b6a06f9133c6f598d4c06817..e189f16bc026bbf2e670a991a365fd86dd281123 100644 --- a/drivers/gpu/drm/amd/display/dc/dc_link.h +++ b/drivers/gpu/drm/amd/display/dc/dc_link.h @@ -103,6 +103,8 @@ struct dc_link { bool lttpr_non_transparent_mode; bool is_internal_display; + bool edp_sink_present; + /* caps is the same as reported_link_cap. link_traing use * reported_link_cap. Will clean up. TODO */ @@ -259,6 +261,13 @@ enum dc_status dc_link_reallocate_mst_payload(struct dc_link *link); bool dc_link_handle_hpd_rx_irq(struct dc_link *dc_link, union hpd_irq_data *hpd_irq_dpcd_data, bool *out_link_loss); +/* + * On eDP links this function call will stall until T12 has elapsed. + * If the panel is not in power off state, this function will return + * immediately. + */ +bool dc_link_wait_for_t12(struct dc_link *link); + enum dc_status read_hpd_rx_irq_data( struct dc_link *link, union hpd_irq_data *irq_data); @@ -369,5 +378,6 @@ uint32_t dc_bandwidth_in_kbps_from_timing( const struct dc_crtc_timing *timing); bool dc_link_is_fec_supported(const struct dc_link *link); +bool dc_link_should_enable_fec(const struct dc_link *link); #endif /* DC_LINK_H_ */ diff --git a/drivers/gpu/drm/amd/display/dc/dc_stream.h b/drivers/gpu/drm/amd/display/dc/dc_stream.h index e243c01b9672eee14e92e384e300d61213c4a601..80b67b860091eeaedc82aa6760e9a1a8b694f8a8 100644 --- a/drivers/gpu/drm/amd/display/dc/dc_stream.h +++ b/drivers/gpu/drm/amd/display/dc/dc_stream.h @@ -130,6 +130,14 @@ union stream_update_flags { uint32_t raw; }; +struct test_pattern { + enum dp_test_pattern type; + enum dp_test_pattern_color_space color_space; + struct link_training_settings const *p_link_settings; + unsigned char const *p_custom_pattern; + unsigned int cust_pattern_size; +}; + struct dc_stream_state { // sink is deprecated, new code should not reference // this pointer @@ -227,6 +235,8 @@ struct dc_stream_state { uint32_t stream_id; bool is_dsc_enabled; + + struct test_pattern test_pattern; union stream_update_flags update_flags; }; @@ -261,6 +271,7 @@ struct dc_stream_update { struct dc_dsc_config *dsc_config; struct dc_transfer_func *func_shaper; struct dc_3dlut *lut3d_func; + struct test_pattern *pending_test_pattern; }; bool dc_is_stream_unchanged( @@ -283,7 +294,8 @@ void dc_commit_updates_for_stream(struct dc *dc, struct dc_surface_update *srf_updates, int surface_count, struct dc_stream_state *stream, - struct dc_stream_update *stream_update); + struct dc_stream_update *stream_update, + struct dc_state *state); /* * Log the current stream state. */ diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_audio.c b/drivers/gpu/drm/amd/display/dc/dce/dce_audio.c index 2a2a0fdb925393d2aa97788d1eb843fceccb1f39..7866cf2a668fa2ae0501b0038d52fbae33905b06 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dce_audio.c +++ b/drivers/gpu/drm/amd/display/dc/dce/dce_audio.c @@ -868,7 +868,7 @@ void dce_aud_wall_dto_setup( } #if defined(CONFIG_DRM_AMD_DC_SI) -void dce60_aud_wall_dto_setup( +static void dce60_aud_wall_dto_setup( struct audio *audio, enum signal_type signal, const struct audio_crtc_info *crtc_info, diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_aux.c b/drivers/gpu/drm/amd/display/dc/dce/dce_aux.c index cda5fd0464bc5309cc1ffa3c19b766be2d1ba196..d51b5fe91287ddbe015e4aba12a23652e19ac2a3 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dce_aux.c +++ b/drivers/gpu/drm/amd/display/dc/dce/dce_aux.c @@ -388,12 +388,6 @@ static enum aux_channel_operation_result get_channel_status( } } -enum i2caux_engine_type get_engine_type( - const struct dce_aux *engine) -{ - return I2CAUX_ENGINE_TYPE_AUX; -} - static bool acquire( struct dce_aux *engine, struct ddc *ddc) @@ -582,7 +576,7 @@ int dce_aux_transfer_raw(struct ddc_service *ddc, *operation_result = get_channel_status(aux_engine, &returned_bytes); if (*operation_result == AUX_CHANNEL_OPERATION_SUCCEEDED) { - int bytes_replied = 0; + int __maybe_unused bytes_replied = 0; bytes_replied = read_channel_reply(aux_engine, payload->length, payload->data, payload->reply, &status); diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_aux.h b/drivers/gpu/drm/amd/display/dc/dce/dce_aux.h index 382465862f2972482be42d6fe0935735910c25bc..277484cf853e483663df86a61bf7d1cf77747b1e 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dce_aux.h +++ b/drivers/gpu/drm/amd/display/dc/dce/dce_aux.h @@ -124,7 +124,6 @@ struct dce110_aux_registers { AUX_SF(AUX_SW_CONTROL, AUX_SW_GO, mask_sh),\ AUX_SF(AUX_SW_DATA, AUX_SW_AUTOINCREMENT_DISABLE, mask_sh),\ AUX_SF(AUX_SW_DATA, AUX_SW_DATA_RW, mask_sh),\ - AUX_SF(AUX_SW_DATA, AUX_SW_AUTOINCREMENT_DISABLE, mask_sh),\ AUX_SF(AUX_SW_DATA, AUX_SW_INDEX, mask_sh),\ AUX_SF(AUX_SW_DATA, AUX_SW_DATA, mask_sh),\ AUX_SF(AUX_SW_STATUS, AUX_SW_REPLY_BYTE_COUNT, mask_sh),\ diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_clock_source.c b/drivers/gpu/drm/amd/display/dc/dce/dce_clock_source.c index fb733f573715e05f8b496d52beb26015de2eb616..dec58b3c42e4d52badf8054933f43a18f5b19706 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dce_clock_source.c +++ b/drivers/gpu/drm/amd/display/dc/dce/dce_clock_source.c @@ -113,20 +113,19 @@ static const struct spread_spectrum_data *get_ss_data_entry( } /** - * Function: calculate_fb_and_fractional_fb_divider + * calculate_fb_and_fractional_fb_divider - Calculates feedback and fractional + * feedback dividers values * - * * DESCRIPTION: Calculates feedback and fractional feedback dividers values + * @calc_pll_cs: Pointer to clock source information + * @target_pix_clk_100hz: Desired frequency in 100 Hz + * @ref_divider: Reference divider (already known) + * @post_divider: Post Divider (already known) + * @feedback_divider_param: Pointer where to store + * calculated feedback divider value + * @fract_feedback_divider_param: Pointer where to store + * calculated fract feedback divider value * - *PARAMETERS: - * targetPixelClock Desired frequency in 100 Hz - * ref_divider Reference divider (already known) - * postDivider Post Divider (already known) - * feedback_divider_param Pointer where to store - * calculated feedback divider value - * fract_feedback_divider_param Pointer where to store - * calculated fract feedback divider value - * - *RETURNS: + * return: * It fills the locations pointed by feedback_divider_param * and fract_feedback_divider_param * It returns - true if feedback divider not 0 @@ -175,22 +174,22 @@ static bool calculate_fb_and_fractional_fb_divider( } /** -*calc_fb_divider_checking_tolerance -* -*DESCRIPTION: Calculates Feedback and Fractional Feedback divider values -* for passed Reference and Post divider, checking for tolerance. -*PARAMETERS: -* pll_settings Pointer to structure -* ref_divider Reference divider (already known) -* postDivider Post Divider (already known) -* tolerance Tolerance for Calculated Pixel Clock to be within -* -*RETURNS: -* It fills the PLLSettings structure with PLL Dividers values -* if calculated values are within required tolerance -* It returns - true if error is within tolerance -* - false if error is not within tolerance -*/ + * calc_fb_divider_checking_tolerance - Calculates Feedback and + * Fractional Feedback divider values + * for passed Reference and Post divider, + * checking for tolerance. + * @calc_pll_cs: Pointer to clock source information + * @pll_settings: Pointer to PLL settings + * @ref_divider: Reference divider (already known) + * @post_divider: Post Divider (already known) + * @tolerance: Tolerance for Calculated Pixel Clock to be within + * + * return: + * It fills the PLLSettings structure with PLL Dividers values + * if calculated values are within required tolerance + * It returns - true if error is within tolerance + * - false if error is not within tolerance + */ static bool calc_fb_divider_checking_tolerance( struct calc_pll_clock_source *calc_pll_cs, struct pll_settings *pll_settings, @@ -241,7 +240,7 @@ static bool calc_fb_divider_checking_tolerance( pll_settings->calculated_pix_clk_100hz = actual_calculated_clock_100hz; pll_settings->vco_freq = - actual_calculated_clock_100hz * post_divider / 10; + div_u64((u64)actual_calculated_clock_100hz * post_divider, 10); return true; } return false; @@ -460,7 +459,7 @@ static bool pll_adjust_pix_clk( return false; } -/** +/* * Calculate PLL Dividers for given Clock Value. * First will call VBIOS Adjust Exec table to check if requested Pixel clock * will be Adjusted based on usage. @@ -871,6 +870,20 @@ static bool dce110_program_pix_clk( bp_pc_params.flags.SET_EXTERNAL_REF_DIV_SRC = pll_settings->use_external_clk; + switch (pix_clk_params->color_depth) { + case COLOR_DEPTH_101010: + bp_pc_params.color_depth = TRANSMITTER_COLOR_DEPTH_30; + break; + case COLOR_DEPTH_121212: + bp_pc_params.color_depth = TRANSMITTER_COLOR_DEPTH_36; + break; + case COLOR_DEPTH_161616: + bp_pc_params.color_depth = TRANSMITTER_COLOR_DEPTH_48; + break; + default: + break; + } + if (clk_src->bios->funcs->set_pixel_clock( clk_src->bios, &bp_pc_params) != BP_RESULT_OK) return false; diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_dmcu.c b/drivers/gpu/drm/amd/display/dc/dce/dce_dmcu.c index f3ed8b619cafd5b84b05ecc5c6e4cd64d059097a..ddc789daf3b13a38ad67c6d05509902f156cdadd 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dce_dmcu.c +++ b/drivers/gpu/drm/amd/display/dc/dce/dce_dmcu.c @@ -57,6 +57,9 @@ #define MCP_SYNC_PHY_LOCK 0x90 #define MCP_SYNC_PHY_UNLOCK 0x91 #define MCP_BL_SET_PWM_FRAC 0x6A /* Enable or disable Fractional PWM */ +#define MCP_SEND_EDID_CEA 0xA0 +#define EDID_CEA_CMD_ACK 1 +#define EDID_CEA_CMD_NACK 2 #define MASTER_COMM_CNTL_REG__MASTER_COMM_INTERRUPT_MASK 0x00000001L // PSP FW version @@ -65,13 +68,17 @@ //Register access policy version #define mmMP0_SMN_C2PMSG_91 0x1609B +#if defined(CONFIG_DRM_AMD_DC_DCN) +static const uint32_t abm_gain_stepsize = 0x0060; +#endif + static bool dce_dmcu_init(struct dmcu *dmcu) { // Do nothing return true; } -bool dce_dmcu_load_iram(struct dmcu *dmcu, +static bool dce_dmcu_load_iram(struct dmcu *dmcu, unsigned int start_offset, const char *src, unsigned int bytes) @@ -807,6 +814,120 @@ static bool dcn20_unlock_phy(struct dmcu *dmcu) return true; } +static bool dcn10_send_edid_cea(struct dmcu *dmcu, + int offset, + int total_length, + uint8_t *data, + int length) +{ + struct dce_dmcu *dmcu_dce = TO_DCE_DMCU(dmcu); + uint32_t header, data1, data2; + + /* If microcontroller is not running, do nothing */ + if (dmcu->dmcu_state != DMCU_RUNNING) + return false; + + if (length > 8 || length <= 0) + return false; + + header = ((uint32_t)offset & 0xFFFF) << 16 | (total_length & 0xFFFF); + data1 = (((uint32_t)data[0]) << 24) | (((uint32_t)data[1]) << 16) | + (((uint32_t)data[2]) << 8) | ((uint32_t)data[3]); + data2 = (((uint32_t)data[4]) << 24) | (((uint32_t)data[5]) << 16) | + (((uint32_t)data[6]) << 8) | ((uint32_t)data[7]); + + /* waitDMCUReadyForCmd */ + REG_WAIT(MASTER_COMM_CNTL_REG, MASTER_COMM_INTERRUPT, 0, 1, 10000); + + /* setDMCUParam_Cmd */ + REG_UPDATE(MASTER_COMM_CMD_REG, MASTER_COMM_CMD_REG_BYTE0, MCP_SEND_EDID_CEA); + + REG_WRITE(MASTER_COMM_DATA_REG1, header); + REG_WRITE(MASTER_COMM_DATA_REG2, data1); + REG_WRITE(MASTER_COMM_DATA_REG3, data2); + + /* notifyDMCUMsg */ + REG_UPDATE(MASTER_COMM_CNTL_REG, MASTER_COMM_INTERRUPT, 1); + + /* waitDMCUReadyForCmd */ + REG_WAIT(MASTER_COMM_CNTL_REG, MASTER_COMM_INTERRUPT, 0, 1, 10000); + + return true; +} + +static bool dcn10_get_scp_results(struct dmcu *dmcu, + uint32_t *cmd, + uint32_t *data1, + uint32_t *data2, + uint32_t *data3) +{ + struct dce_dmcu *dmcu_dce = TO_DCE_DMCU(dmcu); + + /* If microcontroller is not running, do nothing */ + if (dmcu->dmcu_state != DMCU_RUNNING) + return false; + + *cmd = REG_READ(SLAVE_COMM_CMD_REG); + *data1 = REG_READ(SLAVE_COMM_DATA_REG1); + *data2 = REG_READ(SLAVE_COMM_DATA_REG2); + *data3 = REG_READ(SLAVE_COMM_DATA_REG3); + + /* clear SCP interrupt */ + REG_UPDATE(SLAVE_COMM_CNTL_REG, SLAVE_COMM_INTERRUPT, 0); + + return true; +} + +static bool dcn10_recv_amd_vsdb(struct dmcu *dmcu, + int *version, + int *min_frame_rate, + int *max_frame_rate) +{ + uint32_t data[4]; + int cmd, ack, len; + + if (!dcn10_get_scp_results(dmcu, &data[0], &data[1], &data[2], &data[3])) + return false; + + cmd = data[0] & 0x3FF; + len = (data[0] >> 10) & 0x3F; + ack = data[1]; + + if (cmd != MCP_SEND_EDID_CEA || ack != EDID_CEA_CMD_ACK || len != 12) + return false; + + if ((data[2] & 0xFF)) { + *version = (data[2] >> 8) & 0xFF; + *min_frame_rate = (data[3] >> 16) & 0xFFFF; + *max_frame_rate = data[3] & 0xFFFF; + return true; + } + + return false; +} + +static bool dcn10_recv_edid_cea_ack(struct dmcu *dmcu, int *offset) +{ + uint32_t data[4]; + int cmd, ack; + + if (!dcn10_get_scp_results(dmcu, + &data[0], &data[1], &data[2], &data[3])) + return false; + + cmd = data[0] & 0x3FF; + ack = data[1]; + + if (cmd != MCP_SEND_EDID_CEA) + return false; + + if (ack == EDID_CEA_CMD_ACK) + return true; + + *offset = data[2]; /* nack */ + return false; +} + #endif //(CONFIG_DRM_AMD_DC_DCN) static const struct dmcu_funcs dce_funcs = { @@ -829,6 +950,9 @@ static const struct dmcu_funcs dcn10_funcs = { .get_psr_state = dcn10_get_dmcu_psr_state, .set_psr_wait_loop = dcn10_psr_wait_loop, .get_psr_wait_loop = dcn10_get_psr_wait_loop, + .send_edid_cea = dcn10_send_edid_cea, + .recv_amd_vsdb = dcn10_recv_amd_vsdb, + .recv_edid_cea_ack = dcn10_recv_edid_cea_ack, .is_dmcu_initialized = dcn10_is_dmcu_initialized }; diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_dmcu.h b/drivers/gpu/drm/amd/display/dc/dce/dce_dmcu.h index 93e7f34d4775e164c02db133233790b9cba2487d..ff726b35ef6abd5abbd09477f7f0aabc3744abc3 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dce_dmcu.h +++ b/drivers/gpu/drm/amd/display/dc/dce/dce_dmcu.h @@ -40,6 +40,10 @@ SR(MASTER_COMM_DATA_REG3), \ SR(MASTER_COMM_CMD_REG), \ SR(MASTER_COMM_CNTL_REG), \ + SR(SLAVE_COMM_DATA_REG1), \ + SR(SLAVE_COMM_DATA_REG2), \ + SR(SLAVE_COMM_DATA_REG3), \ + SR(SLAVE_COMM_CMD_REG), \ SR(DMCU_IRAM_RD_CTRL), \ SR(DMCU_IRAM_RD_DATA), \ SR(DMCU_INTERRUPT_TO_UC_EN_MASK), \ @@ -112,6 +116,7 @@ DMCU_SF(MASTER_COMM_CMD_REG, \ MASTER_COMM_CMD_REG_BYTE0, mask_sh), \ DMCU_SF(MASTER_COMM_CNTL_REG, MASTER_COMM_INTERRUPT, mask_sh), \ + DMCU_SF(SLAVE_COMM_CNTL_REG, SLAVE_COMM_INTERRUPT, mask_sh), \ DMCU_SF(DMCU_INTERRUPT_TO_UC_EN_MASK, \ STATIC_SCREEN1_INT_TO_UC_EN, mask_sh), \ DMCU_SF(DMCU_INTERRUPT_TO_UC_EN_MASK, \ @@ -179,6 +184,7 @@ type UC_IN_RESET; \ type MASTER_COMM_CMD_REG_BYTE0; \ type MASTER_COMM_INTERRUPT; \ + type SLAVE_COMM_INTERRUPT; \ type DPHY_RX_FAST_TRAINING_CAPABLE; \ type DPHY_LOAD_BS_COUNT; \ type STATIC_SCREEN1_INT_TO_UC_EN; \ @@ -211,6 +217,11 @@ struct dce_dmcu_registers { uint32_t MASTER_COMM_DATA_REG3; uint32_t MASTER_COMM_CMD_REG; uint32_t MASTER_COMM_CNTL_REG; + uint32_t SLAVE_COMM_DATA_REG1; + uint32_t SLAVE_COMM_DATA_REG2; + uint32_t SLAVE_COMM_DATA_REG3; + uint32_t SLAVE_COMM_CMD_REG; + uint32_t SLAVE_COMM_CNTL_REG; uint32_t DMCU_IRAM_RD_CTRL; uint32_t DMCU_IRAM_RD_DATA; uint32_t DMCU_INTERRUPT_TO_UC_EN_MASK; @@ -317,6 +328,4 @@ struct dmcu *dcn21_dmcu_create( void dce_dmcu_destroy(struct dmcu **dmcu); -static const uint32_t abm_gain_stepsize = 0x0060; - #endif /* _DCE_ABM_H_ */ diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_i2c_hw.c b/drivers/gpu/drm/amd/display/dc/dce/dce_i2c_hw.c index 7fbd92fbc63a98e9eb453e1c3030dbcdd521a526..a524f471e0d75c70bcf7222f59d7927ba8185d6b 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dce_i2c_hw.c +++ b/drivers/gpu/drm/amd/display/dc/dce/dce_i2c_hw.c @@ -435,7 +435,7 @@ struct dce_i2c_hw *acquire_i2c_hw_engine( return dce_i2c_hw; } -enum i2c_channel_operation_result dce_i2c_hw_engine_wait_on_operation_result( +static enum i2c_channel_operation_result dce_i2c_hw_engine_wait_on_operation_result( struct dce_i2c_hw *dce_i2c_hw, uint32_t timeout, enum i2c_channel_operation_result expected_result) @@ -502,7 +502,7 @@ static uint32_t get_transaction_timeout_hw( return period_timeout * num_of_clock_stretches; } -bool dce_i2c_hw_engine_submit_payload( +static bool dce_i2c_hw_engine_submit_payload( struct dce_i2c_hw *dce_i2c_hw, struct i2c_payload *payload, bool middle_of_transaction, diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_i2c_sw.c b/drivers/gpu/drm/amd/display/dc/dce/dce_i2c_sw.c index 87d8428df6c46a19ee1a229626d3be16f66ce17f..6846afd83701ba5f68af2a49cebaa9e94ac3dc0f 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dce_i2c_sw.c +++ b/drivers/gpu/drm/amd/display/dc/dce/dce_i2c_sw.c @@ -339,7 +339,7 @@ static bool start_sync_sw( return false; } -void dce_i2c_sw_engine_set_speed( +static void dce_i2c_sw_engine_set_speed( struct dce_i2c_sw *engine, uint32_t speed) { @@ -353,7 +353,7 @@ void dce_i2c_sw_engine_set_speed( engine->clock_delay = 12; } -bool dce_i2c_sw_engine_acquire_engine( +static bool dce_i2c_sw_engine_acquire_engine( struct dce_i2c_sw *engine, struct ddc *ddc) { @@ -397,7 +397,7 @@ bool dce_i2c_engine_acquire_sw( -void dce_i2c_sw_engine_submit_channel_request( +static void dce_i2c_sw_engine_submit_channel_request( struct dce_i2c_sw *engine, struct i2c_request_transaction_data *req) { @@ -440,7 +440,8 @@ void dce_i2c_sw_engine_submit_channel_request( I2C_CHANNEL_OPERATION_SUCCEEDED : I2C_CHANNEL_OPERATION_FAILED; } -bool dce_i2c_sw_engine_submit_payload( + +static bool dce_i2c_sw_engine_submit_payload( struct dce_i2c_sw *engine, struct i2c_payload *payload, bool middle_of_transaction) diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.c b/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.c index 210466b2d8631f0aadec3160b969cac8897e4604..1e77ffee71b30752006fc1202a2356d427f147f3 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.c +++ b/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.c @@ -1197,7 +1197,7 @@ void dce110_link_encoder_enable_dp_mst_output( #if defined(CONFIG_DRM_AMD_DC_SI) /* enables DP PHY output */ -void dce60_link_encoder_enable_dp_output( +static void dce60_link_encoder_enable_dp_output( struct link_encoder *enc, const struct dc_link_settings *link_settings, enum clock_source_id clock_source) @@ -1236,7 +1236,7 @@ void dce60_link_encoder_enable_dp_output( } /* enables DP PHY output in MST mode */ -void dce60_link_encoder_enable_dp_mst_output( +static void dce60_link_encoder_enable_dp_mst_output( struct link_encoder *enc, const struct dc_link_settings *link_settings, enum clock_source_id clock_source) @@ -1426,7 +1426,7 @@ void dce110_link_encoder_dp_set_phy_pattern( #if defined(CONFIG_DRM_AMD_DC_SI) /* set DP PHY test and training patterns */ -void dce60_link_encoder_dp_set_phy_pattern( +static void dce60_link_encoder_dp_set_phy_pattern( struct link_encoder *enc, const struct encoder_set_dp_phy_pattern_param *param) { @@ -1503,7 +1503,6 @@ void dce110_link_encoder_update_mst_stream_allocation_table( const struct link_mst_stream_allocation_table *table) { struct dce110_link_encoder *enc110 = TO_DCE110_LINK_ENC(enc); - uint32_t value0 = 0; uint32_t value1 = 0; uint32_t value2 = 0; uint32_t slots = 0; @@ -1604,7 +1603,7 @@ void dce110_link_encoder_update_mst_stream_allocation_table( do { udelay(10); - value0 = REG_READ(DP_MSE_SAT_UPDATE); + REG_READ(DP_MSE_SAT_UPDATE); REG_GET(DP_MSE_SAT_UPDATE, DP_MSE_SAT_UPDATE, &value1); diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_opp.c b/drivers/gpu/drm/amd/display/dc/dce/dce_opp.c index e459ae65aaf7641368eb3465caa078470820f83a..4600231da6cbb2637aa0469710ca1740ec604e33 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dce_opp.c +++ b/drivers/gpu/drm/amd/display/dc/dce/dce_opp.c @@ -97,7 +97,7 @@ enum { -/** +/* * set_truncation * 1) set truncation depth: 0 for 18 bpp or 1 for 24 bpp * 2) enable truncation @@ -142,7 +142,7 @@ static void set_truncation( } #if defined(CONFIG_DRM_AMD_DC_SI) -/** +/* * dce60_set_truncation * 1) set truncation depth: 0 for 18 bpp or 1 for 24 bpp * 2) enable truncation @@ -183,7 +183,7 @@ static void dce60_set_truncation( } #endif -/** +/* * set_spatial_dither * 1) set spatial dithering mode: pattern of seed * 2) set spatial dithering depth: 0 for 18bpp or 1 for 24bpp @@ -291,7 +291,7 @@ static void set_spatial_dither( FMT_SPATIAL_DITHER_EN, 1); } -/** +/* * SetTemporalDither (Frame Modulation) * 1) set temporal dither depth * 2) select pattern: from hard-coded pattern or programmable pattern @@ -355,7 +355,7 @@ static void set_temporal_dither( FMT_TEMPORAL_DITHER_EN, 1); } -/** +/* * Set Clamping * 1) Set clamping format based on bpc - 0 for 6bpc (No clamping) * 1 for 8 bpc @@ -415,7 +415,7 @@ void dce110_opp_set_clamping( } #if defined(CONFIG_DRM_AMD_DC_SI) -/** +/* * Set Clamping for DCE6 parts * 1) Set clamping format based on bpc - 0 for 6bpc (No clamping) * 1 for 8 bpc @@ -424,7 +424,7 @@ void dce110_opp_set_clamping( * 7 for programable * 2) Enable clamp if Limited range requested */ -void dce60_opp_set_clamping( +static void dce60_opp_set_clamping( struct dce110_opp *opp110, const struct clamping_and_pixel_encoding_params *params) { @@ -465,7 +465,7 @@ void dce60_opp_set_clamping( } #endif -/** +/* * set_pixel_encoding * * Set Pixel Encoding @@ -501,7 +501,7 @@ static void set_pixel_encoding( } #if defined(CONFIG_DRM_AMD_DC_SI) -/** +/* * dce60_set_pixel_encoding * DCE6 has no FMT_SUBSAMPLING_{MODE,ORDER} bits in FMT_CONTROL reg * Set Pixel Encoding @@ -545,7 +545,7 @@ void dce110_opp_program_bit_depth_reduction( } #if defined(CONFIG_DRM_AMD_DC_SI) -void dce60_opp_program_bit_depth_reduction( +static void dce60_opp_program_bit_depth_reduction( struct output_pixel_processor *opp, const struct bit_depth_reduction_params *params) { @@ -568,7 +568,7 @@ void dce110_opp_program_clamping_and_pixel_encoding( } #if defined(CONFIG_DRM_AMD_DC_SI) -void dce60_opp_program_clamping_and_pixel_encoding( +static void dce60_opp_program_clamping_and_pixel_encoding( struct output_pixel_processor *opp, const struct clamping_and_pixel_encoding_params *params) { @@ -678,7 +678,7 @@ void dce110_opp_program_fmt( } #if defined(CONFIG_DRM_AMD_DC_SI) -void dce60_opp_program_fmt( +static void dce60_opp_program_fmt( struct output_pixel_processor *opp, struct bit_depth_reduction_params *fmt_bit_depth, struct clamping_and_pixel_encoding_params *clamping) diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_opp.h b/drivers/gpu/drm/amd/display/dc/dce/dce_opp.h index 4d484ef60f35727a41e56ea225ccdb337f617d28..bf1ffc3629c7feab74e462ee5e14a36d41010098 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dce_opp.h +++ b/drivers/gpu/drm/amd/display/dc/dce/dce_opp.h @@ -111,7 +111,6 @@ enum dce110_opp_reg_type { OPP_SF(FMT_DITHER_RAND_R_SEED, FMT_RAND_R_SEED, mask_sh),\ OPP_SF(FMT_DITHER_RAND_G_SEED, FMT_RAND_G_SEED, mask_sh),\ OPP_SF(FMT_DITHER_RAND_B_SEED, FMT_RAND_B_SEED, mask_sh),\ - OPP_SF(FMT_BIT_DEPTH_CONTROL, FMT_TEMPORAL_DITHER_EN, mask_sh),\ OPP_SF(FMT_BIT_DEPTH_CONTROL, FMT_TEMPORAL_DITHER_RESET, mask_sh),\ OPP_SF(FMT_BIT_DEPTH_CONTROL, FMT_TEMPORAL_DITHER_OFFSET, mask_sh),\ OPP_SF(FMT_BIT_DEPTH_CONTROL, FMT_TEMPORAL_DITHER_DEPTH, mask_sh),\ @@ -219,7 +218,6 @@ enum dce110_opp_reg_type { OPP_SF(FMT_DITHER_RAND_R_SEED, FMT_RAND_R_SEED, mask_sh),\ OPP_SF(FMT_DITHER_RAND_G_SEED, FMT_RAND_G_SEED, mask_sh),\ OPP_SF(FMT_DITHER_RAND_B_SEED, FMT_RAND_B_SEED, mask_sh),\ - OPP_SF(FMT_BIT_DEPTH_CONTROL, FMT_TEMPORAL_DITHER_EN, mask_sh),\ OPP_SF(FMT_BIT_DEPTH_CONTROL, FMT_TEMPORAL_DITHER_RESET, mask_sh),\ OPP_SF(FMT_BIT_DEPTH_CONTROL, FMT_TEMPORAL_DITHER_OFFSET, mask_sh),\ OPP_SF(FMT_BIT_DEPTH_CONTROL, FMT_TEMPORAL_DITHER_DEPTH, mask_sh),\ diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_panel_cntl.c b/drivers/gpu/drm/amd/display/dc/dce/dce_panel_cntl.c index 761fdfc1f5bd04effe3edfae9f6ee6379d6726bb..e9233923586316349952ad5dc9914be364044a74 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dce_panel_cntl.c +++ b/drivers/gpu/drm/amd/display/dc/dce/dce_panel_cntl.c @@ -50,16 +50,16 @@ static unsigned int dce_get_16_bit_backlight_from_pwm(struct panel_cntl *panel_c { uint64_t current_backlight; uint32_t round_result; - uint32_t pwm_period_cntl, bl_period, bl_int_count; - uint32_t bl_pwm_cntl, bl_pwm, fractional_duty_cycle_en; + uint32_t bl_period, bl_int_count; + uint32_t bl_pwm, fractional_duty_cycle_en; uint32_t bl_period_mask, bl_pwm_mask; struct dce_panel_cntl *dce_panel_cntl = TO_DCE_PANEL_CNTL(panel_cntl); - pwm_period_cntl = REG_READ(BL_PWM_PERIOD_CNTL); + REG_READ(BL_PWM_PERIOD_CNTL); REG_GET(BL_PWM_PERIOD_CNTL, BL_PWM_PERIOD, &bl_period); REG_GET(BL_PWM_PERIOD_CNTL, BL_PWM_PERIOD_BITCNT, &bl_int_count); - bl_pwm_cntl = REG_READ(BL_PWM_CNTL); + REG_READ(BL_PWM_CNTL); REG_GET(BL_PWM_CNTL, BL_ACTIVE_INT_FRAC_CNT, (uint32_t *)(&bl_pwm)); REG_GET(BL_PWM_CNTL, BL_PWM_FRACTIONAL_EN, &fractional_duty_cycle_en); diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_stream_encoder.c b/drivers/gpu/drm/amd/display/dc/dce/dce_stream_encoder.c index ada57f745fd764e87a325327446ce7c9b887778d..8d4263da59f2542c2df8f630dbe176ca9f82970c 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dce_stream_encoder.c +++ b/drivers/gpu/drm/amd/display/dc/dce/dce_stream_encoder.c @@ -67,7 +67,6 @@ static void dce110_update_generic_info_packet( uint32_t packet_index, const struct dc_info_packet *info_packet) { - uint32_t regval; /* TODOFPGA Figure out a proper number for max_retries polling for lock * use 50 for now. */ @@ -99,7 +98,7 @@ static void dce110_update_generic_info_packet( } /* choose which generic packet to use */ { - regval = REG_READ(AFMT_VBI_PACKET_CONTROL); + REG_READ(AFMT_VBI_PACKET_CONTROL); REG_UPDATE(AFMT_VBI_PACKET_CONTROL, AFMT_GENERIC_INDEX, packet_index); } @@ -564,6 +563,7 @@ static void dce110_stream_encoder_hdmi_set_stream_attribute( cntl.enable_dp_audio = enable_audio; cntl.pixel_clock = actual_pix_clk_khz; cntl.lanes_number = LANE_COUNT_FOUR; + cntl.color_depth = crtc_timing->display_color_depth; if (enc110->base.bp->funcs->encoder_control( enc110->base.bp, &cntl) != BP_RESULT_OK) diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_transform.c b/drivers/gpu/drm/amd/display/dc/dce/dce_transform.c index 130a0a0c8332918de5ce9541cad2cf4ca2537604..151dc7bf6d235676f196c2ca5837455499ffdfb6 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dce_transform.c +++ b/drivers/gpu/drm/amd/display/dc/dce/dce_transform.c @@ -493,7 +493,6 @@ static void dce60_transform_set_scaler( { struct dce_transform *xfm_dce = TO_DCE_TRANSFORM(xfm); bool is_scaling_required; - bool filter_updated = false; const uint16_t *coeffs_v, *coeffs_h; /*Use whole line buffer memory always*/ @@ -558,7 +557,6 @@ static void dce60_transform_set_scaler( xfm_dce->filter_v = coeffs_v; xfm_dce->filter_h = coeffs_h; - filter_updated = true; } } @@ -601,12 +599,12 @@ static void set_clamp( clamp_max = 0x3FC0; break; case COLOR_DEPTH_101010: - /* 10bit MSB aligned on 14 bit bus '11 1111 1111 1100' */ - clamp_max = 0x3FFC; + /* 10bit MSB aligned on 14 bit bus '11 1111 1111 0000' */ + clamp_max = 0x3FF0; break; case COLOR_DEPTH_121212: - /* 12bit MSB aligned on 14 bit bus '11 1111 1111 1111' */ - clamp_max = 0x3FFF; + /* 12bit MSB aligned on 14 bit bus '11 1111 1111 1100' */ + clamp_max = 0x3FFC; break; default: clamp_max = 0x3FC0; @@ -1037,34 +1035,23 @@ static void dce60_transform_set_pixel_storage_depth( const struct bit_depth_reduction_params *bit_depth_params) { struct dce_transform *xfm_dce = TO_DCE_TRANSFORM(xfm); - int pixel_depth, expan_mode; enum dc_color_depth color_depth; switch (depth) { case LB_PIXEL_DEPTH_18BPP: color_depth = COLOR_DEPTH_666; - pixel_depth = 2; - expan_mode = 1; break; case LB_PIXEL_DEPTH_24BPP: color_depth = COLOR_DEPTH_888; - pixel_depth = 1; - expan_mode = 1; break; case LB_PIXEL_DEPTH_30BPP: color_depth = COLOR_DEPTH_101010; - pixel_depth = 0; - expan_mode = 1; break; case LB_PIXEL_DEPTH_36BPP: color_depth = COLOR_DEPTH_121212; - pixel_depth = 3; - expan_mode = 0; break; default: color_depth = COLOR_DEPTH_101010; - pixel_depth = 0; - expan_mode = 1; BREAK_TO_DEBUGGER(); break; } @@ -1113,7 +1100,7 @@ static void program_gamut_remap( } -/** +/* ***************************************************************************** * Function: dal_transform_wide_gamut_set_gamut_remap * diff --git a/drivers/gpu/drm/amd/display/dc/dce/dmub_abm.c b/drivers/gpu/drm/amd/display/dc/dce/dmub_abm.c index 0cf130dc4e520cae0310388b4290d12fbd6777df..453aaa5757bd003b90610dd331b051f858350c16 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dmub_abm.c +++ b/drivers/gpu/drm/amd/display/dc/dce/dmub_abm.c @@ -57,6 +57,7 @@ static void dmub_abm_enable_fractional_pwm(struct dc_context *dc) union dmub_rb_cmd cmd; uint32_t fractional_pwm = (dc->dc->config.disable_fractional_pwm == false) ? 1 : 0; + memset(&cmd, 0, sizeof(cmd)); cmd.abm_set_pwm_frac.header.type = DMUB_CMD__ABM; cmd.abm_set_pwm_frac.header.sub_type = DMUB_CMD__ABM_SET_PWM_FRAC; cmd.abm_set_pwm_frac.abm_set_pwm_frac_data.fractional_pwm = fractional_pwm; @@ -135,6 +136,7 @@ static bool dmub_abm_set_level(struct abm *abm, uint32_t level) union dmub_rb_cmd cmd; struct dc_context *dc = abm->ctx; + memset(&cmd, 0, sizeof(cmd)); cmd.abm_set_level.header.type = DMUB_CMD__ABM; cmd.abm_set_level.header.sub_type = DMUB_CMD__ABM_SET_LEVEL; cmd.abm_set_level.abm_set_level_data.level = level; @@ -160,6 +162,7 @@ static bool dmub_abm_init_config(struct abm *abm, // Copy iramtable into cw7 memcpy(dc->dmub_srv->dmub->scratch_mem_fb.cpu_addr, (void *)src, bytes); + memset(&cmd, 0, sizeof(cmd)); // Fw will copy from cw7 to fw_state cmd.abm_init_config.header.type = DMUB_CMD__ABM; cmd.abm_init_config.header.sub_type = DMUB_CMD__ABM_INIT_CONFIG; diff --git a/drivers/gpu/drm/amd/display/dc/dce/dmub_hw_lock_mgr.c b/drivers/gpu/drm/amd/display/dc/dce/dmub_hw_lock_mgr.c index d399270fd17e05c33792483f37d2d5e3fc5294ce..c97ee5abc0efd9f442964bff80857a04d78d132c 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dmub_hw_lock_mgr.c +++ b/drivers/gpu/drm/amd/display/dc/dce/dmub_hw_lock_mgr.c @@ -33,8 +33,9 @@ void dmub_hw_lock_mgr_cmd(struct dc_dmub_srv *dmub_srv, union dmub_hw_lock_flags *hw_locks, struct dmub_hw_lock_inst_flags *inst_flags) { - union dmub_rb_cmd cmd = { 0 }; + union dmub_rb_cmd cmd; + memset(&cmd, 0, sizeof(cmd)); cmd.lock_hw.header.type = DMUB_CMD__HW_LOCK; cmd.lock_hw.header.sub_type = 0; cmd.lock_hw.header.payload_bytes = sizeof(struct dmub_cmd_lock_hw_data); diff --git a/drivers/gpu/drm/amd/display/dc/dce/dmub_psr.c b/drivers/gpu/drm/amd/display/dc/dce/dmub_psr.c index 17e84f34ceba1d8c708670ffb3c63f9c4702f7df..69e34bef274c1a28c177e0c5824247e2fc3524be 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dmub_psr.c +++ b/drivers/gpu/drm/amd/display/dc/dce/dmub_psr.c @@ -31,7 +31,7 @@ #define MAX_PIPES 6 -/** +/* * Convert dmcub psr state to dmcu psr state. */ static enum dc_psr_state convert_psr_state(uint32_t raw_state) @@ -74,7 +74,7 @@ static enum dc_psr_state convert_psr_state(uint32_t raw_state) return state; } -/** +/* * Get PSR state from firmware. */ static void dmub_psr_get_state(struct dmub_psr *dmub, enum dc_psr_state *state) @@ -90,7 +90,7 @@ static void dmub_psr_get_state(struct dmub_psr *dmub, enum dc_psr_state *state) *state = convert_psr_state(raw_state); } -/** +/* * Set PSR version. */ static bool dmub_psr_set_version(struct dmub_psr *dmub, struct dc_stream_state *stream) @@ -101,6 +101,7 @@ static bool dmub_psr_set_version(struct dmub_psr *dmub, struct dc_stream_state * if (stream->link->psr_settings.psr_version == DC_PSR_VERSION_UNSUPPORTED) return false; + memset(&cmd, 0, sizeof(cmd)); cmd.psr_set_version.header.type = DMUB_CMD__PSR; cmd.psr_set_version.header.sub_type = DMUB_CMD__PSR_SET_VERSION; switch (stream->link->psr_settings.psr_version) { @@ -121,7 +122,7 @@ static bool dmub_psr_set_version(struct dmub_psr *dmub, struct dc_stream_state * return true; } -/** +/* * Enable/Disable PSR. */ static void dmub_psr_enable(struct dmub_psr *dmub, bool enable, bool wait) @@ -131,7 +132,7 @@ static void dmub_psr_enable(struct dmub_psr *dmub, bool enable, bool wait) uint32_t retry_count; enum dc_psr_state state = PSR_STATE0; - + memset(&cmd, 0, sizeof(cmd)); cmd.psr_enable.header.type = DMUB_CMD__PSR; if (enable) @@ -170,7 +171,7 @@ static void dmub_psr_enable(struct dmub_psr *dmub, bool enable, bool wait) } } -/** +/* * Set PSR level. */ static void dmub_psr_set_level(struct dmub_psr *dmub, uint16_t psr_level) @@ -184,6 +185,7 @@ static void dmub_psr_set_level(struct dmub_psr *dmub, uint16_t psr_level) if (state == PSR_STATE0) return; + memset(&cmd, 0, sizeof(cmd)); cmd.psr_set_level.header.type = DMUB_CMD__PSR; cmd.psr_set_level.header.sub_type = DMUB_CMD__PSR_SET_LEVEL; cmd.psr_set_level.header.payload_bytes = sizeof(struct dmub_cmd_psr_set_level_data); @@ -194,7 +196,7 @@ static void dmub_psr_set_level(struct dmub_psr *dmub, uint16_t psr_level) dc_dmub_srv_wait_idle(dc->dmub_srv); } -/** +/* * Setup PSR by programming phy registers and sending psr hw context values to firmware. */ static bool dmub_psr_copy_settings(struct dmub_psr *dmub, @@ -233,6 +235,7 @@ static bool dmub_psr_copy_settings(struct dmub_psr *dmub, link->link_enc->funcs->psr_program_secondary_packet(link->link_enc, psr_context->sdpTransmitLineNumDeadline); + memset(&cmd, 0, sizeof(cmd)); cmd.psr_copy_settings.header.type = DMUB_CMD__PSR; cmd.psr_copy_settings.header.sub_type = DMUB_CMD__PSR_COPY_SETTINGS; cmd.psr_copy_settings.header.payload_bytes = sizeof(struct dmub_cmd_psr_copy_settings_data); @@ -277,7 +280,7 @@ static bool dmub_psr_copy_settings(struct dmub_psr *dmub, return true; } -/** +/* * Send command to PSR to force static ENTER and ignore all state changes until exit */ static void dmub_psr_force_static(struct dmub_psr *dmub) @@ -285,6 +288,7 @@ static void dmub_psr_force_static(struct dmub_psr *dmub) union dmub_rb_cmd cmd; struct dc_context *dc = dmub->ctx; + memset(&cmd, 0, sizeof(cmd)); cmd.psr_force_static.header.type = DMUB_CMD__PSR; cmd.psr_force_static.header.sub_type = DMUB_CMD__PSR_FORCE_STATIC; cmd.psr_enable.header.payload_bytes = 0; @@ -294,7 +298,7 @@ static void dmub_psr_force_static(struct dmub_psr *dmub) dc_dmub_srv_wait_idle(dc->dmub_srv); } -/** +/* * Get PSR residency from firmware. */ static void dmub_psr_get_residency(struct dmub_psr *dmub, uint32_t *residency) @@ -316,7 +320,7 @@ static const struct dmub_psr_funcs psr_funcs = { .psr_get_residency = dmub_psr_get_residency, }; -/** +/* * Construct PSR object. */ static void dmub_psr_construct(struct dmub_psr *psr, struct dc_context *ctx) @@ -325,7 +329,7 @@ static void dmub_psr_construct(struct dmub_psr *psr, struct dc_context *ctx) psr->funcs = &psr_funcs; } -/** +/* * Allocate and initialize PSR object. */ struct dmub_psr *dmub_psr_create(struct dc_context *ctx) @@ -342,7 +346,7 @@ struct dmub_psr *dmub_psr_create(struct dc_context *ctx) return psr; } -/** +/* * Deallocate PSR object. */ void dmub_psr_destroy(struct dmub_psr **dmub) diff --git a/drivers/gpu/drm/amd/display/dc/dce100/Makefile b/drivers/gpu/drm/amd/display/dc/dce100/Makefile index a822d4e2a1693881f090d776f6fbc11ab68d5f24..ff20c47f559e309fe0b2c7a689e129d091f49d66 100644 --- a/drivers/gpu/drm/amd/display/dc/dce100/Makefile +++ b/drivers/gpu/drm/amd/display/dc/dce100/Makefile @@ -23,6 +23,8 @@ # Makefile for the 'controller' sub-component of DAL. # It provides the control and status of HW CRTC block. +CFLAGS_$(AMDDALPATH)/dc/dce100/dce100_resource.o = $(call cc-disable-warning, override-init) + DCE100 = dce100_resource.o dce100_hw_sequencer.o AMD_DAL_DCE100 = $(addprefix $(AMDDALPATH)/dc/dce100/,$(DCE100)) diff --git a/drivers/gpu/drm/amd/display/dc/dce100/dce100_resource.c b/drivers/gpu/drm/amd/display/dc/dce100/dce100_resource.c index 8ab9d6c7980817e01e9edef156f6cf67080b00ef..635ef0e7c7826deb152b81329aba91b292ce4858 100644 --- a/drivers/gpu/drm/amd/display/dc/dce100/dce100_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dce100/dce100_resource.c @@ -58,6 +58,8 @@ #include "dce/dce_abm.h" #include "dce/dce_i2c.h" +#include "dce100_resource.h" + #ifndef mmMC_HUB_RDREQ_DMIF_LIMIT #include "gmc/gmc_8_2_d.h" #include "gmc/gmc_8_2_sh_mask.h" @@ -385,7 +387,7 @@ static const struct dc_plane_cap plane_cap = { .pixel_format_support = { .argb8888 = true, .nv12 = false, - .fp16 = false + .fp16 = true }, .max_upscale_factor = { @@ -611,7 +613,7 @@ static const struct encoder_feature_support link_enc_feature = { .flags.bits.IS_TPS3_CAPABLE = true }; -struct link_encoder *dce100_link_encoder_create( +static struct link_encoder *dce100_link_encoder_create( const struct encoder_init_data *enc_init_data) { struct dce110_link_encoder *enc110 = @@ -650,7 +652,7 @@ static struct panel_cntl *dce100_panel_cntl_create(const struct panel_cntl_init_ return &panel_cntl->base; } -struct output_pixel_processor *dce100_opp_create( +static struct output_pixel_processor *dce100_opp_create( struct dc_context *ctx, uint32_t inst) { @@ -665,7 +667,7 @@ struct output_pixel_processor *dce100_opp_create( return &opp->base; } -struct dce_aux *dce100_aux_engine_create( +static struct dce_aux *dce100_aux_engine_create( struct dc_context *ctx, uint32_t inst) { @@ -703,7 +705,7 @@ static const struct dce_i2c_mask i2c_masks = { I2C_COMMON_MASK_SH_LIST_DCE_COMMON_BASE(_MASK) }; -struct dce_i2c_hw *dce100_i2c_hw_create( +static struct dce_i2c_hw *dce100_i2c_hw_create( struct dc_context *ctx, uint32_t inst) { @@ -718,7 +720,7 @@ struct dce_i2c_hw *dce100_i2c_hw_create( return dce_i2c_hw; } -struct clock_source *dce100_clock_source_create( +static struct clock_source *dce100_clock_source_create( struct dc_context *ctx, struct dc_bios *bios, enum clock_source_id id, @@ -742,7 +744,7 @@ struct clock_source *dce100_clock_source_create( return NULL; } -void dce100_clock_source_destroy(struct clock_source **clk_src) +static void dce100_clock_source_destroy(struct clock_source **clk_src) { kfree(TO_DCE110_CLK_SRC(*clk_src)); *clk_src = NULL; @@ -831,7 +833,7 @@ static enum dc_status build_mapped_resource( return DC_OK; } -bool dce100_validate_bandwidth( +static bool dce100_validate_bandwidth( struct dc *dc, struct dc_state *context, bool fast_validate) @@ -876,7 +878,7 @@ static bool dce100_validate_surface_sets( return true; } -enum dc_status dce100_validate_global( +static enum dc_status dce100_validate_global( struct dc *dc, struct dc_state *context) { diff --git a/drivers/gpu/drm/amd/display/dc/dce110/Makefile b/drivers/gpu/drm/amd/display/dc/dce110/Makefile index d564c0eb8b045393ae235c6169b6c85b6af94693..84ab48df0c26178c47a1aea3529657fbad19a4ce 100644 --- a/drivers/gpu/drm/amd/display/dc/dce110/Makefile +++ b/drivers/gpu/drm/amd/display/dc/dce110/Makefile @@ -23,6 +23,8 @@ # Makefile for the 'controller' sub-component of DAL. # It provides the control and status of HW CRTC block. +CFLAGS_$(AMDDALPATH)/dc/dce110/dce110_resource.o = $(call cc-disable-warning, override-init) + DCE110 = dce110_timing_generator.o \ dce110_compressor.o dce110_hw_sequencer.o dce110_resource.o \ dce110_opp_regamma_v.o dce110_opp_csc_v.o dce110_timing_generator_v.o \ diff --git a/drivers/gpu/drm/amd/display/dc/dce110/dce110_compressor.c b/drivers/gpu/drm/amd/display/dc/dce110/dce110_compressor.c index 72b580a4eb856830b278cd917e56f3cd36453ae9..44564a4742b52331cea63bd7ae3143623edfe4cc 100644 --- a/drivers/gpu/drm/amd/display/dc/dce110/dce110_compressor.c +++ b/drivers/gpu/drm/amd/display/dc/dce110/dce110_compressor.c @@ -412,36 +412,6 @@ void dce110_compressor_destroy(struct compressor **compressor) *compressor = NULL; } -bool dce110_get_required_compressed_surfacesize(struct fbc_input_info fbc_input_info, - struct fbc_requested_compressed_size size) -{ - bool result = false; - - unsigned int max_x = FBC_MAX_X, max_y = FBC_MAX_Y; - - get_max_support_fbc_buffersize(&max_x, &max_y); - - if (fbc_input_info.dynamic_fbc_buffer_alloc == 0) { - /* - * For DCE11 here use Max HW supported size: HW Support up to 3840x2400 resolution - * or 18000 chunks. - */ - size.preferred_size = size.min_size = align_to_chunks_number_per_line(max_x) * max_y * 4; /* (For FBC when LPT not supported). */ - size.preferred_size_alignment = size.min_size_alignment = 0x100; /* For FBC when LPT not supported */ - size.bits.preferred_must_be_framebuffer_pool = 1; - size.bits.min_must_be_framebuffer_pool = 1; - - result = true; - } - /* - * Maybe to add registry key support with optional size here to override above - * for debugging purposes - */ - - return result; -} - - void get_max_support_fbc_buffersize(unsigned int *max_x, unsigned int *max_y) { *max_x = FBC_MAX_X; @@ -455,31 +425,6 @@ void get_max_support_fbc_buffersize(unsigned int *max_x, unsigned int *max_y) */ } - -unsigned int controller_id_to_index(enum controller_id controller_id) -{ - unsigned int index = 0; - - switch (controller_id) { - case CONTROLLER_ID_D0: - index = 0; - break; - case CONTROLLER_ID_D1: - index = 1; - break; - case CONTROLLER_ID_D2: - index = 2; - break; - case CONTROLLER_ID_D3: - index = 3; - break; - default: - break; - } - return index; -} - - static const struct compressor_funcs dce110_compressor_funcs = { .power_up_fbc = dce110_compressor_power_up_fbc, .enable_fbc = dce110_compressor_enable_fbc, diff --git a/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.c b/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.c index 4c230f1de9a30e7daf1c1c828036213aa4cdb7c8..caee1c9f54bdb2c873913b5632efc2fc9de384fa 100644 --- a/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.c +++ b/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.c @@ -921,6 +921,37 @@ void dce110_edp_power_control( } } +void dce110_edp_wait_for_T12( + struct dc_link *link) +{ + struct dc_context *ctx = link->ctx; + + if (dal_graphics_object_id_get_connector_id(link->link_enc->connector) + != CONNECTOR_ID_EDP) { + BREAK_TO_DEBUGGER(); + return; + } + + if (!link->panel_cntl) + return; + + if (!link->panel_cntl->funcs->is_panel_powered_on(link->panel_cntl) && + link->link_trace.time_stamp.edp_poweroff != 0) { + unsigned int t12_duration = 500; // Default T12 as per spec + unsigned long long current_ts = dm_get_timestamp(ctx); + unsigned long long time_since_edp_poweroff_ms = + div64_u64(dm_get_elapse_time_in_ns( + ctx, + current_ts, + link->link_trace.time_stamp.edp_poweroff), 1000000); + + t12_duration += link->local_sink->edid_caps.panel_patch.extra_t12_ms; // Add extra T12 + + if (time_since_edp_poweroff_ms < t12_duration) + msleep(t12_duration - time_since_edp_poweroff_ms); + } +} + /*todo: cloned in stream enc, fix*/ /* * @brief @@ -1628,7 +1659,7 @@ static struct dc_link *get_edp_link_with_sink( return link; } -/** +/* * When ASIC goes from VBIOS/VGA mode to driver/accelerated mode we need: * 1. Power down all DC HW blocks * 2. Disable VGA engine on all controllers diff --git a/drivers/gpu/drm/amd/display/dc/dce110/dce110_mem_input_v.c b/drivers/gpu/drm/amd/display/dc/dce110/dce110_mem_input_v.c index d54172d88f5f3b826f917560a286b2f6d75a5579..8bbb499067f747a168178e54bb5859200dd8d65f 100644 --- a/drivers/gpu/drm/amd/display/dc/dce110/dce110_mem_input_v.c +++ b/drivers/gpu/drm/amd/display/dc/dce110/dce110_mem_input_v.c @@ -34,6 +34,7 @@ #include "inc/dce_calcs.h" #include "dce/dce_mem_input.h" +#include "dce110_mem_input_v.h" static void set_flip_control( struct dce_mem_input *mem_input110, @@ -468,7 +469,7 @@ static void program_pixel_format( } } -bool dce_mem_input_v_is_surface_pending(struct mem_input *mem_input) +static bool dce_mem_input_v_is_surface_pending(struct mem_input *mem_input) { struct dce_mem_input *mem_input110 = TO_DCE_MEM_INPUT(mem_input); uint32_t value; @@ -483,7 +484,7 @@ bool dce_mem_input_v_is_surface_pending(struct mem_input *mem_input) return false; } -bool dce_mem_input_v_program_surface_flip_and_addr( +static bool dce_mem_input_v_program_surface_flip_and_addr( struct mem_input *mem_input, const struct dc_plane_address *address, bool flip_immediate) @@ -560,7 +561,7 @@ static const unsigned int *get_dvmm_hw_setting( } } -void dce_mem_input_v_program_pte_vm( +static void dce_mem_input_v_program_pte_vm( struct mem_input *mem_input, enum surface_pixel_format format, union dc_tiling_info *tiling_info, @@ -633,7 +634,7 @@ void dce_mem_input_v_program_pte_vm( dm_write_reg(mem_input110->base.ctx, mmUNP_DVMM_PTE_ARB_CONTROL_C, value); } -void dce_mem_input_v_program_surface_config( +static void dce_mem_input_v_program_surface_config( struct mem_input *mem_input, enum surface_pixel_format format, union dc_tiling_info *tiling_info, @@ -919,7 +920,7 @@ static void program_nbp_watermark_c( marks); } -void dce_mem_input_v_program_display_marks( +static void dce_mem_input_v_program_display_marks( struct mem_input *mem_input, struct dce_watermarks nbp, struct dce_watermarks stutter, @@ -942,7 +943,7 @@ void dce_mem_input_v_program_display_marks( } -void dce_mem_input_program_chroma_display_marks( +static void dce_mem_input_program_chroma_display_marks( struct mem_input *mem_input, struct dce_watermarks nbp, struct dce_watermarks stutter, @@ -963,7 +964,7 @@ void dce_mem_input_program_chroma_display_marks( stutter); } -void dce110_allocate_mem_input_v( +static void dce110_allocate_mem_input_v( struct mem_input *mi, uint32_t h_total,/* for current stream */ uint32_t v_total,/* for current stream */ @@ -1005,7 +1006,7 @@ void dce110_allocate_mem_input_v( } -void dce110_free_mem_input_v( +static void dce110_free_mem_input_v( struct mem_input *mi, uint32_t total_stream_num) { diff --git a/drivers/gpu/drm/amd/display/dc/dce110/dce110_resource.c b/drivers/gpu/drm/amd/display/dc/dce110/dce110_resource.c index 3f63822b8e28c6f7b696eb938ce952b298d46a7e..d7fcc5cccdce11a6f7a8d65f3db108916754618d 100644 --- a/drivers/gpu/drm/amd/display/dc/dce110/dce110_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dce110/dce110_resource.c @@ -410,7 +410,7 @@ static const struct dc_plane_cap plane_cap = { .pixel_format_support = { .argb8888 = true, .nv12 = false, - .fp16 = false + .fp16 = true }, .max_upscale_factor = { @@ -715,7 +715,7 @@ static struct output_pixel_processor *dce110_opp_create( return &opp->base; } -struct dce_aux *dce110_aux_engine_create( +static struct dce_aux *dce110_aux_engine_create( struct dc_context *ctx, uint32_t inst) { @@ -753,7 +753,7 @@ static const struct dce_i2c_mask i2c_masks = { I2C_COMMON_MASK_SH_LIST_DCE110(_MASK) }; -struct dce_i2c_hw *dce110_i2c_hw_create( +static struct dce_i2c_hw *dce110_i2c_hw_create( struct dc_context *ctx, uint32_t inst) { @@ -768,7 +768,7 @@ struct dce_i2c_hw *dce110_i2c_hw_create( return dce_i2c_hw; } -struct clock_source *dce110_clock_source_create( +static struct clock_source *dce110_clock_source_create( struct dc_context *ctx, struct dc_bios *bios, enum clock_source_id id, @@ -792,7 +792,7 @@ struct clock_source *dce110_clock_source_create( return NULL; } -void dce110_clock_source_destroy(struct clock_source **clk_src) +static void dce110_clock_source_destroy(struct clock_source **clk_src) { struct dce110_clk_src *dce110_clk_src; @@ -1034,8 +1034,8 @@ static bool dce110_validate_bandwidth( return result; } -enum dc_status dce110_validate_plane(const struct dc_plane_state *plane_state, - struct dc_caps *caps) +static enum dc_status dce110_validate_plane(const struct dc_plane_state *plane_state, + struct dc_caps *caps) { if (((plane_state->dst_rect.width * 2) < plane_state->src_rect.width) || ((plane_state->dst_rect.height * 2) < plane_state->src_rect.height)) @@ -1089,7 +1089,7 @@ static bool dce110_validate_surface_sets( return true; } -enum dc_status dce110_validate_global( +static enum dc_status dce110_validate_global( struct dc *dc, struct dc_state *context) { @@ -1272,7 +1272,6 @@ static bool underlay_create(struct dc_context *ctx, struct resource_pool *pool) /* update the public caps to indicate an underlay is available */ ctx->dc->caps.max_slave_planes = 1; - ctx->dc->caps.max_slave_planes = 1; return true; } @@ -1333,7 +1332,7 @@ static void bw_calcs_data_update_from_pplib(struct dc *dc) 1000); } -const struct resource_caps *dce110_resource_cap( +static const struct resource_caps *dce110_resource_cap( struct hw_asic_id *asic_id) { if (ASIC_REV_IS_STONEY(asic_id->hw_internal_rev)) diff --git a/drivers/gpu/drm/amd/display/dc/dce110/dce110_timing_generator.c b/drivers/gpu/drm/amd/display/dc/dce110/dce110_timing_generator.c index 1ea7db8eeb988469fd13fe4d3ca4be2f94cf1a03..d88a74559edd75ae59a10b248aefd95cc2455460 100644 --- a/drivers/gpu/drm/amd/display/dc/dce110/dce110_timing_generator.c +++ b/drivers/gpu/drm/amd/display/dc/dce110/dce110_timing_generator.c @@ -75,7 +75,7 @@ static void dce110_timing_generator_apply_front_porch_workaround( } } -/** +/* ***************************************************************************** * Function: is_in_vertical_blank * @@ -116,7 +116,7 @@ void dce110_timing_generator_set_early_control( dm_write_reg(tg->ctx, address, regval); } -/** +/* * Enable CRTC * Enable CRTC - call ASIC Control Object to enable Timing generator. */ @@ -175,7 +175,7 @@ void dce110_timing_generator_program_blank_color( dm_write_reg(tg->ctx, addr, value); } -/** +/* ***************************************************************************** * Function: disable_stereo * @@ -226,7 +226,7 @@ static void disable_stereo(struct timing_generator *tg) } #endif -/** +/* * disable_crtc - call ASIC Control Object to disable Timing generator. */ bool dce110_timing_generator_disable_crtc(struct timing_generator *tg) @@ -247,11 +247,10 @@ bool dce110_timing_generator_disable_crtc(struct timing_generator *tg) return result == BP_RESULT_OK; } -/** -* program_horz_count_by_2 -* Programs DxCRTC_HORZ_COUNT_BY2_EN - 1 for DVI 30bpp mode, 0 otherwise -* -*/ +/* + * program_horz_count_by_2 + * Programs DxCRTC_HORZ_COUNT_BY2_EN - 1 for DVI 30bpp mode, 0 otherwise + */ static void program_horz_count_by_2( struct timing_generator *tg, const struct dc_crtc_timing *timing) @@ -273,7 +272,7 @@ static void program_horz_count_by_2( CRTC_REG(mmCRTC_COUNT_CONTROL), regval); } -/** +/* * program_timing_generator * Program CRTC Timing Registers - DxCRTC_H_*, DxCRTC_V_*, Pixel repetition. * Call ASIC Control Object to program Timings. @@ -352,7 +351,7 @@ bool dce110_timing_generator_program_timing_generator( return result == BP_RESULT_OK; } -/** +/* ***************************************************************************** * Function: set_drr * @@ -521,7 +520,7 @@ uint32_t dce110_timing_generator_get_vblank_counter(struct timing_generator *tg) return field; } -/** +/* ***************************************************************************** * Function: dce110_timing_generator_get_position * @@ -557,7 +556,7 @@ void dce110_timing_generator_get_position(struct timing_generator *tg, CRTC_VERT_COUNT_NOM); } -/** +/* ***************************************************************************** * Function: get_crtc_scanoutpos * @@ -1106,11 +1105,11 @@ void dce110_timing_generator_set_test_pattern( } } -/** -* dce110_timing_generator_validate_timing -* The timing generators support a maximum display size of is 8192 x 8192 pixels, -* including both active display and blanking periods. Check H Total and V Total. -*/ +/* + * dce110_timing_generator_validate_timing + * The timing generators support a maximum display size of is 8192 x 8192 pixels, + * including both active display and blanking periods. Check H Total and V Total. + */ bool dce110_timing_generator_validate_timing( struct timing_generator *tg, const struct dc_crtc_timing *timing, @@ -1167,9 +1166,9 @@ bool dce110_timing_generator_validate_timing( return true; } -/** -* Wait till we are at the beginning of VBlank. -*/ +/* + * Wait till we are at the beginning of VBlank. + */ void dce110_timing_generator_wait_for_vblank(struct timing_generator *tg) { /* We want to catch beginning of VBlank here, so if the first try are @@ -1191,9 +1190,9 @@ void dce110_timing_generator_wait_for_vblank(struct timing_generator *tg) } } -/** -* Wait till we are in VActive (anywhere in VActive) -*/ +/* + * Wait till we are in VActive (anywhere in VActive) + */ void dce110_timing_generator_wait_for_vactive(struct timing_generator *tg) { while (dce110_timing_generator_is_in_vertical_blank(tg)) { @@ -1204,7 +1203,7 @@ void dce110_timing_generator_wait_for_vactive(struct timing_generator *tg) } } -/** +/* ***************************************************************************** * Function: dce110_timing_generator_setup_global_swap_lock * @@ -1215,7 +1214,6 @@ void dce110_timing_generator_wait_for_vactive(struct timing_generator *tg) * @param [in] gsl_params: setup data ***************************************************************************** */ - void dce110_timing_generator_setup_global_swap_lock( struct timing_generator *tg, const struct dcp_gsl_params *gsl_params) @@ -1351,10 +1349,7 @@ void dce110_timing_generator_tear_down_global_swap_lock( /* Restore DCP_GSL_PURPOSE_SURFACE_FLIP */ { - uint32_t value_crtc_vtotal; - - value_crtc_vtotal = dm_read_reg(tg->ctx, - CRTC_REG(mmCRTC_V_TOTAL)); + dm_read_reg(tg->ctx, CRTC_REG(mmCRTC_V_TOTAL)); set_reg_field_value(value, 0, @@ -1385,7 +1380,7 @@ void dce110_timing_generator_tear_down_global_swap_lock( dm_write_reg(tg->ctx, address, value); } -/** +/* ***************************************************************************** * Function: is_counter_moving * @@ -1767,7 +1762,7 @@ void dce110_timing_generator_disable_reset_trigger( dm_write_reg(tg->ctx, CRTC_REG(mmCRTC_TRIGB_CNTL), value); } -/** +/* ***************************************************************************** * @brief * Checks whether CRTC triggered reset occurred @@ -1794,7 +1789,7 @@ bool dce110_timing_generator_did_triggered_reset_occur( return (force || vert_sync); } -/** +/* * dce110_timing_generator_disable_vga * Turn OFF VGA Mode and Timing - DxVGA_CONTROL * VGA Mode and VGA Timing is used by VBIOS on CRT Monitors; @@ -1840,14 +1835,13 @@ void dce110_timing_generator_disable_vga( dm_write_reg(tg->ctx, addr, value); } -/** -* set_overscan_color_black -* -* @param :black_color is one of the color space -* :this routine will set overscan black color according to the color space. -* @return none -*/ - +/* + * set_overscan_color_black + * + * @param :black_color is one of the color space + * :this routine will set overscan black color according to the color space. + * @return none + */ void dce110_timing_generator_set_overscan_color_black( struct timing_generator *tg, const struct tg_color *color) diff --git a/drivers/gpu/drm/amd/display/dc/dce110/dce110_timing_generator_v.c b/drivers/gpu/drm/amd/display/dc/dce110/dce110_timing_generator_v.c index a13a2f58944e3d99861405dbec30acf3cf85b0ef..c509384fff5439b22ba8484cf72f3ee176526732 100644 --- a/drivers/gpu/drm/amd/display/dc/dce110/dce110_timing_generator_v.c +++ b/drivers/gpu/drm/amd/display/dc/dce110/dce110_timing_generator_v.c @@ -46,17 +46,16 @@ * **********************************************************************************/ -/** -* Enable CRTCV -*/ +/* + * Enable CRTCV + */ static bool dce110_timing_generator_v_enable_crtc(struct timing_generator *tg) { /* -* Set MASTER_UPDATE_MODE to 0 -* This is needed for DRR, and also suggested to be default value by Syed. -*/ - + * Set MASTER_UPDATE_MODE to 0 + * This is needed for DRR, and also suggested to be default value by Syed. + */ uint32_t value; value = 0; @@ -209,9 +208,9 @@ static void dce110_timing_generator_v_wait_for_vblank(struct timing_generator *t } } -/** -* Wait till we are in VActive (anywhere in VActive) -*/ +/* + * Wait till we are in VActive (anywhere in VActive) + */ static void dce110_timing_generator_v_wait_for_vactive(struct timing_generator *tg) { while (dce110_timing_generator_v_is_in_vertical_blank(tg)) { diff --git a/drivers/gpu/drm/amd/display/dc/dce110/dce110_transform_v.c b/drivers/gpu/drm/amd/display/dc/dce110/dce110_transform_v.c index b1aaab5590cc6cd185dd9adec41f83dfe4cf765b..29438c6050dbbf5529f88ea05545d9d69f34eb0f 100644 --- a/drivers/gpu/drm/amd/display/dc/dce110/dce110_transform_v.c +++ b/drivers/gpu/drm/amd/display/dc/dce110/dce110_transform_v.c @@ -217,16 +217,15 @@ static bool setup_scaling_configuration( return is_scaling_needed; } -/** -* Function: -* void program_overscan -* -* Purpose: Programs overscan border -* Input: overscan -* -* Output: - void -*/ +/* + * Function: + * void program_overscan + * + * Purpose: Programs overscan border + * Input: overscan + * + * Output: void + */ static void program_overscan( struct dce_transform *xfm_dce, const struct scaler_data *data) diff --git a/drivers/gpu/drm/amd/display/dc/dce112/Makefile b/drivers/gpu/drm/amd/display/dc/dce112/Makefile index 8e090446d511937361db77ed56471a44d6ed897b..9de6501702d2c559e5ad6f195ce8c4aa20be662f 100644 --- a/drivers/gpu/drm/amd/display/dc/dce112/Makefile +++ b/drivers/gpu/drm/amd/display/dc/dce112/Makefile @@ -23,6 +23,8 @@ # Makefile for the 'controller' sub-component of DAL. # It provides the control and status of HW CRTC block. +CFLAGS_$(AMDDALPATH)/dc/dce112/dce112_resource.o = $(call cc-disable-warning, override-init) + DCE112 = dce112_compressor.o dce112_hw_sequencer.o \ dce112_resource.o diff --git a/drivers/gpu/drm/amd/display/dc/dce112/dce112_resource.c b/drivers/gpu/drm/amd/display/dc/dce112/dce112_resource.c index f99b1c0845908819ce7c28ee15189a70ea185c0f..ee55cda854bf41178db799e7cf6e152b15c99486 100644 --- a/drivers/gpu/drm/amd/display/dc/dce112/dce112_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dce112/dce112_resource.c @@ -59,7 +59,9 @@ #include "dce/dce_11_2_sh_mask.h" #include "dce100/dce100_resource.h" -#define DC_LOGGER \ +#include "dce112_resource.h" + +#define DC_LOGGER \ dc->ctx->logger #ifndef mmDP_DPHY_INTERNAL_CTRL @@ -617,7 +619,7 @@ static const struct encoder_feature_support link_enc_feature = { .flags.bits.IS_TPS4_CAPABLE = true }; -struct link_encoder *dce112_link_encoder_create( +static struct link_encoder *dce112_link_encoder_create( const struct encoder_init_data *enc_init_data) { struct dce110_link_encoder *enc110 = @@ -671,7 +673,7 @@ static struct input_pixel_processor *dce112_ipp_create( return &ipp->base; } -struct output_pixel_processor *dce112_opp_create( +static struct output_pixel_processor *dce112_opp_create( struct dc_context *ctx, uint32_t inst) { @@ -686,7 +688,7 @@ struct output_pixel_processor *dce112_opp_create( return &opp->base; } -struct dce_aux *dce112_aux_engine_create( +static struct dce_aux *dce112_aux_engine_create( struct dc_context *ctx, uint32_t inst) { @@ -724,7 +726,7 @@ static const struct dce_i2c_mask i2c_masks = { I2C_COMMON_MASK_SH_LIST_DCE110(_MASK) }; -struct dce_i2c_hw *dce112_i2c_hw_create( +static struct dce_i2c_hw *dce112_i2c_hw_create( struct dc_context *ctx, uint32_t inst) { @@ -739,7 +741,7 @@ struct dce_i2c_hw *dce112_i2c_hw_create( return dce_i2c_hw; } -struct clock_source *dce112_clock_source_create( +static struct clock_source *dce112_clock_source_create( struct dc_context *ctx, struct dc_bios *bios, enum clock_source_id id, @@ -763,7 +765,7 @@ struct clock_source *dce112_clock_source_create( return NULL; } -void dce112_clock_source_destroy(struct clock_source **clk_src) +static void dce112_clock_source_destroy(struct clock_source **clk_src) { kfree(TO_DCE110_CLK_SRC(*clk_src)); *clk_src = NULL; @@ -1024,7 +1026,7 @@ enum dc_status dce112_add_stream_to_ctx( return result; } -enum dc_status dce112_validate_global( +static enum dc_status dce112_validate_global( struct dc *dc, struct dc_state *context) { @@ -1202,7 +1204,7 @@ static void bw_calcs_data_update_from_pplib(struct dc *dc) dm_pp_notify_wm_clock_changes(dc->ctx, &clk_ranges); } -const struct resource_caps *dce112_resource_cap( +static const struct resource_caps *dce112_resource_cap( struct hw_asic_id *asic_id) { if (ASIC_REV_IS_POLARIS11_M(asic_id->hw_internal_rev) || diff --git a/drivers/gpu/drm/amd/display/dc/dce120/Makefile b/drivers/gpu/drm/amd/display/dc/dce120/Makefile index 37db1f8d45ea547862b07b2503c4a55d6485c8ff..a9cc4b73270bb22330fd1af9fe48940cfdeecbb8 100644 --- a/drivers/gpu/drm/amd/display/dc/dce120/Makefile +++ b/drivers/gpu/drm/amd/display/dc/dce120/Makefile @@ -24,6 +24,8 @@ # It provides the control and status of HW CRTC block. +CFLAGS_$(AMDDALPATH)/dc/dce120/dce120_resource.o = $(call cc-disable-warning, override-init) + DCE120 = dce120_resource.o dce120_timing_generator.o \ dce120_hw_sequencer.o diff --git a/drivers/gpu/drm/amd/display/dc/dce120/dce120_hw_sequencer.c b/drivers/gpu/drm/amd/display/dc/dce120/dce120_hw_sequencer.c index 66a13aa39c951ddcf696e6e44694c022d7b75f7c..d4afe6c824d2cb75974056f11ff21f874bdfb05b 100644 --- a/drivers/gpu/drm/amd/display/dc/dce120/dce120_hw_sequencer.c +++ b/drivers/gpu/drm/amd/display/dc/dce120/dce120_hw_sequencer.c @@ -50,6 +50,7 @@ struct dce120_hw_seq_reg_offsets { uint32_t crtc; }; +#if 0 static const struct dce120_hw_seq_reg_offsets reg_offsets[] = { { .crtc = (mmCRTC0_CRTC_GSL_CONTROL - mmCRTC0_CRTC_GSL_CONTROL), @@ -79,7 +80,6 @@ static const struct dce120_hw_seq_reg_offsets reg_offsets[] = { /******************************************************************************* * Private definitions ******************************************************************************/ -#if 0 static void dce120_init_pte(struct dc_context *ctx, uint8_t controller_id) { uint32_t addr; diff --git a/drivers/gpu/drm/amd/display/dc/dce120/dce120_resource.c b/drivers/gpu/drm/amd/display/dc/dce120/dce120_resource.c index f1e3d2888eacc2887e64663fad5dc11aaa2c0314..c65e4d125c8e2e00ac5ea12aaae62695c0f3b506 100644 --- a/drivers/gpu/drm/amd/display/dc/dce120/dce120_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dce120/dce120_resource.c @@ -423,7 +423,7 @@ static const struct dce110_clk_src_mask cs_mask = { CS_COMMON_MASK_SH_LIST_DCE_112(_MASK) }; -struct output_pixel_processor *dce120_opp_create( +static struct output_pixel_processor *dce120_opp_create( struct dc_context *ctx, uint32_t inst) { @@ -437,7 +437,7 @@ struct output_pixel_processor *dce120_opp_create( ctx, inst, &opp_regs[inst], &opp_shift, &opp_mask); return &opp->base; } -struct dce_aux *dce120_aux_engine_create( +static struct dce_aux *dce120_aux_engine_create( struct dc_context *ctx, uint32_t inst) { @@ -475,7 +475,7 @@ static const struct dce_i2c_mask i2c_masks = { I2C_COMMON_MASK_SH_LIST_DCE110(_MASK) }; -struct dce_i2c_hw *dce120_i2c_hw_create( +static struct dce_i2c_hw *dce120_i2c_hw_create( struct dc_context *ctx, uint32_t inst) { diff --git a/drivers/gpu/drm/amd/display/dc/dce120/dce120_timing_generator.c b/drivers/gpu/drm/amd/display/dc/dce120/dce120_timing_generator.c index 915fbb8e8168c54472f46ee35bc8978affbd834a..b57c466124e765c2543aba8bdd7cca253da148b6 100644 --- a/drivers/gpu/drm/amd/display/dc/dce120/dce120_timing_generator.c +++ b/drivers/gpu/drm/amd/display/dc/dce120/dce120_timing_generator.c @@ -69,7 +69,7 @@ #define CRTC_REG_SET_3(reg, field1, val1, field2, val2, field3, val3) \ CRTC_REG_SET_N(reg, 3, FD(reg##__##field1), val1, FD(reg##__##field2), val2, FD(reg##__##field3), val3) -/** +/* ***************************************************************************** * Function: is_in_vertical_blank * @@ -98,7 +98,7 @@ static bool dce120_timing_generator_is_in_vertical_blank( /* determine if given timing can be supported by TG */ -bool dce120_timing_generator_validate_timing( +static bool dce120_timing_generator_validate_timing( struct timing_generator *tg, const struct dc_crtc_timing *timing, enum signal_type signal) @@ -125,7 +125,7 @@ bool dce120_timing_generator_validate_timing( return true; } -bool dce120_tg_validate_timing(struct timing_generator *tg, +static bool dce120_tg_validate_timing(struct timing_generator *tg, const struct dc_crtc_timing *timing) { return dce120_timing_generator_validate_timing(tg, timing, SIGNAL_TYPE_NONE); @@ -133,7 +133,7 @@ bool dce120_tg_validate_timing(struct timing_generator *tg, /******** HW programming ************/ /* Disable/Enable Timing Generator */ -bool dce120_timing_generator_enable_crtc(struct timing_generator *tg) +static bool dce120_timing_generator_enable_crtc(struct timing_generator *tg) { enum bp_result result; struct dce110_timing_generator *tg110 = DCE110TG_FROM_TG(tg); @@ -153,7 +153,7 @@ bool dce120_timing_generator_enable_crtc(struct timing_generator *tg) return result == BP_RESULT_OK; } -void dce120_timing_generator_set_early_control( +static void dce120_timing_generator_set_early_control( struct timing_generator *tg, uint32_t early_cntl) { @@ -166,7 +166,7 @@ void dce120_timing_generator_set_early_control( /**************** TG current status ******************/ /* return the current frame counter. Used by Linux kernel DRM */ -uint32_t dce120_timing_generator_get_vblank_counter( +static uint32_t dce120_timing_generator_get_vblank_counter( struct timing_generator *tg) { struct dce110_timing_generator *tg110 = DCE110TG_FROM_TG(tg); @@ -181,7 +181,7 @@ uint32_t dce120_timing_generator_get_vblank_counter( } /* Get current H and V position */ -void dce120_timing_generator_get_crtc_position( +static void dce120_timing_generator_get_crtc_position( struct timing_generator *tg, struct crtc_position *position) { @@ -207,7 +207,7 @@ void dce120_timing_generator_get_crtc_position( } /* wait until TG is in beginning of vertical blank region */ -void dce120_timing_generator_wait_for_vblank(struct timing_generator *tg) +static void dce120_timing_generator_wait_for_vblank(struct timing_generator *tg) { /* We want to catch beginning of VBlank here, so if the first try are * in VBlank, we might be very close to Active, in this case wait for @@ -229,7 +229,7 @@ void dce120_timing_generator_wait_for_vblank(struct timing_generator *tg) } /* wait until TG is in beginning of active region */ -void dce120_timing_generator_wait_for_vactive(struct timing_generator *tg) +static void dce120_timing_generator_wait_for_vactive(struct timing_generator *tg) { while (dce120_timing_generator_is_in_vertical_blank(tg)) { if (!tg->funcs->is_counter_moving(tg)) { @@ -242,7 +242,7 @@ void dce120_timing_generator_wait_for_vactive(struct timing_generator *tg) /*********** Timing Generator Synchronization routines ****/ /* Setups Global Swap Lock group, TimingServer or TimingClient*/ -void dce120_timing_generator_setup_global_swap_lock( +static void dce120_timing_generator_setup_global_swap_lock( struct timing_generator *tg, const struct dcp_gsl_params *gsl_params) { @@ -279,7 +279,7 @@ void dce120_timing_generator_setup_global_swap_lock( } /* Clear all the register writes done by setup_global_swap_lock */ -void dce120_timing_generator_tear_down_global_swap_lock( +static void dce120_timing_generator_tear_down_global_swap_lock( struct timing_generator *tg) { struct dce110_timing_generator *tg110 = DCE110TG_FROM_TG(tg); @@ -300,7 +300,7 @@ void dce120_timing_generator_tear_down_global_swap_lock( } /* Reset slave controllers on master VSync */ -void dce120_timing_generator_enable_reset_trigger( +static void dce120_timing_generator_enable_reset_trigger( struct timing_generator *tg, int source) { @@ -347,7 +347,7 @@ void dce120_timing_generator_enable_reset_trigger( } /* disabling trigger-reset */ -void dce120_timing_generator_disable_reset_trigger( +static void dce120_timing_generator_disable_reset_trigger( struct timing_generator *tg) { struct dce110_timing_generator *tg110 = DCE110TG_FROM_TG(tg); @@ -367,7 +367,7 @@ void dce120_timing_generator_disable_reset_trigger( } /* Checks whether CRTC triggered reset occurred */ -bool dce120_timing_generator_did_triggered_reset_occur( +static bool dce120_timing_generator_did_triggered_reset_occur( struct timing_generator *tg) { struct dce110_timing_generator *tg110 = DCE110TG_FROM_TG(tg); @@ -384,7 +384,7 @@ bool dce120_timing_generator_did_triggered_reset_occur( /******** Stuff to move to other virtual HW objects *****************/ /* Move to enable accelerated mode */ -void dce120_timing_generator_disable_vga(struct timing_generator *tg) +static void dce120_timing_generator_disable_vga(struct timing_generator *tg) { uint32_t offset = 0; uint32_t value = 0; @@ -425,7 +425,7 @@ void dce120_timing_generator_disable_vga(struct timing_generator *tg) } /* TODO: Should we move it to transform */ /* Fully program CRTC timing in timing generator */ -void dce120_timing_generator_program_blanking( +static void dce120_timing_generator_program_blanking( struct timing_generator *tg, const struct dc_crtc_timing *timing) { @@ -485,7 +485,7 @@ void dce120_timing_generator_program_blanking( /* TODO: Should we move it to opp? */ /* Combine with below and move YUV/RGB color conversion to SW layer */ -void dce120_timing_generator_program_blank_color( +static void dce120_timing_generator_program_blank_color( struct timing_generator *tg, const struct tg_color *black_color) { @@ -498,7 +498,7 @@ void dce120_timing_generator_program_blank_color( CRTC_BLACK_COLOR_R_CR, black_color->color_r_cr); } /* Combine with above and move YUV/RGB color conversion to SW layer */ -void dce120_timing_generator_set_overscan_color_black( +static void dce120_timing_generator_set_overscan_color_black( struct timing_generator *tg, const struct tg_color *color) { @@ -540,7 +540,7 @@ void dce120_timing_generator_set_overscan_color_black( */ } -void dce120_timing_generator_set_drr( +static void dce120_timing_generator_set_drr( struct timing_generator *tg, const struct drr_params *params) { @@ -589,50 +589,7 @@ void dce120_timing_generator_set_drr( } } -/** - ***************************************************************************** - * Function: dce120_timing_generator_get_position - * - * @brief - * Returns CRTC vertical/horizontal counters - * - * @param [out] position - ***************************************************************************** - */ -void dce120_timing_generator_get_position(struct timing_generator *tg, - struct crtc_position *position) -{ - uint32_t value; - struct dce110_timing_generator *tg110 = DCE110TG_FROM_TG(tg); - - value = dm_read_reg_soc15( - tg->ctx, - mmCRTC0_CRTC_STATUS_POSITION, - tg110->offsets.crtc); - - position->horizontal_count = get_reg_field_value( - value, - CRTC0_CRTC_STATUS_POSITION, - CRTC_HORZ_COUNT); - - position->vertical_count = get_reg_field_value( - value, - CRTC0_CRTC_STATUS_POSITION, - CRTC_VERT_COUNT); - - value = dm_read_reg_soc15( - tg->ctx, - mmCRTC0_CRTC_NOM_VERT_POSITION, - tg110->offsets.crtc); - - position->nominal_vcount = get_reg_field_value( - value, - CRTC0_CRTC_NOM_VERT_POSITION, - CRTC_VERT_COUNT_NOM); -} - - -void dce120_timing_generator_get_crtc_scanoutpos( +static void dce120_timing_generator_get_crtc_scanoutpos( struct timing_generator *tg, uint32_t *v_blank_start, uint32_t *v_blank_end, @@ -661,7 +618,7 @@ void dce120_timing_generator_get_crtc_scanoutpos( *v_position = position.vertical_count; } -void dce120_timing_generator_enable_advanced_request( +static void dce120_timing_generator_enable_advanced_request( struct timing_generator *tg, bool enable, const struct dc_crtc_timing *timing) @@ -699,7 +656,7 @@ void dce120_timing_generator_enable_advanced_request( value); } -void dce120_tg_program_blank_color(struct timing_generator *tg, +static void dce120_tg_program_blank_color(struct timing_generator *tg, const struct tg_color *black_color) { struct dce110_timing_generator *tg110 = DCE110TG_FROM_TG(tg); @@ -722,7 +679,7 @@ void dce120_tg_program_blank_color(struct timing_generator *tg, value); } -void dce120_tg_set_overscan_color(struct timing_generator *tg, +static void dce120_tg_set_overscan_color(struct timing_generator *tg, const struct tg_color *overscan_color) { struct dce110_timing_generator *tg110 = DCE110TG_FROM_TG(tg); @@ -749,7 +706,7 @@ static void dce120_tg_program_timing(struct timing_generator *tg, dce120_timing_generator_program_blanking(tg, timing); } -bool dce120_tg_is_blanked(struct timing_generator *tg) +static bool dce120_tg_is_blanked(struct timing_generator *tg) { struct dce110_timing_generator *tg110 = DCE110TG_FROM_TG(tg); uint32_t value = dm_read_reg_soc15( @@ -770,7 +727,7 @@ bool dce120_tg_is_blanked(struct timing_generator *tg) return false; } -void dce120_tg_set_blank(struct timing_generator *tg, +static void dce120_tg_set_blank(struct timing_generator *tg, bool enable_blanking) { struct dce110_timing_generator *tg110 = DCE110TG_FROM_TG(tg); @@ -789,7 +746,7 @@ void dce120_tg_set_blank(struct timing_generator *tg, bool dce120_tg_validate_timing(struct timing_generator *tg, const struct dc_crtc_timing *timing); -void dce120_tg_wait_for_state(struct timing_generator *tg, +static void dce120_tg_wait_for_state(struct timing_generator *tg, enum crtc_state state) { switch (state) { @@ -806,7 +763,7 @@ void dce120_tg_wait_for_state(struct timing_generator *tg, } } -void dce120_tg_set_colors(struct timing_generator *tg, +static void dce120_tg_set_colors(struct timing_generator *tg, const struct tg_color *blank_color, const struct tg_color *overscan_color) { @@ -833,7 +790,7 @@ static void dce120_timing_generator_set_static_screen_control( CRTC_STATIC_SCREEN_FRAME_COUNT, num_frames); } -void dce120_timing_generator_set_test_pattern( +static void dce120_timing_generator_set_test_pattern( struct timing_generator *tg, /* TODO: replace 'controller_dp_test_pattern' by 'test_pattern_mode' * because this is not DP-specific (which is probably somewhere in DP diff --git a/drivers/gpu/drm/amd/display/dc/dce60/Makefile b/drivers/gpu/drm/amd/display/dc/dce60/Makefile index 7036c3bd0f871d8689af6a0a3b58cf247f9214f9..dda596fa1cd763e14ffb5f3e7cfe2c199d5a1344 100644 --- a/drivers/gpu/drm/amd/display/dc/dce60/Makefile +++ b/drivers/gpu/drm/amd/display/dc/dce60/Makefile @@ -23,6 +23,8 @@ # Makefile for the 'controller' sub-component of DAL. # It provides the control and status of HW CRTC block. +CFLAGS_AMDDALPATH)/dc/dce60/dce60_resource.o = $(call cc-disable-warning, override-init) + DCE60 = dce60_timing_generator.o dce60_hw_sequencer.o \ dce60_resource.o diff --git a/drivers/gpu/drm/amd/display/dc/dce60/dce60_resource.c b/drivers/gpu/drm/amd/display/dc/dce60/dce60_resource.c index e9dd78c484d6e5fa6cfcbedcce73c6a9908b55de..dcfa0a3efa00d17510d558d6e0ddb5bc883126c5 100644 --- a/drivers/gpu/drm/amd/display/dc/dce60/dce60_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dce60/dce60_resource.c @@ -60,6 +60,8 @@ #include "dce/dce_i2c.h" /* TODO remove this include */ +#include "dce60_resource.h" + #ifndef mmMC_HUB_RDREQ_DMIF_LIMIT #include "gmc/gmc_6_0_d.h" #include "gmc/gmc_6_0_sh_mask.h" @@ -519,7 +521,7 @@ static struct output_pixel_processor *dce60_opp_create( return &opp->base; } -struct dce_aux *dce60_aux_engine_create( +static struct dce_aux *dce60_aux_engine_create( struct dc_context *ctx, uint32_t inst) { @@ -557,7 +559,7 @@ static const struct dce_i2c_mask i2c_masks = { I2C_COMMON_MASK_SH_LIST_DCE_COMMON_BASE(_MASK) }; -struct dce_i2c_hw *dce60_i2c_hw_create( +static struct dce_i2c_hw *dce60_i2c_hw_create( struct dc_context *ctx, uint32_t inst) { @@ -573,7 +575,7 @@ struct dce_i2c_hw *dce60_i2c_hw_create( return dce_i2c_hw; } -struct dce_i2c_sw *dce60_i2c_sw_create( +static struct dce_i2c_sw *dce60_i2c_sw_create( struct dc_context *ctx) { struct dce_i2c_sw *dce_i2c_sw = @@ -707,7 +709,7 @@ static const struct encoder_feature_support link_enc_feature = { .flags.bits.IS_TPS3_CAPABLE = true }; -struct link_encoder *dce60_link_encoder_create( +static struct link_encoder *dce60_link_encoder_create( const struct encoder_init_data *enc_init_data) { struct dce110_link_encoder *enc110 = @@ -746,7 +748,7 @@ static struct panel_cntl *dce60_panel_cntl_create(const struct panel_cntl_init_d return &panel_cntl->base; } -struct clock_source *dce60_clock_source_create( +static struct clock_source *dce60_clock_source_create( struct dc_context *ctx, struct dc_bios *bios, enum clock_source_id id, @@ -770,7 +772,7 @@ struct clock_source *dce60_clock_source_create( return NULL; } -void dce60_clock_source_destroy(struct clock_source **clk_src) +static void dce60_clock_source_destroy(struct clock_source **clk_src) { kfree(TO_DCE110_CLK_SRC(*clk_src)); *clk_src = NULL; @@ -860,7 +862,7 @@ static void dce60_resource_destruct(struct dce110_resource_pool *pool) } } -bool dce60_validate_bandwidth( +static bool dce60_validate_bandwidth( struct dc *dc, struct dc_state *context, bool fast_validate) @@ -905,7 +907,7 @@ static bool dce60_validate_surface_sets( return true; } -enum dc_status dce60_validate_global( +static enum dc_status dce60_validate_global( struct dc *dc, struct dc_state *context) { diff --git a/drivers/gpu/drm/amd/display/dc/dce60/dce60_timing_generator.c b/drivers/gpu/drm/amd/display/dc/dce60/dce60_timing_generator.c index fc1af0ff0ca4c5de361121f72cb545108cb31eba..c1a85ee374d9dab850a9f102217f2b3cb2cad4a5 100644 --- a/drivers/gpu/drm/amd/display/dc/dce60/dce60_timing_generator.c +++ b/drivers/gpu/drm/amd/display/dc/dce60/dce60_timing_generator.c @@ -189,8 +189,8 @@ static bool dce60_is_tg_enabled(struct timing_generator *tg) return field == 1; } -bool dce60_configure_crc(struct timing_generator *tg, - const struct crc_params *params) +static bool dce60_configure_crc(struct timing_generator *tg, + const struct crc_params *params) { /* Cannot configure crc on a CRTC that is disabled */ if (!dce60_is_tg_enabled(tg)) diff --git a/drivers/gpu/drm/amd/display/dc/dce80/Makefile b/drivers/gpu/drm/amd/display/dc/dce80/Makefile index 666fcb2bdbba00091a16170f7058a227f51d99f5..0a9d1a350d8bdf4b33c88aa63887cbcd0b9eed40 100644 --- a/drivers/gpu/drm/amd/display/dc/dce80/Makefile +++ b/drivers/gpu/drm/amd/display/dc/dce80/Makefile @@ -23,6 +23,8 @@ # Makefile for the 'controller' sub-component of DAL. # It provides the control and status of HW CRTC block. +CFLAGS_$(AMDDALPATH)/dc/dce80/dce80_resource.o = $(call cc-disable-warning, override-init) + DCE80 = dce80_timing_generator.o dce80_hw_sequencer.o \ dce80_resource.o diff --git a/drivers/gpu/drm/amd/display/dc/dce80/dce80_resource.c b/drivers/gpu/drm/amd/display/dc/dce80/dce80_resource.c index 390a0fa37239cab6440c49b1211a49e524389402..612450f992782ac9aae6602f568928ba89b26914 100644 --- a/drivers/gpu/drm/amd/display/dc/dce80/dce80_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dce80/dce80_resource.c @@ -60,6 +60,8 @@ #include "dce/dce_i2c.h" /* TODO remove this include */ +#include "dce80_resource.h" + #ifndef mmMC_HUB_RDREQ_DMIF_LIMIT #include "gmc/gmc_7_1_d.h" #include "gmc/gmc_7_1_sh_mask.h" @@ -402,7 +404,7 @@ static const struct dc_plane_cap plane_cap = { .pixel_format_support = { .argb8888 = true, .nv12 = false, - .fp16 = false + .fp16 = true }, .max_upscale_factor = { diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp_cm.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp_cm.c index 4d3f7d5e1473afa1f61b379729f6528eb5702c64..904c2d2789987bce097fa805182a49dc9675a71e 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp_cm.c +++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp_cm.c @@ -577,7 +577,7 @@ void dpp1_power_on_degamma_lut( struct dcn10_dpp *dpp = TO_DCN10_DPP(dpp_base); REG_SET(CM_MEM_PWR_CTRL, 0, - SHARED_MEM_PWR_DIS, power_on == true ? 0:1); + SHARED_MEM_PWR_DIS, power_on ? 0:1); } diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.c index 017b67b830e669a05acd66619484787960f97383..89912bb5014f86fe5c57d756b6c209ea8eb02155 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.c +++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.c @@ -1224,6 +1224,7 @@ void dcn10_init_pipes(struct dc *dc, struct dc_state *context) // signals when OTG blanked. This is to prevent pipe from // requesting data while in PSR. tg->funcs->tg_init(tg); + hubp->power_gated = true; continue; } @@ -2634,7 +2635,7 @@ static void dcn10_update_dchubp_dpp( hws->funcs.update_plane_addr(dc, pipe_ctx); if (is_pipe_tree_visible(pipe_ctx)) - hubp->funcs->set_blank(hubp, false); + dc->hwss.set_hubp_blank(dc, pipe_ctx, false); } void dcn10_blank_pixel_data( @@ -3134,7 +3135,7 @@ void dcn10_setup_stereo(struct pipe_ctx *pipe_ctx, struct dc *dc) pipe_ctx->stream_res.opp->funcs->opp_program_stereo( pipe_ctx->stream_res.opp, - flags.PROGRAM_STEREO == 1 ? true:false, + flags.PROGRAM_STEREO == 1, &stream->timing); pipe_ctx->stream_res.tg->funcs->program_stereo( @@ -3145,13 +3146,16 @@ void dcn10_setup_stereo(struct pipe_ctx *pipe_ctx, struct dc *dc) return; } -static struct hubp *get_hubp_by_inst(struct resource_pool *res_pool, int mpcc_inst) +static struct pipe_ctx *get_pipe_ctx_by_hubp_inst(struct dc_state *context, int mpcc_inst) { int i; - for (i = 0; i < res_pool->pipe_count; i++) { - if (res_pool->hubps[i]->inst == mpcc_inst) - return res_pool->hubps[i]; + for (i = 0; i < MAX_PIPES; i++) { + if (context->res_ctx.pipe_ctx[i].plane_res.hubp + && context->res_ctx.pipe_ctx[i].plane_res.hubp->inst == mpcc_inst) { + return &context->res_ctx.pipe_ctx[i]; + } + } ASSERT(false); return NULL; @@ -3174,11 +3178,23 @@ void dcn10_wait_for_mpcc_disconnect( for (mpcc_inst = 0; mpcc_inst < MAX_PIPES; mpcc_inst++) { if (pipe_ctx->stream_res.opp->mpcc_disconnect_pending[mpcc_inst]) { - struct hubp *hubp = get_hubp_by_inst(res_pool, mpcc_inst); + struct pipe_ctx *restore_bottom_pipe; + struct pipe_ctx *restore_top_pipe; + struct pipe_ctx *inst_pipe_ctx = get_pipe_ctx_by_hubp_inst(dc->current_state, mpcc_inst); + ASSERT(inst_pipe_ctx); res_pool->mpc->funcs->wait_for_idle(res_pool->mpc, mpcc_inst); pipe_ctx->stream_res.opp->mpcc_disconnect_pending[mpcc_inst] = false; - hubp->funcs->set_blank(hubp, true); + /* + * Set top and bottom pipes NULL, as we don't want + * to blank those pipes when disconnecting from MPCC + */ + restore_bottom_pipe = inst_pipe_ctx->bottom_pipe; + restore_top_pipe = inst_pipe_ctx->top_pipe; + inst_pipe_ctx->top_pipe = inst_pipe_ctx->bottom_pipe = NULL; + dc->hwss.set_hubp_blank(dc, inst_pipe_ctx, true); + inst_pipe_ctx->top_pipe = restore_top_pipe; + inst_pipe_ctx->bottom_pipe = restore_bottom_pipe; } } @@ -3731,3 +3747,10 @@ void dcn10_get_clock(struct dc *dc, dc->clk_mgr->funcs->get_clock(dc->clk_mgr, context, clock_type, clock_cfg); } + +void dcn10_set_hubp_blank(const struct dc *dc, + struct pipe_ctx *pipe_ctx, + bool blank_enable) +{ + pipe_ctx->plane_res.hubp->funcs->set_blank(pipe_ctx->plane_res.hubp, blank_enable); +} diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.h b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.h index e5691e49902315f0165dc017ab68ff51b781c067..89e6dfb63da0c1990ef4705aaa4f132ca7aec725 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.h +++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.h @@ -163,6 +163,8 @@ void dcn10_wait_for_mpcc_disconnect( void dce110_edp_backlight_control( struct dc_link *link, bool enable); +void dce110_edp_wait_for_T12( + struct dc_link *link); void dce110_edp_power_control( struct dc_link *link, bool power_up); @@ -202,5 +204,8 @@ void dcn10_wait_for_pending_cleared(struct dc *dc, struct dc_state *context); void dcn10_set_hdr_multiplier(struct pipe_ctx *pipe_ctx); void dcn10_verify_allow_pstate_change_high(struct dc *dc); +void dcn10_set_hubp_blank(const struct dc *dc, + struct pipe_ctx *pipe_ctx, + bool blank_enable); #endif /* __DC_HWSS_DCN10_H__ */ diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer_debug.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer_debug.c index 7f4766e45dff0d37ff08503ace356fdd161fd0d2..e8b6065fffad4f70f3b6522ce1a0966748372301 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer_debug.c +++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer_debug.c @@ -47,7 +47,7 @@ unsigned int snprintf_count(char *pBuf, unsigned int bufSize, char *fmt, ...) { - unsigned int ret_vsnprintf; + int ret_vsnprintf; unsigned int chars_printed; va_list args; diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_init.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_init.c index 254300b06b434b0332df1f93e5c56a18d17fcc58..2f1b802e66a13e7afab10a6fa1ca1f104e643188 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_init.c +++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_init.c @@ -79,6 +79,7 @@ static const struct hw_sequencer_funcs dcn10_funcs = { .set_backlight_level = dce110_set_backlight_level, .set_abm_immediate_disable = dce110_set_abm_immediate_disable, .set_pipe = dce110_set_pipe, + .set_hubp_blank = dcn10_set_hubp_blank, }; static const struct hwseq_private_funcs dcn10_private_funcs = { diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_link_encoder.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_link_encoder.c index 81db0179f7ea8ac381b976c0ccbd28cbc36baa43..e4701825b5a035ddeb36a7e07f761598312fbf02 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_link_encoder.c +++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_link_encoder.c @@ -480,7 +480,6 @@ unsigned int dcn10_get_dig_frontend(struct link_encoder *enc) break; default: // invalid source select DIG - ASSERT(false); result = ENGINE_ID_UNKNOWN; } @@ -956,6 +955,21 @@ void dcn10_link_encoder_enable_tmds_output( } } +void dcn10_link_encoder_enable_tmds_output_with_clk_pattern_wa( + struct link_encoder *enc, + enum clock_source_id clock_source, + enum dc_color_depth color_depth, + enum signal_type signal, + uint32_t pixel_clock) +{ + struct dcn10_link_encoder *enc10 = TO_DCN10_LINK_ENC(enc); + + dcn10_link_encoder_enable_tmds_output( + enc, clock_source, color_depth, signal, pixel_clock); + + REG_UPDATE(DIG_CLOCK_PATTERN, DIG_CLOCK_PATTERN, 0x1F); +} + /* enables DP PHY output */ void dcn10_link_encoder_enable_dp_output( struct link_encoder *enc, diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_link_encoder.h b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_link_encoder.h index d4caad67085547eafb75d9b4e7d90691fd57dccd..3e1a582e4b88fd58708ad663fde21159ea8e4200 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_link_encoder.h +++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_link_encoder.h @@ -42,6 +42,7 @@ #define LE_DCN_COMMON_REG_LIST(id) \ SRI(DIG_BE_CNTL, DIG, id), \ SRI(DIG_BE_EN_CNTL, DIG, id), \ + SRI(DIG_CLOCK_PATTERN, DIG, id), \ SRI(TMDS_CTL_BITS, DIG, id), \ SRI(DP_CONFIG, DP, id), \ SRI(DP_DPHY_CNTL, DP, id), \ @@ -83,6 +84,7 @@ struct dcn10_link_enc_hpd_registers { struct dcn10_link_enc_registers { uint32_t DIG_BE_CNTL; uint32_t DIG_BE_EN_CNTL; + uint32_t DIG_CLOCK_PATTERN; uint32_t DP_CONFIG; uint32_t DP_DPHY_CNTL; uint32_t DP_DPHY_INTERNAL_CTRL; @@ -168,6 +170,7 @@ struct dcn10_link_enc_registers { LE_SF(DIG0_DIG_BE_CNTL, DIG_HPD_SELECT, mask_sh),\ LE_SF(DIG0_DIG_BE_CNTL, DIG_MODE, mask_sh),\ LE_SF(DIG0_DIG_BE_CNTL, DIG_FE_SOURCE_SELECT, mask_sh),\ + LE_SF(DIG0_DIG_CLOCK_PATTERN, DIG_CLOCK_PATTERN, mask_sh),\ LE_SF(DIG0_TMDS_CTL_BITS, TMDS_CTL0, mask_sh), \ LE_SF(DP0_DP_DPHY_CNTL, DPHY_BYPASS, mask_sh),\ LE_SF(DP0_DP_DPHY_CNTL, DPHY_ATEST_SEL_LANE0, mask_sh),\ @@ -218,6 +221,7 @@ struct dcn10_link_enc_registers { type DIG_HPD_SELECT;\ type DIG_MODE;\ type DIG_FE_SOURCE_SELECT;\ + type DIG_CLOCK_PATTERN;\ type DPHY_BYPASS;\ type DPHY_ATEST_SEL_LANE0;\ type DPHY_ATEST_SEL_LANE1;\ @@ -536,6 +540,13 @@ void dcn10_link_encoder_enable_tmds_output( enum signal_type signal, uint32_t pixel_clock); +void dcn10_link_encoder_enable_tmds_output_with_clk_pattern_wa( + struct link_encoder *enc, + enum clock_source_id clock_source, + enum dc_color_depth color_depth, + enum signal_type signal, + uint32_t pixel_clock); + /* enables DP PHY output */ void dcn10_link_encoder_enable_dp_output( struct link_encoder *enc, diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.c index f033397a84e9a2eeb03ef3c320024ee641879e29..6138f4887de7fa597d385059e2801de54370c4bf 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.c +++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.c @@ -659,6 +659,16 @@ void optc1_unlock(struct timing_generator *optc) OTG_MASTER_UPDATE_LOCK, 0); } +bool optc1_is_locked(struct timing_generator *optc) +{ + struct optc *optc1 = DCN10TG_FROM_TG(optc); + uint32_t locked; + + REG_GET(OTG_MASTER_UPDATE_LOCK, UPDATE_LOCK_STATUS, &locked); + + return (locked == 1); +} + void optc1_get_position(struct timing_generator *optc, struct crtc_position *position) { @@ -1513,6 +1523,7 @@ static const struct timing_generator_funcs dcn10_tg_funcs = { .enable_crtc_reset = optc1_enable_crtc_reset, .disable_reset_trigger = optc1_disable_reset_trigger, .lock = optc1_lock, + .is_locked = optc1_is_locked, .unlock = optc1_unlock, .enable_optc_clock = optc1_enable_optc_clock, .set_drr = optc1_set_drr, diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.h b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.h index b12bd9aae52f0159197dbece6f60c47756a95839..b222c67973d469e0c65a56bdee9a2ef743cd3f2c 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.h +++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.h @@ -638,6 +638,7 @@ void optc1_set_blank(struct timing_generator *optc, bool enable_blanking); bool optc1_is_blanked(struct timing_generator *optc); +bool optc1_is_locked(struct timing_generator *optc); void optc1_program_blank_color( struct timing_generator *optc, diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hwseq.c b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hwseq.c index 480d928cb1ca6edb6a3c1dd9e19ac9575521c199..0726fb435e2a3c175138262f9f3151223aabf77e 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hwseq.c +++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hwseq.c @@ -1575,8 +1575,8 @@ static void dcn20_update_dchubp_dpp( - if (pipe_ctx->update_flags.bits.enable) - hubp->funcs->set_blank(hubp, false); + if (is_pipe_tree_visible(pipe_ctx)) + dc->hwss.set_hubp_blank(dc, pipe_ctx, false); } @@ -1770,6 +1770,14 @@ void dcn20_post_unlock_program_front_end( } } + for (i = 0; i < dc->res_pool->pipe_count; i++) { + struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i]; + + if (pipe->vtp_locked) { + dc->hwss.set_hubp_blank(dc, pipe, true); + pipe->vtp_locked = false; + } + } /* WA to apply WM setting*/ if (hwseq->wa.DEGVIDCN21) dc->res_pool->hubbub->funcs->apply_DEDCN21_147_wa(dc->res_pool->hubbub); diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_init.c b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_init.c index de9dcbeea150d4a395d9f33eedbe05017bb455cf..51a4166e975079caa604c2d9414d59d80a15a9fe 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_init.c +++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_init.c @@ -94,6 +94,7 @@ static const struct hw_sequencer_funcs dcn20_funcs = { .optimize_timing_for_fsft = dcn20_optimize_timing_for_fsft, #endif .set_disp_pattern_generator = dcn20_set_disp_pattern_generator, + .set_hubp_blank = dcn10_set_hubp_blank, }; static const struct hwseq_private_funcs dcn20_private_funcs = { diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_link_encoder.c b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_link_encoder.c index 15c2ff264ff601f559a9fba0eabcf15ae68f9c36..fa013496e26baf8ae1aa817d6c9fcf389633420b 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_link_encoder.c +++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_link_encoder.c @@ -363,7 +363,7 @@ static const struct link_encoder_funcs dcn20_link_enc_funcs = { dcn10_link_encoder_validate_output_with_stream, .hw_init = enc2_hw_init, .setup = dcn10_link_encoder_setup, - .enable_tmds_output = dcn10_link_encoder_enable_tmds_output, + .enable_tmds_output = dcn10_link_encoder_enable_tmds_output_with_clk_pattern_wa, .enable_dp_output = dcn20_link_encoder_enable_dp_output, .enable_dp_mst_output = dcn10_link_encoder_enable_dp_mst_output, .disable_output = dcn10_link_encoder_disable_output, diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_resource.c b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_resource.c index d6b4885618713e2c964279e89a0907b14580c223..2c2dbfcd89571089741f14f84e366343aeb34657 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_resource.c @@ -2097,6 +2097,7 @@ int dcn20_populate_dml_pipes_from_context( pipes[pipe_cnt].pipe.dest.pixel_rate_mhz *= 2; pipes[pipe_cnt].pipe.dest.otg_inst = res_ctx->pipe_ctx[i].stream_res.tg->inst; pipes[pipe_cnt].dout.dp_lanes = 4; + pipes[pipe_cnt].dout.is_virtual = 0; pipes[pipe_cnt].pipe.dest.vtotal_min = res_ctx->pipe_ctx[i].stream->adjust.v_total_min; pipes[pipe_cnt].pipe.dest.vtotal_max = res_ctx->pipe_ctx[i].stream->adjust.v_total_max; switch (get_num_odm_splits(&res_ctx->pipe_ctx[i])) { @@ -2150,6 +2151,7 @@ int dcn20_populate_dml_pipes_from_context( break; default: /* In case there is no signal, set dp with 4 lanes to allow max config */ + pipes[pipe_cnt].dout.is_virtual = 1; pipes[pipe_cnt].dout.output_type = dm_dp; pipes[pipe_cnt].dout.dp_lanes = 4; } @@ -3245,7 +3247,7 @@ static noinline bool dcn20_validate_bandwidth_fp(struct dc *dc, bool dcn20_validate_bandwidth(struct dc *dc, struct dc_state *context, bool fast_validate) { - bool voltage_supported = false; + bool voltage_supported; DC_FP_START(); voltage_supported = dcn20_validate_bandwidth_fp(dc, context, fast_validate); DC_FP_END(); @@ -3506,7 +3508,8 @@ void dcn20_update_bounding_box(struct dc *dc, struct _vcs_dpi_soc_bounding_box_s calculated_states[i].dram_speed_mts = uclk_states[i] * 16 / 1000; // FCLK:UCLK ratio is 1.08 - min_fclk_required_by_uclk = mul_u64_u32_shr(BIT_ULL(32) * 1080 / 1000000, uclk_states[i], 32); + min_fclk_required_by_uclk = div_u64(((unsigned long long)uclk_states[i]) * 1080, + 1000000); calculated_states[i].fabricclk_mhz = (min_fclk_required_by_uclk < min_dcfclk) ? min_dcfclk : min_fclk_required_by_uclk; @@ -3606,7 +3609,6 @@ static enum dml_project get_dml_project_version(uint32_t hw_internal_rev) static bool init_soc_bounding_box(struct dc *dc, struct dcn20_resource_pool *pool) { - const struct gpu_info_soc_bounding_box_v1_0 *bb = dc->soc_bounding_box; struct _vcs_dpi_soc_bounding_box_st *loaded_bb = get_asic_rev_soc_bb(dc->ctx->asic_id.hw_internal_rev); struct _vcs_dpi_ip_params_st *loaded_ip = @@ -3614,116 +3616,6 @@ static bool init_soc_bounding_box(struct dc *dc, DC_LOGGER_INIT(dc->ctx->logger); - /* TODO: upstream NV12 bounding box when its launched */ - if (!bb && ASICREV_IS_NAVI12_P(dc->ctx->asic_id.hw_internal_rev)) { - DC_LOG_ERROR("%s: not valid soc bounding box/n", __func__); - return false; - } - - if (bb && ASICREV_IS_NAVI12_P(dc->ctx->asic_id.hw_internal_rev)) { - int i; - - dcn2_0_nv12_soc.sr_exit_time_us = - fixed16_to_double_to_cpu(bb->sr_exit_time_us); - dcn2_0_nv12_soc.sr_enter_plus_exit_time_us = - fixed16_to_double_to_cpu(bb->sr_enter_plus_exit_time_us); - dcn2_0_nv12_soc.urgent_latency_us = - fixed16_to_double_to_cpu(bb->urgent_latency_us); - dcn2_0_nv12_soc.urgent_latency_pixel_data_only_us = - fixed16_to_double_to_cpu(bb->urgent_latency_pixel_data_only_us); - dcn2_0_nv12_soc.urgent_latency_pixel_mixed_with_vm_data_us = - fixed16_to_double_to_cpu(bb->urgent_latency_pixel_mixed_with_vm_data_us); - dcn2_0_nv12_soc.urgent_latency_vm_data_only_us = - fixed16_to_double_to_cpu(bb->urgent_latency_vm_data_only_us); - dcn2_0_nv12_soc.urgent_out_of_order_return_per_channel_pixel_only_bytes = - le32_to_cpu(bb->urgent_out_of_order_return_per_channel_pixel_only_bytes); - dcn2_0_nv12_soc.urgent_out_of_order_return_per_channel_pixel_and_vm_bytes = - le32_to_cpu(bb->urgent_out_of_order_return_per_channel_pixel_and_vm_bytes); - dcn2_0_nv12_soc.urgent_out_of_order_return_per_channel_vm_only_bytes = - le32_to_cpu(bb->urgent_out_of_order_return_per_channel_vm_only_bytes); - dcn2_0_nv12_soc.pct_ideal_dram_sdp_bw_after_urgent_pixel_only = - fixed16_to_double_to_cpu(bb->pct_ideal_dram_sdp_bw_after_urgent_pixel_only); - dcn2_0_nv12_soc.pct_ideal_dram_sdp_bw_after_urgent_pixel_and_vm = - fixed16_to_double_to_cpu(bb->pct_ideal_dram_sdp_bw_after_urgent_pixel_and_vm); - dcn2_0_nv12_soc.pct_ideal_dram_sdp_bw_after_urgent_vm_only = - fixed16_to_double_to_cpu(bb->pct_ideal_dram_sdp_bw_after_urgent_vm_only); - dcn2_0_nv12_soc.max_avg_sdp_bw_use_normal_percent = - fixed16_to_double_to_cpu(bb->max_avg_sdp_bw_use_normal_percent); - dcn2_0_nv12_soc.max_avg_dram_bw_use_normal_percent = - fixed16_to_double_to_cpu(bb->max_avg_dram_bw_use_normal_percent); - dcn2_0_nv12_soc.writeback_latency_us = - fixed16_to_double_to_cpu(bb->writeback_latency_us); - dcn2_0_nv12_soc.ideal_dram_bw_after_urgent_percent = - fixed16_to_double_to_cpu(bb->ideal_dram_bw_after_urgent_percent); - dcn2_0_nv12_soc.max_request_size_bytes = - le32_to_cpu(bb->max_request_size_bytes); - dcn2_0_nv12_soc.dram_channel_width_bytes = - le32_to_cpu(bb->dram_channel_width_bytes); - dcn2_0_nv12_soc.fabric_datapath_to_dcn_data_return_bytes = - le32_to_cpu(bb->fabric_datapath_to_dcn_data_return_bytes); - dcn2_0_nv12_soc.dcn_downspread_percent = - fixed16_to_double_to_cpu(bb->dcn_downspread_percent); - dcn2_0_nv12_soc.downspread_percent = - fixed16_to_double_to_cpu(bb->downspread_percent); - dcn2_0_nv12_soc.dram_page_open_time_ns = - fixed16_to_double_to_cpu(bb->dram_page_open_time_ns); - dcn2_0_nv12_soc.dram_rw_turnaround_time_ns = - fixed16_to_double_to_cpu(bb->dram_rw_turnaround_time_ns); - dcn2_0_nv12_soc.dram_return_buffer_per_channel_bytes = - le32_to_cpu(bb->dram_return_buffer_per_channel_bytes); - dcn2_0_nv12_soc.round_trip_ping_latency_dcfclk_cycles = - le32_to_cpu(bb->round_trip_ping_latency_dcfclk_cycles); - dcn2_0_nv12_soc.urgent_out_of_order_return_per_channel_bytes = - le32_to_cpu(bb->urgent_out_of_order_return_per_channel_bytes); - dcn2_0_nv12_soc.channel_interleave_bytes = - le32_to_cpu(bb->channel_interleave_bytes); - dcn2_0_nv12_soc.num_banks = - le32_to_cpu(bb->num_banks); - dcn2_0_nv12_soc.num_chans = - le32_to_cpu(bb->num_chans); - dcn2_0_nv12_soc.vmm_page_size_bytes = - le32_to_cpu(bb->vmm_page_size_bytes); - dcn2_0_nv12_soc.dram_clock_change_latency_us = - fixed16_to_double_to_cpu(bb->dram_clock_change_latency_us); - // HACK!! Lower uclock latency switch time so we don't switch - dcn2_0_nv12_soc.dram_clock_change_latency_us = 10; - dcn2_0_nv12_soc.writeback_dram_clock_change_latency_us = - fixed16_to_double_to_cpu(bb->writeback_dram_clock_change_latency_us); - dcn2_0_nv12_soc.return_bus_width_bytes = - le32_to_cpu(bb->return_bus_width_bytes); - dcn2_0_nv12_soc.dispclk_dppclk_vco_speed_mhz = - le32_to_cpu(bb->dispclk_dppclk_vco_speed_mhz); - dcn2_0_nv12_soc.xfc_bus_transport_time_us = - le32_to_cpu(bb->xfc_bus_transport_time_us); - dcn2_0_nv12_soc.xfc_xbuf_latency_tolerance_us = - le32_to_cpu(bb->xfc_xbuf_latency_tolerance_us); - dcn2_0_nv12_soc.use_urgent_burst_bw = - le32_to_cpu(bb->use_urgent_burst_bw); - dcn2_0_nv12_soc.num_states = - le32_to_cpu(bb->num_states); - - for (i = 0; i < dcn2_0_nv12_soc.num_states; i++) { - dcn2_0_nv12_soc.clock_limits[i].state = - le32_to_cpu(bb->clock_limits[i].state); - dcn2_0_nv12_soc.clock_limits[i].dcfclk_mhz = - fixed16_to_double_to_cpu(bb->clock_limits[i].dcfclk_mhz); - dcn2_0_nv12_soc.clock_limits[i].fabricclk_mhz = - fixed16_to_double_to_cpu(bb->clock_limits[i].fabricclk_mhz); - dcn2_0_nv12_soc.clock_limits[i].dispclk_mhz = - fixed16_to_double_to_cpu(bb->clock_limits[i].dispclk_mhz); - dcn2_0_nv12_soc.clock_limits[i].dppclk_mhz = - fixed16_to_double_to_cpu(bb->clock_limits[i].dppclk_mhz); - dcn2_0_nv12_soc.clock_limits[i].phyclk_mhz = - fixed16_to_double_to_cpu(bb->clock_limits[i].phyclk_mhz); - dcn2_0_nv12_soc.clock_limits[i].socclk_mhz = - fixed16_to_double_to_cpu(bb->clock_limits[i].socclk_mhz); - dcn2_0_nv12_soc.clock_limits[i].dscclk_mhz = - fixed16_to_double_to_cpu(bb->clock_limits[i].dscclk_mhz); - dcn2_0_nv12_soc.clock_limits[i].dram_speed_mts = - fixed16_to_double_to_cpu(bb->clock_limits[i].dram_speed_mts); - } - } - if (pool->base.pp_smu) { struct pp_smu_nv_clock_table max_clocks = {0}; unsigned int uclk_states[8] = {0}; diff --git a/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_hwseq.c b/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_hwseq.c index 96ee0b82f45809f38e740669fc9154ee649388fa..d3b643089603ea40f58cf3532d70a17939bbc878 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_hwseq.c +++ b/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_hwseq.c @@ -123,7 +123,7 @@ void dcn21_optimize_pwr_state( * PHY will hang on the next mode set attempt. * if enable PLL follow by disable PLL (without executing lane enable/disable), * RDPCS_PHY_DP_MPLLB_STATE remains 1, - * which indicate that PLL disable attempt actually didn’t go through. + * which indicate that PLL disable attempt actually didn't go through. * As a workaround, insert PHY lane enable/disable before PLL disable. */ void dcn21_PLAT_58856_wa(struct dc_state *context, struct pipe_ctx *pipe_ctx) @@ -143,6 +143,7 @@ static bool dmub_abm_set_pipe(struct abm *abm, uint32_t otg_inst, uint32_t optio struct dc_context *dc = abm->ctx; uint32_t ramping_boundary = 0xFFFF; + memset(&cmd, 0, sizeof(cmd)); cmd.abm_set_pipe.header.type = DMUB_CMD__ABM; cmd.abm_set_pipe.header.sub_type = DMUB_CMD__ABM_SET_PIPE; cmd.abm_set_pipe.abm_set_pipe_data.otg_inst = otg_inst; @@ -212,6 +213,7 @@ bool dcn21_set_backlight_level(struct pipe_ctx *pipe_ctx, if (abm && panel_cntl) dmub_abm_set_pipe(abm, otg_inst, SET_ABM_PIPE_NORMAL, panel_cntl->inst); + memset(&cmd, 0, sizeof(cmd)); cmd.abm_set_backlight.header.type = DMUB_CMD__ABM; cmd.abm_set_backlight.header.sub_type = DMUB_CMD__ABM_SET_BACKLIGHT; cmd.abm_set_backlight.abm_set_backlight_data.frame_ramp = frame_ramp; diff --git a/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_init.c b/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_init.c index 074e2713257f179dbc8cc642665e3d50765db686..0597391b21716bd89a138a9930c6a5f5415974e7 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_init.c +++ b/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_init.c @@ -99,6 +99,7 @@ static const struct hw_sequencer_funcs dcn21_funcs = { #endif .is_abm_supported = dcn21_is_abm_supported, .set_disp_pattern_generator = dcn20_set_disp_pattern_generator, + .set_hubp_blank = dcn10_set_hubp_blank, }; static const struct hwseq_private_funcs dcn21_private_funcs = { diff --git a/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_resource.c b/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_resource.c index 6743764289167a06639268284dba98356c883f8f..072f8c880924366ffd42477a3ffd79886611748a 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_resource.c @@ -1329,8 +1329,8 @@ static bool dcn21_fast_validate_bw( return out; } -bool dcn21_validate_bandwidth(struct dc *dc, struct dc_state *context, - bool fast_validate) +static noinline bool dcn21_validate_bandwidth_fp(struct dc *dc, + struct dc_state *context, bool fast_validate) { bool out = false; @@ -1383,6 +1383,22 @@ bool dcn21_validate_bandwidth(struct dc *dc, struct dc_state *context, return out; } + +/* + * Some of the functions further below use the FPU, so we need to wrap this + * with DC_FP_START()/DC_FP_END(). Use the same approach as for + * dcn20_validate_bandwidth in dcn20_resource.c. + */ +bool dcn21_validate_bandwidth(struct dc *dc, struct dc_state *context, + bool fast_validate) +{ + bool voltage_supported; + DC_FP_START(); + voltage_supported = dcn21_validate_bandwidth_fp(dc, context, fast_validate); + DC_FP_END(); + return voltage_supported; +} + static void dcn21_destroy_resource_pool(struct resource_pool **pool) { struct dcn21_resource_pool *dcn21_pool = TO_DCN21_RES_POOL(*pool); diff --git a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_dpp_cm.c b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_dpp_cm.c index 9da66e4911165d406c182707d417faf788fec542..33985401f25cba88b822bc15ad3d26c9cd70a09e 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_dpp_cm.c +++ b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_dpp_cm.c @@ -133,7 +133,6 @@ static void dpp3_power_on_gamcor_lut( struct dpp *dpp_base, bool power_on) { - uint32_t power_status; struct dcn3_dpp *dpp = TO_DCN30_DPP(dpp_base); if (dpp_base->ctx->dc->debug.enable_mem_low_power.bits.cm) { @@ -143,12 +142,6 @@ static void dpp3_power_on_gamcor_lut( } else REG_SET(CM_MEM_PWR_CTRL, 0, GAMCOR_MEM_PWR_DIS, power_on == true ? 0:1); - - REG_GET(CM_MEM_PWR_STATUS, GAMCOR_MEM_PWR_STATE, &power_status); - if (power_status != 0) - BREAK_TO_DEBUGGER(); - - } void dpp3_program_cm_dealpha( diff --git a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_hubp.h b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_hubp.h index 5fa150f34c60093122a482cb30d7656290c18440..705fbfc375029f7c1d089f0dca60b4da9b27f6fc 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_hubp.h +++ b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_hubp.h @@ -62,6 +62,7 @@ HUBP_SF(HUBP0_DCHUBP_CNTL, HUBP_NO_OUTSTANDING_REQ, mask_sh),\ HUBP_SF(HUBP0_DCHUBP_CNTL, HUBP_VTG_SEL, mask_sh),\ HUBP_SF(HUBP0_DCHUBP_CNTL, HUBP_DISABLE, mask_sh),\ + HUBP_SF(HUBP0_DCHUBP_CNTL, HUBP_IN_BLANK, mask_sh),\ HUBP_SF(HUBP0_DCSURF_ADDR_CONFIG, NUM_PIPES, mask_sh),\ HUBP_SF(HUBP0_DCSURF_ADDR_CONFIG, PIPE_INTERLEAVE, mask_sh),\ HUBP_SF(HUBP0_DCSURF_ADDR_CONFIG, MAX_COMPRESSED_FRAGS, mask_sh),\ diff --git a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_hwseq.c b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_hwseq.c index 3deb3fb1724dc3caaef08cda6252184f71be8cbf..06dc1e2e8383acdb2bc8d3819e07e3df8dbf763e 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_hwseq.c +++ b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_hwseq.c @@ -539,6 +539,8 @@ void dcn30_init_hw(struct dc *dc) fe = dc->links[i]->link_enc->funcs->get_dig_frontend( dc->links[i]->link_enc); + if (fe == ENGINE_ID_UNKNOWN) + continue; for (j = 0; j < dc->res_pool->stream_enc_count; j++) { if (fe == dc->res_pool->stream_enc[j]->id) { @@ -710,8 +712,11 @@ void dcn30_program_dmdata_engine(struct pipe_ctx *pipe_ctx) bool dcn30_apply_idle_power_optimizations(struct dc *dc, bool enable) { union dmub_rb_cmd cmd; - unsigned int surface_size, refresh_hz, denom; uint32_t tmr_delay = 0, tmr_scale = 0; + struct dc_cursor_attributes cursor_attr; + bool cursor_cache_enable = false; + struct dc_stream_state *stream = NULL; + struct dc_plane_state *plane = NULL; if (!dc->ctx->dmub_srv) return false; @@ -722,72 +727,150 @@ bool dcn30_apply_idle_power_optimizations(struct dc *dc, bool enable) /* First, check no-memory-requests case */ for (i = 0; i < dc->current_state->stream_count; i++) { - if (dc->current_state->stream_status[i] - .plane_count) + if (dc->current_state->stream_status[i].plane_count) /* Fail eligibility on a visible stream */ break; } - if (dc->current_state->stream_count == 1 // single display only - && dc->current_state->stream_status[0].plane_count == 1 // single surface only - && dc->current_state->stream_status[0].plane_states[0]->address.page_table_base.quad_part == 0 // no VM - // Only 8 and 16 bit formats - && dc->current_state->stream_status[0].plane_states[0]->format <= SURFACE_PIXEL_FORMAT_GRPH_ABGR16161616F - && dc->current_state->stream_status[0].plane_states[0]->format >= SURFACE_PIXEL_FORMAT_GRPH_ARGB8888) { - surface_size = dc->current_state->stream_status[0].plane_states[0]->plane_size.surface_pitch * - dc->current_state->stream_status[0].plane_states[0]->plane_size.surface_size.height * - (dc->current_state->stream_status[0].plane_states[0]->format >= SURFACE_PIXEL_FORMAT_GRPH_ARGB16161616 ? - 8 : 4); - } else { - // TODO: remove hard code size - surface_size = 128 * 1024 * 1024; + if (i == dc->current_state->stream_count) { + /* Enable no-memory-requests case */ + memset(&cmd, 0, sizeof(cmd)); + cmd.mall.header.type = DMUB_CMD__MALL; + cmd.mall.header.sub_type = DMUB_CMD__MALL_ACTION_NO_DF_REQ; + cmd.mall.header.payload_bytes = sizeof(cmd.mall) - sizeof(cmd.mall.header); + + dc_dmub_srv_cmd_queue(dc->ctx->dmub_srv, &cmd); + dc_dmub_srv_cmd_execute(dc->ctx->dmub_srv); + + return true; + } + + stream = dc->current_state->streams[0]; + plane = (stream ? dc->current_state->stream_status[0].plane_states[0] : NULL); + + if (stream && plane) { + cursor_cache_enable = stream->cursor_position.enable && + plane->address.grph.cursor_cache_addr.quad_part; + cursor_attr = stream->cursor_attributes; } - // TODO: remove hard code size - if (surface_size < 128 * 1024 * 1024) { - refresh_hz = div_u64((unsigned long long) dc->current_state->streams[0]->timing.pix_clk_100hz * - 100LL, - (dc->current_state->streams[0]->timing.v_total * - dc->current_state->streams[0]->timing.h_total)); + /* + * Second, check MALL eligibility + * + * single display only, single surface only, 8 and 16 bit formats only, no VM, + * do not use MALL for displays that support PSR as they use D0i3.2 in DMCUB FW + * + * TODO: When we implement multi-display, PSR displays will be allowed if there is + * a non-PSR display present, since in that case we can't do D0i3.2 + */ + if (dc->current_state->stream_count == 1 && + stream->link->psr_settings.psr_version == DC_PSR_VERSION_UNSUPPORTED && + dc->current_state->stream_status[0].plane_count == 1 && + plane->format <= SURFACE_PIXEL_FORMAT_GRPH_ABGR16161616F && + plane->format >= SURFACE_PIXEL_FORMAT_GRPH_ARGB8888 && + plane->address.page_table_base.quad_part == 0 && + dc->hwss.does_plane_fit_in_mall && + dc->hwss.does_plane_fit_in_mall(dc, plane, + cursor_cache_enable ? &cursor_attr : NULL)) { + unsigned int v_total = stream->adjust.v_total_max ? + stream->adjust.v_total_max : stream->timing.v_total; + unsigned int refresh_hz = div_u64((unsigned long long) stream->timing.pix_clk_100hz * + 100LL, (v_total * stream->timing.h_total)); /* - * Delay_Us = 65.28 * (64 + MallFrameCacheTmrDly) * 2^MallFrameCacheTmrScale - * Delay_Us / 65.28 = (64 + MallFrameCacheTmrDly) * 2^MallFrameCacheTmrScale - * (Delay_Us / 65.28) / 2^MallFrameCacheTmrScale = 64 + MallFrameCacheTmrDly - * MallFrameCacheTmrDly = ((Delay_Us / 65.28) / 2^MallFrameCacheTmrScale) - 64 - * = (1000000 / refresh) / 65.28 / 2^MallFrameCacheTmrScale - 64 - * = 1000000 / (refresh * 65.28 * 2^MallFrameCacheTmrScale) - 64 - * = (1000000 * 100) / (refresh * 6528 * 2^MallFrameCacheTmrScale) - 64 + * one frame time in microsec: + * Delay_Us = 1000000 / refresh + * dynamic_delay_us = 1000000 / refresh + 2 * stutter_period + * + * one frame time modified by 'additional timer percent' (p): + * Delay_Us_modified = dynamic_delay_us + dynamic_delay_us * p / 100 + * = dynamic_delay_us * (1 + p / 100) + * = (1000000 / refresh + 2 * stutter_period) * (100 + p) / 100 + * = (1000000 + 2 * stutter_period * refresh) * (100 + p) / (100 * refresh) + * + * formula for timer duration based on parameters, from regspec: + * dynamic_delay_us = 65.28 * (64 + MallFrameCacheTmrDly) * 2^MallFrameCacheTmrScale + * + * dynamic_delay_us / 65.28 = (64 + MallFrameCacheTmrDly) * 2^MallFrameCacheTmrScale + * (dynamic_delay_us / 65.28) / 2^MallFrameCacheTmrScale = 64 + MallFrameCacheTmrDly + * MallFrameCacheTmrDly = ((dynamic_delay_us / 65.28) / 2^MallFrameCacheTmrScale) - 64 + * = (1000000 + 2 * stutter_period * refresh) * (100 + p) / (100 * refresh) / 65.28 / 2^MallFrameCacheTmrScale - 64 + * = (1000000 + 2 * stutter_period * refresh) * (100 + p) / (refresh * 6528 * 2^MallFrameCacheTmrScale) - 64 * * need to round up the result of the division before the subtraction */ - denom = refresh_hz * 6528; - tmr_delay = div_u64((100000000LL + denom - 1), denom) - 64LL; + unsigned int denom = refresh_hz * 6528; + unsigned int stutter_period = dc->current_state->perf_params.stutter_period_us; + + tmr_delay = div_u64(((1000000LL + 2 * stutter_period * refresh_hz) * + (100LL + dc->debug.mall_additional_timer_percent) + denom - 1), + denom) - 64LL; /* scale should be increased until it fits into 6 bits */ while (tmr_delay & ~0x3F) { tmr_scale++; if (tmr_scale > 3) { - /* The delay exceeds the range of the hystersis timer */ + /* Delay exceeds range of hysteresis timer */ ASSERT(false); return false; } denom *= 2; - tmr_delay = div_u64((100000000LL + denom - 1), denom) - 64LL; + tmr_delay = div_u64(((1000000LL + 2 * stutter_period * refresh_hz) * + (100LL + dc->debug.mall_additional_timer_percent) + denom - 1), + denom) - 64LL; + } + + /* Copy HW cursor */ + if (cursor_cache_enable) { + memset(&cmd, 0, sizeof(cmd)); + cmd.mall.header.type = DMUB_CMD__MALL; + cmd.mall.header.sub_type = DMUB_CMD__MALL_ACTION_COPY_CURSOR; + cmd.mall.header.payload_bytes = + sizeof(cmd.mall) - sizeof(cmd.mall.header); + + switch (cursor_attr.color_format) { + case CURSOR_MODE_MONO: + cmd.mall.cursor_bpp = 2; + break; + case CURSOR_MODE_COLOR_1BIT_AND: + case CURSOR_MODE_COLOR_PRE_MULTIPLIED_ALPHA: + case CURSOR_MODE_COLOR_UN_PRE_MULTIPLIED_ALPHA: + cmd.mall.cursor_bpp = 32; + break; + + case CURSOR_MODE_COLOR_64BIT_FP_PRE_MULTIPLIED: + case CURSOR_MODE_COLOR_64BIT_FP_UN_PRE_MULTIPLIED: + cmd.mall.cursor_bpp = 64; + break; + } + + cmd.mall.cursor_copy_src.quad_part = cursor_attr.address.quad_part; + cmd.mall.cursor_copy_dst.quad_part = + plane->address.grph.cursor_cache_addr.quad_part; + cmd.mall.cursor_width = cursor_attr.width; + cmd.mall.cursor_height = cursor_attr.height; + cmd.mall.cursor_pitch = cursor_attr.pitch; + + dc_dmub_srv_cmd_queue(dc->ctx->dmub_srv, &cmd); + dc_dmub_srv_cmd_execute(dc->ctx->dmub_srv); + dc_dmub_srv_wait_idle(dc->ctx->dmub_srv); + + /* Use copied cursor, and it's okay to not switch back */ + cursor_attr.address.quad_part = + plane->address.grph.cursor_cache_addr.quad_part; + dc_stream_set_cursor_attributes(stream, &cursor_attr); } /* Enable MALL */ memset(&cmd, 0, sizeof(cmd)); cmd.mall.header.type = DMUB_CMD__MALL; - cmd.mall.header.sub_type = - DMUB_CMD__MALL_ACTION_ALLOW; - cmd.mall.header.payload_bytes = - sizeof(cmd.mall) - - sizeof(cmd.mall.header); + cmd.mall.header.sub_type = DMUB_CMD__MALL_ACTION_ALLOW; + cmd.mall.header.payload_bytes = sizeof(cmd.mall) - sizeof(cmd.mall.header); cmd.mall.tmr_delay = tmr_delay; cmd.mall.tmr_scale = tmr_scale; + cmd.mall.debug_bits = dc->debug.mall_error_as_fatal; dc_dmub_srv_cmd_queue(dc->ctx->dmub_srv, &cmd); dc_dmub_srv_cmd_execute(dc->ctx->dmub_srv); @@ -814,6 +897,40 @@ bool dcn30_apply_idle_power_optimizations(struct dc *dc, bool enable) return true; } +bool dcn30_does_plane_fit_in_mall(struct dc *dc, struct dc_plane_state *plane, struct dc_cursor_attributes *cursor_attr) +{ + // add meta size? + unsigned int surface_size = plane->plane_size.surface_pitch * plane->plane_size.surface_size.height * + (plane->format >= SURFACE_PIXEL_FORMAT_GRPH_ARGB16161616 ? 8 : 4); + unsigned int mall_size = dc->caps.mall_size_total; + unsigned int cursor_size = 0; + + if (dc->debug.mall_size_override) + mall_size = 1024 * 1024 * dc->debug.mall_size_override; + + if (cursor_attr) { + cursor_size = dc->caps.max_cursor_size * dc->caps.max_cursor_size; + + switch (cursor_attr->color_format) { + case CURSOR_MODE_MONO: + cursor_size /= 2; + break; + case CURSOR_MODE_COLOR_1BIT_AND: + case CURSOR_MODE_COLOR_PRE_MULTIPLIED_ALPHA: + case CURSOR_MODE_COLOR_UN_PRE_MULTIPLIED_ALPHA: + cursor_size *= 4; + break; + + case CURSOR_MODE_COLOR_64BIT_FP_PRE_MULTIPLIED: + case CURSOR_MODE_COLOR_64BIT_FP_UN_PRE_MULTIPLIED: + cursor_size *= 8; + break; + } + } + + return (surface_size + cursor_size) < mall_size; +} + void dcn30_hardware_release(struct dc *dc) { /* if pstate unsupported, force it supported */ @@ -823,6 +940,53 @@ void dcn30_hardware_release(struct dc *dc) dc->res_pool->hubbub, true, true); } +void dcn30_set_hubp_blank(const struct dc *dc, + struct pipe_ctx *pipe_ctx, + bool blank_enable) +{ + struct pipe_ctx *mpcc_pipe; + struct pipe_ctx *odm_pipe; + + if (blank_enable) { + struct plane_resource *plane_res = &pipe_ctx->plane_res; + struct stream_resource *stream_res = &pipe_ctx->stream_res; + + /* Wait for enter vblank */ + stream_res->tg->funcs->wait_for_state(stream_res->tg, CRTC_STATE_VBLANK); + + /* Blank HUBP to allow p-state during blank on all timings */ + pipe_ctx->plane_res.hubp->funcs->set_blank(pipe_ctx->plane_res.hubp, true); + /* Confirm hubp in blank */ + ASSERT(plane_res->hubp->funcs->hubp_in_blank(plane_res->hubp)); + /* Toggle HUBP_DISABLE */ + plane_res->hubp->funcs->hubp_soft_reset(plane_res->hubp, true); + plane_res->hubp->funcs->hubp_soft_reset(plane_res->hubp, false); + for (mpcc_pipe = pipe_ctx->bottom_pipe; mpcc_pipe; mpcc_pipe = mpcc_pipe->bottom_pipe) { + mpcc_pipe->plane_res.hubp->funcs->set_blank(mpcc_pipe->plane_res.hubp, true); + /* Confirm hubp in blank */ + ASSERT(mpcc_pipe->plane_res.hubp->funcs->hubp_in_blank(mpcc_pipe->plane_res.hubp)); + /* Toggle HUBP_DISABLE */ + mpcc_pipe->plane_res.hubp->funcs->hubp_soft_reset(mpcc_pipe->plane_res.hubp, true); + mpcc_pipe->plane_res.hubp->funcs->hubp_soft_reset(mpcc_pipe->plane_res.hubp, false); + + } + for (odm_pipe = pipe_ctx->next_odm_pipe; odm_pipe; odm_pipe = odm_pipe->next_odm_pipe) { + odm_pipe->plane_res.hubp->funcs->set_blank(odm_pipe->plane_res.hubp, true); + /* Confirm hubp in blank */ + ASSERT(odm_pipe->plane_res.hubp->funcs->hubp_in_blank(odm_pipe->plane_res.hubp)); + /* Toggle HUBP_DISABLE */ + odm_pipe->plane_res.hubp->funcs->hubp_soft_reset(odm_pipe->plane_res.hubp, true); + odm_pipe->plane_res.hubp->funcs->hubp_soft_reset(odm_pipe->plane_res.hubp, false); + } + } else { + pipe_ctx->plane_res.hubp->funcs->set_blank(pipe_ctx->plane_res.hubp, false); + for (mpcc_pipe = pipe_ctx->bottom_pipe; mpcc_pipe; mpcc_pipe = mpcc_pipe->bottom_pipe) + mpcc_pipe->plane_res.hubp->funcs->set_blank(mpcc_pipe->plane_res.hubp, false); + for (odm_pipe = pipe_ctx->next_odm_pipe; odm_pipe; odm_pipe = odm_pipe->next_odm_pipe) + odm_pipe->plane_res.hubp->funcs->set_blank(odm_pipe->plane_res.hubp, false); + } +} + void dcn30_set_disp_pattern_generator(const struct dc *dc, struct pipe_ctx *pipe_ctx, enum controller_dp_test_pattern test_pattern, @@ -831,6 +995,25 @@ void dcn30_set_disp_pattern_generator(const struct dc *dc, const struct tg_color *solid_color, int width, int height, int offset) { - pipe_ctx->stream_res.opp->funcs->opp_set_disp_pattern_generator(pipe_ctx->stream_res.opp, test_pattern, - color_space, color_depth, solid_color, width, height, offset); + struct stream_resource *stream_res = &pipe_ctx->stream_res; + + if (test_pattern != CONTROLLER_DP_TEST_PATTERN_VIDEOMODE) { + pipe_ctx->vtp_locked = false; + /* turning on DPG */ + stream_res->opp->funcs->opp_set_disp_pattern_generator(stream_res->opp, test_pattern, color_space, + color_depth, solid_color, width, height, offset); + + /* Defer hubp blank if tg is locked */ + if (stream_res->tg->funcs->is_tg_enabled(stream_res->tg)) { + if (stream_res->tg->funcs->is_locked(stream_res->tg)) + pipe_ctx->vtp_locked = true; + else + dc->hwss.set_hubp_blank(dc, pipe_ctx, true); + } + } else { + dc->hwss.set_hubp_blank(dc, pipe_ctx, false); + /* turning off DPG */ + stream_res->opp->funcs->opp_set_disp_pattern_generator(stream_res->opp, test_pattern, color_space, + color_depth, solid_color, width, height, offset); + } } diff --git a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_hwseq.h b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_hwseq.h index 7d32c43aafe0e65aa54a863b9f29fa612f2d89cf..3b7d4812e3119b1008f081f8da156a2aa6318c1a 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_hwseq.h +++ b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_hwseq.h @@ -65,6 +65,9 @@ void dcn30_set_avmute(struct pipe_ctx *pipe_ctx, bool enable); void dcn30_update_info_frame(struct pipe_ctx *pipe_ctx); void dcn30_program_dmdata_engine(struct pipe_ctx *pipe_ctx); +bool dcn30_does_plane_fit_in_mall(struct dc *dc, struct dc_plane_state *plane, + struct dc_cursor_attributes *cursor_attr); + bool dcn30_apply_idle_power_optimizations(struct dc *dc, bool enable); void dcn30_hardware_release(struct dc *dc); @@ -77,4 +80,8 @@ void dcn30_set_disp_pattern_generator(const struct dc *dc, const struct tg_color *solid_color, int width, int height, int offset); +void dcn30_set_hubp_blank(const struct dc *dc, + struct pipe_ctx *pipe_ctx, + bool blank_enable); + #endif /* __DC_HWSS_DCN30_H__ */ diff --git a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_init.c b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_init.c index 6125fe440ad0ee44d81efac972656478bf4b9d43..204444fead9777cb4399fd97244380b0c7979165 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_init.c +++ b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_init.c @@ -71,6 +71,7 @@ static const struct hw_sequencer_funcs dcn30_funcs = { .edp_backlight_control = dce110_edp_backlight_control, .edp_power_control = dce110_edp_power_control, .edp_wait_for_hpd_ready = dce110_edp_wait_for_hpd_ready, + .edp_wait_for_T12 = dce110_edp_wait_for_T12, .set_cursor_position = dcn10_set_cursor_position, .set_cursor_attribute = dcn10_set_cursor_attribute, .set_cursor_sdr_white_level = dcn10_set_cursor_sdr_white_level, @@ -91,11 +92,13 @@ static const struct hw_sequencer_funcs dcn30_funcs = { .get_vupdate_offset_from_vsync = dcn10_get_vupdate_offset_from_vsync, .calc_vupdate_position = dcn10_calc_vupdate_position, .apply_idle_power_optimizations = dcn30_apply_idle_power_optimizations, + .does_plane_fit_in_mall = dcn30_does_plane_fit_in_mall, .set_backlight_level = dcn21_set_backlight_level, .set_abm_immediate_disable = dcn21_set_abm_immediate_disable, .hardware_release = dcn30_hardware_release, .set_pipe = dcn21_set_pipe, .set_disp_pattern_generator = dcn30_set_disp_pattern_generator, + .set_hubp_blank = dcn30_set_hubp_blank, }; static const struct hwseq_private_funcs dcn30_private_funcs = { diff --git a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_optc.c b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_optc.c index 3ba3991ee6128b489eaded6262ab04376c25c915..8980c90b2277a242ea8c59a9481fdddb8927ba7d 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_optc.c +++ b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_optc.c @@ -309,6 +309,7 @@ static struct timing_generator_funcs dcn30_tg_funcs = { .enable_crtc_reset = optc1_enable_crtc_reset, .disable_reset_trigger = optc1_disable_reset_trigger, .lock = optc3_lock, + .is_locked = optc1_is_locked, .unlock = optc1_unlock, .lock_doublebuffer_enable = optc3_lock_doublebuffer_enable, .lock_doublebuffer_disable = optc3_lock_doublebuffer_disable, diff --git a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_resource.c b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_resource.c index 5e126fdf6ec105e0a5971d84e0a4b30b2f728eff..8d0f663489ac5a915330f7f1d689d299daed647f 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_resource.c @@ -1716,125 +1716,22 @@ static bool is_soc_bounding_box_valid(struct dc *dc) static bool init_soc_bounding_box(struct dc *dc, struct dcn30_resource_pool *pool) { - const struct gpu_info_soc_bounding_box_v1_0 *bb = dc->soc_bounding_box; struct _vcs_dpi_soc_bounding_box_st *loaded_bb = &dcn3_0_soc; struct _vcs_dpi_ip_params_st *loaded_ip = &dcn3_0_ip; DC_LOGGER_INIT(dc->ctx->logger); - if (!bb && !is_soc_bounding_box_valid(dc)) { + if (!is_soc_bounding_box_valid(dc)) { DC_LOG_ERROR("%s: not valid soc bounding box/n", __func__); return false; } - if (bb && !is_soc_bounding_box_valid(dc)) { - int i; - - dcn3_0_soc.sr_exit_time_us = - fixed16_to_double_to_cpu(bb->sr_exit_time_us); - dcn3_0_soc.sr_enter_plus_exit_time_us = - fixed16_to_double_to_cpu(bb->sr_enter_plus_exit_time_us); - dcn3_0_soc.urgent_latency_us = - fixed16_to_double_to_cpu(bb->urgent_latency_us); - dcn3_0_soc.urgent_latency_pixel_data_only_us = - fixed16_to_double_to_cpu(bb->urgent_latency_pixel_data_only_us); - dcn3_0_soc.urgent_latency_pixel_mixed_with_vm_data_us = - fixed16_to_double_to_cpu(bb->urgent_latency_pixel_mixed_with_vm_data_us); - dcn3_0_soc.urgent_latency_vm_data_only_us = - fixed16_to_double_to_cpu(bb->urgent_latency_vm_data_only_us); - dcn3_0_soc.urgent_out_of_order_return_per_channel_pixel_only_bytes = - le32_to_cpu(bb->urgent_out_of_order_return_per_channel_pixel_only_bytes); - dcn3_0_soc.urgent_out_of_order_return_per_channel_pixel_and_vm_bytes = - le32_to_cpu(bb->urgent_out_of_order_return_per_channel_pixel_and_vm_bytes); - dcn3_0_soc.urgent_out_of_order_return_per_channel_vm_only_bytes = - le32_to_cpu(bb->urgent_out_of_order_return_per_channel_vm_only_bytes); - dcn3_0_soc.pct_ideal_dram_sdp_bw_after_urgent_pixel_only = - fixed16_to_double_to_cpu(bb->pct_ideal_dram_sdp_bw_after_urgent_pixel_only); - dcn3_0_soc.pct_ideal_dram_sdp_bw_after_urgent_pixel_and_vm = - fixed16_to_double_to_cpu(bb->pct_ideal_dram_sdp_bw_after_urgent_pixel_and_vm); - dcn3_0_soc.pct_ideal_dram_sdp_bw_after_urgent_vm_only = - fixed16_to_double_to_cpu(bb->pct_ideal_dram_sdp_bw_after_urgent_vm_only); - dcn3_0_soc.max_avg_sdp_bw_use_normal_percent = - fixed16_to_double_to_cpu(bb->max_avg_sdp_bw_use_normal_percent); - dcn3_0_soc.max_avg_dram_bw_use_normal_percent = - fixed16_to_double_to_cpu(bb->max_avg_dram_bw_use_normal_percent); - dcn3_0_soc.writeback_latency_us = - fixed16_to_double_to_cpu(bb->writeback_latency_us); - dcn3_0_soc.ideal_dram_bw_after_urgent_percent = - fixed16_to_double_to_cpu(bb->ideal_dram_bw_after_urgent_percent); - dcn3_0_soc.max_request_size_bytes = - le32_to_cpu(bb->max_request_size_bytes); - dcn3_0_soc.dram_channel_width_bytes = - le32_to_cpu(bb->dram_channel_width_bytes); - dcn3_0_soc.fabric_datapath_to_dcn_data_return_bytes = - le32_to_cpu(bb->fabric_datapath_to_dcn_data_return_bytes); - dcn3_0_soc.dcn_downspread_percent = - fixed16_to_double_to_cpu(bb->dcn_downspread_percent); - dcn3_0_soc.downspread_percent = - fixed16_to_double_to_cpu(bb->downspread_percent); - dcn3_0_soc.dram_page_open_time_ns = - fixed16_to_double_to_cpu(bb->dram_page_open_time_ns); - dcn3_0_soc.dram_rw_turnaround_time_ns = - fixed16_to_double_to_cpu(bb->dram_rw_turnaround_time_ns); - dcn3_0_soc.dram_return_buffer_per_channel_bytes = - le32_to_cpu(bb->dram_return_buffer_per_channel_bytes); - dcn3_0_soc.round_trip_ping_latency_dcfclk_cycles = - le32_to_cpu(bb->round_trip_ping_latency_dcfclk_cycles); - dcn3_0_soc.urgent_out_of_order_return_per_channel_bytes = - le32_to_cpu(bb->urgent_out_of_order_return_per_channel_bytes); - dcn3_0_soc.channel_interleave_bytes = - le32_to_cpu(bb->channel_interleave_bytes); - dcn3_0_soc.num_banks = - le32_to_cpu(bb->num_banks); - dcn3_0_soc.num_chans = - le32_to_cpu(bb->num_chans); - dcn3_0_soc.gpuvm_min_page_size_bytes = - le32_to_cpu(bb->vmm_page_size_bytes); - dcn3_0_soc.dram_clock_change_latency_us = - fixed16_to_double_to_cpu(bb->dram_clock_change_latency_us); - dcn3_0_soc.writeback_dram_clock_change_latency_us = - fixed16_to_double_to_cpu(bb->writeback_dram_clock_change_latency_us); - dcn3_0_soc.return_bus_width_bytes = - le32_to_cpu(bb->return_bus_width_bytes); - dcn3_0_soc.dispclk_dppclk_vco_speed_mhz = - le32_to_cpu(bb->dispclk_dppclk_vco_speed_mhz); - dcn3_0_soc.xfc_bus_transport_time_us = - le32_to_cpu(bb->xfc_bus_transport_time_us); - dcn3_0_soc.xfc_xbuf_latency_tolerance_us = - le32_to_cpu(bb->xfc_xbuf_latency_tolerance_us); - dcn3_0_soc.use_urgent_burst_bw = - le32_to_cpu(bb->use_urgent_burst_bw); - dcn3_0_soc.num_states = - le32_to_cpu(bb->num_states); - - for (i = 0; i < dcn3_0_soc.num_states; i++) { - dcn3_0_soc.clock_limits[i].state = - le32_to_cpu(bb->clock_limits[i].state); - dcn3_0_soc.clock_limits[i].dcfclk_mhz = - fixed16_to_double_to_cpu(bb->clock_limits[i].dcfclk_mhz); - dcn3_0_soc.clock_limits[i].fabricclk_mhz = - fixed16_to_double_to_cpu(bb->clock_limits[i].fabricclk_mhz); - dcn3_0_soc.clock_limits[i].dispclk_mhz = - fixed16_to_double_to_cpu(bb->clock_limits[i].dispclk_mhz); - dcn3_0_soc.clock_limits[i].dppclk_mhz = - fixed16_to_double_to_cpu(bb->clock_limits[i].dppclk_mhz); - dcn3_0_soc.clock_limits[i].phyclk_mhz = - fixed16_to_double_to_cpu(bb->clock_limits[i].phyclk_mhz); - dcn3_0_soc.clock_limits[i].socclk_mhz = - fixed16_to_double_to_cpu(bb->clock_limits[i].socclk_mhz); - dcn3_0_soc.clock_limits[i].dscclk_mhz = - fixed16_to_double_to_cpu(bb->clock_limits[i].dscclk_mhz); - dcn3_0_soc.clock_limits[i].dram_speed_mts = - fixed16_to_double_to_cpu(bb->clock_limits[i].dram_speed_mts); - } - } - loaded_ip->max_num_otg = pool->base.res_cap->num_timing_generator; loaded_ip->max_num_dpp = pool->base.pipe_count; loaded_ip->clamp_min_dcfclk = dc->config.clamp_min_dcfclk; dcn20_patch_bounding_box(dc, loaded_bb); - if (!bb && dc->ctx->dc_bios->funcs->get_soc_bb_info) { + if (dc->ctx->dc_bios->funcs->get_soc_bb_info) { struct bp_soc_bb_info bb_info = {0}; if (dc->ctx->dc_bios->funcs->get_soc_bb_info(dc->ctx->dc_bios, &bb_info) == BP_RESULT_OK) { @@ -2292,17 +2189,15 @@ static noinline void dcn30_calculate_wm_and_dlg_fp( unsigned int min_dram_speed_mts = context->bw_ctx.dml.vba.DRAMSpeed; unsigned int min_dram_speed_mts_margin = 160; - context->bw_ctx.dml.soc.dram_clock_change_latency_us = dc->clk_mgr->bw_params->dummy_pstate_table[0].dummy_pstate_latency_us; - if (context->bw_ctx.dml.vba.DRAMClockChangeSupport[vlevel][context->bw_ctx.dml.vba.maxMpcComb] == dm_dram_clock_change_unsupported) min_dram_speed_mts = dc->clk_mgr->bw_params->clk_table.entries[dc->clk_mgr->bw_params->clk_table.num_entries - 1].memclk_mhz * 16; - for (i = 3; i > 0; i--) { - if ((min_dram_speed_mts + min_dram_speed_mts_margin > dc->clk_mgr->bw_params->dummy_pstate_table[i].dram_speed_mts) && - (min_dram_speed_mts - min_dram_speed_mts_margin < dc->clk_mgr->bw_params->dummy_pstate_table[i].dram_speed_mts)) - context->bw_ctx.dml.soc.dram_clock_change_latency_us = dc->clk_mgr->bw_params->dummy_pstate_table[i].dummy_pstate_latency_us; - } + /* find largest table entry that is lower than dram speed, but lower than DPM0 still uses DPM0 */ + for (i = 3; i > 0; i--) + if (min_dram_speed_mts + min_dram_speed_mts_margin > dc->clk_mgr->bw_params->dummy_pstate_table[i].dram_speed_mts) + break; + context->bw_ctx.dml.soc.dram_clock_change_latency_us = dc->clk_mgr->bw_params->dummy_pstate_table[i].dummy_pstate_latency_us; context->bw_ctx.dml.soc.sr_enter_plus_exit_time_us = dc->clk_mgr->bw_params->wm_table.nv_entries[WM_C].dml_input.sr_enter_plus_exit_time_us; context->bw_ctx.dml.soc.sr_exit_time_us = dc->clk_mgr->bw_params->wm_table.nv_entries[WM_C].dml_input.sr_exit_time_us; } @@ -2437,16 +2332,28 @@ bool dcn30_validate_bandwidth(struct dc *dc, return out; } -static noinline void get_optimal_dcfclk_fclk_for_uclk(unsigned int uclk_mts, - unsigned int *optimal_dcfclk, - unsigned int *optimal_fclk) +/* + * This must be noinline to ensure anything that deals with FP registers + * is contained within this call; previously our compiling with hard-float + * would result in fp instructions being emitted outside of the boundaries + * of the DC_FP_START/END macros, which makes sense as the compiler has no + * idea about what is wrapped and what is not + * + * This is largely just a workaround to avoid breakage introduced with 5.6, + * ideally all fp-using code should be moved into its own file, only that + * should be compiled with hard-float, and all code exported from there + * should be strictly wrapped with DC_FP_START/END + */ +static noinline void dcn30_get_optimal_dcfclk_fclk_for_uclk(unsigned int uclk_mts, + unsigned int *optimal_dcfclk, + unsigned int *optimal_fclk) { double bw_from_dram, bw_from_dram1, bw_from_dram2; bw_from_dram1 = uclk_mts * dcn3_0_soc.num_chans * - dcn3_0_soc.dram_channel_width_bytes * (dcn3_0_soc.max_avg_dram_bw_use_normal_percent / 100); + dcn3_0_soc.dram_channel_width_bytes * (dcn3_0_soc.max_avg_dram_bw_use_normal_percent / 100); bw_from_dram2 = uclk_mts * dcn3_0_soc.num_chans * - dcn3_0_soc.dram_channel_width_bytes * (dcn3_0_soc.max_avg_sdp_bw_use_normal_percent / 100); + dcn3_0_soc.dram_channel_width_bytes * (dcn3_0_soc.max_avg_sdp_bw_use_normal_percent / 100); bw_from_dram = (bw_from_dram1 < bw_from_dram2) ? bw_from_dram1 : bw_from_dram2; @@ -2505,7 +2412,7 @@ void dcn30_update_bw_bounding_box(struct dc *dc, struct clk_bw_params *bw_params // Calculate optimal dcfclk for each uclk for (i = 0; i < num_uclk_states; i++) { DC_FP_START(); - get_optimal_dcfclk_fclk_for_uclk(bw_params->clk_table.entries[i].memclk_mhz * 16, + dcn30_get_optimal_dcfclk_fclk_for_uclk(bw_params->clk_table.entries[i].memclk_mhz * 16, &optimal_dcfclk_for_uclk[i], NULL); DC_FP_END(); if (optimal_dcfclk_for_uclk[i] < bw_params->clk_table.entries[0].dcfclk_mhz) { @@ -2631,6 +2538,10 @@ static bool dcn30_resource_construct( dc->caps.max_cursor_size = 256; dc->caps.min_horizontal_blanking_period = 80; dc->caps.dmdata_alloc_size = 2048; + dc->caps.mall_size_per_mem_channel = 8; + /* total size = mall per channel * num channels * 1024 * 1024 */ + dc->caps.mall_size_total = dc->caps.mall_size_per_mem_channel * dc->ctx->dc_bios->vram_info.num_chans * 1048576; + dc->caps.cursor_cache_size = dc->caps.max_cursor_size * dc->caps.max_cursor_size * 8; dc->caps.max_slave_planes = 1; dc->caps.post_blend_color_processing = true; diff --git a/drivers/gpu/drm/amd/display/dc/dcn301/dcn301_init.c b/drivers/gpu/drm/amd/display/dc/dcn301/dcn301_init.c index bdad72140cbcec03fe2b61a6b544ec150904d4f4..b8bf6d61005b070cf66923b01be618753dd3fce3 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn301/dcn301_init.c +++ b/drivers/gpu/drm/amd/display/dc/dcn301/dcn301_init.c @@ -98,6 +98,7 @@ static const struct hw_sequencer_funcs dcn301_funcs = { .set_abm_immediate_disable = dcn21_set_abm_immediate_disable, .set_pipe = dcn21_set_pipe, .set_disp_pattern_generator = dcn30_set_disp_pattern_generator, + .set_hubp_blank = dcn30_set_hubp_blank, }; static const struct hwseq_private_funcs dcn301_private_funcs = { diff --git a/drivers/gpu/drm/amd/display/dc/dcn301/dcn301_resource.c b/drivers/gpu/drm/amd/display/dc/dcn301/dcn301_resource.c index 35f5bf08ae96e2570d22282da46f1f499236d4a4..5d4b2c60192ee49763f318777ce4271cf5554feb 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn301/dcn301_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dcn301/dcn301_resource.c @@ -1489,124 +1489,21 @@ static bool is_soc_bounding_box_valid(struct dc *dc) static bool init_soc_bounding_box(struct dc *dc, struct dcn301_resource_pool *pool) { - const struct gpu_info_soc_bounding_box_v1_0 *bb = dc->soc_bounding_box; struct _vcs_dpi_soc_bounding_box_st *loaded_bb = &dcn3_01_soc; struct _vcs_dpi_ip_params_st *loaded_ip = &dcn3_01_ip; DC_LOGGER_INIT(dc->ctx->logger); - if (!bb && !is_soc_bounding_box_valid(dc)) { + if (!is_soc_bounding_box_valid(dc)) { DC_LOG_ERROR("%s: not valid soc bounding box/n", __func__); return false; } - if (bb && !is_soc_bounding_box_valid(dc)) { - int i; - - dcn3_01_soc.sr_exit_time_us = - fixed16_to_double_to_cpu(bb->sr_exit_time_us); - dcn3_01_soc.sr_enter_plus_exit_time_us = - fixed16_to_double_to_cpu(bb->sr_enter_plus_exit_time_us); - dcn3_01_soc.urgent_latency_us = - fixed16_to_double_to_cpu(bb->urgent_latency_us); - dcn3_01_soc.urgent_latency_pixel_data_only_us = - fixed16_to_double_to_cpu(bb->urgent_latency_pixel_data_only_us); - dcn3_01_soc.urgent_latency_pixel_mixed_with_vm_data_us = - fixed16_to_double_to_cpu(bb->urgent_latency_pixel_mixed_with_vm_data_us); - dcn3_01_soc.urgent_latency_vm_data_only_us = - fixed16_to_double_to_cpu(bb->urgent_latency_vm_data_only_us); - dcn3_01_soc.urgent_out_of_order_return_per_channel_pixel_only_bytes = - le32_to_cpu(bb->urgent_out_of_order_return_per_channel_pixel_only_bytes); - dcn3_01_soc.urgent_out_of_order_return_per_channel_pixel_and_vm_bytes = - le32_to_cpu(bb->urgent_out_of_order_return_per_channel_pixel_and_vm_bytes); - dcn3_01_soc.urgent_out_of_order_return_per_channel_vm_only_bytes = - le32_to_cpu(bb->urgent_out_of_order_return_per_channel_vm_only_bytes); - dcn3_01_soc.pct_ideal_dram_sdp_bw_after_urgent_pixel_only = - fixed16_to_double_to_cpu(bb->pct_ideal_dram_sdp_bw_after_urgent_pixel_only); - dcn3_01_soc.pct_ideal_dram_sdp_bw_after_urgent_pixel_and_vm = - fixed16_to_double_to_cpu(bb->pct_ideal_dram_sdp_bw_after_urgent_pixel_and_vm); - dcn3_01_soc.pct_ideal_dram_sdp_bw_after_urgent_vm_only = - fixed16_to_double_to_cpu(bb->pct_ideal_dram_sdp_bw_after_urgent_vm_only); - dcn3_01_soc.max_avg_sdp_bw_use_normal_percent = - fixed16_to_double_to_cpu(bb->max_avg_sdp_bw_use_normal_percent); - dcn3_01_soc.max_avg_dram_bw_use_normal_percent = - fixed16_to_double_to_cpu(bb->max_avg_dram_bw_use_normal_percent); - dcn3_01_soc.writeback_latency_us = - fixed16_to_double_to_cpu(bb->writeback_latency_us); - dcn3_01_soc.ideal_dram_bw_after_urgent_percent = - fixed16_to_double_to_cpu(bb->ideal_dram_bw_after_urgent_percent); - dcn3_01_soc.max_request_size_bytes = - le32_to_cpu(bb->max_request_size_bytes); - dcn3_01_soc.dram_channel_width_bytes = - le32_to_cpu(bb->dram_channel_width_bytes); - dcn3_01_soc.fabric_datapath_to_dcn_data_return_bytes = - le32_to_cpu(bb->fabric_datapath_to_dcn_data_return_bytes); - dcn3_01_soc.dcn_downspread_percent = - fixed16_to_double_to_cpu(bb->dcn_downspread_percent); - dcn3_01_soc.downspread_percent = - fixed16_to_double_to_cpu(bb->downspread_percent); - dcn3_01_soc.dram_page_open_time_ns = - fixed16_to_double_to_cpu(bb->dram_page_open_time_ns); - dcn3_01_soc.dram_rw_turnaround_time_ns = - fixed16_to_double_to_cpu(bb->dram_rw_turnaround_time_ns); - dcn3_01_soc.dram_return_buffer_per_channel_bytes = - le32_to_cpu(bb->dram_return_buffer_per_channel_bytes); - dcn3_01_soc.round_trip_ping_latency_dcfclk_cycles = - le32_to_cpu(bb->round_trip_ping_latency_dcfclk_cycles); - dcn3_01_soc.urgent_out_of_order_return_per_channel_bytes = - le32_to_cpu(bb->urgent_out_of_order_return_per_channel_bytes); - dcn3_01_soc.channel_interleave_bytes = - le32_to_cpu(bb->channel_interleave_bytes); - dcn3_01_soc.num_banks = - le32_to_cpu(bb->num_banks); - dcn3_01_soc.num_chans = - le32_to_cpu(bb->num_chans); - dcn3_01_soc.gpuvm_min_page_size_bytes = - le32_to_cpu(bb->vmm_page_size_bytes); - dcn3_01_soc.dram_clock_change_latency_us = - fixed16_to_double_to_cpu(bb->dram_clock_change_latency_us); - dcn3_01_soc.writeback_dram_clock_change_latency_us = - fixed16_to_double_to_cpu(bb->writeback_dram_clock_change_latency_us); - dcn3_01_soc.return_bus_width_bytes = - le32_to_cpu(bb->return_bus_width_bytes); - dcn3_01_soc.dispclk_dppclk_vco_speed_mhz = - le32_to_cpu(bb->dispclk_dppclk_vco_speed_mhz); - dcn3_01_soc.xfc_bus_transport_time_us = - le32_to_cpu(bb->xfc_bus_transport_time_us); - dcn3_01_soc.xfc_xbuf_latency_tolerance_us = - le32_to_cpu(bb->xfc_xbuf_latency_tolerance_us); - dcn3_01_soc.use_urgent_burst_bw = - le32_to_cpu(bb->use_urgent_burst_bw); - dcn3_01_soc.num_states = - le32_to_cpu(bb->num_states); - - for (i = 0; i < dcn3_01_soc.num_states; i++) { - dcn3_01_soc.clock_limits[i].state = - le32_to_cpu(bb->clock_limits[i].state); - dcn3_01_soc.clock_limits[i].dcfclk_mhz = - fixed16_to_double_to_cpu(bb->clock_limits[i].dcfclk_mhz); - dcn3_01_soc.clock_limits[i].fabricclk_mhz = - fixed16_to_double_to_cpu(bb->clock_limits[i].fabricclk_mhz); - dcn3_01_soc.clock_limits[i].dispclk_mhz = - fixed16_to_double_to_cpu(bb->clock_limits[i].dispclk_mhz); - dcn3_01_soc.clock_limits[i].dppclk_mhz = - fixed16_to_double_to_cpu(bb->clock_limits[i].dppclk_mhz); - dcn3_01_soc.clock_limits[i].phyclk_mhz = - fixed16_to_double_to_cpu(bb->clock_limits[i].phyclk_mhz); - dcn3_01_soc.clock_limits[i].socclk_mhz = - fixed16_to_double_to_cpu(bb->clock_limits[i].socclk_mhz); - dcn3_01_soc.clock_limits[i].dscclk_mhz = - fixed16_to_double_to_cpu(bb->clock_limits[i].dscclk_mhz); - dcn3_01_soc.clock_limits[i].dram_speed_mts = - fixed16_to_double_to_cpu(bb->clock_limits[i].dram_speed_mts); - } - } - loaded_ip->max_num_otg = pool->base.res_cap->num_timing_generator; loaded_ip->max_num_dpp = pool->base.pipe_count; dcn20_patch_bounding_box(dc, loaded_bb); - if (!bb && dc->ctx->dc_bios->funcs->get_soc_bb_info) { + if (dc->ctx->dc_bios->funcs->get_soc_bb_info) { struct bp_soc_bb_info bb_info = {0}; if (dc->ctx->dc_bios->funcs->get_soc_bb_info(dc->ctx->dc_bios, &bb_info) == BP_RESULT_OK) { diff --git a/drivers/gpu/drm/amd/display/dc/dcn302/dcn302_resource.c b/drivers/gpu/drm/amd/display/dc/dcn302/dcn302_resource.c index 808c4dcdb3ac830c8d9f596f1866438d6210f4fc..4b659b63f75b3d3aea15df00bb218d25957904a9 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn302/dcn302_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dcn302/dcn302_resource.c @@ -53,6 +53,8 @@ #include "dce/dce_i2c_hw.h" #include "dce/dce_panel_cntl.h" #include "dce/dmub_abm.h" +#include "dce/dmub_psr.h" +#include "clk_mgr.h" #include "hw_sequencer_private.h" #include "reg_helper.h" @@ -162,8 +164,8 @@ struct _vcs_dpi_soc_bounding_box_st dcn3_02_soc = { .min_dcfclk = 500.0, /* TODO: set this to actual min DCFCLK */ .num_states = 1, - .sr_exit_time_us = 5.20, - .sr_enter_plus_exit_time_us = 9.60, + .sr_exit_time_us = 12, + .sr_enter_plus_exit_time_us = 20, .urgent_latency_us = 4.0, .urgent_latency_pixel_data_only_us = 4.0, .urgent_latency_pixel_mixed_with_vm_data_us = 4.0, @@ -190,7 +192,7 @@ struct _vcs_dpi_soc_bounding_box_st dcn3_02_soc = { .num_banks = 8, .gpuvm_min_page_size_bytes = 4096, .hostvm_min_page_size_bytes = 4096, - .dram_clock_change_latency_us = 350, + .dram_clock_change_latency_us = 404, .dummy_pstate_latency_us = 5, .writeback_dram_clock_change_latency_us = 23.0, .return_bus_width_bytes = 64, @@ -238,6 +240,7 @@ static const struct dc_debug_options debug_defaults_diags = { .dwb_fi_phase = -1, // -1 = disable .dmub_command_table = true, .enable_tri_buf = true, + .disable_psr = true, }; enum dcn302_clk_src_array_id { @@ -1213,6 +1216,9 @@ static void dcn302_resource_destruct(struct resource_pool *pool) dce_abm_destroy(&pool->multiple_abms[i]); } + if (pool->psr != NULL) + dmub_psr_destroy(&pool->psr); + if (pool->dccg != NULL) dcn_dccg_destroy(&pool->dccg); } @@ -1224,6 +1230,165 @@ static void dcn302_destroy_resource_pool(struct resource_pool **pool) *pool = NULL; } +static void dcn302_get_optimal_dcfclk_fclk_for_uclk(unsigned int uclk_mts, + unsigned int *optimal_dcfclk, + unsigned int *optimal_fclk) +{ + double bw_from_dram, bw_from_dram1, bw_from_dram2; + + bw_from_dram1 = uclk_mts * dcn3_02_soc.num_chans * + dcn3_02_soc.dram_channel_width_bytes * (dcn3_02_soc.max_avg_dram_bw_use_normal_percent / 100); + bw_from_dram2 = uclk_mts * dcn3_02_soc.num_chans * + dcn3_02_soc.dram_channel_width_bytes * (dcn3_02_soc.max_avg_sdp_bw_use_normal_percent / 100); + + bw_from_dram = (bw_from_dram1 < bw_from_dram2) ? bw_from_dram1 : bw_from_dram2; + + if (optimal_fclk) + *optimal_fclk = bw_from_dram / + (dcn3_02_soc.fabric_datapath_to_dcn_data_return_bytes * (dcn3_02_soc.max_avg_sdp_bw_use_normal_percent / 100)); + + if (optimal_dcfclk) + *optimal_dcfclk = bw_from_dram / + (dcn3_02_soc.return_bus_width_bytes * (dcn3_02_soc.max_avg_sdp_bw_use_normal_percent / 100)); +} + +void dcn302_update_bw_bounding_box(struct dc *dc, struct clk_bw_params *bw_params) +{ + unsigned int i, j; + unsigned int num_states = 0; + + unsigned int dcfclk_mhz[DC__VOLTAGE_STATES] = {0}; + unsigned int dram_speed_mts[DC__VOLTAGE_STATES] = {0}; + unsigned int optimal_uclk_for_dcfclk_sta_targets[DC__VOLTAGE_STATES] = {0}; + unsigned int optimal_dcfclk_for_uclk[DC__VOLTAGE_STATES] = {0}; + + unsigned int dcfclk_sta_targets[DC__VOLTAGE_STATES] = {694, 875, 1000, 1200}; + unsigned int num_dcfclk_sta_targets = 4; + unsigned int num_uclk_states; + + + if (dc->ctx->dc_bios->vram_info.num_chans) + dcn3_02_soc.num_chans = dc->ctx->dc_bios->vram_info.num_chans; + + if (dc->ctx->dc_bios->vram_info.dram_channel_width_bytes) + dcn3_02_soc.dram_channel_width_bytes = dc->ctx->dc_bios->vram_info.dram_channel_width_bytes; + + dcn3_02_soc.dispclk_dppclk_vco_speed_mhz = dc->clk_mgr->dentist_vco_freq_khz / 1000.0; + dc->dml.soc.dispclk_dppclk_vco_speed_mhz = dc->clk_mgr->dentist_vco_freq_khz / 1000.0; + + if (bw_params->clk_table.entries[0].memclk_mhz) { + int max_dcfclk_mhz = 0, max_dispclk_mhz = 0, max_dppclk_mhz = 0, max_phyclk_mhz = 0; + + for (i = 0; i < MAX_NUM_DPM_LVL; i++) { + if (bw_params->clk_table.entries[i].dcfclk_mhz > max_dcfclk_mhz) + max_dcfclk_mhz = bw_params->clk_table.entries[i].dcfclk_mhz; + if (bw_params->clk_table.entries[i].dispclk_mhz > max_dispclk_mhz) + max_dispclk_mhz = bw_params->clk_table.entries[i].dispclk_mhz; + if (bw_params->clk_table.entries[i].dppclk_mhz > max_dppclk_mhz) + max_dppclk_mhz = bw_params->clk_table.entries[i].dppclk_mhz; + if (bw_params->clk_table.entries[i].phyclk_mhz > max_phyclk_mhz) + max_phyclk_mhz = bw_params->clk_table.entries[i].phyclk_mhz; + } + if (!max_dcfclk_mhz) + max_dcfclk_mhz = dcn3_02_soc.clock_limits[0].dcfclk_mhz; + if (!max_dispclk_mhz) + max_dispclk_mhz = dcn3_02_soc.clock_limits[0].dispclk_mhz; + if (!max_dppclk_mhz) + max_dppclk_mhz = dcn3_02_soc.clock_limits[0].dppclk_mhz; + if (!max_phyclk_mhz) + max_phyclk_mhz = dcn3_02_soc.clock_limits[0].phyclk_mhz; + + if (max_dcfclk_mhz > dcfclk_sta_targets[num_dcfclk_sta_targets-1]) { + /* If max DCFCLK is greater than the max DCFCLK STA target, insert into the DCFCLK STA target array */ + dcfclk_sta_targets[num_dcfclk_sta_targets] = max_dcfclk_mhz; + num_dcfclk_sta_targets++; + } else if (max_dcfclk_mhz < dcfclk_sta_targets[num_dcfclk_sta_targets-1]) { + /* If max DCFCLK is less than the max DCFCLK STA target, cap values and remove duplicates */ + for (i = 0; i < num_dcfclk_sta_targets; i++) { + if (dcfclk_sta_targets[i] > max_dcfclk_mhz) { + dcfclk_sta_targets[i] = max_dcfclk_mhz; + break; + } + } + /* Update size of array since we "removed" duplicates */ + num_dcfclk_sta_targets = i + 1; + } + + num_uclk_states = bw_params->clk_table.num_entries; + + /* Calculate optimal dcfclk for each uclk */ + for (i = 0; i < num_uclk_states; i++) { + dcn302_get_optimal_dcfclk_fclk_for_uclk(bw_params->clk_table.entries[i].memclk_mhz * 16, + &optimal_dcfclk_for_uclk[i], NULL); + if (optimal_dcfclk_for_uclk[i] < bw_params->clk_table.entries[0].dcfclk_mhz) { + optimal_dcfclk_for_uclk[i] = bw_params->clk_table.entries[0].dcfclk_mhz; + } + } + + /* Calculate optimal uclk for each dcfclk sta target */ + for (i = 0; i < num_dcfclk_sta_targets; i++) { + for (j = 0; j < num_uclk_states; j++) { + if (dcfclk_sta_targets[i] < optimal_dcfclk_for_uclk[j]) { + optimal_uclk_for_dcfclk_sta_targets[i] = + bw_params->clk_table.entries[j].memclk_mhz * 16; + break; + } + } + } + + i = 0; + j = 0; + /* create the final dcfclk and uclk table */ + while (i < num_dcfclk_sta_targets && j < num_uclk_states && num_states < DC__VOLTAGE_STATES) { + if (dcfclk_sta_targets[i] < optimal_dcfclk_for_uclk[j] && i < num_dcfclk_sta_targets) { + dcfclk_mhz[num_states] = dcfclk_sta_targets[i]; + dram_speed_mts[num_states++] = optimal_uclk_for_dcfclk_sta_targets[i++]; + } else { + if (j < num_uclk_states && optimal_dcfclk_for_uclk[j] <= max_dcfclk_mhz) { + dcfclk_mhz[num_states] = optimal_dcfclk_for_uclk[j]; + dram_speed_mts[num_states++] = bw_params->clk_table.entries[j++].memclk_mhz * 16; + } else { + j = num_uclk_states; + } + } + } + + while (i < num_dcfclk_sta_targets && num_states < DC__VOLTAGE_STATES) { + dcfclk_mhz[num_states] = dcfclk_sta_targets[i]; + dram_speed_mts[num_states++] = optimal_uclk_for_dcfclk_sta_targets[i++]; + } + + while (j < num_uclk_states && num_states < DC__VOLTAGE_STATES && + optimal_dcfclk_for_uclk[j] <= max_dcfclk_mhz) { + dcfclk_mhz[num_states] = optimal_dcfclk_for_uclk[j]; + dram_speed_mts[num_states++] = bw_params->clk_table.entries[j++].memclk_mhz * 16; + } + + dcn3_02_soc.num_states = num_states; + for (i = 0; i < dcn3_02_soc.num_states; i++) { + dcn3_02_soc.clock_limits[i].state = i; + dcn3_02_soc.clock_limits[i].dcfclk_mhz = dcfclk_mhz[i]; + dcn3_02_soc.clock_limits[i].fabricclk_mhz = dcfclk_mhz[i]; + dcn3_02_soc.clock_limits[i].dram_speed_mts = dram_speed_mts[i]; + + /* Fill all states with max values of all other clocks */ + dcn3_02_soc.clock_limits[i].dispclk_mhz = max_dispclk_mhz; + dcn3_02_soc.clock_limits[i].dppclk_mhz = max_dppclk_mhz; + dcn3_02_soc.clock_limits[i].phyclk_mhz = max_phyclk_mhz; + dcn3_02_soc.clock_limits[i].dtbclk_mhz = dcn3_02_soc.clock_limits[0].dtbclk_mhz; + /* These clocks cannot come from bw_params, always fill from dcn3_02_soc[1] */ + /* FCLK, PHYCLK_D18, SOCCLK, DSCCLK */ + dcn3_02_soc.clock_limits[i].phyclk_d18_mhz = dcn3_02_soc.clock_limits[0].phyclk_d18_mhz; + dcn3_02_soc.clock_limits[i].socclk_mhz = dcn3_02_soc.clock_limits[0].socclk_mhz; + dcn3_02_soc.clock_limits[i].dscclk_mhz = dcn3_02_soc.clock_limits[0].dscclk_mhz; + } + /* re-init DML with updated bb */ + dml_init_instance(&dc->dml, &dcn3_02_soc, &dcn3_02_ip, DML_PROJECT_DCN30); + if (dc->current_state) + dml_init_instance(&dc->current_state->bw_ctx.dml, &dcn3_02_soc, &dcn3_02_ip, DML_PROJECT_DCN30); + } +} + static struct resource_funcs dcn302_res_pool_funcs = { .destroy = dcn302_destroy_resource_pool, .link_enc_create = dcn302_link_encoder_create, @@ -1240,7 +1405,7 @@ static struct resource_funcs dcn302_res_pool_funcs = { .find_first_free_match_stream_enc_for_link = dcn10_find_first_free_match_stream_enc_for_link, .acquire_post_bldn_3dlut = dcn30_acquire_post_bldn_3dlut, .release_post_bldn_3dlut = dcn30_release_post_bldn_3dlut, - .update_bw_bounding_box = dcn30_update_bw_bounding_box, + .update_bw_bounding_box = dcn302_update_bw_bounding_box, .patch_unknown_plane_state = dcn20_patch_unknown_plane_state, }; @@ -1311,7 +1476,10 @@ static bool dcn302_resource_construct( dc->caps.max_cursor_size = 256; dc->caps.min_horizontal_blanking_period = 80; dc->caps.dmdata_alloc_size = 2048; - + dc->caps.mall_size_per_mem_channel = 4; + /* total size = mall per channel * num channels * 1024 * 1024 */ + dc->caps.mall_size_total = dc->caps.mall_size_per_mem_channel * dc->ctx->dc_bios->vram_info.num_chans * 1048576; + dc->caps.cursor_cache_size = dc->caps.max_cursor_size * dc->caps.max_cursor_size * 8; dc->caps.max_slave_planes = 1; dc->caps.post_blend_color_processing = true; dc->caps.force_dp_tps4_for_cp2520 = true; @@ -1354,8 +1522,6 @@ static bool dcn302_resource_construct( if (dc->ctx->dce_environment == DCE_ENV_PRODUCTION_DRV) dc->debug = debug_defaults_drv; - else if (dc->ctx->dce_environment == DCE_ENV_FPGA_MAXIMUS) - dc->debug = debug_defaults_diags; else dc->debug = debug_defaults_diags; @@ -1469,6 +1635,14 @@ static bool dcn302_resource_construct( } pool->timing_generator_count = i; + /* PSR */ + pool->psr = dmub_psr_create(ctx); + if (pool->psr == NULL) { + dm_error("DC: failed to create psr!\n"); + BREAK_TO_DEBUGGER(); + goto create_fail; + } + /* ABMs */ for (i = 0; i < pool->res_cap->num_timing_generator; i++) { pool->multiple_abms[i] = dmub_abm_create(ctx, &abm_regs[i], &abm_shift, &abm_mask); diff --git a/drivers/gpu/drm/amd/display/dc/dcn302/dcn302_resource.h b/drivers/gpu/drm/amd/display/dc/dcn302/dcn302_resource.h index 71f7deed18e34aae16b813e67de50f8e491e1797..42d2c73e30bc163801ed58e27dc1c7ccf1cfbc41 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn302/dcn302_resource.h +++ b/drivers/gpu/drm/amd/display/dc/dcn302/dcn302_resource.h @@ -30,4 +30,6 @@ struct resource_pool *dcn302_create_resource_pool(const struct dc_init_data *init_data, struct dc *dc); +void dcn302_update_bw_bounding_box(struct dc *dc, struct clk_bw_params *bw_params); + #endif /* _DCN302_RESOURCE_H_ */ diff --git a/drivers/gpu/drm/amd/display/dc/dm_cp_psp.h b/drivers/gpu/drm/amd/display/dc/dm_cp_psp.h index 5da7677627a171638524191cbb4bcea3d229054a..cac0b2c0d31b23a5935ec69de039b08ff563da23 100644 --- a/drivers/gpu/drm/amd/display/dc/dm_cp_psp.h +++ b/drivers/gpu/drm/amd/display/dc/dm_cp_psp.h @@ -30,9 +30,10 @@ struct dc_link; struct cp_psp_stream_config { uint8_t otg_inst; - uint8_t link_enc_inst; - uint8_t stream_enc_inst; - uint8_t mst_supported; + uint8_t dig_be; + uint8_t dig_fe; + uint8_t assr_enabled; + uint8_t mst_enabled; void *dm_stream_ctx; bool dpms_off; }; diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_mode_vba_20.c b/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_mode_vba_20.c index 45f028986a8dbfe0fb4d4ad2b4b24d8dc757958c..0f3f510fd83b8a0b54542145df9c44a90507eed7 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_mode_vba_20.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_mode_vba_20.c @@ -3138,7 +3138,7 @@ static void CalculateFlipSchedule( 4.0 * (TimeForFetchingMetaPTEImmediateFlip / LineTime + 0.125), 1) / 4.0; - if ((GPUVMEnable == true || DCCEnable == true)) { + if ((GPUVMEnable || DCCEnable)) { mode_lib->vba.ImmediateFlipBW[0] = BandwidthAvailableForImmediateFlip * ImmediateFlipBytes / TotImmediateFlipBytes; TimeForFetchingRowInVBlankImmediateFlip = dml_max( @@ -4168,10 +4168,11 @@ void dml20_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l for (i = 0; i <= mode_lib->vba.soc.num_states; i++) { locals->DIOSupport[i] = true; for (k = 0; k <= mode_lib->vba.NumberOfActivePlanes - 1; k++) { - if (locals->OutputBppPerState[i][k] == BPP_INVALID - || (mode_lib->vba.OutputFormat[k] == dm_420 + if (!mode_lib->vba.skip_dio_check[k] + && (locals->OutputBppPerState[i][k] == BPP_INVALID + || (mode_lib->vba.OutputFormat[k] == dm_420 && mode_lib->vba.Interlace[k] == true - && mode_lib->vba.ProgressiveToInterlaceUnitInOPP == true)) { + && mode_lib->vba.ProgressiveToInterlaceUnitInOPP == true))) { locals->DIOSupport[i] = false; } } diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_mode_vba_20v2.c b/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_mode_vba_20v2.c index 80170f9721ce949330d72408b81c48575af13101..210c96cd5b03b5bfe4277f84e3562c4f51e42fc4 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_mode_vba_20v2.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_mode_vba_20v2.c @@ -3263,6 +3263,7 @@ static void CalculateFlipSchedule( static unsigned int TruncToValidBPP( double DecimalBPP, + double DesiredBPP, bool DSCEnabled, enum output_encoder_class Output, enum output_format_class Format, @@ -3270,31 +3271,31 @@ static unsigned int TruncToValidBPP( { if (Output == dm_hdmi) { if (Format == dm_420) { - if (DecimalBPP >= 18) + if (DecimalBPP >= 18 && (DesiredBPP == 0 || DesiredBPP == 18)) return 18; - else if (DecimalBPP >= 15) + else if (DecimalBPP >= 15 && (DesiredBPP == 0 || DesiredBPP == 15)) return 15; - else if (DecimalBPP >= 12) + else if (DecimalBPP >= 12 && (DesiredBPP == 0 || DesiredBPP == 12)) return 12; else return BPP_INVALID; } else if (Format == dm_444) { - if (DecimalBPP >= 36) + if (DecimalBPP >= 36 && (DesiredBPP == 0 || DesiredBPP == 36)) return 36; - else if (DecimalBPP >= 30) + else if (DecimalBPP >= 30 && (DesiredBPP == 0 || DesiredBPP == 30)) return 30; - else if (DecimalBPP >= 24) + else if (DecimalBPP >= 24 && (DesiredBPP == 0 || DesiredBPP == 24)) return 24; - else if (DecimalBPP >= 18) + else if (DecimalBPP >= 18 && (DesiredBPP == 0 || DesiredBPP == 18)) return 18; else return BPP_INVALID; } else { - if (DecimalBPP / 1.5 >= 24) + if (DecimalBPP / 1.5 >= 24 && (DesiredBPP == 0 || DesiredBPP == 24)) return 24; - else if (DecimalBPP / 1.5 >= 20) + else if (DecimalBPP / 1.5 >= 20 && (DesiredBPP == 0 || DesiredBPP == 20)) return 20; - else if (DecimalBPP / 1.5 >= 16) + else if (DecimalBPP / 1.5 >= 16 && (DesiredBPP == 0 || DesiredBPP == 16)) return 16; else return BPP_INVALID; @@ -3302,53 +3303,86 @@ static unsigned int TruncToValidBPP( } else { if (DSCEnabled) { if (Format == dm_420) { - if (DecimalBPP < 6) - return BPP_INVALID; - else if (DecimalBPP >= 1.5 * DSCInputBitPerComponent - 1 / 16) - return 1.5 * DSCInputBitPerComponent - 1 / 16; - else - return dml_floor(16 * DecimalBPP, 1) / 16; + if (DesiredBPP == 0) { + if (DecimalBPP < 6) + return BPP_INVALID; + else if (DecimalBPP >= 1.5 * DSCInputBitPerComponent - 1.0 / 16.0) + return 1.5 * DSCInputBitPerComponent - 1.0 / 16.0; + else + return dml_floor(16 * DecimalBPP, 1) / 16.0; + } else { + if (DecimalBPP < 6 + || DesiredBPP < 6 + || DesiredBPP > 1.5 * DSCInputBitPerComponent - 1.0 / 16.0 + || DecimalBPP < DesiredBPP) { + return BPP_INVALID; + } else { + return DesiredBPP; + } + } } else if (Format == dm_n422) { - if (DecimalBPP < 7) - return BPP_INVALID; - else if (DecimalBPP >= 2 * DSCInputBitPerComponent - 1 / 16) - return 2 * DSCInputBitPerComponent - 1 / 16; - else - return dml_floor(16 * DecimalBPP, 1) / 16; + if (DesiredBPP == 0) { + if (DecimalBPP < 7) + return BPP_INVALID; + else if (DecimalBPP >= 2 * DSCInputBitPerComponent - 1.0 / 16.0) + return 2 * DSCInputBitPerComponent - 1.0 / 16.0; + else + return dml_floor(16 * DecimalBPP, 1) / 16.0; + } else { + if (DecimalBPP < 7 + || DesiredBPP < 7 + || DesiredBPP > 2 * DSCInputBitPerComponent - 1.0 / 16.0 + || DecimalBPP < DesiredBPP) { + return BPP_INVALID; + } else { + return DesiredBPP; + } + } } else { - if (DecimalBPP < 8) - return BPP_INVALID; - else if (DecimalBPP >= 3 * DSCInputBitPerComponent - 1 / 16) - return 3 * DSCInputBitPerComponent - 1 / 16; - else - return dml_floor(16 * DecimalBPP, 1) / 16; + if (DesiredBPP == 0) { + if (DecimalBPP < 8) + return BPP_INVALID; + else if (DecimalBPP >= 3 * DSCInputBitPerComponent - 1.0 / 16.0) + return 3 * DSCInputBitPerComponent - 1.0 / 16.0; + else + return dml_floor(16 * DecimalBPP, 1) / 16.0; + } else { + if (DecimalBPP < 8 + || DesiredBPP < 8 + || DesiredBPP > 3 * DSCInputBitPerComponent - 1.0 / 16.0 + || DecimalBPP < DesiredBPP) { + return BPP_INVALID; + } else { + return DesiredBPP; + } + } } } else if (Format == dm_420) { - if (DecimalBPP >= 18) + if (DecimalBPP >= 18 && (DesiredBPP == 0 || DesiredBPP == 18)) return 18; - else if (DecimalBPP >= 15) + else if (DecimalBPP >= 15 && (DesiredBPP == 0 || DesiredBPP == 15)) return 15; - else if (DecimalBPP >= 12) + else if (DecimalBPP >= 12 && (DesiredBPP == 0 || DesiredBPP == 12)) return 12; else return BPP_INVALID; } else if (Format == dm_s422 || Format == dm_n422) { - if (DecimalBPP >= 24) + if (DecimalBPP >= 24 && (DesiredBPP == 0 || DesiredBPP == 24)) return 24; - else if (DecimalBPP >= 20) + else if (DecimalBPP >= 20 && (DesiredBPP == 0 || DesiredBPP == 20)) return 20; - else if (DecimalBPP >= 16) + else if (DecimalBPP >= 16 && (DesiredBPP == 0 || DesiredBPP == 16)) return 16; else return BPP_INVALID; } else { - if (DecimalBPP >= 36) + if (DecimalBPP >= 36 && (DesiredBPP == 0 || DesiredBPP == 36)) return 36; - else if (DecimalBPP >= 30) + else if (DecimalBPP >= 30 && (DesiredBPP == 0 || DesiredBPP == 30)) return 30; - else if (DecimalBPP >= 24) + else if (DecimalBPP >= 24 && (DesiredBPP == 0 || DesiredBPP == 24)) return 24; - else if (DecimalBPP >= 18) + else if (DecimalBPP >= 18 && (DesiredBPP == 0 || DesiredBPP == 18)) return 18; else return BPP_INVALID; @@ -4137,6 +4171,7 @@ void dml20v2_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode locals->RequiresFEC[i][k] = 0; locals->OutputBppPerState[i][k] = TruncToValidBPP( dml_min(600.0, mode_lib->vba.PHYCLKPerState[i]) / mode_lib->vba.PixelClockBackEnd[k] * 24, + mode_lib->vba.ForcedOutputLinkBPP[k], false, mode_lib->vba.Output[k], mode_lib->vba.OutputFormat[k], @@ -4153,6 +4188,7 @@ void dml20v2_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode mode_lib->vba.Outbpp = TruncToValidBPP( (1.0 - mode_lib->vba.Downspreading / 100.0) * 270.0 * mode_lib->vba.OutputLinkDPLanes[k] / mode_lib->vba.PixelClockBackEnd[k] * 8.0, + mode_lib->vba.ForcedOutputLinkBPP[k], false, mode_lib->vba.Output[k], mode_lib->vba.OutputFormat[k], @@ -4160,6 +4196,7 @@ void dml20v2_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode mode_lib->vba.OutbppDSC = TruncToValidBPP( (1.0 - mode_lib->vba.Downspreading / 100.0) * (1.0 - mode_lib->vba.EffectiveFECOverhead / 100.0) * 270.0 * mode_lib->vba.OutputLinkDPLanes[k] / mode_lib->vba.PixelClockBackEnd[k] * 8.0, + mode_lib->vba.ForcedOutputLinkBPP[k], true, mode_lib->vba.Output[k], mode_lib->vba.OutputFormat[k], @@ -4182,6 +4219,7 @@ void dml20v2_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode mode_lib->vba.Outbpp = TruncToValidBPP( (1.0 - mode_lib->vba.Downspreading / 100.0) * 540.0 * mode_lib->vba.OutputLinkDPLanes[k] / mode_lib->vba.PixelClockBackEnd[k] * 8.0, + mode_lib->vba.ForcedOutputLinkBPP[k], false, mode_lib->vba.Output[k], mode_lib->vba.OutputFormat[k], @@ -4189,6 +4227,7 @@ void dml20v2_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode mode_lib->vba.OutbppDSC = TruncToValidBPP( (1.0 - mode_lib->vba.Downspreading / 100.0) * (1.0 - mode_lib->vba.EffectiveFECOverhead / 100.0) * 540.0 * mode_lib->vba.OutputLinkDPLanes[k] / mode_lib->vba.PixelClockBackEnd[k] * 8.0, + mode_lib->vba.ForcedOutputLinkBPP[k], true, mode_lib->vba.Output[k], mode_lib->vba.OutputFormat[k], @@ -4213,6 +4252,7 @@ void dml20v2_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode mode_lib->vba.Outbpp = TruncToValidBPP( (1.0 - mode_lib->vba.Downspreading / 100.0) * 810.0 * mode_lib->vba.OutputLinkDPLanes[k] / mode_lib->vba.PixelClockBackEnd[k] * 8.0, + mode_lib->vba.ForcedOutputLinkBPP[k], false, mode_lib->vba.Output[k], mode_lib->vba.OutputFormat[k], @@ -4220,6 +4260,7 @@ void dml20v2_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode mode_lib->vba.OutbppDSC = TruncToValidBPP( (1.0 - mode_lib->vba.Downspreading / 100.0) * (1.0 - mode_lib->vba.EffectiveFECOverhead / 100.0) * 810.0 * mode_lib->vba.OutputLinkDPLanes[k] / mode_lib->vba.PixelClockBackEnd[k] * 8.0, + mode_lib->vba.ForcedOutputLinkBPP[k], true, mode_lib->vba.Output[k], mode_lib->vba.OutputFormat[k], @@ -4248,10 +4289,11 @@ void dml20v2_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode for (i = 0; i <= mode_lib->vba.soc.num_states; i++) { locals->DIOSupport[i] = true; for (k = 0; k <= mode_lib->vba.NumberOfActivePlanes - 1; k++) { - if (locals->OutputBppPerState[i][k] == BPP_INVALID - || (mode_lib->vba.OutputFormat[k] == dm_420 + if (!mode_lib->vba.skip_dio_check[k] + && (locals->OutputBppPerState[i][k] == BPP_INVALID + || (mode_lib->vba.OutputFormat[k] == dm_420 && mode_lib->vba.Interlace[k] == true - && mode_lib->vba.ProgressiveToInterlaceUnitInOPP == true)) { + && mode_lib->vba.ProgressiveToInterlaceUnitInOPP == true))) { locals->DIOSupport[i] = false; } } diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn21/display_mode_vba_21.c b/drivers/gpu/drm/amd/display/dc/dml/dcn21/display_mode_vba_21.c index 86ff24dffc3efbc7e4830c53015d0f979b8f0249..398210d1af34fcb8eaf3270833d71bfb30ba30f0 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn21/display_mode_vba_21.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn21/display_mode_vba_21.c @@ -4257,10 +4257,11 @@ void dml21_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l for (i = 0; i <= mode_lib->vba.soc.num_states; i++) { locals->DIOSupport[i] = true; for (k = 0; k <= mode_lib->vba.NumberOfActivePlanes - 1; k++) { - if (locals->OutputBppPerState[i][k] == BPP_INVALID - || (mode_lib->vba.OutputFormat[k] == dm_420 + if (!mode_lib->vba.skip_dio_check[k] + && (locals->OutputBppPerState[i][k] == BPP_INVALID + || (mode_lib->vba.OutputFormat[k] == dm_420 && mode_lib->vba.Interlace[k] == true - && mode_lib->vba.ProgressiveToInterlaceUnitInOPP == true)) { + && mode_lib->vba.ProgressiveToInterlaceUnitInOPP == true))) { locals->DIOSupport[i] = false; } } @@ -5121,48 +5122,48 @@ void dml21_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l for (j = 0; j < 2; j++) { enum dm_validation_status status = DML_VALIDATION_OK; - if (mode_lib->vba.ScaleRatioAndTapsSupport != true) { + if (!mode_lib->vba.ScaleRatioAndTapsSupport) { status = DML_FAIL_SCALE_RATIO_TAP; - } else if (mode_lib->vba.SourceFormatPixelAndScanSupport != true) { + } else if (!mode_lib->vba.SourceFormatPixelAndScanSupport) { status = DML_FAIL_SOURCE_PIXEL_FORMAT; - } else if (locals->ViewportSizeSupport[i][0] != true) { + } else if (!locals->ViewportSizeSupport[i][0]) { status = DML_FAIL_VIEWPORT_SIZE; - } else if (locals->DIOSupport[i] != true) { + } else if (!locals->DIOSupport[i]) { status = DML_FAIL_DIO_SUPPORT; - } else if (locals->NotEnoughDSCUnits[i] != false) { + } else if (locals->NotEnoughDSCUnits[i]) { status = DML_FAIL_NOT_ENOUGH_DSC; - } else if (locals->DSCCLKRequiredMoreThanSupported[i] != false) { + } else if (locals->DSCCLKRequiredMoreThanSupported[i]) { status = DML_FAIL_DSC_CLK_REQUIRED; - } else if (locals->ROBSupport[i][0] != true) { + } else if (!locals->ROBSupport[i][0]) { status = DML_FAIL_REORDERING_BUFFER; - } else if (locals->DISPCLK_DPPCLK_Support[i][j] != true) { + } else if (!locals->DISPCLK_DPPCLK_Support[i][j]) { status = DML_FAIL_DISPCLK_DPPCLK; - } else if (locals->TotalAvailablePipesSupport[i][j] != true) { + } else if (!locals->TotalAvailablePipesSupport[i][j]) { status = DML_FAIL_TOTAL_AVAILABLE_PIPES; - } else if (mode_lib->vba.NumberOfOTGSupport != true) { + } else if (!mode_lib->vba.NumberOfOTGSupport) { status = DML_FAIL_NUM_OTG; - } else if (mode_lib->vba.WritebackModeSupport != true) { + } else if (!mode_lib->vba.WritebackModeSupport) { status = DML_FAIL_WRITEBACK_MODE; - } else if (mode_lib->vba.WritebackLatencySupport != true) { + } else if (!mode_lib->vba.WritebackLatencySupport) { status = DML_FAIL_WRITEBACK_LATENCY; - } else if (mode_lib->vba.WritebackScaleRatioAndTapsSupport != true) { + } else if (!mode_lib->vba.WritebackScaleRatioAndTapsSupport) { status = DML_FAIL_WRITEBACK_SCALE_RATIO_TAP; - } else if (mode_lib->vba.CursorSupport != true) { + } else if (!mode_lib->vba.CursorSupport) { status = DML_FAIL_CURSOR_SUPPORT; - } else if (mode_lib->vba.PitchSupport != true) { + } else if (!mode_lib->vba.PitchSupport) { status = DML_FAIL_PITCH_SUPPORT; - } else if (locals->TotalVerticalActiveBandwidthSupport[i][0] != true) { + } else if (!locals->TotalVerticalActiveBandwidthSupport[i][0]) { status = DML_FAIL_TOTAL_V_ACTIVE_BW; - } else if (locals->PTEBufferSizeNotExceeded[i][j] != true) { + } else if (!locals->PTEBufferSizeNotExceeded[i][j]) { status = DML_FAIL_PTE_BUFFER_SIZE; - } else if (mode_lib->vba.NonsupportedDSCInputBPC != false) { + } else if (mode_lib->vba.NonsupportedDSCInputBPC) { status = DML_FAIL_DSC_INPUT_BPC; - } else if ((mode_lib->vba.HostVMEnable != false - && locals->ImmediateFlipSupportedForState[i][j] != true)) { + } else if ((mode_lib->vba.HostVMEnable + && !locals->ImmediateFlipSupportedForState[i][j])) { status = DML_FAIL_HOST_VM_IMMEDIATE_FLIP; - } else if (locals->PrefetchSupported[i][j] != true) { + } else if (!locals->PrefetchSupported[i][j]) { status = DML_FAIL_PREFETCH_SUPPORT; - } else if (locals->VRatioInPrefetchSupported[i][j] != true) { + } else if (!locals->VRatioInPrefetchSupported[i][j]) { status = DML_FAIL_V_RATIO_PREFETCH; } diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn30/display_mode_vba_30.c b/drivers/gpu/drm/amd/display/dc/dml/dcn30/display_mode_vba_30.c index 319dec59bcd1e585d82ea351560ee5d7d80a6a9e..bc07082c135755950e31b63af9bec7356907db1d 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn30/display_mode_vba_30.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn30/display_mode_vba_30.c @@ -1219,13 +1219,13 @@ static bool CalculatePrefetchSchedule( dml_print("DML: prefetch_bw_equ: %f\n", prefetch_bw_equ); if (prefetch_bw_equ > 0) { - if (GPUVMEnable == true) { + if (GPUVMEnable) { Tvm_equ = dml_max3(*Tno_bw + PDEAndMetaPTEBytesFrame * HostVMInefficiencyFactor / prefetch_bw_equ, Tvm_trips, LineTime / 4); } else { Tvm_equ = LineTime / 4; } - if ((GPUVMEnable == true || myPipe->DCCEnable == true)) { + if ((GPUVMEnable || myPipe->DCCEnable)) { Tr0_equ = dml_max4( (MetaRowByte + PixelPTEBytesPerRow * HostVMInefficiencyFactor) / prefetch_bw_equ, Tr0_trips, @@ -4263,7 +4263,7 @@ void dml30_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l for (i = 0; i < v->soc.num_states; i++) { v->DIOSupport[i] = true; for (k = 0; k <= v->NumberOfActivePlanes - 1; k++) { - if (v->BlendingAndTiming[k] == k && (v->Output[k] == dm_dp || v->Output[k] == dm_edp || v->Output[k] == dm_hdmi) + if (!v->skip_dio_check[k] && v->BlendingAndTiming[k] == k && (v->Output[k] == dm_dp || v->Output[k] == dm_edp || v->Output[k] == dm_hdmi) && (v->OutputBppPerState[i][k] == 0 || (v->OutputFormat[k] == dm_420 && v->Interlace[k] == true && v->ProgressiveToInterlaceUnitInOPP == true))) { v->DIOSupport[i] = false; @@ -5558,7 +5558,7 @@ static void CalculateWatermarksAndDRAMSpeedChangeSupport( } } - if (mode_lib->vba.MinActiveDRAMClockChangeMargin > 0 && PrefetchMode == 0) { + if (mode_lib->vba.MinActiveDRAMClockChangeMargin > 0) { *DRAMClockChangeSupport = dm_dram_clock_change_vactive; } else if (((mode_lib->vba.SynchronizedVBlank == true || mode_lib->vba.TotalNumberOfActiveOTG == 1 || SecondMinActiveDRAMClockChangeMarginOneDisplayInVBLank > 0) && PrefetchMode == 0)) { *DRAMClockChangeSupport = dm_dram_clock_change_vblank; diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn30/display_rq_dlg_calc_30.c b/drivers/gpu/drm/amd/display/dc/dml/dcn30/display_rq_dlg_calc_30.c index 5b5916b5bc7107aa240beb70258f8bb8a08dd815..0f14f205ebe593c3ab3a25d1091de8395fba7a71 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn30/display_rq_dlg_calc_30.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn30/display_rq_dlg_calc_30.c @@ -165,8 +165,8 @@ static void handle_det_buf_split(struct display_mode_lib *mode_lib, unsigned int swath_bytes_c = 0; unsigned int full_swath_bytes_packed_l = 0; unsigned int full_swath_bytes_packed_c = 0; - bool req128_l = 0; - bool req128_c = 0; + bool req128_l = false; + bool req128_c = false; bool surf_linear = (pipe_src_param.sw_mode == dm_sw_linear); bool surf_vert = (pipe_src_param.source_scan == dm_vert); unsigned int log2_swath_height_l = 0; @@ -191,37 +191,37 @@ static void handle_det_buf_split(struct display_mode_lib *mode_lib, total_swath_bytes = 2 * full_swath_bytes_packed_l; if (total_swath_bytes <= detile_buf_size_in_bytes) { //full 256b request - req128_l = 0; - req128_c = 0; + req128_l = false; + req128_c = false; swath_bytes_l = full_swath_bytes_packed_l; swath_bytes_c = full_swath_bytes_packed_c; } else if (!rq_param->yuv420) { - req128_l = 1; - req128_c = 0; + req128_l = true; + req128_c = false; swath_bytes_c = full_swath_bytes_packed_c; swath_bytes_l = full_swath_bytes_packed_l / 2; } else if ((double)full_swath_bytes_packed_l / (double)full_swath_bytes_packed_c < 1.5) { - req128_l = 0; - req128_c = 1; + req128_l = false; + req128_c = true; swath_bytes_l = full_swath_bytes_packed_l; swath_bytes_c = full_swath_bytes_packed_c / 2; total_swath_bytes = 2 * swath_bytes_l + 2 * swath_bytes_c; if (total_swath_bytes > detile_buf_size_in_bytes) { - req128_l = 1; + req128_l = true; swath_bytes_l = full_swath_bytes_packed_l / 2; } } else { - req128_l = 1; - req128_c = 0; + req128_l = true; + req128_c = false; swath_bytes_l = full_swath_bytes_packed_l/2; swath_bytes_c = full_swath_bytes_packed_c; total_swath_bytes = 2 * swath_bytes_l + 2 * swath_bytes_c; if (total_swath_bytes > detile_buf_size_in_bytes) { - req128_c = 1; + req128_c = true; swath_bytes_c = full_swath_bytes_packed_c/2; } } @@ -1006,8 +1006,8 @@ static void dml_rq_dlg_get_dlg_params(struct display_mode_lib *mode_lib, double min_dst_y_ttu_vblank = 0; unsigned int dlg_vblank_start = 0; - bool dual_plane = 0; - bool mode_422 = 0; + bool dual_plane = false; + bool mode_422 = false; unsigned int access_dir = 0; unsigned int vp_height_l = 0; unsigned int vp_width_l = 0; @@ -1021,7 +1021,7 @@ static void dml_rq_dlg_get_dlg_params(struct display_mode_lib *mode_lib, double hratio_c = 0; double vratio_l = 0; double vratio_c = 0; - bool scl_enable = 0; + bool scl_enable = false; double line_time_in_us = 0; // double vinit_l; @@ -1156,7 +1156,7 @@ static void dml_rq_dlg_get_dlg_params(struct display_mode_lib *mode_lib, // Source // dcc_en = src.dcc; dual_plane = is_dual_plane((enum source_format_class)(src->source_format)); - mode_422 = 0; // TODO + mode_422 = false; // TODO access_dir = (src->source_scan == dm_vert); // vp access direction: horizontal or vertical accessed vp_height_l = src->viewport_height; vp_width_l = src->viewport_width; diff --git a/drivers/gpu/drm/amd/display/dc/dml/display_mode_structs.h b/drivers/gpu/drm/amd/display/dc/dml/display_mode_structs.h index dd0c3b1780d702967ad3ef5a7db79cc14df1a6d7..0c5128187e0896868134d3436e41fc9256effc0a 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/display_mode_structs.h +++ b/drivers/gpu/drm/amd/display/dc/dml/display_mode_structs.h @@ -297,6 +297,7 @@ struct _vcs_dpi_display_output_params_st { int num_active_wb; int output_bpc; int output_type; + int is_virtual; int output_format; int dsc_slices; int max_audio_sample_rate; diff --git a/drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.c b/drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.c index c9fbb33f05a32e8c918ad30098186aae6b92c469..bc0485a59018ad2622e3c19c2895b340cbe44c2f 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.c +++ b/drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.c @@ -451,6 +451,8 @@ static void fetch_pipe_params(struct display_mode_lib *mode_lib) dout->output_bpp; mode_lib->vba.Output[mode_lib->vba.NumberOfActivePlanes] = (enum output_encoder_class) (dout->output_type); + mode_lib->vba.skip_dio_check[mode_lib->vba.NumberOfActivePlanes] = + dout->is_virtual; if (!dout->dsc_enable) mode_lib->vba.ForcedOutputLinkBPP[mode_lib->vba.NumberOfActivePlanes] = dout->output_bpp; diff --git a/drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.h b/drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.h index 3529fedc4c523c1ac37eeb763ef7c8360ae3ff8d..025aa5bd8ea0a0191d37343b06a69a4e93257861 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.h +++ b/drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.h @@ -340,6 +340,7 @@ struct vba_vars_st { unsigned int DSCInputBitPerComponent[DC__NUM_DPP__MAX]; enum output_format_class OutputFormat[DC__NUM_DPP__MAX]; enum output_encoder_class Output[DC__NUM_DPP__MAX]; + bool skip_dio_check[DC__NUM_DPP__MAX]; unsigned int BlendingAndTiming[DC__NUM_DPP__MAX]; bool SynchronizedVBlank; unsigned int NumberOfCursors[DC__NUM_DPP__MAX]; diff --git a/drivers/gpu/drm/amd/display/dc/gpio/diagnostics/hw_factory_diag.c b/drivers/gpu/drm/amd/display/dc/gpio/diagnostics/hw_factory_diag.c index df68430aeb0c2e08ce2e05daa63ddff6838d18c3..c6e28f6bf1a2791eb9fa19c70d02addde88d3249 100644 --- a/drivers/gpu/drm/amd/display/dc/gpio/diagnostics/hw_factory_diag.c +++ b/drivers/gpu/drm/amd/display/dc/gpio/diagnostics/hw_factory_diag.c @@ -28,6 +28,7 @@ */ #include "dm_services.h" +#include "hw_factory_diag.h" #include "include/gpio_types.h" #include "../hw_factory.h" diff --git a/drivers/gpu/drm/amd/display/dc/gpio/diagnostics/hw_factory_diag.h b/drivers/gpu/drm/amd/display/dc/gpio/diagnostics/hw_factory_diag.h index 8a74f6adb8eeeec433d553643ab19af5c44a3adc..bf68eb1d9a1d2fc7fa6dd5e659c29ed589179252 100644 --- a/drivers/gpu/drm/amd/display/dc/gpio/diagnostics/hw_factory_diag.h +++ b/drivers/gpu/drm/amd/display/dc/gpio/diagnostics/hw_factory_diag.h @@ -26,6 +26,8 @@ #ifndef __DAL_HW_FACTORY_DIAG_FPGA_H__ #define __DAL_HW_FACTORY_DIAG_FPGA_H__ +struct hw_factory; + /* Initialize HW factory function pointers and pin info */ void dal_hw_factory_diag_fpga_init(struct hw_factory *factory); diff --git a/drivers/gpu/drm/amd/display/dc/gpio/diagnostics/hw_translate_diag.c b/drivers/gpu/drm/amd/display/dc/gpio/diagnostics/hw_translate_diag.c index bf906884692714a17ae7516fa660c007b6da8894..e5138a5a8eb5afaef039e45a3270c01d939e8033 100644 --- a/drivers/gpu/drm/amd/display/dc/gpio/diagnostics/hw_translate_diag.c +++ b/drivers/gpu/drm/amd/display/dc/gpio/diagnostics/hw_translate_diag.c @@ -24,6 +24,7 @@ */ #include "dm_services.h" +#include "hw_translate_diag.h" #include "include/gpio_types.h" #include "../hw_translate.h" diff --git a/drivers/gpu/drm/amd/display/dc/gpio/hw_ddc.c b/drivers/gpu/drm/amd/display/dc/gpio/hw_ddc.c index 1ae153eab31d7e02765599ddf0ff6ee0dcf47347..7a8cec2d7a902ac7e3bebbf6ac2c2d096ecacf7f 100644 --- a/drivers/gpu/drm/amd/display/dc/gpio/hw_ddc.c +++ b/drivers/gpu/drm/amd/display/dc/gpio/hw_ddc.c @@ -107,13 +107,12 @@ static enum gpio_result set_config( msleep(3); } } else { - uint32_t reg2; uint32_t sda_pd_dis = 0; uint32_t scl_pd_dis = 0; - reg2 = REG_GET_2(gpio.MASK_reg, - DC_GPIO_SDA_PD_DIS, &sda_pd_dis, - DC_GPIO_SCL_PD_DIS, &scl_pd_dis); + REG_GET_2(gpio.MASK_reg, + DC_GPIO_SDA_PD_DIS, &sda_pd_dis, + DC_GPIO_SCL_PD_DIS, &scl_pd_dis); if (sda_pd_dis) { REG_SET(gpio.MASK_reg, regval, diff --git a/drivers/gpu/drm/amd/display/dc/gpio/hw_factory.c b/drivers/gpu/drm/amd/display/dc/gpio/hw_factory.c index da73bfb3cacd04dce9528125750f20e793fb5b7a..92c65d2fa7d713d2e92c4798fd9a314fd39a26f7 100644 --- a/drivers/gpu/drm/amd/display/dc/gpio/hw_factory.c +++ b/drivers/gpu/drm/amd/display/dc/gpio/hw_factory.c @@ -119,17 +119,3 @@ bool dal_hw_factory_init( return false; } } - -void dal_hw_factory_destroy( - struct dc_context *ctx, - struct hw_factory **factory) -{ - if (!factory || !*factory) { - BREAK_TO_DEBUGGER(); - return; - } - - kfree(*factory); - - *factory = NULL; -} diff --git a/drivers/gpu/drm/amd/display/dc/inc/core_types.h b/drivers/gpu/drm/amd/display/dc/inc/core_types.h index 2d77eac66cb036cfdb85e5ff9cf379f59d7140d9..8efa1b80546deaa03f0a93c7d610a45d6830f3ca 100644 --- a/drivers/gpu/drm/amd/display/dc/inc/core_types.h +++ b/drivers/gpu/drm/amd/display/dc/inc/core_types.h @@ -333,6 +333,7 @@ struct pipe_ctx { union pipe_update_flags update_flags; struct dwbc *dwbc; struct mcif_wb *mcif_wb; + bool vtp_locked; }; struct resource_context { diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/clk_mgr_internal.h b/drivers/gpu/drm/amd/display/dc/inc/hw/clk_mgr_internal.h index ffd37696b6b9adbe161215fc96c5b599fd02f3ae..316301fc1e305eeb17eb24c29e11253f50fa3b50 100644 --- a/drivers/gpu/drm/amd/display/dc/inc/hw/clk_mgr_internal.h +++ b/drivers/gpu/drm/amd/display/dc/inc/hw/clk_mgr_internal.h @@ -309,9 +309,9 @@ static inline bool should_set_clock(bool safe_to_lower, int calc_clk, int cur_cl static inline bool should_update_pstate_support(bool safe_to_lower, bool calc_support, bool cur_support) { if (cur_support != calc_support) { - if (calc_support == true && safe_to_lower) + if (calc_support && safe_to_lower) return true; - else if (calc_support == false && !safe_to_lower) + else if (!calc_support && !safe_to_lower) return true; } diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/dmcu.h b/drivers/gpu/drm/amd/display/dc/inc/hw/dmcu.h index 69d9fbfb4bec18a183fc604b315ad2c4a8e430ac..cd1c0dc32bf8a8dd5e695ee2f67aebde062d5f07 100644 --- a/drivers/gpu/drm/amd/display/dc/inc/hw/dmcu.h +++ b/drivers/gpu/drm/amd/display/dc/inc/hw/dmcu.h @@ -74,6 +74,16 @@ struct dmcu_funcs { bool (*is_dmcu_initialized)(struct dmcu *dmcu); bool (*lock_phy)(struct dmcu *dmcu); bool (*unlock_phy)(struct dmcu *dmcu); + bool (*send_edid_cea)(struct dmcu *dmcu, + int offset, + int total_length, + uint8_t *data, + int length); + bool (*recv_amd_vsdb)(struct dmcu *dmcu, + int *version, + int *min_frame_rate, + int *max_frame_rate); + bool (*recv_edid_cea_ack)(struct dmcu *dmcu, int *offset); }; #endif diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/timing_generator.h b/drivers/gpu/drm/amd/display/dc/inc/hw/timing_generator.h index f7632fe25976d399eb220668c4790432d25b26ab..754832d216fdabbdfa5c65fede5f12e7285964ab 100644 --- a/drivers/gpu/drm/amd/display/dc/inc/hw/timing_generator.h +++ b/drivers/gpu/drm/amd/display/dc/inc/hw/timing_generator.h @@ -190,6 +190,7 @@ struct timing_generator_funcs { void (*set_blank)(struct timing_generator *tg, bool enable_blanking); bool (*is_blanked)(struct timing_generator *tg); + bool (*is_locked)(struct timing_generator *tg); void (*set_overscan_blank_color) (struct timing_generator *tg, const struct tg_color *color); void (*set_blank_color)(struct timing_generator *tg, const struct tg_color *color); void (*set_colors)(struct timing_generator *tg, diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw_sequencer.h b/drivers/gpu/drm/amd/display/dc/inc/hw_sequencer.h index 62804dc7b6981c0928c4b70b3a3baeb8e84c6c6b..0586ab2ffd6ab46d9001f0ed9fa0cdc244676f7e 100644 --- a/drivers/gpu/drm/amd/display/dc/inc/hw_sequencer.h +++ b/drivers/gpu/drm/amd/display/dc/inc/hw_sequencer.h @@ -54,6 +54,7 @@ struct hw_sequencer_funcs { /* Embedded Display Related */ void (*edp_power_control)(struct dc_link *link, bool enable); void (*edp_wait_for_hpd_ready)(struct dc_link *link, bool power_up); + void (*edp_wait_for_T12)(struct dc_link *link); /* Pipe Programming Related */ void (*init_hw)(struct dc *dc); @@ -217,6 +218,9 @@ struct hw_sequencer_funcs { /* Idle Optimization Related */ bool (*apply_idle_power_optimizations)(struct dc *dc, bool enable); + bool (*does_plane_fit_in_mall)(struct dc *dc, struct dc_plane_state *plane, + struct dc_cursor_attributes *cursor_attr); + bool (*is_abm_supported)(struct dc *dc, struct dc_state *context, struct dc_stream_state *stream); @@ -227,6 +231,10 @@ struct hw_sequencer_funcs { enum dc_color_depth color_depth, const struct tg_color *solid_color, int width, int height, int offset); + + void (*set_hubp_blank)(const struct dc *dc, + struct pipe_ctx *pipe_ctx, + bool blank_enable); }; void color_space_to_black_color( diff --git a/drivers/gpu/drm/amd/display/dc/irq/dcn10/irq_service_dcn10.c b/drivers/gpu/drm/amd/display/dc/irq/dcn10/irq_service_dcn10.c index f956b3bde680dfee271115d21587748e945ac4fd..34f43cb650f8c7c135225a4488e697380b0f3783 100644 --- a/drivers/gpu/drm/amd/display/dc/irq/dcn10/irq_service_dcn10.c +++ b/drivers/gpu/drm/amd/display/dc/irq/dcn10/irq_service_dcn10.c @@ -58,6 +58,18 @@ enum dc_irq_source to_dal_irq_source_dcn10( return DC_IRQ_SOURCE_VBLANK5; case DCN_1_0__SRCID__DC_D6_OTG_VSTARTUP: return DC_IRQ_SOURCE_VBLANK6; + case DCN_1_0__SRCID__OTG1_VERTICAL_INTERRUPT0_CONTROL: + return DC_IRQ_SOURCE_DC1_VLINE0; + case DCN_1_0__SRCID__OTG2_VERTICAL_INTERRUPT0_CONTROL: + return DC_IRQ_SOURCE_DC2_VLINE0; + case DCN_1_0__SRCID__OTG3_VERTICAL_INTERRUPT0_CONTROL: + return DC_IRQ_SOURCE_DC3_VLINE0; + case DCN_1_0__SRCID__OTG4_VERTICAL_INTERRUPT0_CONTROL: + return DC_IRQ_SOURCE_DC4_VLINE0; + case DCN_1_0__SRCID__OTG5_VERTICAL_INTERRUPT0_CONTROL: + return DC_IRQ_SOURCE_DC5_VLINE0; + case DCN_1_0__SRCID__OTG6_VERTICAL_INTERRUPT0_CONTROL: + return DC_IRQ_SOURCE_DC6_VLINE0; case DCN_1_0__SRCID__OTG0_IHC_V_UPDATE_NO_LOCK_INTERRUPT: return DC_IRQ_SOURCE_VUPDATE1; case DCN_1_0__SRCID__OTG1_IHC_V_UPDATE_NO_LOCK_INTERRUPT: @@ -167,6 +179,11 @@ static const struct irq_source_info_funcs vblank_irq_info_funcs = { .ack = NULL }; +static const struct irq_source_info_funcs vline0_irq_info_funcs = { + .set = NULL, + .ack = NULL +}; + static const struct irq_source_info_funcs vupdate_no_lock_irq_info_funcs = { .set = NULL, .ack = NULL @@ -241,6 +258,14 @@ static const struct irq_source_info_funcs vupdate_no_lock_irq_info_funcs = { .funcs = &vblank_irq_info_funcs\ } +#define vline0_int_entry(reg_num)\ + [DC_IRQ_SOURCE_DC1_VLINE0 + reg_num] = {\ + IRQ_REG_ENTRY(OTG, reg_num,\ + OTG_VERTICAL_INTERRUPT0_CONTROL, OTG_VERTICAL_INTERRUPT0_INT_ENABLE,\ + OTG_VERTICAL_INTERRUPT0_CONTROL, OTG_VERTICAL_INTERRUPT0_CLEAR),\ + .funcs = &vline0_irq_info_funcs\ + } + #define dummy_irq_entry() \ {\ .funcs = &dummy_irq_info_funcs\ @@ -349,6 +374,12 @@ irq_source_info_dcn10[DAL_IRQ_SOURCES_NUMBER] = { vblank_int_entry(3), vblank_int_entry(4), vblank_int_entry(5), + vline0_int_entry(0), + vline0_int_entry(1), + vline0_int_entry(2), + vline0_int_entry(3), + vline0_int_entry(4), + vline0_int_entry(5), }; static const struct irq_service_funcs irq_service_funcs_dcn10 = { diff --git a/drivers/gpu/drm/amd/display/dc/irq/dcn21/irq_service_dcn21.c b/drivers/gpu/drm/amd/display/dc/irq/dcn21/irq_service_dcn21.c index 1b971265418b6667b60f6ecd9db90b72b9231186..0e0f494fbb5e138b1f739cd664252d2899c2fc12 100644 --- a/drivers/gpu/drm/amd/display/dc/irq/dcn21/irq_service_dcn21.c +++ b/drivers/gpu/drm/amd/display/dc/irq/dcn21/irq_service_dcn21.c @@ -168,6 +168,11 @@ static const struct irq_source_info_funcs vblank_irq_info_funcs = { .ack = NULL }; +static const struct irq_source_info_funcs vupdate_no_lock_irq_info_funcs = { + .set = NULL, + .ack = NULL +}; + #undef BASE_INNER #define BASE_INNER(seg) DMU_BASE__INST0_SEG ## seg @@ -230,6 +235,17 @@ static const struct irq_source_info_funcs vblank_irq_info_funcs = { .funcs = &vblank_irq_info_funcs\ } +/* vupdate_no_lock_int_entry maps to DC_IRQ_SOURCE_VUPDATEx, to match semantic + * of DCE's DC_IRQ_SOURCE_VUPDATEx. + */ +#define vupdate_no_lock_int_entry(reg_num)\ + [DC_IRQ_SOURCE_VUPDATE1 + reg_num] = {\ + IRQ_REG_ENTRY(OTG, reg_num,\ + OTG_GLOBAL_SYNC_STATUS, VUPDATE_NO_LOCK_INT_EN,\ + OTG_GLOBAL_SYNC_STATUS, VUPDATE_NO_LOCK_EVENT_CLEAR),\ + .funcs = &vupdate_no_lock_irq_info_funcs\ + } + #define vblank_int_entry(reg_num)\ [DC_IRQ_SOURCE_VBLANK1 + reg_num] = {\ IRQ_REG_ENTRY(OTG, reg_num,\ @@ -338,6 +354,12 @@ irq_source_info_dcn21[DAL_IRQ_SOURCES_NUMBER] = { vupdate_int_entry(3), vupdate_int_entry(4), vupdate_int_entry(5), + vupdate_no_lock_int_entry(0), + vupdate_no_lock_int_entry(1), + vupdate_no_lock_int_entry(2), + vupdate_no_lock_int_entry(3), + vupdate_no_lock_int_entry(4), + vupdate_no_lock_int_entry(5), vblank_int_entry(0), vblank_int_entry(1), vblank_int_entry(2), diff --git a/drivers/gpu/drm/amd/display/dc/irq/irq_service.c b/drivers/gpu/drm/amd/display/dc/irq/irq_service.c index 6bf27bde87240b1b4b87565d9e0c9c1b856d67b0..5f245bde54ff7664cf749a8d759f47b260ff657d 100644 --- a/drivers/gpu/drm/amd/display/dc/irq/irq_service.c +++ b/drivers/gpu/drm/amd/display/dc/irq/irq_service.c @@ -79,7 +79,7 @@ void dal_irq_service_destroy(struct irq_service **irq_service) *irq_service = NULL; } -const struct irq_source_info *find_irq_source_info( +static const struct irq_source_info *find_irq_source_info( struct irq_service *irq_service, enum dc_irq_source source) { diff --git a/drivers/gpu/drm/amd/display/dc/irq_types.h b/drivers/gpu/drm/amd/display/dc/irq_types.h index d0ccd81ad5b4d8f04c297665bc963cd97677df7f..87812d81fed3c9501bc220547172795acebfbfd1 100644 --- a/drivers/gpu/drm/amd/display/dc/irq_types.h +++ b/drivers/gpu/drm/amd/display/dc/irq_types.h @@ -160,6 +160,7 @@ enum irq_type IRQ_TYPE_PFLIP = DC_IRQ_SOURCE_PFLIP1, IRQ_TYPE_VUPDATE = DC_IRQ_SOURCE_VUPDATE1, IRQ_TYPE_VBLANK = DC_IRQ_SOURCE_VBLANK1, + IRQ_TYPE_VLINE0 = DC_IRQ_SOURCE_DC1_VLINE0, }; #define DAL_VALID_IRQ_SRC_NUM(src) \ diff --git a/drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h b/drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h index 249a076d6f69c573eece0e3a9d23fe10bc7d7638..072b4e7e624b006aa0eced5f99fd3824b73e8f9a 100644 --- a/drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h +++ b/drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h @@ -47,10 +47,10 @@ /* Firmware versioning. */ #ifdef DMUB_EXPOSE_VERSION -#define DMUB_FW_VERSION_GIT_HASH 0xf51b86a +#define DMUB_FW_VERSION_GIT_HASH 0x6444c02e7 #define DMUB_FW_VERSION_MAJOR 0 #define DMUB_FW_VERSION_MINOR 0 -#define DMUB_FW_VERSION_REVISION 47 +#define DMUB_FW_VERSION_REVISION 51 #define DMUB_FW_VERSION_TEST 0 #define DMUB_FW_VERSION_VBIOS 0 #define DMUB_FW_VERSION_HOTFIX 0 @@ -458,6 +458,10 @@ struct dmub_rb_cmd_mall { uint16_t cursor_pitch; uint16_t cursor_height; uint8_t cursor_bpp; + uint8_t debug_bits; + + uint8_t reserved1; + uint8_t reserved2; }; struct dmub_cmd_digx_encoder_control_data { @@ -487,13 +491,34 @@ struct dmub_rb_cmd_enable_disp_power_gating { struct dmub_cmd_enable_disp_power_gating_data power_gating; }; -struct dmub_cmd_dig1_transmitter_control_data { +struct dmub_dig_transmitter_control_data_v1_7 { + uint8_t phyid; /**< 0=UNIPHYA, 1=UNIPHYB, 2=UNIPHYC, 3=UNIPHYD, 4=UNIPHYE, 5=UNIPHYF */ + uint8_t action; /**< Defined as ATOM_TRANSMITER_ACTION_xxx */ + union { + uint8_t digmode; /**< enum atom_encode_mode_def */ + uint8_t dplaneset; /**< DP voltage swing and pre-emphasis value, "DP_LANE_SET__xDB_y_zV" */ + } mode_laneset; + uint8_t lanenum; /**< Number of lanes */ + union { + uint32_t symclk_10khz; /**< Symbol Clock in 10Khz */ + } symclk_units; + uint8_t hpdsel; /**< =1: HPD1, =2: HPD2, ..., =6: HPD6, =0: HPD is not assigned */ + uint8_t digfe_sel; /**< DIG front-end selection, bit0 means DIG0 FE is enabled */ + uint8_t connobj_id; /**< Connector Object Id defined in ObjectId.h */ + uint8_t reserved0; /**< For future use */ + uint8_t reserved1; /**< For future use */ + uint8_t reserved2[3]; /**< For future use */ + uint32_t reserved3[11]; /**< For future use */ +}; + +union dmub_cmd_dig1_transmitter_control_data { struct dig_transmitter_control_parameters_v1_6 dig; + struct dmub_dig_transmitter_control_data_v1_7 dig_v1_7; }; struct dmub_rb_cmd_dig1_transmitter_control { struct dmub_cmd_header header; - struct dmub_cmd_dig1_transmitter_control_data transmitter_control; + union dmub_cmd_dig1_transmitter_control_data transmitter_control; }; struct dmub_rb_cmd_dpphy_init { @@ -624,6 +649,7 @@ enum dmub_cmd_mall_type { DMUB_CMD__MALL_ACTION_ALLOW = 0, DMUB_CMD__MALL_ACTION_DISALLOW = 1, DMUB_CMD__MALL_ACTION_COPY_CURSOR = 2, + DMUB_CMD__MALL_ACTION_NO_DF_REQ = 3, }; struct dmub_cmd_psr_copy_settings_data { @@ -648,6 +674,7 @@ struct dmub_cmd_psr_copy_settings_data { uint8_t multi_disp_optimizations_en; uint16_t init_sdp_deadline; uint16_t pad2; + uint32_t line_time_in_us; }; struct dmub_rb_cmd_psr_copy_settings { diff --git a/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn20.c b/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn20.c index cafba1d23c6a60837d2192da17b155a51def9770..8e8e65fa83c020619bd048dc4b191e16a3bb321d 100644 --- a/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn20.c +++ b/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn20.c @@ -81,6 +81,13 @@ static inline void dmub_dcn20_translate_addr(const union dmub_addr *addr_in, addr_out->quad_part = addr_in->quad_part - fb_base + fb_offset; } +bool dmub_dcn20_use_cached_inbox(struct dmub_srv *dmub) +{ + /* Cached inbox is not supported in this fw version range */ + return !(dmub->fw_version >= DMUB_FW_VERSION(1, 0, 0) && + dmub->fw_version <= DMUB_FW_VERSION(1, 10, 0)); +} + void dmub_dcn20_reset(struct dmub_srv *dmub) { union dmub_gpint_data_register cmd; @@ -216,7 +223,7 @@ void dmub_dcn20_setup_windows(struct dmub_srv *dmub, dmub_dcn20_translate_addr(&cw4->offset, fb_base, fb_offset, &offset); /* New firmware can support CW4. */ - if (dmub->fw_version > DMUB_FW_VERSION(1, 0, 10)) { + if (dmub_dcn20_use_cached_inbox(dmub)) { REG_WRITE(DMCUB_REGION3_CW4_OFFSET, offset.u.low_part); REG_WRITE(DMCUB_REGION3_CW4_OFFSET_HIGH, offset.u.high_part); REG_WRITE(DMCUB_REGION3_CW4_BASE_ADDRESS, cw4->region.base); @@ -255,7 +262,7 @@ void dmub_dcn20_setup_mailbox(struct dmub_srv *dmub, const struct dmub_region *inbox1) { /* New firmware can support CW4 for the inbox. */ - if (dmub->fw_version > DMUB_FW_VERSION(1, 0, 10)) + if (dmub_dcn20_use_cached_inbox(dmub)) REG_WRITE(DMCUB_INBOX1_BASE_ADDRESS, inbox1->base); else REG_WRITE(DMCUB_INBOX1_BASE_ADDRESS, 0x80000000); diff --git a/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn20.h b/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn20.h index d438f365cbb08f5405bc09902787dc9d47f62fc3..a62be9c0652e3ec19332b8ae3ffbcb5677c5a695 100644 --- a/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn20.h +++ b/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn20.h @@ -198,4 +198,6 @@ void dmub_dcn20_skip_dmub_panel_power_sequence(struct dmub_srv *dmub, bool skip) union dmub_fw_boot_status dmub_dcn20_get_fw_boot_status(struct dmub_srv *dmub); +bool dmub_dcn20_use_cached_inbox(struct dmub_srv *dmub); + #endif /* _DMUB_DCN20_H_ */ diff --git a/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn30.c b/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn30.c index f00df02ded81bc9c1b054144d15323effebc83a4..b4bc0df2f14a3e66ff169c449f43fbb888d89004 100644 --- a/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn30.c +++ b/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn30.c @@ -26,6 +26,7 @@ #include "../dmub_srv.h" #include "dmub_reg.h" #include "dmub_dcn20.h" +#include "dmub_dcn30.h" #include "sienna_cichlid_ip_offset.h" #include "dcn/dcn_3_0_0_offset.h" @@ -154,7 +155,7 @@ void dmub_dcn30_setup_windows(struct dmub_srv *dmub, offset = cw4->offset; /* New firmware can support CW4. */ - if (dmub->fw_version > DMUB_FW_VERSION(1, 0, 10)) { + if (dmub_dcn20_use_cached_inbox(dmub)) { REG_WRITE(DMCUB_REGION3_CW4_OFFSET, offset.u.low_part); REG_WRITE(DMCUB_REGION3_CW4_OFFSET_HIGH, offset.u.high_part); REG_WRITE(DMCUB_REGION3_CW4_BASE_ADDRESS, cw4->region.base); diff --git a/drivers/gpu/drm/amd/display/dmub/src/dmub_srv.c b/drivers/gpu/drm/amd/display/dmub/src/dmub_srv.c index f388d36af0b6c1e0505a5ef3758182c33d96bfde..61f64a295f06aa82c4dfb6092eb3d0be295dc067 100644 --- a/drivers/gpu/drm/amd/display/dmub/src/dmub_srv.c +++ b/drivers/gpu/drm/amd/display/dmub/src/dmub_srv.c @@ -406,6 +406,9 @@ enum dmub_status dmub_srv_hw_init(struct dmub_srv *dmub, dmub->fb_offset = params->fb_offset; dmub->psp_version = params->psp_version; + if (dmub->hw_funcs.reset) + dmub->hw_funcs.reset(dmub); + if (inst_fb && data_fb) { cw0.offset.quad_part = inst_fb->gpu_addr; cw0.region.base = DMUB_CW0_BASE; @@ -427,9 +430,6 @@ enum dmub_status dmub_srv_hw_init(struct dmub_srv *dmub, } - if (dmub->hw_funcs.reset) - dmub->hw_funcs.reset(dmub); - if (inst_fb && data_fb && bios_fb && mail_fb && tracebuff_fb && fw_state_fb && scratch_mem_fb) { cw2.offset.quad_part = data_fb->gpu_addr; @@ -489,9 +489,6 @@ enum dmub_status dmub_srv_hw_reset(struct dmub_srv *dmub) if (!dmub->sw_init) return DMUB_STATUS_INVALID; - if (dmub->hw_init == false) - return DMUB_STATUS_OK; - if (dmub->hw_funcs.reset) dmub->hw_funcs.reset(dmub); diff --git a/drivers/gpu/drm/amd/display/modules/color/color_table.c b/drivers/gpu/drm/amd/display/modules/color/color_table.c index 692e536e7d05702694343e100499608f0b45f8de..410f2a82b9a2ebbe3a2e3872f55047f5106a7b36 100644 --- a/drivers/gpu/drm/amd/display/modules/color/color_table.c +++ b/drivers/gpu/drm/amd/display/modules/color/color_table.c @@ -1,10 +1,26 @@ /* - * Copyright (c) 2019 Advanced Micro Devices, Inc. (unpublished) + * Copyright 2019 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: AMD * - * All rights reserved. This notice is intended as a precaution against - * inadvertent publication and does not imply publication or any waiver - * of confidentiality. The year included in the foregoing notice is the - * year of creation of the work. */ #include "color_table.h" diff --git a/drivers/gpu/drm/amd/display/modules/hdcp/hdcp.h b/drivers/gpu/drm/amd/display/modules/hdcp/hdcp.h index 6c678cfb82e39898f7e5134b9dc96d9d739e107e..5c22cf7e6118fb41960e762bf2e41bcdd83f327c 100644 --- a/drivers/gpu/drm/amd/display/modules/hdcp/hdcp.h +++ b/drivers/gpu/drm/amd/display/modules/hdcp/hdcp.h @@ -397,7 +397,7 @@ static inline uint8_t is_dp_hdcp(struct mod_hdcp *hdcp) static inline uint8_t is_dp_mst_hdcp(struct mod_hdcp *hdcp) { return (hdcp->connection.link.mode == MOD_HDCP_MODE_DP && - hdcp->connection.link.dp.mst_supported); + hdcp->connection.link.dp.mst_enabled); } static inline uint8_t is_hdmi_dvi_sl_hdcp(struct mod_hdcp *hdcp) diff --git a/drivers/gpu/drm/amd/display/modules/hdcp/hdcp_psp.c b/drivers/gpu/drm/amd/display/modules/hdcp/hdcp_psp.c index 3a367a5968ae19c5e297a7337dacf0c04e5ec397..904ce9b88088eae800e7e5e21681aedac7e12c0b 100644 --- a/drivers/gpu/drm/amd/display/modules/hdcp/hdcp_psp.c +++ b/drivers/gpu/drm/amd/display/modules/hdcp/hdcp_psp.c @@ -106,7 +106,7 @@ enum mod_hdcp_status mod_hdcp_add_display_to_topology(struct mod_hdcp *hdcp, dtm_cmd->dtm_in_message.topology_update_v2.dig_be = link->dig_be; dtm_cmd->dtm_in_message.topology_update_v2.dig_fe = display->dig_fe; if (is_dp_hdcp(hdcp)) - dtm_cmd->dtm_in_message.topology_update_v2.is_assr = link->dp.assr_supported; + dtm_cmd->dtm_in_message.topology_update_v2.is_assr = link->dp.assr_enabled; dtm_cmd->dtm_in_message.topology_update_v2.dp_mst_vcid = display->vc_id; dtm_cmd->dtm_in_message.topology_update_v2.max_hdcp_supported_version = @@ -548,6 +548,8 @@ enum mod_hdcp_status mod_hdcp_hdcp2_validate_ake_cert(struct mod_hdcp *hdcp) TA_HDCP2_MSG_AUTHENTICATION_STATUS__RECEIVERID_REVOKED) { hdcp->connection.is_hdcp2_revoked = 1; status = MOD_HDCP_STATUS_HDCP2_AKE_CERT_REVOKED; + } else { + status = MOD_HDCP_STATUS_HDCP2_VALIDATE_AKE_CERT_FAILURE; } } mutex_unlock(&psp->hdcp_context.mutex); diff --git a/drivers/gpu/drm/amd/display/modules/inc/mod_hdcp.h b/drivers/gpu/drm/amd/display/modules/inc/mod_hdcp.h index eed560eecbab498dce293105ca1add9add9a5dc9..d223ed3be5d3deca64218cc5752c75472c80d9aa 100644 --- a/drivers/gpu/drm/amd/display/modules/inc/mod_hdcp.h +++ b/drivers/gpu/drm/amd/display/modules/inc/mod_hdcp.h @@ -101,8 +101,8 @@ enum mod_hdcp_status { struct mod_hdcp_displayport { uint8_t rev; - uint8_t assr_supported; - uint8_t mst_supported; + uint8_t assr_enabled; + uint8_t mst_enabled; }; struct mod_hdcp_hdmi { diff --git a/drivers/gpu/drm/amd/display/modules/info_packet/info_packet.c b/drivers/gpu/drm/amd/display/modules/info_packet/info_packet.c index 0fdf7a3e96deaca5c5e5a0f0f9fb3cf1b04d6d95..57f198de5e2cb3cdef09656ecc525ad73a34666d 100644 --- a/drivers/gpu/drm/amd/display/modules/info_packet/info_packet.c +++ b/drivers/gpu/drm/amd/display/modules/info_packet/info_packet.c @@ -409,16 +409,11 @@ void mod_build_vsc_infopacket(const struct dc_stream_state *stream, } /** - ***************************************************************************** - * Function: mod_build_hf_vsif_infopacket + * mod_build_hf_vsif_infopacket - Prepare HDMI Vendor Specific info frame. + * Follows HDMI Spec to build up Vendor Specific info frame * - * @brief - * Prepare HDMI Vendor Specific info frame. - * Follows HDMI Spec to build up Vendor Specific info frame - * - * @param [in] stream: contains data we may need to construct VSIF (i.e. timing_3d_format, etc.) - * @param [out] info_packet: output structure where to store VSIF - ***************************************************************************** + * @stream: contains data we may need to construct VSIF (i.e. timing_3d_format, etc.) + * @info_packet: output structure where to store VSIF */ void mod_build_hf_vsif_infopacket(const struct dc_stream_state *stream, struct dc_info_packet *info_packet) diff --git a/drivers/gpu/drm/amd/display/modules/power/power_helpers.c b/drivers/gpu/drm/amd/display/modules/power/power_helpers.c index 4fd8bce95d843140341bae3db3acd99ce9dad9ba..6270ecbd24389b8f47f684aa2142e7e312cb3585 100644 --- a/drivers/gpu/drm/amd/display/modules/power/power_helpers.c +++ b/drivers/gpu/drm/amd/display/modules/power/power_helpers.c @@ -266,7 +266,7 @@ static void fill_backlight_transform_table_v_2_2(struct dmcu_iram_parameters par * format U4.10. */ for (i = 1; i+1 < num_entries; i++) { - lut_index = (params.backlight_lut_array_size - 1) * i / (num_entries - 1); + lut_index = DIV_ROUNDUP((i * params.backlight_lut_array_size), num_entries); ASSERT(lut_index < params.backlight_lut_array_size); table->backlight_thresholds[i] = (big_endian) ? @@ -278,7 +278,7 @@ static void fill_backlight_transform_table_v_2_2(struct dmcu_iram_parameters par } } -void fill_iram_v_2(struct iram_table_v_2 *ram_table, struct dmcu_iram_parameters params) +static void fill_iram_v_2(struct iram_table_v_2 *ram_table, struct dmcu_iram_parameters params) { unsigned int set = params.set; @@ -452,7 +452,7 @@ void fill_iram_v_2(struct iram_table_v_2 *ram_table, struct dmcu_iram_parameters params, ram_table); } -void fill_iram_v_2_2(struct iram_table_v_2_2 *ram_table, struct dmcu_iram_parameters params) +static void fill_iram_v_2_2(struct iram_table_v_2_2 *ram_table, struct dmcu_iram_parameters params) { unsigned int set = params.set; @@ -598,7 +598,7 @@ void fill_iram_v_2_2(struct iram_table_v_2_2 *ram_table, struct dmcu_iram_parame params, ram_table, true); } -void fill_iram_v_2_3(struct iram_table_v_2_2 *ram_table, struct dmcu_iram_parameters params, bool big_endian) +static void fill_iram_v_2_3(struct iram_table_v_2_2 *ram_table, struct dmcu_iram_parameters params, bool big_endian) { unsigned int i, j; unsigned int set = params.set; diff --git a/drivers/gpu/drm/amd/include/amd_pcie.h b/drivers/gpu/drm/amd/include/amd_pcie.h index 9cb9ceb4d74dc28ae48b8acb80b659d03b8e5e09..a1ece3eecdf5e0af5fa3679b5d3d89f8334c657f 100644 --- a/drivers/gpu/drm/amd/include/amd_pcie.h +++ b/drivers/gpu/drm/amd/include/amd_pcie.h @@ -28,6 +28,7 @@ #define CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 0x00020000 #define CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3 0x00040000 #define CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4 0x00080000 +#define CAIL_PCIE_LINK_SPEED_SUPPORT_GEN5 0x00100000 #define CAIL_PCIE_LINK_SPEED_SUPPORT_MASK 0xFFFF0000 #define CAIL_PCIE_LINK_SPEED_SUPPORT_SHIFT 16 @@ -36,6 +37,7 @@ #define CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 0x00000002 #define CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3 0x00000004 #define CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN4 0x00000008 +#define CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN5 0x00000010 #define CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_MASK 0x0000FFFF #define CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_SHIFT 0 diff --git a/drivers/gpu/drm/amd/include/amd_shared.h b/drivers/gpu/drm/amd/include/amd_shared.h index 9676016a37ce906f4eef84c4f8b5cf8fabc8450e..43ed6291b2b89acea6e0f736043afde1fa1e2a68 100644 --- a/drivers/gpu/drm/amd/include/amd_shared.h +++ b/drivers/gpu/drm/amd/include/amd_shared.h @@ -213,6 +213,7 @@ enum PP_FEATURE_MASK { PP_ACG_MASK = 0x10000, PP_STUTTER_MODE = 0x20000, PP_AVFS_MASK = 0x40000, + PP_GFX_DCS_MASK = 0x80000, }; enum DC_FEATURE_MASK { diff --git a/drivers/gpu/drm/amd/include/asic_reg/oss/osssys_4_2_0_offset.h b/drivers/gpu/drm/amd/include/asic_reg/oss/osssys_4_2_0_offset.h new file mode 100644 index 0000000000000000000000000000000000000000..bd129266ebfd10d9418f4f72beb86be4376d912e --- /dev/null +++ b/drivers/gpu/drm/amd/include/asic_reg/oss/osssys_4_2_0_offset.h @@ -0,0 +1,345 @@ +/* + * Copyright 2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ +#ifndef _osssys_4_2_0_OFFSET_HEADER +#define _osssys_4_2_0_OFFSET_HEADER + + + +// addressBlock: osssys_osssysdec +// base address: 0x4280 +#define mmIH_VMID_0_LUT 0x0000 +#define mmIH_VMID_0_LUT_BASE_IDX 0 +#define mmIH_VMID_1_LUT 0x0001 +#define mmIH_VMID_1_LUT_BASE_IDX 0 +#define mmIH_VMID_2_LUT 0x0002 +#define mmIH_VMID_2_LUT_BASE_IDX 0 +#define mmIH_VMID_3_LUT 0x0003 +#define mmIH_VMID_3_LUT_BASE_IDX 0 +#define mmIH_VMID_4_LUT 0x0004 +#define mmIH_VMID_4_LUT_BASE_IDX 0 +#define mmIH_VMID_5_LUT 0x0005 +#define mmIH_VMID_5_LUT_BASE_IDX 0 +#define mmIH_VMID_6_LUT 0x0006 +#define mmIH_VMID_6_LUT_BASE_IDX 0 +#define mmIH_VMID_7_LUT 0x0007 +#define mmIH_VMID_7_LUT_BASE_IDX 0 +#define mmIH_VMID_8_LUT 0x0008 +#define mmIH_VMID_8_LUT_BASE_IDX 0 +#define mmIH_VMID_9_LUT 0x0009 +#define mmIH_VMID_9_LUT_BASE_IDX 0 +#define mmIH_VMID_10_LUT 0x000a +#define mmIH_VMID_10_LUT_BASE_IDX 0 +#define mmIH_VMID_11_LUT 0x000b +#define mmIH_VMID_11_LUT_BASE_IDX 0 +#define mmIH_VMID_12_LUT 0x000c +#define mmIH_VMID_12_LUT_BASE_IDX 0 +#define mmIH_VMID_13_LUT 0x000d +#define mmIH_VMID_13_LUT_BASE_IDX 0 +#define mmIH_VMID_14_LUT 0x000e +#define mmIH_VMID_14_LUT_BASE_IDX 0 +#define mmIH_VMID_15_LUT 0x000f +#define mmIH_VMID_15_LUT_BASE_IDX 0 +#define mmIH_VMID_0_LUT_MM 0x0010 +#define mmIH_VMID_0_LUT_MM_BASE_IDX 0 +#define mmIH_VMID_1_LUT_MM 0x0011 +#define mmIH_VMID_1_LUT_MM_BASE_IDX 0 +#define mmIH_VMID_2_LUT_MM 0x0012 +#define mmIH_VMID_2_LUT_MM_BASE_IDX 0 +#define mmIH_VMID_3_LUT_MM 0x0013 +#define mmIH_VMID_3_LUT_MM_BASE_IDX 0 +#define mmIH_VMID_4_LUT_MM 0x0014 +#define mmIH_VMID_4_LUT_MM_BASE_IDX 0 +#define mmIH_VMID_5_LUT_MM 0x0015 +#define mmIH_VMID_5_LUT_MM_BASE_IDX 0 +#define mmIH_VMID_6_LUT_MM 0x0016 +#define mmIH_VMID_6_LUT_MM_BASE_IDX 0 +#define mmIH_VMID_7_LUT_MM 0x0017 +#define mmIH_VMID_7_LUT_MM_BASE_IDX 0 +#define mmIH_VMID_8_LUT_MM 0x0018 +#define mmIH_VMID_8_LUT_MM_BASE_IDX 0 +#define mmIH_VMID_9_LUT_MM 0x0019 +#define mmIH_VMID_9_LUT_MM_BASE_IDX 0 +#define mmIH_VMID_10_LUT_MM 0x001a +#define mmIH_VMID_10_LUT_MM_BASE_IDX 0 +#define mmIH_VMID_11_LUT_MM 0x001b +#define mmIH_VMID_11_LUT_MM_BASE_IDX 0 +#define mmIH_VMID_12_LUT_MM 0x001c +#define mmIH_VMID_12_LUT_MM_BASE_IDX 0 +#define mmIH_VMID_13_LUT_MM 0x001d +#define mmIH_VMID_13_LUT_MM_BASE_IDX 0 +#define mmIH_VMID_14_LUT_MM 0x001e +#define mmIH_VMID_14_LUT_MM_BASE_IDX 0 +#define mmIH_VMID_15_LUT_MM 0x001f +#define mmIH_VMID_15_LUT_MM_BASE_IDX 0 +#define mmIH_COOKIE_0 0x0020 +#define mmIH_COOKIE_0_BASE_IDX 0 +#define mmIH_COOKIE_1 0x0021 +#define mmIH_COOKIE_1_BASE_IDX 0 +#define mmIH_COOKIE_2 0x0022 +#define mmIH_COOKIE_2_BASE_IDX 0 +#define mmIH_COOKIE_3 0x0023 +#define mmIH_COOKIE_3_BASE_IDX 0 +#define mmIH_COOKIE_4 0x0024 +#define mmIH_COOKIE_4_BASE_IDX 0 +#define mmIH_COOKIE_5 0x0025 +#define mmIH_COOKIE_5_BASE_IDX 0 +#define mmIH_COOKIE_6 0x0026 +#define mmIH_COOKIE_6_BASE_IDX 0 +#define mmIH_COOKIE_7 0x0027 +#define mmIH_COOKIE_7_BASE_IDX 0 +#define mmIH_REGISTER_LAST_PART0 0x003f +#define mmIH_REGISTER_LAST_PART0_BASE_IDX 0 +#define mmSEM_REQ_INPUT_0 0x0040 +#define mmSEM_REQ_INPUT_0_BASE_IDX 0 +#define mmSEM_REQ_INPUT_1 0x0041 +#define mmSEM_REQ_INPUT_1_BASE_IDX 0 +#define mmSEM_REQ_INPUT_2 0x0042 +#define mmSEM_REQ_INPUT_2_BASE_IDX 0 +#define mmSEM_REQ_INPUT_3 0x0043 +#define mmSEM_REQ_INPUT_3_BASE_IDX 0 +#define mmSEM_REGISTER_LAST_PART0 0x007f +#define mmSEM_REGISTER_LAST_PART0_BASE_IDX 0 +#define mmIH_RB_CNTL 0x0080 +#define mmIH_RB_CNTL_BASE_IDX 0 +#define mmIH_RB_BASE 0x0081 +#define mmIH_RB_BASE_BASE_IDX 0 +#define mmIH_RB_BASE_HI 0x0082 +#define mmIH_RB_BASE_HI_BASE_IDX 0 +#define mmIH_RB_RPTR 0x0083 +#define mmIH_RB_RPTR_BASE_IDX 0 +#define mmIH_RB_WPTR 0x0084 +#define mmIH_RB_WPTR_BASE_IDX 0 +#define mmIH_RB_WPTR_ADDR_HI 0x0085 +#define mmIH_RB_WPTR_ADDR_HI_BASE_IDX 0 +#define mmIH_RB_WPTR_ADDR_LO 0x0086 +#define mmIH_RB_WPTR_ADDR_LO_BASE_IDX 0 +#define mmIH_DOORBELL_RPTR 0x0087 +#define mmIH_DOORBELL_RPTR_BASE_IDX 0 +#define mmIH_RB_CNTL_RING1 0x008c +#define mmIH_RB_CNTL_RING1_BASE_IDX 0 +#define mmIH_RB_BASE_RING1 0x008d +#define mmIH_RB_BASE_RING1_BASE_IDX 0 +#define mmIH_RB_BASE_HI_RING1 0x008e +#define mmIH_RB_BASE_HI_RING1_BASE_IDX 0 +#define mmIH_RB_RPTR_RING1 0x008f +#define mmIH_RB_RPTR_RING1_BASE_IDX 0 +#define mmIH_RB_WPTR_RING1 0x0090 +#define mmIH_RB_WPTR_RING1_BASE_IDX 0 +#define mmIH_DOORBELL_RPTR_RING1 0x0093 +#define mmIH_DOORBELL_RPTR_RING1_BASE_IDX 0 +#define mmIH_RB_CNTL_RING2 0x0098 +#define mmIH_RB_CNTL_RING2_BASE_IDX 0 +#define mmIH_RB_BASE_RING2 0x0099 +#define mmIH_RB_BASE_RING2_BASE_IDX 0 +#define mmIH_RB_BASE_HI_RING2 0x009a +#define mmIH_RB_BASE_HI_RING2_BASE_IDX 0 +#define mmIH_RB_RPTR_RING2 0x009b +#define mmIH_RB_RPTR_RING2_BASE_IDX 0 +#define mmIH_RB_WPTR_RING2 0x009c +#define mmIH_RB_WPTR_RING2_BASE_IDX 0 +#define mmIH_DOORBELL_RPTR_RING2 0x009f +#define mmIH_DOORBELL_RPTR_RING2_BASE_IDX 0 +#define mmIH_VERSION 0x00a5 +#define mmIH_VERSION_BASE_IDX 0 +#define mmIH_CNTL 0x00c0 +#define mmIH_CNTL_BASE_IDX 0 +#define mmIH_CNTL2 0x00c1 +#define mmIH_CNTL2_BASE_IDX 0 +#define mmIH_STATUS 0x00c2 +#define mmIH_STATUS_BASE_IDX 0 +#define mmIH_PERFMON_CNTL 0x00c3 +#define mmIH_PERFMON_CNTL_BASE_IDX 0 +#define mmIH_PERFCOUNTER0_RESULT 0x00c4 +#define mmIH_PERFCOUNTER0_RESULT_BASE_IDX 0 +#define mmIH_PERFCOUNTER1_RESULT 0x00c5 +#define mmIH_PERFCOUNTER1_RESULT_BASE_IDX 0 +#define mmIH_DSM_MATCH_VALUE_BIT_31_0 0x00c7 +#define mmIH_DSM_MATCH_VALUE_BIT_31_0_BASE_IDX 0 +#define mmIH_DSM_MATCH_VALUE_BIT_63_32 0x00c8 +#define mmIH_DSM_MATCH_VALUE_BIT_63_32_BASE_IDX 0 +#define mmIH_DSM_MATCH_VALUE_BIT_95_64 0x00c9 +#define mmIH_DSM_MATCH_VALUE_BIT_95_64_BASE_IDX 0 +#define mmIH_DSM_MATCH_FIELD_CONTROL 0x00ca +#define mmIH_DSM_MATCH_FIELD_CONTROL_BASE_IDX 0 +#define mmIH_DSM_MATCH_DATA_CONTROL 0x00cb +#define mmIH_DSM_MATCH_DATA_CONTROL_BASE_IDX 0 +#define mmIH_DSM_MATCH_FCN_ID 0x00cc +#define mmIH_DSM_MATCH_FCN_ID_BASE_IDX 0 +#define mmIH_LIMIT_INT_RATE_CNTL 0x00cd +#define mmIH_LIMIT_INT_RATE_CNTL_BASE_IDX 0 +#define mmIH_VF_RB_STATUS 0x00ce +#define mmIH_VF_RB_STATUS_BASE_IDX 0 +#define mmIH_VF_RB_STATUS2 0x00cf +#define mmIH_VF_RB_STATUS2_BASE_IDX 0 +#define mmIH_VF_RB1_STATUS 0x00d0 +#define mmIH_VF_RB1_STATUS_BASE_IDX 0 +#define mmIH_VF_RB1_STATUS2 0x00d1 +#define mmIH_VF_RB1_STATUS2_BASE_IDX 0 +#define mmIH_VF_RB2_STATUS 0x00d2 +#define mmIH_VF_RB2_STATUS_BASE_IDX 0 +#define mmIH_VF_RB2_STATUS2 0x00d3 +#define mmIH_VF_RB2_STATUS2_BASE_IDX 0 +#define mmIH_INT_FLOOD_CNTL 0x00d5 +#define mmIH_INT_FLOOD_CNTL_BASE_IDX 0 +#define mmIH_RB0_INT_FLOOD_STATUS 0x00d6 +#define mmIH_RB0_INT_FLOOD_STATUS_BASE_IDX 0 +#define mmIH_RB1_INT_FLOOD_STATUS 0x00d7 +#define mmIH_RB1_INT_FLOOD_STATUS_BASE_IDX 0 +#define mmIH_RB2_INT_FLOOD_STATUS 0x00d8 +#define mmIH_RB2_INT_FLOOD_STATUS_BASE_IDX 0 +#define mmIH_INT_FLOOD_STATUS 0x00d9 +#define mmIH_INT_FLOOD_STATUS_BASE_IDX 0 +#define mmIH_STORM_CLIENT_LIST_CNTL 0x00da +#define mmIH_STORM_CLIENT_LIST_CNTL_BASE_IDX 0 +#define mmIH_CLK_CTRL 0x00db +#define mmIH_CLK_CTRL_BASE_IDX 0 +#define mmIH_INT_FLAGS 0x00dc +#define mmIH_INT_FLAGS_BASE_IDX 0 +#define mmIH_LAST_INT_INFO0 0x00dd +#define mmIH_LAST_INT_INFO0_BASE_IDX 0 +#define mmIH_LAST_INT_INFO1 0x00de +#define mmIH_LAST_INT_INFO1_BASE_IDX 0 +#define mmIH_LAST_INT_INFO2 0x00df +#define mmIH_LAST_INT_INFO2_BASE_IDX 0 +#define mmIH_SCRATCH 0x00e0 +#define mmIH_SCRATCH_BASE_IDX 0 +#define mmIH_CLIENT_CREDIT_ERROR 0x00e1 +#define mmIH_CLIENT_CREDIT_ERROR_BASE_IDX 0 +#define mmIH_GPU_IOV_VIOLATION_LOG 0x00e2 +#define mmIH_GPU_IOV_VIOLATION_LOG_BASE_IDX 0 +#define mmIH_COOKIE_REC_VIOLATION_LOG 0x00e3 +#define mmIH_COOKIE_REC_VIOLATION_LOG_BASE_IDX 0 +#define mmIH_CREDIT_STATUS 0x00e4 +#define mmIH_CREDIT_STATUS_BASE_IDX 0 +#define mmIH_MMHUB_ERROR 0x00e5 +#define mmIH_MMHUB_ERROR_BASE_IDX 0 +#define mmIH_MEM_POWER_CTRL 0x00e8 +#define mmIH_MEM_POWER_CTRL_BASE_IDX 0 +#define mmIH_REGISTER_LAST_PART2 0x00ff +#define mmIH_REGISTER_LAST_PART2_BASE_IDX 0 +#define mmSEM_CLK_CTRL 0x0100 +#define mmSEM_CLK_CTRL_BASE_IDX 0 +#define mmSEM_UTC_CREDIT 0x0101 +#define mmSEM_UTC_CREDIT_BASE_IDX 0 +#define mmSEM_UTC_CONFIG 0x0102 +#define mmSEM_UTC_CONFIG_BASE_IDX 0 +#define mmSEM_UTCL2_TRAN_EN_LUT 0x0103 +#define mmSEM_UTCL2_TRAN_EN_LUT_BASE_IDX 0 +#define mmSEM_MCIF_CONFIG 0x0104 +#define mmSEM_MCIF_CONFIG_BASE_IDX 0 +#define mmSEM_PERFMON_CNTL 0x0105 +#define mmSEM_PERFMON_CNTL_BASE_IDX 0 +#define mmSEM_PERFCOUNTER0_RESULT 0x0106 +#define mmSEM_PERFCOUNTER0_RESULT_BASE_IDX 0 +#define mmSEM_PERFCOUNTER1_RESULT 0x0107 +#define mmSEM_PERFCOUNTER1_RESULT_BASE_IDX 0 +#define mmSEM_STATUS 0x0108 +#define mmSEM_STATUS_BASE_IDX 0 +#define mmSEM_MAILBOX_CLIENTCONFIG 0x0109 +#define mmSEM_MAILBOX_CLIENTCONFIG_BASE_IDX 0 +#define mmSEM_MAILBOX 0x010a +#define mmSEM_MAILBOX_BASE_IDX 0 +#define mmSEM_MAILBOX_CONTROL 0x010b +#define mmSEM_MAILBOX_CONTROL_BASE_IDX 0 +#define mmSEM_CHICKEN_BITS 0x010c +#define mmSEM_CHICKEN_BITS_BASE_IDX 0 +#define mmSEM_MAILBOX_CLIENTCONFIG_EXTRA 0x010d +#define mmSEM_MAILBOX_CLIENTCONFIG_EXTRA_BASE_IDX 0 +#define mmSEM_GPU_IOV_VIOLATION_LOG 0x010e +#define mmSEM_GPU_IOV_VIOLATION_LOG_BASE_IDX 0 +#define mmSEM_OUTSTANDING_THRESHOLD 0x010f +#define mmSEM_OUTSTANDING_THRESHOLD_BASE_IDX 0 +#define mmSEM_MEM_POWER_CTRL 0x0110 +#define mmSEM_MEM_POWER_CTRL_BASE_IDX 0 +#define mmSEM_REGISTER_LAST_PART2 0x017f +#define mmSEM_REGISTER_LAST_PART2_BASE_IDX 0 +#define mmIH_ACTIVE_FCN_ID 0x0180 +#define mmIH_ACTIVE_FCN_ID_BASE_IDX 0 +#define mmIH_VIRT_RESET_REQ 0x0181 +#define mmIH_VIRT_RESET_REQ_BASE_IDX 0 +#define mmIH_CLIENT_CFG 0x0184 +#define mmIH_CLIENT_CFG_BASE_IDX 0 +#define mmIH_CLIENT_CFG_INDEX 0x0188 +#define mmIH_CLIENT_CFG_INDEX_BASE_IDX 0 +#define mmIH_CLIENT_CFG_DATA 0x0189 +#define mmIH_CLIENT_CFG_DATA_BASE_IDX 0 +#define mmIH_CID_REMAP_INDEX 0x018a +#define mmIH_CID_REMAP_INDEX_BASE_IDX 0 +#define mmIH_CID_REMAP_DATA 0x018b +#define mmIH_CID_REMAP_DATA_BASE_IDX 0 +#define mmIH_CHICKEN 0x018c +#define mmIH_CHICKEN_BASE_IDX 0 +#define mmIH_MMHUB_CNTL 0x018d +#define mmIH_MMHUB_CNTL_BASE_IDX 0 +#define mmIH_INT_DROP_CNTL 0x018e +#define mmIH_INT_DROP_CNTL_BASE_IDX 0 +#define mmIH_INT_DROP_MATCH_VALUE0 0x018f +#define mmIH_INT_DROP_MATCH_VALUE0_BASE_IDX 0 +#define mmIH_INT_DROP_MATCH_VALUE1 0x0190 +#define mmIH_INT_DROP_MATCH_VALUE1_BASE_IDX 0 +#define mmIH_INT_DROP_MATCH_MASK0 0x0191 +#define mmIH_INT_DROP_MATCH_MASK0_BASE_IDX 0 +#define mmIH_INT_DROP_MATCH_MASK1 0x0192 +#define mmIH_INT_DROP_MATCH_MASK1_BASE_IDX 0 +#define mmIH_REGISTER_LAST_PART1 0x019f +#define mmIH_REGISTER_LAST_PART1_BASE_IDX 0 +#define mmSEM_ACTIVE_FCN_ID 0x01a0 +#define mmSEM_ACTIVE_FCN_ID_BASE_IDX 0 +#define mmSEM_VIRT_RESET_REQ 0x01a1 +#define mmSEM_VIRT_RESET_REQ_BASE_IDX 0 +#define mmSEM_RESP_SDMA0 0x01a4 +#define mmSEM_RESP_SDMA0_BASE_IDX 0 +#define mmSEM_RESP_SDMA1 0x01a5 +#define mmSEM_RESP_SDMA1_BASE_IDX 0 +#define mmSEM_RESP_UVD 0x01a6 +#define mmSEM_RESP_UVD_BASE_IDX 0 +#define mmSEM_RESP_VCE_0 0x01a7 +#define mmSEM_RESP_VCE_0_BASE_IDX 0 +#define mmSEM_RESP_ACP 0x01a8 +#define mmSEM_RESP_ACP_BASE_IDX 0 +#define mmSEM_RESP_ISP 0x01a9 +#define mmSEM_RESP_ISP_BASE_IDX 0 +#define mmSEM_RESP_VCE_1 0x01aa +#define mmSEM_RESP_VCE_1_BASE_IDX 0 +#define mmSEM_RESP_VP8 0x01ab +#define mmSEM_RESP_VP8_BASE_IDX 0 +#define mmSEM_RESP_GC 0x01ac +#define mmSEM_RESP_GC_BASE_IDX 0 +#define mmSEM_RESP_UVD_1 0x01ad +#define mmSEM_RESP_UVD_1_BASE_IDX 0 +#define mmSEM_CID_REMAP_INDEX 0x01b0 +#define mmSEM_CID_REMAP_INDEX_BASE_IDX 0 +#define mmSEM_CID_REMAP_DATA 0x01b1 +#define mmSEM_CID_REMAP_DATA_BASE_IDX 0 +#define mmSEM_ATOMIC_OP_LUT 0x01b2 +#define mmSEM_ATOMIC_OP_LUT_BASE_IDX 0 +#define mmSEM_EDC_CONFIG 0x01b3 +#define mmSEM_EDC_CONFIG_BASE_IDX 0 +#define mmSEM_CHICKEN_BITS2 0x01b4 +#define mmSEM_CHICKEN_BITS2_BASE_IDX 0 +#define mmSEM_MMHUB_CNTL 0x01b5 +#define mmSEM_MMHUB_CNTL_BASE_IDX 0 +#define mmSEM_REGISTER_LAST_PART1 0x01bf +#define mmSEM_REGISTER_LAST_PART1_BASE_IDX 0 + +#endif diff --git a/drivers/gpu/drm/amd/include/asic_reg/oss/osssys_4_2_0_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/oss/osssys_4_2_0_sh_mask.h new file mode 100644 index 0000000000000000000000000000000000000000..3ea83ea9ce3a4bc88c9eece8a2ef537245413360 --- /dev/null +++ b/drivers/gpu/drm/amd/include/asic_reg/oss/osssys_4_2_0_sh_mask.h @@ -0,0 +1,1300 @@ +/* + * Copyright 2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ +#ifndef _osssys_4_2_0_SH_MASK_HEADER +#define _osssys_4_2_0_SH_MASK_HEADER + + +// addressBlock: osssys_osssysdec +//IH_VMID_0_LUT +#define IH_VMID_0_LUT__PASID__SHIFT 0x0 +#define IH_VMID_0_LUT__PASID_MASK 0x0000FFFFL +//IH_VMID_1_LUT +#define IH_VMID_1_LUT__PASID__SHIFT 0x0 +#define IH_VMID_1_LUT__PASID_MASK 0x0000FFFFL +//IH_VMID_2_LUT +#define IH_VMID_2_LUT__PASID__SHIFT 0x0 +#define IH_VMID_2_LUT__PASID_MASK 0x0000FFFFL +//IH_VMID_3_LUT +#define IH_VMID_3_LUT__PASID__SHIFT 0x0 +#define IH_VMID_3_LUT__PASID_MASK 0x0000FFFFL +//IH_VMID_4_LUT +#define IH_VMID_4_LUT__PASID__SHIFT 0x0 +#define IH_VMID_4_LUT__PASID_MASK 0x0000FFFFL +//IH_VMID_5_LUT +#define IH_VMID_5_LUT__PASID__SHIFT 0x0 +#define IH_VMID_5_LUT__PASID_MASK 0x0000FFFFL +//IH_VMID_6_LUT +#define IH_VMID_6_LUT__PASID__SHIFT 0x0 +#define IH_VMID_6_LUT__PASID_MASK 0x0000FFFFL +//IH_VMID_7_LUT +#define IH_VMID_7_LUT__PASID__SHIFT 0x0 +#define IH_VMID_7_LUT__PASID_MASK 0x0000FFFFL +//IH_VMID_8_LUT +#define IH_VMID_8_LUT__PASID__SHIFT 0x0 +#define IH_VMID_8_LUT__PASID_MASK 0x0000FFFFL +//IH_VMID_9_LUT +#define IH_VMID_9_LUT__PASID__SHIFT 0x0 +#define IH_VMID_9_LUT__PASID_MASK 0x0000FFFFL +//IH_VMID_10_LUT +#define IH_VMID_10_LUT__PASID__SHIFT 0x0 +#define IH_VMID_10_LUT__PASID_MASK 0x0000FFFFL +//IH_VMID_11_LUT +#define IH_VMID_11_LUT__PASID__SHIFT 0x0 +#define IH_VMID_11_LUT__PASID_MASK 0x0000FFFFL +//IH_VMID_12_LUT +#define IH_VMID_12_LUT__PASID__SHIFT 0x0 +#define IH_VMID_12_LUT__PASID_MASK 0x0000FFFFL +//IH_VMID_13_LUT +#define IH_VMID_13_LUT__PASID__SHIFT 0x0 +#define IH_VMID_13_LUT__PASID_MASK 0x0000FFFFL +//IH_VMID_14_LUT +#define IH_VMID_14_LUT__PASID__SHIFT 0x0 +#define IH_VMID_14_LUT__PASID_MASK 0x0000FFFFL +//IH_VMID_15_LUT +#define IH_VMID_15_LUT__PASID__SHIFT 0x0 +#define IH_VMID_15_LUT__PASID_MASK 0x0000FFFFL +//IH_VMID_0_LUT_MM +#define IH_VMID_0_LUT_MM__PASID__SHIFT 0x0 +#define IH_VMID_0_LUT_MM__PASID_MASK 0x0000FFFFL +//IH_VMID_1_LUT_MM +#define IH_VMID_1_LUT_MM__PASID__SHIFT 0x0 +#define IH_VMID_1_LUT_MM__PASID_MASK 0x0000FFFFL +//IH_VMID_2_LUT_MM +#define IH_VMID_2_LUT_MM__PASID__SHIFT 0x0 +#define IH_VMID_2_LUT_MM__PASID_MASK 0x0000FFFFL +//IH_VMID_3_LUT_MM +#define IH_VMID_3_LUT_MM__PASID__SHIFT 0x0 +#define IH_VMID_3_LUT_MM__PASID_MASK 0x0000FFFFL +//IH_VMID_4_LUT_MM +#define IH_VMID_4_LUT_MM__PASID__SHIFT 0x0 +#define IH_VMID_4_LUT_MM__PASID_MASK 0x0000FFFFL +//IH_VMID_5_LUT_MM +#define IH_VMID_5_LUT_MM__PASID__SHIFT 0x0 +#define IH_VMID_5_LUT_MM__PASID_MASK 0x0000FFFFL +//IH_VMID_6_LUT_MM +#define IH_VMID_6_LUT_MM__PASID__SHIFT 0x0 +#define IH_VMID_6_LUT_MM__PASID_MASK 0x0000FFFFL +//IH_VMID_7_LUT_MM +#define IH_VMID_7_LUT_MM__PASID__SHIFT 0x0 +#define IH_VMID_7_LUT_MM__PASID_MASK 0x0000FFFFL +//IH_VMID_8_LUT_MM +#define IH_VMID_8_LUT_MM__PASID__SHIFT 0x0 +#define IH_VMID_8_LUT_MM__PASID_MASK 0x0000FFFFL +//IH_VMID_9_LUT_MM +#define IH_VMID_9_LUT_MM__PASID__SHIFT 0x0 +#define IH_VMID_9_LUT_MM__PASID_MASK 0x0000FFFFL +//IH_VMID_10_LUT_MM +#define IH_VMID_10_LUT_MM__PASID__SHIFT 0x0 +#define IH_VMID_10_LUT_MM__PASID_MASK 0x0000FFFFL +//IH_VMID_11_LUT_MM +#define IH_VMID_11_LUT_MM__PASID__SHIFT 0x0 +#define IH_VMID_11_LUT_MM__PASID_MASK 0x0000FFFFL +//IH_VMID_12_LUT_MM +#define IH_VMID_12_LUT_MM__PASID__SHIFT 0x0 +#define IH_VMID_12_LUT_MM__PASID_MASK 0x0000FFFFL +//IH_VMID_13_LUT_MM +#define IH_VMID_13_LUT_MM__PASID__SHIFT 0x0 +#define IH_VMID_13_LUT_MM__PASID_MASK 0x0000FFFFL +//IH_VMID_14_LUT_MM +#define IH_VMID_14_LUT_MM__PASID__SHIFT 0x0 +#define IH_VMID_14_LUT_MM__PASID_MASK 0x0000FFFFL +//IH_VMID_15_LUT_MM +#define IH_VMID_15_LUT_MM__PASID__SHIFT 0x0 +#define IH_VMID_15_LUT_MM__PASID_MASK 0x0000FFFFL +//IH_COOKIE_0 +#define IH_COOKIE_0__CLIENT_ID__SHIFT 0x0 +#define IH_COOKIE_0__SOURCE_ID__SHIFT 0x8 +#define IH_COOKIE_0__RING_ID__SHIFT 0x10 +#define IH_COOKIE_0__VM_ID__SHIFT 0x18 +#define IH_COOKIE_0__RESERVED__SHIFT 0x1c +#define IH_COOKIE_0__VMID_TYPE__SHIFT 0x1f +#define IH_COOKIE_0__CLIENT_ID_MASK 0x000000FFL +#define IH_COOKIE_0__SOURCE_ID_MASK 0x0000FF00L +#define IH_COOKIE_0__RING_ID_MASK 0x00FF0000L +#define IH_COOKIE_0__VM_ID_MASK 0x0F000000L +#define IH_COOKIE_0__RESERVED_MASK 0x70000000L +#define IH_COOKIE_0__VMID_TYPE_MASK 0x80000000L +//IH_COOKIE_1 +#define IH_COOKIE_1__TIMESTAMP_31_0__SHIFT 0x0 +#define IH_COOKIE_1__TIMESTAMP_31_0_MASK 0xFFFFFFFFL +//IH_COOKIE_2 +#define IH_COOKIE_2__TIMESTAMP_47_32__SHIFT 0x0 +#define IH_COOKIE_2__RESERVED__SHIFT 0x10 +#define IH_COOKIE_2__TIMESTAMP_SRC__SHIFT 0x1f +#define IH_COOKIE_2__TIMESTAMP_47_32_MASK 0x0000FFFFL +#define IH_COOKIE_2__RESERVED_MASK 0x7FFF0000L +#define IH_COOKIE_2__TIMESTAMP_SRC_MASK 0x80000000L +//IH_COOKIE_3 +#define IH_COOKIE_3__PAS_ID__SHIFT 0x0 +#define IH_COOKIE_3__RESERVED__SHIFT 0x10 +#define IH_COOKIE_3__PASID_SRC__SHIFT 0x1f +#define IH_COOKIE_3__PAS_ID_MASK 0x0000FFFFL +#define IH_COOKIE_3__RESERVED_MASK 0x7FFF0000L +#define IH_COOKIE_3__PASID_SRC_MASK 0x80000000L +//IH_COOKIE_4 +#define IH_COOKIE_4__CONTEXT_ID_31_0__SHIFT 0x0 +#define IH_COOKIE_4__CONTEXT_ID_31_0_MASK 0xFFFFFFFFL +//IH_COOKIE_5 +#define IH_COOKIE_5__CONTEXT_ID_63_32__SHIFT 0x0 +#define IH_COOKIE_5__CONTEXT_ID_63_32_MASK 0xFFFFFFFFL +//IH_COOKIE_6 +#define IH_COOKIE_6__CONTEXT_ID_95_64__SHIFT 0x0 +#define IH_COOKIE_6__CONTEXT_ID_95_64_MASK 0xFFFFFFFFL +//IH_COOKIE_7 +#define IH_COOKIE_7__CONTEXT_ID_128_96__SHIFT 0x0 +#define IH_COOKIE_7__CONTEXT_ID_128_96_MASK 0xFFFFFFFFL +//IH_REGISTER_LAST_PART0 +#define IH_REGISTER_LAST_PART0__RESERVED__SHIFT 0x0 +#define IH_REGISTER_LAST_PART0__RESERVED_MASK 0xFFFFFFFFL +//SEM_REQ_INPUT_0 +#define SEM_REQ_INPUT_0__DATA__SHIFT 0x0 +#define SEM_REQ_INPUT_0__DATA_MASK 0xFFFFFFFFL +//SEM_REQ_INPUT_1 +#define SEM_REQ_INPUT_1__DATA__SHIFT 0x0 +#define SEM_REQ_INPUT_1__DATA_MASK 0xFFFFFFFFL +//SEM_REQ_INPUT_2 +#define SEM_REQ_INPUT_2__DATA__SHIFT 0x0 +#define SEM_REQ_INPUT_2__DATA_MASK 0xFFFFFFFFL +//SEM_REQ_INPUT_3 +#define SEM_REQ_INPUT_3__DATA__SHIFT 0x0 +#define SEM_REQ_INPUT_3__DATA_MASK 0xFFFFFFFFL +//SEM_REGISTER_LAST_PART0 +#define SEM_REGISTER_LAST_PART0__RESERVED__SHIFT 0x0 +#define SEM_REGISTER_LAST_PART0__RESERVED_MASK 0xFFFFFFFFL +//IH_RB_CNTL +#define IH_RB_CNTL__RB_ENABLE__SHIFT 0x0 +#define IH_RB_CNTL__RB_SIZE__SHIFT 0x1 +#define IH_RB_CNTL__RB_GPU_TS_ENABLE__SHIFT 0x7 +#define IH_RB_CNTL__WPTR_WRITEBACK_ENABLE__SHIFT 0x8 +#define IH_RB_CNTL__RB_FULL_DRAIN_ENABLE__SHIFT 0x9 +#define IH_RB_CNTL__FULL_DRAIN_CLEAR__SHIFT 0xa +#define IH_RB_CNTL__PAGE_RB_CLEAR__SHIFT 0xb +#define IH_RB_CNTL__RB_USED_INT_THRESHOLD__SHIFT 0xc +#define IH_RB_CNTL__WPTR_OVERFLOW_ENABLE__SHIFT 0x10 +#define IH_RB_CNTL__ENABLE_INTR__SHIFT 0x11 +#define IH_RB_CNTL__MC_SWAP__SHIFT 0x12 +#define IH_RB_CNTL__MC_SNOOP__SHIFT 0x14 +#define IH_RB_CNTL__RPTR_REARM__SHIFT 0x15 +#define IH_RB_CNTL__MC_RO__SHIFT 0x16 +#define IH_RB_CNTL__MC_VMID__SHIFT 0x18 +#define IH_RB_CNTL__MC_SPACE__SHIFT 0x1c +#define IH_RB_CNTL__WPTR_OVERFLOW_CLEAR__SHIFT 0x1f +#define IH_RB_CNTL__RB_ENABLE_MASK 0x00000001L +#define IH_RB_CNTL__RB_SIZE_MASK 0x0000003EL +#define IH_RB_CNTL__RB_GPU_TS_ENABLE_MASK 0x00000080L +#define IH_RB_CNTL__WPTR_WRITEBACK_ENABLE_MASK 0x00000100L +#define IH_RB_CNTL__RB_FULL_DRAIN_ENABLE_MASK 0x00000200L +#define IH_RB_CNTL__FULL_DRAIN_CLEAR_MASK 0x00000400L +#define IH_RB_CNTL__PAGE_RB_CLEAR_MASK 0x00000800L +#define IH_RB_CNTL__RB_USED_INT_THRESHOLD_MASK 0x0000F000L +#define IH_RB_CNTL__WPTR_OVERFLOW_ENABLE_MASK 0x00010000L +#define IH_RB_CNTL__ENABLE_INTR_MASK 0x00020000L +#define IH_RB_CNTL__MC_SWAP_MASK 0x000C0000L +#define IH_RB_CNTL__MC_SNOOP_MASK 0x00100000L +#define IH_RB_CNTL__RPTR_REARM_MASK 0x00200000L +#define IH_RB_CNTL__MC_RO_MASK 0x00400000L +#define IH_RB_CNTL__MC_VMID_MASK 0x0F000000L +#define IH_RB_CNTL__MC_SPACE_MASK 0x70000000L +#define IH_RB_CNTL__WPTR_OVERFLOW_CLEAR_MASK 0x80000000L +//IH_RB_BASE +#define IH_RB_BASE__ADDR__SHIFT 0x0 +#define IH_RB_BASE__ADDR_MASK 0xFFFFFFFFL +//IH_RB_BASE_HI +#define IH_RB_BASE_HI__ADDR__SHIFT 0x0 +#define IH_RB_BASE_HI__ADDR_MASK 0x000000FFL +//IH_RB_RPTR +#define IH_RB_RPTR__OFFSET__SHIFT 0x2 +#define IH_RB_RPTR__OFFSET_MASK 0x0003FFFCL +//IH_RB_WPTR +#define IH_RB_WPTR__RB_OVERFLOW__SHIFT 0x0 +#define IH_RB_WPTR__OFFSET__SHIFT 0x2 +#define IH_RB_WPTR__RB_LEFT_NONE__SHIFT 0x12 +#define IH_RB_WPTR__RB_MAY_OVERFLOW__SHIFT 0x13 +#define IH_RB_WPTR__RB_OVERFLOW_MASK 0x00000001L +#define IH_RB_WPTR__OFFSET_MASK 0x0003FFFCL +#define IH_RB_WPTR__RB_LEFT_NONE_MASK 0x00040000L +#define IH_RB_WPTR__RB_MAY_OVERFLOW_MASK 0x00080000L +//IH_RB_WPTR_ADDR_HI +#define IH_RB_WPTR_ADDR_HI__ADDR__SHIFT 0x0 +#define IH_RB_WPTR_ADDR_HI__ADDR_MASK 0x0000FFFFL +//IH_RB_WPTR_ADDR_LO +#define IH_RB_WPTR_ADDR_LO__ADDR__SHIFT 0x2 +#define IH_RB_WPTR_ADDR_LO__ADDR_MASK 0xFFFFFFFCL +//IH_DOORBELL_RPTR +#define IH_DOORBELL_RPTR__OFFSET__SHIFT 0x0 +#define IH_DOORBELL_RPTR__ENABLE__SHIFT 0x1c +#define IH_DOORBELL_RPTR__OFFSET_MASK 0x03FFFFFFL +#define IH_DOORBELL_RPTR__ENABLE_MASK 0x10000000L +//IH_RB_CNTL_RING1 +#define IH_RB_CNTL_RING1__RB_ENABLE__SHIFT 0x0 +#define IH_RB_CNTL_RING1__RB_SIZE__SHIFT 0x1 +#define IH_RB_CNTL_RING1__RB_GPU_TS_ENABLE__SHIFT 0x7 +#define IH_RB_CNTL_RING1__RB_FULL_DRAIN_ENABLE__SHIFT 0x9 +#define IH_RB_CNTL_RING1__FULL_DRAIN_CLEAR__SHIFT 0xa +#define IH_RB_CNTL_RING1__PAGE_RB_CLEAR__SHIFT 0xb +#define IH_RB_CNTL_RING1__RB_USED_INT_THRESHOLD__SHIFT 0xc +#define IH_RB_CNTL_RING1__WPTR_OVERFLOW_ENABLE__SHIFT 0x10 +#define IH_RB_CNTL_RING1__MC_SWAP__SHIFT 0x12 +#define IH_RB_CNTL_RING1__MC_SNOOP__SHIFT 0x14 +#define IH_RB_CNTL_RING1__MC_RO__SHIFT 0x16 +#define IH_RB_CNTL_RING1__MC_VMID__SHIFT 0x18 +#define IH_RB_CNTL_RING1__MC_SPACE__SHIFT 0x1c +#define IH_RB_CNTL_RING1__WPTR_OVERFLOW_CLEAR__SHIFT 0x1f +#define IH_RB_CNTL_RING1__RB_ENABLE_MASK 0x00000001L +#define IH_RB_CNTL_RING1__RB_SIZE_MASK 0x0000003EL +#define IH_RB_CNTL_RING1__RB_GPU_TS_ENABLE_MASK 0x00000080L +#define IH_RB_CNTL_RING1__RB_FULL_DRAIN_ENABLE_MASK 0x00000200L +#define IH_RB_CNTL_RING1__FULL_DRAIN_CLEAR_MASK 0x00000400L +#define IH_RB_CNTL_RING1__PAGE_RB_CLEAR_MASK 0x00000800L +#define IH_RB_CNTL_RING1__RB_USED_INT_THRESHOLD_MASK 0x0000F000L +#define IH_RB_CNTL_RING1__WPTR_OVERFLOW_ENABLE_MASK 0x00010000L +#define IH_RB_CNTL_RING1__MC_SWAP_MASK 0x000C0000L +#define IH_RB_CNTL_RING1__MC_SNOOP_MASK 0x00100000L +#define IH_RB_CNTL_RING1__MC_RO_MASK 0x00400000L +#define IH_RB_CNTL_RING1__MC_VMID_MASK 0x0F000000L +#define IH_RB_CNTL_RING1__MC_SPACE_MASK 0x70000000L +#define IH_RB_CNTL_RING1__WPTR_OVERFLOW_CLEAR_MASK 0x80000000L +//IH_RB_BASE_RING1 +#define IH_RB_BASE_RING1__ADDR__SHIFT 0x0 +#define IH_RB_BASE_RING1__ADDR_MASK 0xFFFFFFFFL +//IH_RB_BASE_HI_RING1 +#define IH_RB_BASE_HI_RING1__ADDR__SHIFT 0x0 +#define IH_RB_BASE_HI_RING1__ADDR_MASK 0x000000FFL +//IH_RB_RPTR_RING1 +#define IH_RB_RPTR_RING1__OFFSET__SHIFT 0x2 +#define IH_RB_RPTR_RING1__OFFSET_MASK 0x0003FFFCL +//IH_RB_WPTR_RING1 +#define IH_RB_WPTR_RING1__RB_OVERFLOW__SHIFT 0x0 +#define IH_RB_WPTR_RING1__OFFSET__SHIFT 0x2 +#define IH_RB_WPTR_RING1__RB_LEFT_NONE__SHIFT 0x12 +#define IH_RB_WPTR_RING1__RB_MAY_OVERFLOW__SHIFT 0x13 +#define IH_RB_WPTR_RING1__RB_OVERFLOW_MASK 0x00000001L +#define IH_RB_WPTR_RING1__OFFSET_MASK 0x0003FFFCL +#define IH_RB_WPTR_RING1__RB_LEFT_NONE_MASK 0x00040000L +#define IH_RB_WPTR_RING1__RB_MAY_OVERFLOW_MASK 0x00080000L +//IH_DOORBELL_RPTR_RING1 +#define IH_DOORBELL_RPTR_RING1__OFFSET__SHIFT 0x0 +#define IH_DOORBELL_RPTR_RING1__ENABLE__SHIFT 0x1c +#define IH_DOORBELL_RPTR_RING1__OFFSET_MASK 0x03FFFFFFL +#define IH_DOORBELL_RPTR_RING1__ENABLE_MASK 0x10000000L +//IH_RB_CNTL_RING2 +#define IH_RB_CNTL_RING2__RB_ENABLE__SHIFT 0x0 +#define IH_RB_CNTL_RING2__RB_SIZE__SHIFT 0x1 +#define IH_RB_CNTL_RING2__RB_GPU_TS_ENABLE__SHIFT 0x7 +#define IH_RB_CNTL_RING2__RB_FULL_DRAIN_ENABLE__SHIFT 0x9 +#define IH_RB_CNTL_RING2__FULL_DRAIN_CLEAR__SHIFT 0xa +#define IH_RB_CNTL_RING2__PAGE_RB_CLEAR__SHIFT 0xb +#define IH_RB_CNTL_RING2__RB_USED_INT_THRESHOLD__SHIFT 0xc +#define IH_RB_CNTL_RING2__WPTR_OVERFLOW_ENABLE__SHIFT 0x10 +#define IH_RB_CNTL_RING2__MC_SWAP__SHIFT 0x12 +#define IH_RB_CNTL_RING2__MC_SNOOP__SHIFT 0x14 +#define IH_RB_CNTL_RING2__MC_RO__SHIFT 0x16 +#define IH_RB_CNTL_RING2__MC_VMID__SHIFT 0x18 +#define IH_RB_CNTL_RING2__MC_SPACE__SHIFT 0x1c +#define IH_RB_CNTL_RING2__WPTR_OVERFLOW_CLEAR__SHIFT 0x1f +#define IH_RB_CNTL_RING2__RB_ENABLE_MASK 0x00000001L +#define IH_RB_CNTL_RING2__RB_SIZE_MASK 0x0000003EL +#define IH_RB_CNTL_RING2__RB_GPU_TS_ENABLE_MASK 0x00000080L +#define IH_RB_CNTL_RING2__RB_FULL_DRAIN_ENABLE_MASK 0x00000200L +#define IH_RB_CNTL_RING2__FULL_DRAIN_CLEAR_MASK 0x00000400L +#define IH_RB_CNTL_RING2__PAGE_RB_CLEAR_MASK 0x00000800L +#define IH_RB_CNTL_RING2__RB_USED_INT_THRESHOLD_MASK 0x0000F000L +#define IH_RB_CNTL_RING2__WPTR_OVERFLOW_ENABLE_MASK 0x00010000L +#define IH_RB_CNTL_RING2__MC_SWAP_MASK 0x000C0000L +#define IH_RB_CNTL_RING2__MC_SNOOP_MASK 0x00100000L +#define IH_RB_CNTL_RING2__MC_RO_MASK 0x00400000L +#define IH_RB_CNTL_RING2__MC_VMID_MASK 0x0F000000L +#define IH_RB_CNTL_RING2__MC_SPACE_MASK 0x70000000L +#define IH_RB_CNTL_RING2__WPTR_OVERFLOW_CLEAR_MASK 0x80000000L +//IH_RB_BASE_RING2 +#define IH_RB_BASE_RING2__ADDR__SHIFT 0x0 +#define IH_RB_BASE_RING2__ADDR_MASK 0xFFFFFFFFL +//IH_RB_BASE_HI_RING2 +#define IH_RB_BASE_HI_RING2__ADDR__SHIFT 0x0 +#define IH_RB_BASE_HI_RING2__ADDR_MASK 0x000000FFL +//IH_RB_RPTR_RING2 +#define IH_RB_RPTR_RING2__OFFSET__SHIFT 0x2 +#define IH_RB_RPTR_RING2__OFFSET_MASK 0x0003FFFCL +//IH_RB_WPTR_RING2 +#define IH_RB_WPTR_RING2__RB_OVERFLOW__SHIFT 0x0 +#define IH_RB_WPTR_RING2__OFFSET__SHIFT 0x2 +#define IH_RB_WPTR_RING2__RB_LEFT_NONE__SHIFT 0x12 +#define IH_RB_WPTR_RING2__RB_MAY_OVERFLOW__SHIFT 0x13 +#define IH_RB_WPTR_RING2__RB_OVERFLOW_MASK 0x00000001L +#define IH_RB_WPTR_RING2__OFFSET_MASK 0x0003FFFCL +#define IH_RB_WPTR_RING2__RB_LEFT_NONE_MASK 0x00040000L +#define IH_RB_WPTR_RING2__RB_MAY_OVERFLOW_MASK 0x00080000L +//IH_DOORBELL_RPTR_RING2 +#define IH_DOORBELL_RPTR_RING2__OFFSET__SHIFT 0x0 +#define IH_DOORBELL_RPTR_RING2__ENABLE__SHIFT 0x1c +#define IH_DOORBELL_RPTR_RING2__OFFSET_MASK 0x03FFFFFFL +#define IH_DOORBELL_RPTR_RING2__ENABLE_MASK 0x10000000L +//IH_VERSION +#define IH_VERSION__MINVER__SHIFT 0x0 +#define IH_VERSION__MAJVER__SHIFT 0x8 +#define IH_VERSION__REV__SHIFT 0x10 +#define IH_VERSION__MINVER_MASK 0x0000007FL +#define IH_VERSION__MAJVER_MASK 0x00007F00L +#define IH_VERSION__REV_MASK 0x003F0000L +//IH_CNTL +#define IH_CNTL__WPTR_WRITEBACK_TIMER__SHIFT 0x0 +#define IH_CNTL__IH_IDLE_HYSTERESIS_CNTL__SHIFT 0x6 +#define IH_CNTL__IH_FIFO_HIGHWATER__SHIFT 0x8 +#define IH_CNTL__MC_WR_CLEAN_CNT__SHIFT 0x14 +#define IH_CNTL__WPTR_WRITEBACK_TIMER_MASK 0x0000001FL +#define IH_CNTL__IH_IDLE_HYSTERESIS_CNTL_MASK 0x000000C0L +#define IH_CNTL__IH_FIFO_HIGHWATER_MASK 0x00007F00L +#define IH_CNTL__MC_WR_CLEAN_CNT_MASK 0x01F00000L +//IH_CNTL2 +#define IH_CNTL2__SELF_IV_FORCE_WPTR_UPDATE_TIMEOUT__SHIFT 0x0 +#define IH_CNTL2__SELF_IV_FORCE_WPTR_UPDATE_ENABLE__SHIFT 0x8 +#define IH_CNTL2__SELF_IV_FORCE_WPTR_UPDATE_TIMEOUT_MASK 0x0000001FL +#define IH_CNTL2__SELF_IV_FORCE_WPTR_UPDATE_ENABLE_MASK 0x00000100L +//IH_STATUS +#define IH_STATUS__IDLE__SHIFT 0x0 +#define IH_STATUS__INPUT_IDLE__SHIFT 0x1 +#define IH_STATUS__BUFFER_IDLE__SHIFT 0x2 +#define IH_STATUS__RB_FULL__SHIFT 0x3 +#define IH_STATUS__RB_FULL_DRAIN__SHIFT 0x4 +#define IH_STATUS__RB_OVERFLOW__SHIFT 0x5 +#define IH_STATUS__MC_WR_IDLE__SHIFT 0x6 +#define IH_STATUS__MC_WR_STALL__SHIFT 0x7 +#define IH_STATUS__MC_WR_CLEAN_PENDING__SHIFT 0x8 +#define IH_STATUS__MC_WR_CLEAN_STALL__SHIFT 0x9 +#define IH_STATUS__BIF_INTERRUPT_LINE__SHIFT 0xa +#define IH_STATUS__SWITCH_READY__SHIFT 0xb +#define IH_STATUS__RB1_FULL__SHIFT 0xc +#define IH_STATUS__RB1_FULL_DRAIN__SHIFT 0xd +#define IH_STATUS__RB1_OVERFLOW__SHIFT 0xe +#define IH_STATUS__RB2_FULL__SHIFT 0xf +#define IH_STATUS__RB2_FULL_DRAIN__SHIFT 0x10 +#define IH_STATUS__RB2_OVERFLOW__SHIFT 0x11 +#define IH_STATUS__SELF_INT_GEN_IDLE__SHIFT 0x12 +#define IH_STATUS__IDLE_MASK 0x00000001L +#define IH_STATUS__INPUT_IDLE_MASK 0x00000002L +#define IH_STATUS__BUFFER_IDLE_MASK 0x00000004L +#define IH_STATUS__RB_FULL_MASK 0x00000008L +#define IH_STATUS__RB_FULL_DRAIN_MASK 0x00000010L +#define IH_STATUS__RB_OVERFLOW_MASK 0x00000020L +#define IH_STATUS__MC_WR_IDLE_MASK 0x00000040L +#define IH_STATUS__MC_WR_STALL_MASK 0x00000080L +#define IH_STATUS__MC_WR_CLEAN_PENDING_MASK 0x00000100L +#define IH_STATUS__MC_WR_CLEAN_STALL_MASK 0x00000200L +#define IH_STATUS__BIF_INTERRUPT_LINE_MASK 0x00000400L +#define IH_STATUS__SWITCH_READY_MASK 0x00000800L +#define IH_STATUS__RB1_FULL_MASK 0x00001000L +#define IH_STATUS__RB1_FULL_DRAIN_MASK 0x00002000L +#define IH_STATUS__RB1_OVERFLOW_MASK 0x00004000L +#define IH_STATUS__RB2_FULL_MASK 0x00008000L +#define IH_STATUS__RB2_FULL_DRAIN_MASK 0x00010000L +#define IH_STATUS__RB2_OVERFLOW_MASK 0x00020000L +#define IH_STATUS__SELF_INT_GEN_IDLE_MASK 0x00040000L +//IH_PERFMON_CNTL +#define IH_PERFMON_CNTL__ENABLE0__SHIFT 0x0 +#define IH_PERFMON_CNTL__CLEAR0__SHIFT 0x1 +#define IH_PERFMON_CNTL__PERF_SEL0__SHIFT 0x2 +#define IH_PERFMON_CNTL__ENABLE1__SHIFT 0x10 +#define IH_PERFMON_CNTL__CLEAR1__SHIFT 0x11 +#define IH_PERFMON_CNTL__PERF_SEL1__SHIFT 0x12 +#define IH_PERFMON_CNTL__ENABLE0_MASK 0x00000001L +#define IH_PERFMON_CNTL__CLEAR0_MASK 0x00000002L +#define IH_PERFMON_CNTL__PERF_SEL0_MASK 0x000007FCL +#define IH_PERFMON_CNTL__ENABLE1_MASK 0x00010000L +#define IH_PERFMON_CNTL__CLEAR1_MASK 0x00020000L +#define IH_PERFMON_CNTL__PERF_SEL1_MASK 0x07FC0000L +//IH_PERFCOUNTER0_RESULT +#define IH_PERFCOUNTER0_RESULT__PERF_COUNT__SHIFT 0x0 +#define IH_PERFCOUNTER0_RESULT__PERF_COUNT_MASK 0xFFFFFFFFL +//IH_PERFCOUNTER1_RESULT +#define IH_PERFCOUNTER1_RESULT__PERF_COUNT__SHIFT 0x0 +#define IH_PERFCOUNTER1_RESULT__PERF_COUNT_MASK 0xFFFFFFFFL +//IH_DSM_MATCH_VALUE_BIT_31_0 +#define IH_DSM_MATCH_VALUE_BIT_31_0__VALUE__SHIFT 0x0 +#define IH_DSM_MATCH_VALUE_BIT_31_0__VALUE_MASK 0xFFFFFFFFL +//IH_DSM_MATCH_VALUE_BIT_63_32 +#define IH_DSM_MATCH_VALUE_BIT_63_32__VALUE__SHIFT 0x0 +#define IH_DSM_MATCH_VALUE_BIT_63_32__VALUE_MASK 0xFFFFFFFFL +//IH_DSM_MATCH_VALUE_BIT_95_64 +#define IH_DSM_MATCH_VALUE_BIT_95_64__VALUE__SHIFT 0x0 +#define IH_DSM_MATCH_VALUE_BIT_95_64__VALUE_MASK 0xFFFFFFFFL +//IH_DSM_MATCH_FIELD_CONTROL +#define IH_DSM_MATCH_FIELD_CONTROL__SRC_EN__SHIFT 0x0 +#define IH_DSM_MATCH_FIELD_CONTROL__FCNID_EN__SHIFT 0x1 +#define IH_DSM_MATCH_FIELD_CONTROL__TIMESTAMP_EN__SHIFT 0x2 +#define IH_DSM_MATCH_FIELD_CONTROL__RINGID_EN__SHIFT 0x3 +#define IH_DSM_MATCH_FIELD_CONTROL__VMID_EN__SHIFT 0x4 +#define IH_DSM_MATCH_FIELD_CONTROL__PASID_EN__SHIFT 0x5 +#define IH_DSM_MATCH_FIELD_CONTROL__CLIENT_ID_EN__SHIFT 0x6 +#define IH_DSM_MATCH_FIELD_CONTROL__SRC_EN_MASK 0x00000001L +#define IH_DSM_MATCH_FIELD_CONTROL__FCNID_EN_MASK 0x00000002L +#define IH_DSM_MATCH_FIELD_CONTROL__TIMESTAMP_EN_MASK 0x00000004L +#define IH_DSM_MATCH_FIELD_CONTROL__RINGID_EN_MASK 0x00000008L +#define IH_DSM_MATCH_FIELD_CONTROL__VMID_EN_MASK 0x00000010L +#define IH_DSM_MATCH_FIELD_CONTROL__PASID_EN_MASK 0x00000020L +#define IH_DSM_MATCH_FIELD_CONTROL__CLIENT_ID_EN_MASK 0x00000040L +//IH_DSM_MATCH_DATA_CONTROL +#define IH_DSM_MATCH_DATA_CONTROL__VALUE__SHIFT 0x0 +#define IH_DSM_MATCH_DATA_CONTROL__VALUE_MASK 0x0FFFFFFFL +//IH_DSM_MATCH_FCN_ID +#define IH_DSM_MATCH_FCN_ID__PF_VF__SHIFT 0x0 +#define IH_DSM_MATCH_FCN_ID__VF_ID__SHIFT 0x1 +#define IH_DSM_MATCH_FCN_ID__PF_VF_MASK 0x00000001L +#define IH_DSM_MATCH_FCN_ID__VF_ID_MASK 0x0000001EL +//IH_LIMIT_INT_RATE_CNTL +#define IH_LIMIT_INT_RATE_CNTL__LIMIT_ENABLE__SHIFT 0x0 +#define IH_LIMIT_INT_RATE_CNTL__PERF_INTERVAL__SHIFT 0x1 +#define IH_LIMIT_INT_RATE_CNTL__PERF_THRESHOLD__SHIFT 0x5 +#define IH_LIMIT_INT_RATE_CNTL__RETURN_DELAY__SHIFT 0x11 +#define IH_LIMIT_INT_RATE_CNTL__PERF_RESULT__SHIFT 0x15 +#define IH_LIMIT_INT_RATE_CNTL__LIMIT_ENABLE_MASK 0x00000001L +#define IH_LIMIT_INT_RATE_CNTL__PERF_INTERVAL_MASK 0x0000001EL +#define IH_LIMIT_INT_RATE_CNTL__PERF_THRESHOLD_MASK 0x0000FFE0L +#define IH_LIMIT_INT_RATE_CNTL__RETURN_DELAY_MASK 0x001E0000L +#define IH_LIMIT_INT_RATE_CNTL__PERF_RESULT_MASK 0xFFE00000L +//IH_VF_RB_STATUS +#define IH_VF_RB_STATUS__RB_FULL_DRAIN_VF__SHIFT 0x0 +#define IH_VF_RB_STATUS__RB_OVERFLOW_VF__SHIFT 0x10 +#define IH_VF_RB_STATUS__RB_FULL_DRAIN_VF_MASK 0x0000FFFFL +#define IH_VF_RB_STATUS__RB_OVERFLOW_VF_MASK 0xFFFF0000L +//IH_VF_RB_STATUS2 +#define IH_VF_RB_STATUS2__RB_FULL_VF__SHIFT 0x0 +#define IH_VF_RB_STATUS2__BIF_INTERRUPT_LINE_VF__SHIFT 0x10 +#define IH_VF_RB_STATUS2__RB_FULL_VF_MASK 0x0000FFFFL +#define IH_VF_RB_STATUS2__BIF_INTERRUPT_LINE_VF_MASK 0xFFFF0000L +//IH_VF_RB1_STATUS +#define IH_VF_RB1_STATUS__RB_FULL_DRAIN_VF__SHIFT 0x0 +#define IH_VF_RB1_STATUS__RB_OVERFLOW_VF__SHIFT 0x10 +#define IH_VF_RB1_STATUS__RB_FULL_DRAIN_VF_MASK 0x0000FFFFL +#define IH_VF_RB1_STATUS__RB_OVERFLOW_VF_MASK 0xFFFF0000L +//IH_VF_RB1_STATUS2 +#define IH_VF_RB1_STATUS2__RB_FULL_VF__SHIFT 0x0 +#define IH_VF_RB1_STATUS2__RB_FULL_VF_MASK 0x0000FFFFL +//IH_VF_RB2_STATUS +#define IH_VF_RB2_STATUS__RB_FULL_DRAIN_VF__SHIFT 0x0 +#define IH_VF_RB2_STATUS__RB_OVERFLOW_VF__SHIFT 0x10 +#define IH_VF_RB2_STATUS__RB_FULL_DRAIN_VF_MASK 0x0000FFFFL +#define IH_VF_RB2_STATUS__RB_OVERFLOW_VF_MASK 0xFFFF0000L +//IH_VF_RB2_STATUS2 +#define IH_VF_RB2_STATUS2__RB_FULL_VF__SHIFT 0x0 +#define IH_VF_RB2_STATUS2__RB_FULL_VF_MASK 0x0000FFFFL +//IH_INT_FLOOD_CNTL +#define IH_INT_FLOOD_CNTL__HIGHWATER__SHIFT 0x0 +#define IH_INT_FLOOD_CNTL__FLOOD_CNTL_ENABLE__SHIFT 0x3 +#define IH_INT_FLOOD_CNTL__CLEAR_INT_FLOOD_STATUS__SHIFT 0x4 +#define IH_INT_FLOOD_CNTL__HIGHWATER_MASK 0x00000007L +#define IH_INT_FLOOD_CNTL__FLOOD_CNTL_ENABLE_MASK 0x00000008L +#define IH_INT_FLOOD_CNTL__CLEAR_INT_FLOOD_STATUS_MASK 0x00000010L +//IH_RB0_INT_FLOOD_STATUS +#define IH_RB0_INT_FLOOD_STATUS__RB_INT_DROPPED_VF__SHIFT 0x0 +#define IH_RB0_INT_FLOOD_STATUS__RB_INT_DROPPED__SHIFT 0x1f +#define IH_RB0_INT_FLOOD_STATUS__RB_INT_DROPPED_VF_MASK 0x0000FFFFL +#define IH_RB0_INT_FLOOD_STATUS__RB_INT_DROPPED_MASK 0x80000000L +//IH_RB1_INT_FLOOD_STATUS +#define IH_RB1_INT_FLOOD_STATUS__RB_INT_DROPPED_VF__SHIFT 0x0 +#define IH_RB1_INT_FLOOD_STATUS__RB_INT_DROPPED__SHIFT 0x1f +#define IH_RB1_INT_FLOOD_STATUS__RB_INT_DROPPED_VF_MASK 0x0000FFFFL +#define IH_RB1_INT_FLOOD_STATUS__RB_INT_DROPPED_MASK 0x80000000L +//IH_RB2_INT_FLOOD_STATUS +#define IH_RB2_INT_FLOOD_STATUS__RB_INT_DROPPED_VF__SHIFT 0x0 +#define IH_RB2_INT_FLOOD_STATUS__RB_INT_DROPPED__SHIFT 0x1f +#define IH_RB2_INT_FLOOD_STATUS__RB_INT_DROPPED_VF_MASK 0x0000FFFFL +#define IH_RB2_INT_FLOOD_STATUS__RB_INT_DROPPED_MASK 0x80000000L +//IH_INT_FLOOD_STATUS +#define IH_INT_FLOOD_STATUS__INT_DROP_CNT__SHIFT 0x0 +#define IH_INT_FLOOD_STATUS__FIRST_DROP_INT_CLIENT_ID__SHIFT 0x8 +#define IH_INT_FLOOD_STATUS__FIRST_DROP_INT_SOURCE_ID__SHIFT 0x10 +#define IH_INT_FLOOD_STATUS__FIRST_DROP_INT_VF_ID__SHIFT 0x18 +#define IH_INT_FLOOD_STATUS__FIRST_DROP_INT_VF__SHIFT 0x1c +#define IH_INT_FLOOD_STATUS__INT_DROPPED__SHIFT 0x1e +#define IH_INT_FLOOD_STATUS__INT_DROP_CNT_MASK 0x000000FFL +#define IH_INT_FLOOD_STATUS__FIRST_DROP_INT_CLIENT_ID_MASK 0x0000FF00L +#define IH_INT_FLOOD_STATUS__FIRST_DROP_INT_SOURCE_ID_MASK 0x00FF0000L +#define IH_INT_FLOOD_STATUS__FIRST_DROP_INT_VF_ID_MASK 0x0F000000L +#define IH_INT_FLOOD_STATUS__FIRST_DROP_INT_VF_MASK 0x10000000L +#define IH_INT_FLOOD_STATUS__INT_DROPPED_MASK 0x40000000L +//IH_STORM_CLIENT_LIST_CNTL +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT1_IS_STORM_CLIENT__SHIFT 0x1 +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT2_IS_STORM_CLIENT__SHIFT 0x2 +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT3_IS_STORM_CLIENT__SHIFT 0x3 +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT4_IS_STORM_CLIENT__SHIFT 0x4 +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT5_IS_STORM_CLIENT__SHIFT 0x5 +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT6_IS_STORM_CLIENT__SHIFT 0x6 +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT7_IS_STORM_CLIENT__SHIFT 0x7 +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT8_IS_STORM_CLIENT__SHIFT 0x8 +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT9_IS_STORM_CLIENT__SHIFT 0x9 +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT10_IS_STORM_CLIENT__SHIFT 0xa +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT11_IS_STORM_CLIENT__SHIFT 0xb +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT12_IS_STORM_CLIENT__SHIFT 0xc +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT13_IS_STORM_CLIENT__SHIFT 0xd +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT14_IS_STORM_CLIENT__SHIFT 0xe +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT15_IS_STORM_CLIENT__SHIFT 0xf +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT16_IS_STORM_CLIENT__SHIFT 0x10 +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT17_IS_STORM_CLIENT__SHIFT 0x11 +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT18_IS_STORM_CLIENT__SHIFT 0x12 +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT19_IS_STORM_CLIENT__SHIFT 0x13 +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT20_IS_STORM_CLIENT__SHIFT 0x14 +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT21_IS_STORM_CLIENT__SHIFT 0x15 +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT22_IS_STORM_CLIENT__SHIFT 0x16 +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT23_IS_STORM_CLIENT__SHIFT 0x17 +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT24_IS_STORM_CLIENT__SHIFT 0x18 +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT25_IS_STORM_CLIENT__SHIFT 0x19 +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT26_IS_STORM_CLIENT__SHIFT 0x1a +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT27_IS_STORM_CLIENT__SHIFT 0x1b +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT28_IS_STORM_CLIENT__SHIFT 0x1c +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT29_IS_STORM_CLIENT__SHIFT 0x1d +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT30_IS_STORM_CLIENT__SHIFT 0x1e +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT31_IS_STORM_CLIENT__SHIFT 0x1f +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT1_IS_STORM_CLIENT_MASK 0x00000002L +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT2_IS_STORM_CLIENT_MASK 0x00000004L +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT3_IS_STORM_CLIENT_MASK 0x00000008L +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT4_IS_STORM_CLIENT_MASK 0x00000010L +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT5_IS_STORM_CLIENT_MASK 0x00000020L +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT6_IS_STORM_CLIENT_MASK 0x00000040L +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT7_IS_STORM_CLIENT_MASK 0x00000080L +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT8_IS_STORM_CLIENT_MASK 0x00000100L +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT9_IS_STORM_CLIENT_MASK 0x00000200L +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT10_IS_STORM_CLIENT_MASK 0x00000400L +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT11_IS_STORM_CLIENT_MASK 0x00000800L +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT12_IS_STORM_CLIENT_MASK 0x00001000L +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT13_IS_STORM_CLIENT_MASK 0x00002000L +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT14_IS_STORM_CLIENT_MASK 0x00004000L +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT15_IS_STORM_CLIENT_MASK 0x00008000L +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT16_IS_STORM_CLIENT_MASK 0x00010000L +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT17_IS_STORM_CLIENT_MASK 0x00020000L +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT18_IS_STORM_CLIENT_MASK 0x00040000L +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT19_IS_STORM_CLIENT_MASK 0x00080000L +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT20_IS_STORM_CLIENT_MASK 0x00100000L +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT21_IS_STORM_CLIENT_MASK 0x00200000L +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT22_IS_STORM_CLIENT_MASK 0x00400000L +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT23_IS_STORM_CLIENT_MASK 0x00800000L +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT24_IS_STORM_CLIENT_MASK 0x01000000L +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT25_IS_STORM_CLIENT_MASK 0x02000000L +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT26_IS_STORM_CLIENT_MASK 0x04000000L +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT27_IS_STORM_CLIENT_MASK 0x08000000L +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT28_IS_STORM_CLIENT_MASK 0x10000000L +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT29_IS_STORM_CLIENT_MASK 0x20000000L +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT30_IS_STORM_CLIENT_MASK 0x40000000L +#define IH_STORM_CLIENT_LIST_CNTL__CLIENT31_IS_STORM_CLIENT_MASK 0x80000000L +//IH_CLK_CTRL +#define IH_CLK_CTRL__IH_RETRY_INT_CAM_MEM_CLK_SOFT_OVERRIDE__SHIFT 0x19 +#define IH_CLK_CTRL__IH_BUFFER_MEM_CLK_SOFT_OVERRIDE__SHIFT 0x1a +#define IH_CLK_CTRL__DBUS_MUX_CLK_SOFT_OVERRIDE__SHIFT 0x1b +#define IH_CLK_CTRL__OSSSYS_SHARE_CLK_SOFT_OVERRIDE__SHIFT 0x1c +#define IH_CLK_CTRL__LIMIT_SMN_CLK_SOFT_OVERRIDE__SHIFT 0x1d +#define IH_CLK_CTRL__DYN_CLK_SOFT_OVERRIDE__SHIFT 0x1e +#define IH_CLK_CTRL__REG_CLK_SOFT_OVERRIDE__SHIFT 0x1f +#define IH_CLK_CTRL__IH_RETRY_INT_CAM_MEM_CLK_SOFT_OVERRIDE_MASK 0x02000000L +#define IH_CLK_CTRL__IH_BUFFER_MEM_CLK_SOFT_OVERRIDE_MASK 0x04000000L +#define IH_CLK_CTRL__DBUS_MUX_CLK_SOFT_OVERRIDE_MASK 0x08000000L +#define IH_CLK_CTRL__OSSSYS_SHARE_CLK_SOFT_OVERRIDE_MASK 0x10000000L +#define IH_CLK_CTRL__LIMIT_SMN_CLK_SOFT_OVERRIDE_MASK 0x20000000L +#define IH_CLK_CTRL__DYN_CLK_SOFT_OVERRIDE_MASK 0x40000000L +#define IH_CLK_CTRL__REG_CLK_SOFT_OVERRIDE_MASK 0x80000000L +//IH_INT_FLAGS +#define IH_INT_FLAGS__CLIENT_0_FLAG__SHIFT 0x0 +#define IH_INT_FLAGS__CLIENT_1_FLAG__SHIFT 0x1 +#define IH_INT_FLAGS__CLIENT_2_FLAG__SHIFT 0x2 +#define IH_INT_FLAGS__CLIENT_3_FLAG__SHIFT 0x3 +#define IH_INT_FLAGS__CLIENT_4_FLAG__SHIFT 0x4 +#define IH_INT_FLAGS__CLIENT_5_FLAG__SHIFT 0x5 +#define IH_INT_FLAGS__CLIENT_6_FLAG__SHIFT 0x6 +#define IH_INT_FLAGS__CLIENT_7_FLAG__SHIFT 0x7 +#define IH_INT_FLAGS__CLIENT_8_FLAG__SHIFT 0x8 +#define IH_INT_FLAGS__CLIENT_9_FLAG__SHIFT 0x9 +#define IH_INT_FLAGS__CLIENT_10_FLAG__SHIFT 0xa +#define IH_INT_FLAGS__CLIENT_11_FLAG__SHIFT 0xb +#define IH_INT_FLAGS__CLIENT_12_FLAG__SHIFT 0xc +#define IH_INT_FLAGS__CLIENT_13_FLAG__SHIFT 0xd +#define IH_INT_FLAGS__CLIENT_14_FLAG__SHIFT 0xe +#define IH_INT_FLAGS__CLIENT_15_FLAG__SHIFT 0xf +#define IH_INT_FLAGS__CLIENT_16_FLAG__SHIFT 0x10 +#define IH_INT_FLAGS__CLIENT_17_FLAG__SHIFT 0x11 +#define IH_INT_FLAGS__CLIENT_18_FLAG__SHIFT 0x12 +#define IH_INT_FLAGS__CLIENT_19_FLAG__SHIFT 0x13 +#define IH_INT_FLAGS__CLIENT_20_FLAG__SHIFT 0x14 +#define IH_INT_FLAGS__CLIENT_21_FLAG__SHIFT 0x15 +#define IH_INT_FLAGS__CLIENT_22_FLAG__SHIFT 0x16 +#define IH_INT_FLAGS__CLIENT_23_FLAG__SHIFT 0x17 +#define IH_INT_FLAGS__CLIENT_24_FLAG__SHIFT 0x18 +#define IH_INT_FLAGS__CLIENT_25_FLAG__SHIFT 0x19 +#define IH_INT_FLAGS__CLIENT_26_FLAG__SHIFT 0x1a +#define IH_INT_FLAGS__CLIENT_27_FLAG__SHIFT 0x1b +#define IH_INT_FLAGS__CLIENT_28_FLAG__SHIFT 0x1c +#define IH_INT_FLAGS__CLIENT_29_FLAG__SHIFT 0x1d +#define IH_INT_FLAGS__CLIENT_30_FLAG__SHIFT 0x1e +#define IH_INT_FLAGS__CLIENT_31_FLAG__SHIFT 0x1f +#define IH_INT_FLAGS__CLIENT_0_FLAG_MASK 0x00000001L +#define IH_INT_FLAGS__CLIENT_1_FLAG_MASK 0x00000002L +#define IH_INT_FLAGS__CLIENT_2_FLAG_MASK 0x00000004L +#define IH_INT_FLAGS__CLIENT_3_FLAG_MASK 0x00000008L +#define IH_INT_FLAGS__CLIENT_4_FLAG_MASK 0x00000010L +#define IH_INT_FLAGS__CLIENT_5_FLAG_MASK 0x00000020L +#define IH_INT_FLAGS__CLIENT_6_FLAG_MASK 0x00000040L +#define IH_INT_FLAGS__CLIENT_7_FLAG_MASK 0x00000080L +#define IH_INT_FLAGS__CLIENT_8_FLAG_MASK 0x00000100L +#define IH_INT_FLAGS__CLIENT_9_FLAG_MASK 0x00000200L +#define IH_INT_FLAGS__CLIENT_10_FLAG_MASK 0x00000400L +#define IH_INT_FLAGS__CLIENT_11_FLAG_MASK 0x00000800L +#define IH_INT_FLAGS__CLIENT_12_FLAG_MASK 0x00001000L +#define IH_INT_FLAGS__CLIENT_13_FLAG_MASK 0x00002000L +#define IH_INT_FLAGS__CLIENT_14_FLAG_MASK 0x00004000L +#define IH_INT_FLAGS__CLIENT_15_FLAG_MASK 0x00008000L +#define IH_INT_FLAGS__CLIENT_16_FLAG_MASK 0x00010000L +#define IH_INT_FLAGS__CLIENT_17_FLAG_MASK 0x00020000L +#define IH_INT_FLAGS__CLIENT_18_FLAG_MASK 0x00040000L +#define IH_INT_FLAGS__CLIENT_19_FLAG_MASK 0x00080000L +#define IH_INT_FLAGS__CLIENT_20_FLAG_MASK 0x00100000L +#define IH_INT_FLAGS__CLIENT_21_FLAG_MASK 0x00200000L +#define IH_INT_FLAGS__CLIENT_22_FLAG_MASK 0x00400000L +#define IH_INT_FLAGS__CLIENT_23_FLAG_MASK 0x00800000L +#define IH_INT_FLAGS__CLIENT_24_FLAG_MASK 0x01000000L +#define IH_INT_FLAGS__CLIENT_25_FLAG_MASK 0x02000000L +#define IH_INT_FLAGS__CLIENT_26_FLAG_MASK 0x04000000L +#define IH_INT_FLAGS__CLIENT_27_FLAG_MASK 0x08000000L +#define IH_INT_FLAGS__CLIENT_28_FLAG_MASK 0x10000000L +#define IH_INT_FLAGS__CLIENT_29_FLAG_MASK 0x20000000L +#define IH_INT_FLAGS__CLIENT_30_FLAG_MASK 0x40000000L +#define IH_INT_FLAGS__CLIENT_31_FLAG_MASK 0x80000000L +//IH_LAST_INT_INFO0 +#define IH_LAST_INT_INFO0__CLIENT_ID__SHIFT 0x0 +#define IH_LAST_INT_INFO0__SOURCE_ID__SHIFT 0x8 +#define IH_LAST_INT_INFO0__RING_ID__SHIFT 0x10 +#define IH_LAST_INT_INFO0__VM_ID__SHIFT 0x18 +#define IH_LAST_INT_INFO0__VMID_TYPE__SHIFT 0x1f +#define IH_LAST_INT_INFO0__CLIENT_ID_MASK 0x000000FFL +#define IH_LAST_INT_INFO0__SOURCE_ID_MASK 0x0000FF00L +#define IH_LAST_INT_INFO0__RING_ID_MASK 0x00FF0000L +#define IH_LAST_INT_INFO0__VM_ID_MASK 0x0F000000L +#define IH_LAST_INT_INFO0__VMID_TYPE_MASK 0x80000000L +//IH_LAST_INT_INFO1 +#define IH_LAST_INT_INFO1__CONTEXT_ID__SHIFT 0x0 +#define IH_LAST_INT_INFO1__CONTEXT_ID_MASK 0xFFFFFFFFL +//IH_LAST_INT_INFO2 +#define IH_LAST_INT_INFO2__PAS_ID__SHIFT 0x0 +#define IH_LAST_INT_INFO2__VF_ID__SHIFT 0x10 +#define IH_LAST_INT_INFO2__VF__SHIFT 0x14 +#define IH_LAST_INT_INFO2__PAS_ID_MASK 0x0000FFFFL +#define IH_LAST_INT_INFO2__VF_ID_MASK 0x000F0000L +#define IH_LAST_INT_INFO2__VF_MASK 0x00100000L +//IH_SCRATCH +#define IH_SCRATCH__DATA__SHIFT 0x0 +#define IH_SCRATCH__DATA_MASK 0xFFFFFFFFL +//IH_CLIENT_CREDIT_ERROR +#define IH_CLIENT_CREDIT_ERROR__CLEAR__SHIFT 0x0 +#define IH_CLIENT_CREDIT_ERROR__CLIENT_1_ERROR__SHIFT 0x1 +#define IH_CLIENT_CREDIT_ERROR__CLIENT_2_ERROR__SHIFT 0x2 +#define IH_CLIENT_CREDIT_ERROR__CLIENT_3_ERROR__SHIFT 0x3 +#define IH_CLIENT_CREDIT_ERROR__CLIENT_4_ERROR__SHIFT 0x4 +#define IH_CLIENT_CREDIT_ERROR__CLIENT_5_ERROR__SHIFT 0x5 +#define IH_CLIENT_CREDIT_ERROR__CLIENT_6_ERROR__SHIFT 0x6 +#define IH_CLIENT_CREDIT_ERROR__CLIENT_7_ERROR__SHIFT 0x7 +#define IH_CLIENT_CREDIT_ERROR__CLIENT_8_ERROR__SHIFT 0x8 +#define IH_CLIENT_CREDIT_ERROR__CLIENT_9_ERROR__SHIFT 0x9 +#define IH_CLIENT_CREDIT_ERROR__CLIENT_10_ERROR__SHIFT 0xa +#define IH_CLIENT_CREDIT_ERROR__CLIENT_11_ERROR__SHIFT 0xb +#define IH_CLIENT_CREDIT_ERROR__CLIENT_12_ERROR__SHIFT 0xc +#define IH_CLIENT_CREDIT_ERROR__CLIENT_13_ERROR__SHIFT 0xd +#define IH_CLIENT_CREDIT_ERROR__CLIENT_14_ERROR__SHIFT 0xe +#define IH_CLIENT_CREDIT_ERROR__CLIENT_15_ERROR__SHIFT 0xf +#define IH_CLIENT_CREDIT_ERROR__CLIENT_16_ERROR__SHIFT 0x10 +#define IH_CLIENT_CREDIT_ERROR__CLIENT_17_ERROR__SHIFT 0x11 +#define IH_CLIENT_CREDIT_ERROR__CLIENT_18_ERROR__SHIFT 0x12 +#define IH_CLIENT_CREDIT_ERROR__CLIENT_19_ERROR__SHIFT 0x13 +#define IH_CLIENT_CREDIT_ERROR__CLIENT_20_ERROR__SHIFT 0x14 +#define IH_CLIENT_CREDIT_ERROR__CLIENT_21_ERROR__SHIFT 0x15 +#define IH_CLIENT_CREDIT_ERROR__CLIENT_22_ERROR__SHIFT 0x16 +#define IH_CLIENT_CREDIT_ERROR__CLIENT_23_ERROR__SHIFT 0x17 +#define IH_CLIENT_CREDIT_ERROR__CLIENT_24_ERROR__SHIFT 0x18 +#define IH_CLIENT_CREDIT_ERROR__CLIENT_25_ERROR__SHIFT 0x19 +#define IH_CLIENT_CREDIT_ERROR__CLIENT_26_ERROR__SHIFT 0x1a +#define IH_CLIENT_CREDIT_ERROR__CLIENT_27_ERROR__SHIFT 0x1b +#define IH_CLIENT_CREDIT_ERROR__CLIENT_28_ERROR__SHIFT 0x1c +#define IH_CLIENT_CREDIT_ERROR__CLIENT_29_ERROR__SHIFT 0x1d +#define IH_CLIENT_CREDIT_ERROR__CLIENT_30_ERROR__SHIFT 0x1e +#define IH_CLIENT_CREDIT_ERROR__CLIENT_31_ERROR__SHIFT 0x1f +#define IH_CLIENT_CREDIT_ERROR__CLEAR_MASK 0x00000001L +#define IH_CLIENT_CREDIT_ERROR__CLIENT_1_ERROR_MASK 0x00000002L +#define IH_CLIENT_CREDIT_ERROR__CLIENT_2_ERROR_MASK 0x00000004L +#define IH_CLIENT_CREDIT_ERROR__CLIENT_3_ERROR_MASK 0x00000008L +#define IH_CLIENT_CREDIT_ERROR__CLIENT_4_ERROR_MASK 0x00000010L +#define IH_CLIENT_CREDIT_ERROR__CLIENT_5_ERROR_MASK 0x00000020L +#define IH_CLIENT_CREDIT_ERROR__CLIENT_6_ERROR_MASK 0x00000040L +#define IH_CLIENT_CREDIT_ERROR__CLIENT_7_ERROR_MASK 0x00000080L +#define IH_CLIENT_CREDIT_ERROR__CLIENT_8_ERROR_MASK 0x00000100L +#define IH_CLIENT_CREDIT_ERROR__CLIENT_9_ERROR_MASK 0x00000200L +#define IH_CLIENT_CREDIT_ERROR__CLIENT_10_ERROR_MASK 0x00000400L +#define IH_CLIENT_CREDIT_ERROR__CLIENT_11_ERROR_MASK 0x00000800L +#define IH_CLIENT_CREDIT_ERROR__CLIENT_12_ERROR_MASK 0x00001000L +#define IH_CLIENT_CREDIT_ERROR__CLIENT_13_ERROR_MASK 0x00002000L +#define IH_CLIENT_CREDIT_ERROR__CLIENT_14_ERROR_MASK 0x00004000L +#define IH_CLIENT_CREDIT_ERROR__CLIENT_15_ERROR_MASK 0x00008000L +#define IH_CLIENT_CREDIT_ERROR__CLIENT_16_ERROR_MASK 0x00010000L +#define IH_CLIENT_CREDIT_ERROR__CLIENT_17_ERROR_MASK 0x00020000L +#define IH_CLIENT_CREDIT_ERROR__CLIENT_18_ERROR_MASK 0x00040000L +#define IH_CLIENT_CREDIT_ERROR__CLIENT_19_ERROR_MASK 0x00080000L +#define IH_CLIENT_CREDIT_ERROR__CLIENT_20_ERROR_MASK 0x00100000L +#define IH_CLIENT_CREDIT_ERROR__CLIENT_21_ERROR_MASK 0x00200000L +#define IH_CLIENT_CREDIT_ERROR__CLIENT_22_ERROR_MASK 0x00400000L +#define IH_CLIENT_CREDIT_ERROR__CLIENT_23_ERROR_MASK 0x00800000L +#define IH_CLIENT_CREDIT_ERROR__CLIENT_24_ERROR_MASK 0x01000000L +#define IH_CLIENT_CREDIT_ERROR__CLIENT_25_ERROR_MASK 0x02000000L +#define IH_CLIENT_CREDIT_ERROR__CLIENT_26_ERROR_MASK 0x04000000L +#define IH_CLIENT_CREDIT_ERROR__CLIENT_27_ERROR_MASK 0x08000000L +#define IH_CLIENT_CREDIT_ERROR__CLIENT_28_ERROR_MASK 0x10000000L +#define IH_CLIENT_CREDIT_ERROR__CLIENT_29_ERROR_MASK 0x20000000L +#define IH_CLIENT_CREDIT_ERROR__CLIENT_30_ERROR_MASK 0x40000000L +#define IH_CLIENT_CREDIT_ERROR__CLIENT_31_ERROR_MASK 0x80000000L +//IH_GPU_IOV_VIOLATION_LOG +#define IH_GPU_IOV_VIOLATION_LOG__VIOLATION_STATUS__SHIFT 0x0 +#define IH_GPU_IOV_VIOLATION_LOG__MULTIPLE_VIOLATION_STATUS__SHIFT 0x1 +#define IH_GPU_IOV_VIOLATION_LOG__ADDRESS__SHIFT 0x2 +#define IH_GPU_IOV_VIOLATION_LOG__OPCODE__SHIFT 0x12 +#define IH_GPU_IOV_VIOLATION_LOG__VF__SHIFT 0x13 +#define IH_GPU_IOV_VIOLATION_LOG__VF_ID__SHIFT 0x14 +#define IH_GPU_IOV_VIOLATION_LOG__INITIATOR_ID__SHIFT 0x18 +#define IH_GPU_IOV_VIOLATION_LOG__VIOLATION_STATUS_MASK 0x00000001L +#define IH_GPU_IOV_VIOLATION_LOG__MULTIPLE_VIOLATION_STATUS_MASK 0x00000002L +#define IH_GPU_IOV_VIOLATION_LOG__ADDRESS_MASK 0x0003FFFCL +#define IH_GPU_IOV_VIOLATION_LOG__OPCODE_MASK 0x00040000L +#define IH_GPU_IOV_VIOLATION_LOG__VF_MASK 0x00080000L +#define IH_GPU_IOV_VIOLATION_LOG__VF_ID_MASK 0x00F00000L +#define IH_GPU_IOV_VIOLATION_LOG__INITIATOR_ID_MASK 0xFF000000L +//IH_COOKIE_REC_VIOLATION_LOG +#define IH_COOKIE_REC_VIOLATION_LOG__VIOLATION_STATUS__SHIFT 0x0 +#define IH_COOKIE_REC_VIOLATION_LOG__CLIENT_ID__SHIFT 0x10 +#define IH_COOKIE_REC_VIOLATION_LOG__INITIATOR_ID__SHIFT 0x18 +#define IH_COOKIE_REC_VIOLATION_LOG__VIOLATION_STATUS_MASK 0x00000001L +#define IH_COOKIE_REC_VIOLATION_LOG__CLIENT_ID_MASK 0x00FF0000L +#define IH_COOKIE_REC_VIOLATION_LOG__INITIATOR_ID_MASK 0xFF000000L +//IH_CREDIT_STATUS +#define IH_CREDIT_STATUS__CLIENT_1_CREDIT_RETURNED__SHIFT 0x1 +#define IH_CREDIT_STATUS__CLIENT_2_CREDIT_RETURNED__SHIFT 0x2 +#define IH_CREDIT_STATUS__CLIENT_3_CREDIT_RETURNED__SHIFT 0x3 +#define IH_CREDIT_STATUS__CLIENT_4_CREDIT_RETURNED__SHIFT 0x4 +#define IH_CREDIT_STATUS__CLIENT_5_CREDIT_RETURNED__SHIFT 0x5 +#define IH_CREDIT_STATUS__CLIENT_6_CREDIT_RETURNED__SHIFT 0x6 +#define IH_CREDIT_STATUS__CLIENT_7_CREDIT_RETURNED__SHIFT 0x7 +#define IH_CREDIT_STATUS__CLIENT_8_CREDIT_RETURNED__SHIFT 0x8 +#define IH_CREDIT_STATUS__CLIENT_9_CREDIT_RETURNED__SHIFT 0x9 +#define IH_CREDIT_STATUS__CLIENT_10_CREDIT_RETURNED__SHIFT 0xa +#define IH_CREDIT_STATUS__CLIENT_11_CREDIT_RETURNED__SHIFT 0xb +#define IH_CREDIT_STATUS__CLIENT_12_CREDIT_RETURNED__SHIFT 0xc +#define IH_CREDIT_STATUS__CLIENT_13_CREDIT_RETURNED__SHIFT 0xd +#define IH_CREDIT_STATUS__CLIENT_14_CREDIT_RETURNED__SHIFT 0xe +#define IH_CREDIT_STATUS__CLIENT_15_CREDIT_RETURNED__SHIFT 0xf +#define IH_CREDIT_STATUS__CLIENT_16_CREDIT_RETURNED__SHIFT 0x10 +#define IH_CREDIT_STATUS__CLIENT_17_CREDIT_RETURNED__SHIFT 0x11 +#define IH_CREDIT_STATUS__CLIENT_18_CREDIT_RETURNED__SHIFT 0x12 +#define IH_CREDIT_STATUS__CLIENT_19_CREDIT_RETURNED__SHIFT 0x13 +#define IH_CREDIT_STATUS__CLIENT_20_CREDIT_RETURNED__SHIFT 0x14 +#define IH_CREDIT_STATUS__CLIENT_21_CREDIT_RETURNED__SHIFT 0x15 +#define IH_CREDIT_STATUS__CLIENT_22_CREDIT_RETURNED__SHIFT 0x16 +#define IH_CREDIT_STATUS__CLIENT_23_CREDIT_RETURNED__SHIFT 0x17 +#define IH_CREDIT_STATUS__CLIENT_24_CREDIT_RETURNED__SHIFT 0x18 +#define IH_CREDIT_STATUS__CLIENT_25_CREDIT_RETURNED__SHIFT 0x19 +#define IH_CREDIT_STATUS__CLIENT_26_CREDIT_RETURNED__SHIFT 0x1a +#define IH_CREDIT_STATUS__CLIENT_27_CREDIT_RETURNED__SHIFT 0x1b +#define IH_CREDIT_STATUS__CLIENT_28_CREDIT_RETURNED__SHIFT 0x1c +#define IH_CREDIT_STATUS__CLIENT_29_CREDIT_RETURNED__SHIFT 0x1d +#define IH_CREDIT_STATUS__CLIENT_30_CREDIT_RETURNED__SHIFT 0x1e +#define IH_CREDIT_STATUS__CLIENT_31_CREDIT_RETURNED__SHIFT 0x1f +#define IH_CREDIT_STATUS__CLIENT_1_CREDIT_RETURNED_MASK 0x00000002L +#define IH_CREDIT_STATUS__CLIENT_2_CREDIT_RETURNED_MASK 0x00000004L +#define IH_CREDIT_STATUS__CLIENT_3_CREDIT_RETURNED_MASK 0x00000008L +#define IH_CREDIT_STATUS__CLIENT_4_CREDIT_RETURNED_MASK 0x00000010L +#define IH_CREDIT_STATUS__CLIENT_5_CREDIT_RETURNED_MASK 0x00000020L +#define IH_CREDIT_STATUS__CLIENT_6_CREDIT_RETURNED_MASK 0x00000040L +#define IH_CREDIT_STATUS__CLIENT_7_CREDIT_RETURNED_MASK 0x00000080L +#define IH_CREDIT_STATUS__CLIENT_8_CREDIT_RETURNED_MASK 0x00000100L +#define IH_CREDIT_STATUS__CLIENT_9_CREDIT_RETURNED_MASK 0x00000200L +#define IH_CREDIT_STATUS__CLIENT_10_CREDIT_RETURNED_MASK 0x00000400L +#define IH_CREDIT_STATUS__CLIENT_11_CREDIT_RETURNED_MASK 0x00000800L +#define IH_CREDIT_STATUS__CLIENT_12_CREDIT_RETURNED_MASK 0x00001000L +#define IH_CREDIT_STATUS__CLIENT_13_CREDIT_RETURNED_MASK 0x00002000L +#define IH_CREDIT_STATUS__CLIENT_14_CREDIT_RETURNED_MASK 0x00004000L +#define IH_CREDIT_STATUS__CLIENT_15_CREDIT_RETURNED_MASK 0x00008000L +#define IH_CREDIT_STATUS__CLIENT_16_CREDIT_RETURNED_MASK 0x00010000L +#define IH_CREDIT_STATUS__CLIENT_17_CREDIT_RETURNED_MASK 0x00020000L +#define IH_CREDIT_STATUS__CLIENT_18_CREDIT_RETURNED_MASK 0x00040000L +#define IH_CREDIT_STATUS__CLIENT_19_CREDIT_RETURNED_MASK 0x00080000L +#define IH_CREDIT_STATUS__CLIENT_20_CREDIT_RETURNED_MASK 0x00100000L +#define IH_CREDIT_STATUS__CLIENT_21_CREDIT_RETURNED_MASK 0x00200000L +#define IH_CREDIT_STATUS__CLIENT_22_CREDIT_RETURNED_MASK 0x00400000L +#define IH_CREDIT_STATUS__CLIENT_23_CREDIT_RETURNED_MASK 0x00800000L +#define IH_CREDIT_STATUS__CLIENT_24_CREDIT_RETURNED_MASK 0x01000000L +#define IH_CREDIT_STATUS__CLIENT_25_CREDIT_RETURNED_MASK 0x02000000L +#define IH_CREDIT_STATUS__CLIENT_26_CREDIT_RETURNED_MASK 0x04000000L +#define IH_CREDIT_STATUS__CLIENT_27_CREDIT_RETURNED_MASK 0x08000000L +#define IH_CREDIT_STATUS__CLIENT_28_CREDIT_RETURNED_MASK 0x10000000L +#define IH_CREDIT_STATUS__CLIENT_29_CREDIT_RETURNED_MASK 0x20000000L +#define IH_CREDIT_STATUS__CLIENT_30_CREDIT_RETURNED_MASK 0x40000000L +#define IH_CREDIT_STATUS__CLIENT_31_CREDIT_RETURNED_MASK 0x80000000L +//IH_MMHUB_ERROR +#define IH_MMHUB_ERROR__IH_BRESP_01__SHIFT 0x1 +#define IH_MMHUB_ERROR__IH_BRESP_10__SHIFT 0x2 +#define IH_MMHUB_ERROR__IH_BRESP_11__SHIFT 0x3 +#define IH_MMHUB_ERROR__IH_BUSER_NACK_01__SHIFT 0x5 +#define IH_MMHUB_ERROR__IH_BUSER_NACK_10__SHIFT 0x6 +#define IH_MMHUB_ERROR__IH_BUSER_NACK_11__SHIFT 0x7 +#define IH_MMHUB_ERROR__IH_BRESP_01_MASK 0x00000002L +#define IH_MMHUB_ERROR__IH_BRESP_10_MASK 0x00000004L +#define IH_MMHUB_ERROR__IH_BRESP_11_MASK 0x00000008L +#define IH_MMHUB_ERROR__IH_BUSER_NACK_01_MASK 0x00000020L +#define IH_MMHUB_ERROR__IH_BUSER_NACK_10_MASK 0x00000040L +#define IH_MMHUB_ERROR__IH_BUSER_NACK_11_MASK 0x00000080L +//IH_MEM_POWER_CTRL +#define IH_MEM_POWER_CTRL__IH_BUFFER_MEM_POWER_CTRL_EN__SHIFT 0x0 +#define IH_MEM_POWER_CTRL__IH_BUFFER_MEM_POWER_LS_EN__SHIFT 0x1 +#define IH_MEM_POWER_CTRL__IH_BUFFER_MEM_POWER_DS_EN__SHIFT 0x2 +#define IH_MEM_POWER_CTRL__IH_BUFFER_MEM_POWER_SD_EN__SHIFT 0x3 +#define IH_MEM_POWER_CTRL__IH_BUFFER_MEM_IDLE_HYSTERESIS__SHIFT 0x4 +#define IH_MEM_POWER_CTRL__IH_BUFFER_MEM_POWER_UP_RECOVER_DELAY__SHIFT 0x8 +#define IH_MEM_POWER_CTRL__IH_BUFFER_MEM_POWER_DOWN_LS_ENTER_DELAY__SHIFT 0xe +#define IH_MEM_POWER_CTRL__IH_BUFFER_MEM_POWER_CTRL_EN_MASK 0x00000001L +#define IH_MEM_POWER_CTRL__IH_BUFFER_MEM_POWER_LS_EN_MASK 0x00000002L +#define IH_MEM_POWER_CTRL__IH_BUFFER_MEM_POWER_DS_EN_MASK 0x00000004L +#define IH_MEM_POWER_CTRL__IH_BUFFER_MEM_POWER_SD_EN_MASK 0x00000008L +#define IH_MEM_POWER_CTRL__IH_BUFFER_MEM_IDLE_HYSTERESIS_MASK 0x00000070L +#define IH_MEM_POWER_CTRL__IH_BUFFER_MEM_POWER_UP_RECOVER_DELAY_MASK 0x00003F00L +#define IH_MEM_POWER_CTRL__IH_BUFFER_MEM_POWER_DOWN_LS_ENTER_DELAY_MASK 0x0000C000L +//IH_REGISTER_LAST_PART2 +#define IH_REGISTER_LAST_PART2__RESERVED__SHIFT 0x0 +#define IH_REGISTER_LAST_PART2__RESERVED_MASK 0xFFFFFFFFL +//SEM_CLK_CTRL +#define SEM_CLK_CTRL__ON_DELAY__SHIFT 0x0 +#define SEM_CLK_CTRL__OFF_HYSTERESIS__SHIFT 0x4 +#define SEM_CLK_CTRL__RESERVED__SHIFT 0xc +#define SEM_CLK_CTRL__SOFT_OVERRIDE7__SHIFT 0x18 +#define SEM_CLK_CTRL__SOFT_OVERRIDE6__SHIFT 0x19 +#define SEM_CLK_CTRL__MEM_CLK_SOFT_OVERRIDE__SHIFT 0x1a +#define SEM_CLK_CTRL__SOFT_OVERRIDE4__SHIFT 0x1b +#define SEM_CLK_CTRL__SOFT_OVERRIDE3__SHIFT 0x1c +#define SEM_CLK_CTRL__SOFT_OVERRIDE2__SHIFT 0x1d +#define SEM_CLK_CTRL__DYN_CLK_SOFT_OVERRIDE__SHIFT 0x1e +#define SEM_CLK_CTRL__REG_CLK_SOFT_OVERRIDE__SHIFT 0x1f +#define SEM_CLK_CTRL__ON_DELAY_MASK 0x0000000FL +#define SEM_CLK_CTRL__OFF_HYSTERESIS_MASK 0x00000FF0L +#define SEM_CLK_CTRL__RESERVED_MASK 0x00FFF000L +#define SEM_CLK_CTRL__SOFT_OVERRIDE7_MASK 0x01000000L +#define SEM_CLK_CTRL__SOFT_OVERRIDE6_MASK 0x02000000L +#define SEM_CLK_CTRL__MEM_CLK_SOFT_OVERRIDE_MASK 0x04000000L +#define SEM_CLK_CTRL__SOFT_OVERRIDE4_MASK 0x08000000L +#define SEM_CLK_CTRL__SOFT_OVERRIDE3_MASK 0x10000000L +#define SEM_CLK_CTRL__SOFT_OVERRIDE2_MASK 0x20000000L +#define SEM_CLK_CTRL__DYN_CLK_SOFT_OVERRIDE_MASK 0x40000000L +#define SEM_CLK_CTRL__REG_CLK_SOFT_OVERRIDE_MASK 0x80000000L +//SEM_UTC_CREDIT +#define SEM_UTC_CREDIT__UTCL2_CREDIT__SHIFT 0x0 +#define SEM_UTC_CREDIT__WATERMARK__SHIFT 0x8 +#define SEM_UTC_CREDIT__UTCL2_CREDIT_MASK 0x0000001FL +#define SEM_UTC_CREDIT__WATERMARK_MASK 0x00000F00L +//SEM_UTC_CONFIG +#define SEM_UTC_CONFIG__USE_MTYPE__SHIFT 0x0 +#define SEM_UTC_CONFIG__FORCE_SNOOP__SHIFT 0x3 +#define SEM_UTC_CONFIG__FORCE_GCC__SHIFT 0x4 +#define SEM_UTC_CONFIG__USE_PT_SNOOP__SHIFT 0x5 +#define SEM_UTC_CONFIG__USE_MTYPE_MASK 0x00000007L +#define SEM_UTC_CONFIG__FORCE_SNOOP_MASK 0x00000008L +#define SEM_UTC_CONFIG__FORCE_GCC_MASK 0x00000010L +#define SEM_UTC_CONFIG__USE_PT_SNOOP_MASK 0x00000020L +//SEM_UTCL2_TRAN_EN_LUT +#define SEM_UTCL2_TRAN_EN_LUT__SDMA0_UTCL2_EN__SHIFT 0x0 +#define SEM_UTCL2_TRAN_EN_LUT__SDMA1_UTCL2_EN__SHIFT 0x1 +#define SEM_UTCL2_TRAN_EN_LUT__UVD_UTCL2_EN__SHIFT 0x2 +#define SEM_UTCL2_TRAN_EN_LUT__VCE0_UTCL2_EN__SHIFT 0x3 +#define SEM_UTCL2_TRAN_EN_LUT__ACP_UTCL2_EN__SHIFT 0x4 +#define SEM_UTCL2_TRAN_EN_LUT__ISP_UTCL2_EN__SHIFT 0x5 +#define SEM_UTCL2_TRAN_EN_LUT__VCE1_UTCL2_EN__SHIFT 0x6 +#define SEM_UTCL2_TRAN_EN_LUT__VP8_UTCL2_EN__SHIFT 0x7 +#define SEM_UTCL2_TRAN_EN_LUT__UVD1_UTCL2_EN__SHIFT 0x8 +#define SEM_UTCL2_TRAN_EN_LUT__RESERVED__SHIFT 0x9 +#define SEM_UTCL2_TRAN_EN_LUT__CP_UTCL2_EN__SHIFT 0x1f +#define SEM_UTCL2_TRAN_EN_LUT__SDMA0_UTCL2_EN_MASK 0x00000001L +#define SEM_UTCL2_TRAN_EN_LUT__SDMA1_UTCL2_EN_MASK 0x00000002L +#define SEM_UTCL2_TRAN_EN_LUT__UVD_UTCL2_EN_MASK 0x00000004L +#define SEM_UTCL2_TRAN_EN_LUT__VCE0_UTCL2_EN_MASK 0x00000008L +#define SEM_UTCL2_TRAN_EN_LUT__ACP_UTCL2_EN_MASK 0x00000010L +#define SEM_UTCL2_TRAN_EN_LUT__ISP_UTCL2_EN_MASK 0x00000020L +#define SEM_UTCL2_TRAN_EN_LUT__VCE1_UTCL2_EN_MASK 0x00000040L +#define SEM_UTCL2_TRAN_EN_LUT__VP8_UTCL2_EN_MASK 0x00000080L +#define SEM_UTCL2_TRAN_EN_LUT__UVD1_UTCL2_EN_MASK 0x00000100L +#define SEM_UTCL2_TRAN_EN_LUT__RESERVED_MASK 0x7FFFFE00L +#define SEM_UTCL2_TRAN_EN_LUT__CP_UTCL2_EN_MASK 0x80000000L +//SEM_MCIF_CONFIG +#define SEM_MCIF_CONFIG__MC_REQ_SWAP__SHIFT 0x0 +#define SEM_MCIF_CONFIG__MC_WRREQ_CREDIT__SHIFT 0x2 +#define SEM_MCIF_CONFIG__MC_RDREQ_CREDIT__SHIFT 0x8 +#define SEM_MCIF_CONFIG__MC_REQ_SWAP_MASK 0x00000003L +#define SEM_MCIF_CONFIG__MC_WRREQ_CREDIT_MASK 0x000000FCL +#define SEM_MCIF_CONFIG__MC_RDREQ_CREDIT_MASK 0x00003F00L +//SEM_PERFMON_CNTL +#define SEM_PERFMON_CNTL__PERF_ENABLE0__SHIFT 0x0 +#define SEM_PERFMON_CNTL__PERF_CLEAR0__SHIFT 0x1 +#define SEM_PERFMON_CNTL__PERF_SEL0__SHIFT 0x2 +#define SEM_PERFMON_CNTL__PERF_ENABLE1__SHIFT 0xa +#define SEM_PERFMON_CNTL__PERF_CLEAR1__SHIFT 0xb +#define SEM_PERFMON_CNTL__PERF_SEL1__SHIFT 0xc +#define SEM_PERFMON_CNTL__PERF_ENABLE0_MASK 0x00000001L +#define SEM_PERFMON_CNTL__PERF_CLEAR0_MASK 0x00000002L +#define SEM_PERFMON_CNTL__PERF_SEL0_MASK 0x000003FCL +#define SEM_PERFMON_CNTL__PERF_ENABLE1_MASK 0x00000400L +#define SEM_PERFMON_CNTL__PERF_CLEAR1_MASK 0x00000800L +#define SEM_PERFMON_CNTL__PERF_SEL1_MASK 0x000FF000L +//SEM_PERFCOUNTER0_RESULT +#define SEM_PERFCOUNTER0_RESULT__PERF_COUNT__SHIFT 0x0 +#define SEM_PERFCOUNTER0_RESULT__PERF_COUNT_MASK 0xFFFFFFFFL +//SEM_PERFCOUNTER1_RESULT +#define SEM_PERFCOUNTER1_RESULT__PERF_COUNT__SHIFT 0x0 +#define SEM_PERFCOUNTER1_RESULT__PERF_COUNT_MASK 0xFFFFFFFFL +//SEM_STATUS +#define SEM_STATUS__SEM_IDLE__SHIFT 0x0 +#define SEM_STATUS__SEM_INTERNAL_IDLE__SHIFT 0x1 +#define SEM_STATUS__MC_RDREQ_FIFO_FULL__SHIFT 0x2 +#define SEM_STATUS__MC_WRREQ_FIFO_FULL__SHIFT 0x3 +#define SEM_STATUS__WRITE1_FIFO_FULL__SHIFT 0x4 +#define SEM_STATUS__CHECK0_FIFO_FULL__SHIFT 0x5 +#define SEM_STATUS__MC_RDREQ_PENDING__SHIFT 0x6 +#define SEM_STATUS__MC_WRREQ_PENDING__SHIFT 0x7 +#define SEM_STATUS__SDMA0_MAILBOX_PENDING__SHIFT 0x8 +#define SEM_STATUS__SDMA1_MAILBOX_PENDING__SHIFT 0x9 +#define SEM_STATUS__UVD_MAILBOX_PENDING__SHIFT 0xa +#define SEM_STATUS__VCE_MAILBOX_PENDING__SHIFT 0xb +#define SEM_STATUS__CPG1_MAILBOX_PENDING__SHIFT 0xc +#define SEM_STATUS__CPG2_MAILBOX_PENDING__SHIFT 0xd +#define SEM_STATUS__VCE1_MAILBOX_PENDING__SHIFT 0xe +#define SEM_STATUS__ATC_REQ_PENDING__SHIFT 0xf +#define SEM_STATUS__OUTSTANDING_CLEAN__SHIFT 0x10 +#define SEM_STATUS__INVREQ_FLUSH_VF_MISMATCH__SHIFT 0x11 +#define SEM_STATUS__INVREQ_NONFLUSH_VF_MISMATCH__SHIFT 0x12 +#define SEM_STATUS__INVREQ_CNT_IDLE__SHIFT 0x13 +#define SEM_STATUS__ENTRYLIST_IDLE__SHIFT 0x14 +#define SEM_STATUS__MIF_IDLE__SHIFT 0x15 +#define SEM_STATUS__REGISTER_IDLE__SHIFT 0x16 +#define SEM_STATUS__ATCL2_INVREQ_IDLE__SHIFT 0x17 +#define SEM_STATUS__UVD1_MAILBOX_PENDING__SHIFT 0x18 +#define SEM_STATUS__SWITCH_READY__SHIFT 0x1f +#define SEM_STATUS__SEM_IDLE_MASK 0x00000001L +#define SEM_STATUS__SEM_INTERNAL_IDLE_MASK 0x00000002L +#define SEM_STATUS__MC_RDREQ_FIFO_FULL_MASK 0x00000004L +#define SEM_STATUS__MC_WRREQ_FIFO_FULL_MASK 0x00000008L +#define SEM_STATUS__WRITE1_FIFO_FULL_MASK 0x00000010L +#define SEM_STATUS__CHECK0_FIFO_FULL_MASK 0x00000020L +#define SEM_STATUS__MC_RDREQ_PENDING_MASK 0x00000040L +#define SEM_STATUS__MC_WRREQ_PENDING_MASK 0x00000080L +#define SEM_STATUS__SDMA0_MAILBOX_PENDING_MASK 0x00000100L +#define SEM_STATUS__SDMA1_MAILBOX_PENDING_MASK 0x00000200L +#define SEM_STATUS__UVD_MAILBOX_PENDING_MASK 0x00000400L +#define SEM_STATUS__VCE_MAILBOX_PENDING_MASK 0x00000800L +#define SEM_STATUS__CPG1_MAILBOX_PENDING_MASK 0x00001000L +#define SEM_STATUS__CPG2_MAILBOX_PENDING_MASK 0x00002000L +#define SEM_STATUS__VCE1_MAILBOX_PENDING_MASK 0x00004000L +#define SEM_STATUS__ATC_REQ_PENDING_MASK 0x00008000L +#define SEM_STATUS__OUTSTANDING_CLEAN_MASK 0x00010000L +#define SEM_STATUS__INVREQ_FLUSH_VF_MISMATCH_MASK 0x00020000L +#define SEM_STATUS__INVREQ_NONFLUSH_VF_MISMATCH_MASK 0x00040000L +#define SEM_STATUS__INVREQ_CNT_IDLE_MASK 0x00080000L +#define SEM_STATUS__ENTRYLIST_IDLE_MASK 0x00100000L +#define SEM_STATUS__MIF_IDLE_MASK 0x00200000L +#define SEM_STATUS__REGISTER_IDLE_MASK 0x00400000L +#define SEM_STATUS__ATCL2_INVREQ_IDLE_MASK 0x00800000L +#define SEM_STATUS__UVD1_MAILBOX_PENDING_MASK 0x01000000L +#define SEM_STATUS__SWITCH_READY_MASK 0x80000000L +//SEM_MAILBOX_CLIENTCONFIG +#define SEM_MAILBOX_CLIENTCONFIG__CP_CLIENT0__SHIFT 0x0 +#define SEM_MAILBOX_CLIENTCONFIG__CP_CLIENT1__SHIFT 0x3 +#define SEM_MAILBOX_CLIENTCONFIG__CP_CLIENT2__SHIFT 0x6 +#define SEM_MAILBOX_CLIENTCONFIG__CP_CLIENT3__SHIFT 0x9 +#define SEM_MAILBOX_CLIENTCONFIG__SDMA_CLIENT0__SHIFT 0xc +#define SEM_MAILBOX_CLIENTCONFIG__UVD_CLIENT0__SHIFT 0xf +#define SEM_MAILBOX_CLIENTCONFIG__SDMA1_CLIENT0__SHIFT 0x12 +#define SEM_MAILBOX_CLIENTCONFIG__VCE_CLIENT0__SHIFT 0x15 +#define SEM_MAILBOX_CLIENTCONFIG__CP_CLIENT0_MASK 0x00000007L +#define SEM_MAILBOX_CLIENTCONFIG__CP_CLIENT1_MASK 0x00000038L +#define SEM_MAILBOX_CLIENTCONFIG__CP_CLIENT2_MASK 0x000001C0L +#define SEM_MAILBOX_CLIENTCONFIG__CP_CLIENT3_MASK 0x00000E00L +#define SEM_MAILBOX_CLIENTCONFIG__SDMA_CLIENT0_MASK 0x00007000L +#define SEM_MAILBOX_CLIENTCONFIG__UVD_CLIENT0_MASK 0x00038000L +#define SEM_MAILBOX_CLIENTCONFIG__SDMA1_CLIENT0_MASK 0x001C0000L +#define SEM_MAILBOX_CLIENTCONFIG__VCE_CLIENT0_MASK 0x00E00000L +//SEM_MAILBOX +#define SEM_MAILBOX__HOSTPORT__SHIFT 0x0 +#define SEM_MAILBOX__RESERVED__SHIFT 0x10 +#define SEM_MAILBOX__HOSTPORT_MASK 0x0000FFFFL +#define SEM_MAILBOX__RESERVED_MASK 0xFFFF0000L +//SEM_MAILBOX_CONTROL +#define SEM_MAILBOX_CONTROL__HOSTPORT_ENABLE__SHIFT 0x0 +#define SEM_MAILBOX_CONTROL__RESERVED__SHIFT 0x10 +#define SEM_MAILBOX_CONTROL__HOSTPORT_ENABLE_MASK 0x0000FFFFL +#define SEM_MAILBOX_CONTROL__RESERVED_MASK 0xFFFF0000L +//SEM_CHICKEN_BITS +#define SEM_CHICKEN_BITS__VMID_PIPELINE_EN__SHIFT 0x0 +#define SEM_CHICKEN_BITS__ENTRY_PIPELINE_EN__SHIFT 0x1 +#define SEM_CHICKEN_BITS__CHECK_COUNTER_EN__SHIFT 0x2 +#define SEM_CHICKEN_BITS__ECC_BEHAVIOR__SHIFT 0x3 +#define SEM_CHICKEN_BITS__PHY_TRAN_EN__SHIFT 0x6 +#define SEM_CHICKEN_BITS__ADDR_CMP_UNTRAN_EN__SHIFT 0x7 +#define SEM_CHICKEN_BITS__IDLE_COUNTER_INDEX__SHIFT 0x8 +#define SEM_CHICKEN_BITS__OUTSTANDING_CLEAN_COUNTER_INDEX__SHIFT 0xa +#define SEM_CHICKEN_BITS__ATCL2_BUS_ID__SHIFT 0xc +#define SEM_CHICKEN_BITS__ATOMIC_EN__SHIFT 0xe +#define SEM_CHICKEN_BITS__EXTERNAL_ATOMIC_CHECK__SHIFT 0xf +#define SEM_CHICKEN_BITS__CLEAR_MAILBOX__SHIFT 0x10 +#define SEM_CHICKEN_BITS__INVACK_AFTER_OUTSTANDING_CLEAN__SHIFT 0x12 +#define SEM_CHICKEN_BITS__UTC_TAG_CONFLICT_CHECK__SHIFT 0x13 +#define SEM_CHICKEN_BITS__VMID_PIPELINE_EN_MASK 0x00000001L +#define SEM_CHICKEN_BITS__ENTRY_PIPELINE_EN_MASK 0x00000002L +#define SEM_CHICKEN_BITS__CHECK_COUNTER_EN_MASK 0x00000004L +#define SEM_CHICKEN_BITS__ECC_BEHAVIOR_MASK 0x00000018L +#define SEM_CHICKEN_BITS__PHY_TRAN_EN_MASK 0x00000040L +#define SEM_CHICKEN_BITS__ADDR_CMP_UNTRAN_EN_MASK 0x00000080L +#define SEM_CHICKEN_BITS__IDLE_COUNTER_INDEX_MASK 0x00000300L +#define SEM_CHICKEN_BITS__OUTSTANDING_CLEAN_COUNTER_INDEX_MASK 0x00000C00L +#define SEM_CHICKEN_BITS__ATCL2_BUS_ID_MASK 0x00003000L +#define SEM_CHICKEN_BITS__ATOMIC_EN_MASK 0x00004000L +#define SEM_CHICKEN_BITS__EXTERNAL_ATOMIC_CHECK_MASK 0x00008000L +#define SEM_CHICKEN_BITS__CLEAR_MAILBOX_MASK 0x00030000L +#define SEM_CHICKEN_BITS__INVACK_AFTER_OUTSTANDING_CLEAN_MASK 0x00040000L +#define SEM_CHICKEN_BITS__UTC_TAG_CONFLICT_CHECK_MASK 0x00080000L +//SEM_MAILBOX_CLIENTCONFIG_EXTRA +#define SEM_MAILBOX_CLIENTCONFIG_EXTRA__VCE1_CLIENT0__SHIFT 0x0 +#define SEM_MAILBOX_CLIENTCONFIG_EXTRA__UVD1_CLIENT0__SHIFT 0x4 +#define SEM_MAILBOX_CLIENTCONFIG_EXTRA__VCE1_CLIENT0_MASK 0x0000000FL +#define SEM_MAILBOX_CLIENTCONFIG_EXTRA__UVD1_CLIENT0_MASK 0x000000F0L +//SEM_GPU_IOV_VIOLATION_LOG +#define SEM_GPU_IOV_VIOLATION_LOG__VIOLATION_STATUS__SHIFT 0x0 +#define SEM_GPU_IOV_VIOLATION_LOG__MULTIPLE_VIOLATION_STATUS__SHIFT 0x1 +#define SEM_GPU_IOV_VIOLATION_LOG__ADDRESS__SHIFT 0x2 +#define SEM_GPU_IOV_VIOLATION_LOG__OPCODE__SHIFT 0x12 +#define SEM_GPU_IOV_VIOLATION_LOG__VF__SHIFT 0x13 +#define SEM_GPU_IOV_VIOLATION_LOG__VF_ID__SHIFT 0x14 +#define SEM_GPU_IOV_VIOLATION_LOG__INITIATOR_ID__SHIFT 0x18 +#define SEM_GPU_IOV_VIOLATION_LOG__VIOLATION_STATUS_MASK 0x00000001L +#define SEM_GPU_IOV_VIOLATION_LOG__MULTIPLE_VIOLATION_STATUS_MASK 0x00000002L +#define SEM_GPU_IOV_VIOLATION_LOG__ADDRESS_MASK 0x0003FFFCL +#define SEM_GPU_IOV_VIOLATION_LOG__OPCODE_MASK 0x00040000L +#define SEM_GPU_IOV_VIOLATION_LOG__VF_MASK 0x00080000L +#define SEM_GPU_IOV_VIOLATION_LOG__VF_ID_MASK 0x00F00000L +#define SEM_GPU_IOV_VIOLATION_LOG__INITIATOR_ID_MASK 0xFF000000L +//SEM_OUTSTANDING_THRESHOLD +#define SEM_OUTSTANDING_THRESHOLD__VALUE__SHIFT 0x0 +#define SEM_OUTSTANDING_THRESHOLD__VALUE_MASK 0x000000FFL +//SEM_MEM_POWER_CTRL +#define SEM_MEM_POWER_CTRL__MEM_POWER_CTRL_EN__SHIFT 0x0 +#define SEM_MEM_POWER_CTRL__MEM_POWER_LS_EN__SHIFT 0x1 +#define SEM_MEM_POWER_CTRL__MEM_POWER_DS_EN__SHIFT 0x2 +#define SEM_MEM_POWER_CTRL__MEM_POWER_SD_EN__SHIFT 0x3 +#define SEM_MEM_POWER_CTRL__MEM_IDLE_HYSTERESIS__SHIFT 0x4 +#define SEM_MEM_POWER_CTRL__MEM_POWER_UP_RECOVER_DELAY__SHIFT 0x8 +#define SEM_MEM_POWER_CTRL__MEM_POWER_DOWN_LS_ENTER_DELAY__SHIFT 0xe +#define SEM_MEM_POWER_CTRL__MEM_POWER_CTRL_EN_MASK 0x00000001L +#define SEM_MEM_POWER_CTRL__MEM_POWER_LS_EN_MASK 0x00000002L +#define SEM_MEM_POWER_CTRL__MEM_POWER_DS_EN_MASK 0x00000004L +#define SEM_MEM_POWER_CTRL__MEM_POWER_SD_EN_MASK 0x00000008L +#define SEM_MEM_POWER_CTRL__MEM_IDLE_HYSTERESIS_MASK 0x00000070L +#define SEM_MEM_POWER_CTRL__MEM_POWER_UP_RECOVER_DELAY_MASK 0x00003F00L +#define SEM_MEM_POWER_CTRL__MEM_POWER_DOWN_LS_ENTER_DELAY_MASK 0x0000C000L +//SEM_REGISTER_LAST_PART2 +#define SEM_REGISTER_LAST_PART2__RESERVED__SHIFT 0x0 +#define SEM_REGISTER_LAST_PART2__RESERVED_MASK 0xFFFFFFFFL +//IH_ACTIVE_FCN_ID +#define IH_ACTIVE_FCN_ID__VF_ID__SHIFT 0x0 +#define IH_ACTIVE_FCN_ID__RESERVED__SHIFT 0x4 +#define IH_ACTIVE_FCN_ID__PF_VF__SHIFT 0x1f +#define IH_ACTIVE_FCN_ID__VF_ID_MASK 0x0000000FL +#define IH_ACTIVE_FCN_ID__RESERVED_MASK 0x7FFFFFF0L +#define IH_ACTIVE_FCN_ID__PF_VF_MASK 0x80000000L +//IH_VIRT_RESET_REQ +#define IH_VIRT_RESET_REQ__VF__SHIFT 0x0 +#define IH_VIRT_RESET_REQ__PF__SHIFT 0x1f +#define IH_VIRT_RESET_REQ__VF_MASK 0x0000FFFFL +#define IH_VIRT_RESET_REQ__PF_MASK 0x80000000L +//IH_CLIENT_CFG +#define IH_CLIENT_CFG__TOTAL_CLIENT_NUM__SHIFT 0x0 +#define IH_CLIENT_CFG__TOTAL_CLIENT_NUM_MASK 0x0000001FL +//IH_CLIENT_CFG_INDEX +#define IH_CLIENT_CFG_INDEX__INDEX__SHIFT 0x0 +#define IH_CLIENT_CFG_INDEX__INDEX_MASK 0x0000001FL +//IH_CLIENT_CFG_DATA +#define IH_CLIENT_CFG_DATA__CREDIT_RETURN_ADDR__SHIFT 0x0 +#define IH_CLIENT_CFG_DATA__CLIENT_TYPE__SHIFT 0x12 +#define IH_CLIENT_CFG_DATA__RING_ID__SHIFT 0x14 +#define IH_CLIENT_CFG_DATA__VF_RB_SELECT__SHIFT 0x16 +#define IH_CLIENT_CFG_DATA__OVERWRITE_RING_ID_WITH_ACTIVE_FCN_ID__SHIFT 0x18 +#define IH_CLIENT_CFG_DATA__CREDIT_RETURN_ADDR_MASK 0x0003FFFFL +#define IH_CLIENT_CFG_DATA__CLIENT_TYPE_MASK 0x000C0000L +#define IH_CLIENT_CFG_DATA__RING_ID_MASK 0x00300000L +#define IH_CLIENT_CFG_DATA__VF_RB_SELECT_MASK 0x00C00000L +#define IH_CLIENT_CFG_DATA__OVERWRITE_RING_ID_WITH_ACTIVE_FCN_ID_MASK 0x01000000L +//IH_CID_REMAP_INDEX +#define IH_CID_REMAP_INDEX__INDEX__SHIFT 0x0 +#define IH_CID_REMAP_INDEX__INDEX_MASK 0x00000003L +//IH_CID_REMAP_DATA +#define IH_CID_REMAP_DATA__CLIENT_ID__SHIFT 0x0 +#define IH_CID_REMAP_DATA__INITIATOR_ID__SHIFT 0x8 +#define IH_CID_REMAP_DATA__CLIENT_ID_REMAP__SHIFT 0x10 +#define IH_CID_REMAP_DATA__CLIENT_ID_MASK 0x000000FFL +#define IH_CID_REMAP_DATA__INITIATOR_ID_MASK 0x0000FF00L +#define IH_CID_REMAP_DATA__CLIENT_ID_REMAP_MASK 0x00FF0000L +//IH_CHICKEN +#define IH_CHICKEN__ACTIVE_FCN_ID_PROT_ENABLE__SHIFT 0x0 +#define IH_CHICKEN__MC_SPACE_FBPA_ENABLE__SHIFT 0x3 +#define IH_CHICKEN__MC_SPACE_GPA_ENABLE__SHIFT 0x4 +#define IH_CHICKEN__ACTIVE_FCN_ID_PROT_ENABLE_MASK 0x00000001L +#define IH_CHICKEN__MC_SPACE_FBPA_ENABLE_MASK 0x00000008L +#define IH_CHICKEN__MC_SPACE_GPA_ENABLE_MASK 0x00000010L +//IH_MMHUB_CNTL +#define IH_MMHUB_CNTL__UNITID__SHIFT 0x0 +#define IH_MMHUB_CNTL__IV_TLVL__SHIFT 0x8 +#define IH_MMHUB_CNTL__WPTR_WB_TLVL__SHIFT 0xc +#define IH_MMHUB_CNTL__UNITID_MASK 0x0000003FL +#define IH_MMHUB_CNTL__IV_TLVL_MASK 0x00000700L +#define IH_MMHUB_CNTL__WPTR_WB_TLVL_MASK 0x00007000L +//IH_INT_DROP_CNTL +#define IH_INT_DROP_CNTL__INT_DROP_EN__SHIFT 0x0 +#define IH_INT_DROP_CNTL__CLIENT_ID_MATCH_EN__SHIFT 0x1 +#define IH_INT_DROP_CNTL__SOURCE_ID_MATCH_EN__SHIFT 0x2 +#define IH_INT_DROP_CNTL__VF_ID_MATCH_EN__SHIFT 0x3 +#define IH_INT_DROP_CNTL__VF_MATCH_EN__SHIFT 0x4 +#define IH_INT_DROP_CNTL__CONTEXT_ID_MATCH_EN__SHIFT 0x5 +#define IH_INT_DROP_CNTL__INT_DROP_MODE__SHIFT 0x6 +#define IH_INT_DROP_CNTL__UTCL2_RETRY_INT_DROP_EN__SHIFT 0x8 +#define IH_INT_DROP_CNTL__INT_DROPPED__SHIFT 0x10 +#define IH_INT_DROP_CNTL__INT_DROP_EN_MASK 0x00000001L +#define IH_INT_DROP_CNTL__CLIENT_ID_MATCH_EN_MASK 0x00000002L +#define IH_INT_DROP_CNTL__SOURCE_ID_MATCH_EN_MASK 0x00000004L +#define IH_INT_DROP_CNTL__VF_ID_MATCH_EN_MASK 0x00000008L +#define IH_INT_DROP_CNTL__VF_MATCH_EN_MASK 0x00000010L +#define IH_INT_DROP_CNTL__CONTEXT_ID_MATCH_EN_MASK 0x00000020L +#define IH_INT_DROP_CNTL__INT_DROP_MODE_MASK 0x000000C0L +#define IH_INT_DROP_CNTL__UTCL2_RETRY_INT_DROP_EN_MASK 0x00000100L +#define IH_INT_DROP_CNTL__INT_DROPPED_MASK 0x00010000L +//IH_INT_DROP_MATCH_VALUE0 +#define IH_INT_DROP_MATCH_VALUE0__CLIENT_ID_MATCH_VALUE__SHIFT 0x0 +#define IH_INT_DROP_MATCH_VALUE0__SOURCE_ID_MATCH_VALUE__SHIFT 0x8 +#define IH_INT_DROP_MATCH_VALUE0__VF_ID_MATCH_VALUE__SHIFT 0x10 +#define IH_INT_DROP_MATCH_VALUE0__VF_MATCH_VALUE__SHIFT 0x17 +#define IH_INT_DROP_MATCH_VALUE0__CONTEXT_ID_39_32_MATCH_VALUE__SHIFT 0x18 +#define IH_INT_DROP_MATCH_VALUE0__CLIENT_ID_MATCH_VALUE_MASK 0x000000FFL +#define IH_INT_DROP_MATCH_VALUE0__SOURCE_ID_MATCH_VALUE_MASK 0x0000FF00L +#define IH_INT_DROP_MATCH_VALUE0__VF_ID_MATCH_VALUE_MASK 0x000F0000L +#define IH_INT_DROP_MATCH_VALUE0__VF_MATCH_VALUE_MASK 0x00800000L +#define IH_INT_DROP_MATCH_VALUE0__CONTEXT_ID_39_32_MATCH_VALUE_MASK 0xFF000000L +//IH_INT_DROP_MATCH_VALUE1 +#define IH_INT_DROP_MATCH_VALUE1__CONTEXT_ID_31_0_MATCH_VALUE__SHIFT 0x0 +#define IH_INT_DROP_MATCH_VALUE1__CONTEXT_ID_31_0_MATCH_VALUE_MASK 0xFFFFFFFFL +//IH_INT_DROP_MATCH_MASK0 +#define IH_INT_DROP_MATCH_MASK0__CLIENT_ID_MATCH_MASK__SHIFT 0x0 +#define IH_INT_DROP_MATCH_MASK0__SOURCE_ID_MATCH_MASK__SHIFT 0x8 +#define IH_INT_DROP_MATCH_MASK0__VF_ID_MATCH_MASK__SHIFT 0x10 +#define IH_INT_DROP_MATCH_MASK0__VF_MATCH_MASK__SHIFT 0x17 +#define IH_INT_DROP_MATCH_MASK0__CONTEXT_ID_39_32_MATCH_MASK__SHIFT 0x18 +#define IH_INT_DROP_MATCH_MASK0__CLIENT_ID_MATCH_MASK_MASK 0x000000FFL +#define IH_INT_DROP_MATCH_MASK0__SOURCE_ID_MATCH_MASK_MASK 0x0000FF00L +#define IH_INT_DROP_MATCH_MASK0__VF_ID_MATCH_MASK_MASK 0x000F0000L +#define IH_INT_DROP_MATCH_MASK0__VF_MATCH_MASK_MASK 0x00800000L +#define IH_INT_DROP_MATCH_MASK0__CONTEXT_ID_39_32_MATCH_MASK_MASK 0xFF000000L +//IH_INT_DROP_MATCH_MASK1 +#define IH_INT_DROP_MATCH_MASK1__CONTEXT_ID_31_0_MATCH_MASK__SHIFT 0x0 +#define IH_INT_DROP_MATCH_MASK1__CONTEXT_ID_31_0_MATCH_MASK_MASK 0xFFFFFFFFL +//IH_REGISTER_LAST_PART1 +#define IH_REGISTER_LAST_PART1__RESERVED__SHIFT 0x0 +#define IH_REGISTER_LAST_PART1__RESERVED_MASK 0xFFFFFFFFL +//SEM_ACTIVE_FCN_ID +#define SEM_ACTIVE_FCN_ID__VFID__SHIFT 0x0 +#define SEM_ACTIVE_FCN_ID__VF__SHIFT 0x1f +#define SEM_ACTIVE_FCN_ID__VFID_MASK 0x0000000FL +#define SEM_ACTIVE_FCN_ID__VF_MASK 0x80000000L +//SEM_VIRT_RESET_REQ +#define SEM_VIRT_RESET_REQ__VF__SHIFT 0x0 +#define SEM_VIRT_RESET_REQ__PF__SHIFT 0x1f +#define SEM_VIRT_RESET_REQ__VF_MASK 0x0000FFFFL +#define SEM_VIRT_RESET_REQ__PF_MASK 0x80000000L +//SEM_RESP_SDMA0 +#define SEM_RESP_SDMA0__ADDR__SHIFT 0x2 +#define SEM_RESP_SDMA0__ADDR_MASK 0x000FFFFCL +//SEM_RESP_SDMA1 +#define SEM_RESP_SDMA1__ADDR__SHIFT 0x2 +#define SEM_RESP_SDMA1__ADDR_MASK 0x000FFFFCL +//SEM_RESP_UVD +#define SEM_RESP_UVD__ADDR__SHIFT 0x2 +#define SEM_RESP_UVD__ADDR_MASK 0x000FFFFCL +//SEM_RESP_VCE_0 +#define SEM_RESP_VCE_0__ADDR__SHIFT 0x2 +#define SEM_RESP_VCE_0__ADDR_MASK 0x000FFFFCL +//SEM_RESP_ACP +#define SEM_RESP_ACP__ADDR__SHIFT 0x2 +#define SEM_RESP_ACP__ADDR_MASK 0x000FFFFCL +//SEM_RESP_ISP +#define SEM_RESP_ISP__ADDR__SHIFT 0x2 +#define SEM_RESP_ISP__ADDR_MASK 0x000FFFFCL +//SEM_RESP_VCE_1 +#define SEM_RESP_VCE_1__ADDR__SHIFT 0x2 +#define SEM_RESP_VCE_1__ADDR_MASK 0x000FFFFCL +//SEM_RESP_VP8 +#define SEM_RESP_VP8__ADDR__SHIFT 0x2 +#define SEM_RESP_VP8__ADDR_MASK 0x000FFFFCL +//SEM_RESP_GC +#define SEM_RESP_GC__ADDR__SHIFT 0x2 +#define SEM_RESP_GC__ADDR_MASK 0x000FFFFCL +//SEM_RESP_UVD_1 +#define SEM_RESP_UVD_1__ADDR__SHIFT 0x2 +#define SEM_RESP_UVD_1__ADDR_MASK 0x000FFFFCL +//SEM_CID_REMAP_INDEX +#define SEM_CID_REMAP_INDEX__INDEX__SHIFT 0x0 +#define SEM_CID_REMAP_INDEX__INDEX_MASK 0x00000003L +//SEM_CID_REMAP_DATA +#define SEM_CID_REMAP_DATA__CLIENT_ID__SHIFT 0x0 +#define SEM_CID_REMAP_DATA__INITIATOR_ID__SHIFT 0x8 +#define SEM_CID_REMAP_DATA__CLIENT_ID_REMAP__SHIFT 0x10 +#define SEM_CID_REMAP_DATA__CLIENT_ID_MASK 0x000000FFL +#define SEM_CID_REMAP_DATA__INITIATOR_ID_MASK 0x0000FF00L +#define SEM_CID_REMAP_DATA__CLIENT_ID_REMAP_MASK 0x00FF0000L +//SEM_ATOMIC_OP_LUT +#define SEM_ATOMIC_OP_LUT__SIGNAL_NORMAL__SHIFT 0x0 +#define SEM_ATOMIC_OP_LUT__SIGNAL_WRITE1__SHIFT 0x7 +#define SEM_ATOMIC_OP_LUT__WAIT_NORMAL__SHIFT 0xe +#define SEM_ATOMIC_OP_LUT__WAIT_CHECK0__SHIFT 0x15 +#define SEM_ATOMIC_OP_LUT__SIGNAL_NORMAL_MASK 0x0000007FL +#define SEM_ATOMIC_OP_LUT__SIGNAL_WRITE1_MASK 0x00003F80L +#define SEM_ATOMIC_OP_LUT__WAIT_NORMAL_MASK 0x001FC000L +#define SEM_ATOMIC_OP_LUT__WAIT_CHECK0_MASK 0x0FE00000L +//SEM_EDC_CONFIG +#define SEM_EDC_CONFIG__WRITE_DIS__SHIFT 0x0 +#define SEM_EDC_CONFIG__DIS_EDC__SHIFT 0x1 +#define SEM_EDC_CONFIG__WRITE_DIS_MASK 0x00000001L +#define SEM_EDC_CONFIG__DIS_EDC_MASK 0x00000002L +//SEM_CHICKEN_BITS2 +#define SEM_CHICKEN_BITS2__ACTIVE_FCN_ID_PROT_ENABLE__SHIFT 0x0 +#define SEM_CHICKEN_BITS2__MM_CLIENT_USE_CONFIG_VFID__SHIFT 0x1 +#define SEM_CHICKEN_BITS2__ACTIVE_FCN_ID_PROT_ENABLE_MASK 0x00000001L +#define SEM_CHICKEN_BITS2__MM_CLIENT_USE_CONFIG_VFID_MASK 0x00000002L +//SEM_MMHUB_CNTL +#define SEM_MMHUB_CNTL__UNIT_ID__SHIFT 0x0 +#define SEM_MMHUB_CNTL__TLVL_VALUE__SHIFT 0x8 +#define SEM_MMHUB_CNTL__UNIT_ID_MASK 0x0000003FL +#define SEM_MMHUB_CNTL__TLVL_VALUE_MASK 0x00000700L +//SEM_REGISTER_LAST_PART1 +#define SEM_REGISTER_LAST_PART1__RESERVED__SHIFT 0x0 +#define SEM_REGISTER_LAST_PART1__RESERVED_MASK 0xFFFFFFFFL + +#endif diff --git a/drivers/gpu/drm/amd/include/asic_reg/smuio/smuio_11_0_6_offset.h b/drivers/gpu/drm/amd/include/asic_reg/smuio/smuio_11_0_6_offset.h new file mode 100644 index 0000000000000000000000000000000000000000..55facadea54b9fe66d08b7d3af47480f86f938ed --- /dev/null +++ b/drivers/gpu/drm/amd/include/asic_reg/smuio/smuio_11_0_6_offset.h @@ -0,0 +1,35 @@ +/* + * Copyright (C) 2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +#ifndef _smuio_11_0_6_OFFSET_HEADER +#define _smuio_11_0_6_OFFSET_HEADER + + + +// addressBlock: smuio_smuio_SmuSmuioDec +// base address: 0x5a000 +#define mmCGTT_ROM_CLK_CTRL0 0x00e4 +#define mmCGTT_ROM_CLK_CTRL0_BASE_IDX 0 +#define mmROM_INDEX 0x00e5 +#define mmROM_INDEX_BASE_IDX 0 +#define mmROM_DATA 0x00e6 +#define mmROM_DATA_BASE_IDX 0 + +#endif diff --git a/drivers/gpu/drm/amd/include/asic_reg/smuio/smuio_11_0_6_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/smuio/smuio_11_0_6_sh_mask.h new file mode 100644 index 0000000000000000000000000000000000000000..7d6a2fac28398f63519c8ea42ba41024f75ac417 --- /dev/null +++ b/drivers/gpu/drm/amd/include/asic_reg/smuio/smuio_11_0_6_sh_mask.h @@ -0,0 +1,41 @@ +/* + * Copyright (C) 2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +#ifndef _smuio_11_0_6_SH_MASK_HEADER +#define _smuio_11_0_6_SH_MASK_HEADER + + +//CGTT_ROM_CLK_CTRL0 +#define CGTT_ROM_CLK_CTRL0__ON_DELAY__SHIFT 0x0 +#define CGTT_ROM_CLK_CTRL0__OFF_HYSTERESIS__SHIFT 0x4 +#define CGTT_ROM_CLK_CTRL0__SOFT_OVERRIDE1__SHIFT 0x1e +#define CGTT_ROM_CLK_CTRL0__SOFT_OVERRIDE0__SHIFT 0x1f +#define CGTT_ROM_CLK_CTRL0__ON_DELAY_MASK 0x0000000FL +#define CGTT_ROM_CLK_CTRL0__OFF_HYSTERESIS_MASK 0x00000FF0L +#define CGTT_ROM_CLK_CTRL0__SOFT_OVERRIDE1_MASK 0x40000000L +#define CGTT_ROM_CLK_CTRL0__SOFT_OVERRIDE0_MASK 0x80000000L +//ROM_INDEX +#define ROM_INDEX__ROM_INDEX__SHIFT 0x0 +#define ROM_INDEX__ROM_INDEX_MASK 0x01FFFFFFL +//ROM_DATA +#define ROM_DATA__ROM_DATA__SHIFT 0x0 +#define ROM_DATA__ROM_DATA_MASK 0xFFFFFFFFL + +#endif diff --git a/drivers/gpu/drm/amd/include/kgd_pp_interface.h b/drivers/gpu/drm/amd/include/kgd_pp_interface.h index f775aac6c1bd005660bbde2dd1a92fcfd7ad3993..a41875ac5dfbdaca9564db8a0679f610b870f438 100644 --- a/drivers/gpu/drm/amd/include/kgd_pp_interface.h +++ b/drivers/gpu/drm/amd/include/kgd_pp_interface.h @@ -103,6 +103,7 @@ enum pp_clock_type { enum amd_pp_sensors { AMDGPU_PP_SENSOR_GFX_SCLK = 0, + AMDGPU_PP_SENSOR_CPU_CLK, AMDGPU_PP_SENSOR_VDDNB, AMDGPU_PP_SENSOR_VDDGFX, AMDGPU_PP_SENSOR_UVD_VCLK, @@ -155,9 +156,11 @@ enum { enum PP_OD_DPM_TABLE_COMMAND { PP_OD_EDIT_SCLK_VDDC_TABLE, PP_OD_EDIT_MCLK_VDDC_TABLE, + PP_OD_EDIT_CCLK_VDDC_TABLE, PP_OD_EDIT_VDDC_CURVE, PP_OD_RESTORE_DEFAULT_TABLE, - PP_OD_COMMIT_DPM_TABLE + PP_OD_COMMIT_DPM_TABLE, + PP_OD_EDIT_VDDGFX_OFFSET }; struct pp_states_info { diff --git a/drivers/gpu/drm/amd/include/renoir_ip_offset.h b/drivers/gpu/drm/amd/include/renoir_ip_offset.h index 07633e22e99a15c15debf61b0b79a2ba3b82f266..7dff85c81e5a7ea09cf0ed9905bfb101690f45b3 100644 --- a/drivers/gpu/drm/amd/include/renoir_ip_offset.h +++ b/drivers/gpu/drm/amd/include/renoir_ip_offset.h @@ -33,7 +33,7 @@ struct IP_BASE_INSTANCE struct IP_BASE { struct IP_BASE_INSTANCE instance[MAX_INSTANCE]; -}; +} __maybe_unused; static const struct IP_BASE ACP_BASE ={ { { { 0x02403800, 0x00480000, 0, 0, 0 } }, diff --git a/drivers/gpu/drm/amd/pm/amdgpu_pm.c b/drivers/gpu/drm/amd/pm/amdgpu_pm.c index 7b6ef05a1d35a959627128b3d73b9f38fedae680..5fa65f191a3790071d8b2de786ede25c1bfcabc1 100644 --- a/drivers/gpu/drm/amd/pm/amdgpu_pm.c +++ b/drivers/gpu/drm/amd/pm/amdgpu_pm.c @@ -36,6 +36,7 @@ #include #include #include +#include #include "hwmgr.h" static const struct cg_flag_name clocks[] = { @@ -730,11 +731,18 @@ static ssize_t amdgpu_set_pp_table(struct device *dev, * * - minimum and maximum engine clock labeled OD_SCLK * - * - maximum memory clock labeled OD_MCLK + * - minimum(not available for Vega20 and Navi1x) and maximum memory + * clock labeled OD_MCLK * * - three points labeled OD_VDDC_CURVE. * They can be used to calibrate the sclk voltage curve. * + * - voltage offset(in mV) applied on target voltage calculation. + * This is available for Sienna Cichlid, Navy Flounder and Dimgrey + * Cavefish. For these ASICs, the target voltage calculation can be + * illustrated by "voltage = voltage calculated from v/f curve + + * overdrive vddgfx offset" + * * - a list of valid ranges for sclk, mclk, and voltage curve points * labeled OD_RANGE * @@ -755,6 +763,11 @@ static ssize_t amdgpu_set_pp_table(struct device *dev, * 600mV. "vc 2 1000 1000" will update point3 with clock set * as 1000Mhz and voltage 1000mV. * + * To update the voltage offset applied for gfxclk/voltage calculation, + * enter the new value by writing a string that contains "vo offset". + * This is supported by Sienna Cichlid, Navy Flounder and Dimgrey Cavefish. + * And the offset can be a positive or negative value. + * * - When you have edited all of the states as needed, write "c" (commit) * to the file to commit your changes * @@ -787,6 +800,8 @@ static ssize_t amdgpu_set_pp_od_clk_voltage(struct device *dev, if (*buf == 's') type = PP_OD_EDIT_SCLK_VDDC_TABLE; + else if (*buf == 'p') + type = PP_OD_EDIT_CCLK_VDDC_TABLE; else if (*buf == 'm') type = PP_OD_EDIT_MCLK_VDDC_TABLE; else if(*buf == 'r') @@ -795,6 +810,8 @@ static ssize_t amdgpu_set_pp_od_clk_voltage(struct device *dev, type = PP_OD_COMMIT_DPM_TABLE; else if (!strncmp(buf, "vc", 2)) type = PP_OD_EDIT_VDDC_CURVE; + else if (!strncmp(buf, "vo", 2)) + type = PP_OD_EDIT_VDDGFX_OFFSET; else return -EINVAL; @@ -802,12 +819,14 @@ static ssize_t amdgpu_set_pp_od_clk_voltage(struct device *dev, tmp_str = buf_cpy; - if (type == PP_OD_EDIT_VDDC_CURVE) + if ((type == PP_OD_EDIT_VDDC_CURVE) || + (type == PP_OD_EDIT_VDDGFX_OFFSET)) tmp_str++; while (isspace(*++tmp_str)); - while (tmp_str[0]) { - sub_str = strsep(&tmp_str, delimiter); + while ((sub_str = strsep(&tmp_str, delimiter)) != NULL) { + if (strlen(sub_str) == 0) + continue; ret = kstrtol(sub_str, 0, ¶meter[parameter_size]); if (ret) return -EINVAL; @@ -898,7 +917,9 @@ static ssize_t amdgpu_get_pp_od_clk_voltage(struct device *dev, size = smu_print_clk_levels(&adev->smu, SMU_OD_SCLK, buf); size += smu_print_clk_levels(&adev->smu, SMU_OD_MCLK, buf+size); size += smu_print_clk_levels(&adev->smu, SMU_OD_VDDC_CURVE, buf+size); + size += smu_print_clk_levels(&adev->smu, SMU_OD_VDDGFX_OFFSET, buf+size); size += smu_print_clk_levels(&adev->smu, SMU_OD_RANGE, buf+size); + size += smu_print_clk_levels(&adev->smu, SMU_OD_CCLK, buf+size); } else if (adev->powerplay.pp_funcs->print_clock_levels) { size = amdgpu_dpm_print_clock_levels(adev, OD_SCLK, buf); size += amdgpu_dpm_print_clock_levels(adev, OD_MCLK, buf+size); @@ -1074,7 +1095,7 @@ static ssize_t amdgpu_get_pp_dpm_sclk(struct device *dev, static ssize_t amdgpu_read_mask(const char *buf, size_t count, uint32_t *mask) { int ret; - long level; + unsigned long level; char *sub_str = NULL; char *tmp; char buf_cpy[AMDGPU_MASK_BUF_MAX + 1]; @@ -1087,11 +1108,10 @@ static ssize_t amdgpu_read_mask(const char *buf, size_t count, uint32_t *mask) memcpy(buf_cpy, buf, bytes); buf_cpy[bytes] = '\0'; tmp = buf_cpy; - while (tmp[0]) { - sub_str = strsep(&tmp, delimiter); + while ((sub_str = strsep(&tmp, delimiter)) != NULL) { if (strlen(sub_str)) { - ret = kstrtol(sub_str, 0, &level); - if (ret) + ret = kstrtoul(sub_str, 0, &level); + if (ret || level > 31) return -EINVAL; *mask |= 1 << level; } else @@ -1346,6 +1366,138 @@ static ssize_t amdgpu_set_pp_dpm_fclk(struct device *dev, return count; } +static ssize_t amdgpu_get_pp_dpm_vclk(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = drm_to_adev(ddev); + ssize_t size; + int ret; + + if (amdgpu_in_reset(adev)) + return -EPERM; + + ret = pm_runtime_get_sync(ddev->dev); + if (ret < 0) { + pm_runtime_put_autosuspend(ddev->dev); + return ret; + } + + if (is_support_sw_smu(adev)) + size = smu_print_clk_levels(&adev->smu, SMU_VCLK, buf); + else + size = snprintf(buf, PAGE_SIZE, "\n"); + + pm_runtime_mark_last_busy(ddev->dev); + pm_runtime_put_autosuspend(ddev->dev); + + return size; +} + +static ssize_t amdgpu_set_pp_dpm_vclk(struct device *dev, + struct device_attribute *attr, + const char *buf, + size_t count) +{ + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = drm_to_adev(ddev); + int ret; + uint32_t mask = 0; + + if (amdgpu_in_reset(adev)) + return -EPERM; + + ret = amdgpu_read_mask(buf, count, &mask); + if (ret) + return ret; + + ret = pm_runtime_get_sync(ddev->dev); + if (ret < 0) { + pm_runtime_put_autosuspend(ddev->dev); + return ret; + } + + if (is_support_sw_smu(adev)) + ret = smu_force_clk_levels(&adev->smu, SMU_VCLK, mask); + else + ret = 0; + + pm_runtime_mark_last_busy(ddev->dev); + pm_runtime_put_autosuspend(ddev->dev); + + if (ret) + return -EINVAL; + + return count; +} + +static ssize_t amdgpu_get_pp_dpm_dclk(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = drm_to_adev(ddev); + ssize_t size; + int ret; + + if (amdgpu_in_reset(adev)) + return -EPERM; + + ret = pm_runtime_get_sync(ddev->dev); + if (ret < 0) { + pm_runtime_put_autosuspend(ddev->dev); + return ret; + } + + if (is_support_sw_smu(adev)) + size = smu_print_clk_levels(&adev->smu, SMU_DCLK, buf); + else + size = snprintf(buf, PAGE_SIZE, "\n"); + + pm_runtime_mark_last_busy(ddev->dev); + pm_runtime_put_autosuspend(ddev->dev); + + return size; +} + +static ssize_t amdgpu_set_pp_dpm_dclk(struct device *dev, + struct device_attribute *attr, + const char *buf, + size_t count) +{ + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = drm_to_adev(ddev); + int ret; + uint32_t mask = 0; + + if (amdgpu_in_reset(adev)) + return -EPERM; + + ret = amdgpu_read_mask(buf, count, &mask); + if (ret) + return ret; + + ret = pm_runtime_get_sync(ddev->dev); + if (ret < 0) { + pm_runtime_put_autosuspend(ddev->dev); + return ret; + } + + if (is_support_sw_smu(adev)) + ret = smu_force_clk_levels(&adev->smu, SMU_DCLK, mask); + else + ret = 0; + + pm_runtime_mark_last_busy(ddev->dev); + pm_runtime_put_autosuspend(ddev->dev); + + if (ret) + return -EINVAL; + + return count; +} + static ssize_t amdgpu_get_pp_dpm_dcefclk(struct device *dev, struct device_attribute *attr, char *buf) @@ -1717,8 +1869,9 @@ static ssize_t amdgpu_set_pp_power_profile_mode(struct device *dev, i++; memcpy(buf_cpy, buf, count-i); tmp_str = buf_cpy; - while (tmp_str[0]) { - sub_str = strsep(&tmp_str, delimiter); + while ((sub_str = strsep(&tmp_str, delimiter)) != NULL) { + if (strlen(sub_str) == 0) + continue; ret = kstrtol(sub_str, 0, ¶meter[parameter_size]); if (ret) return -EINVAL; @@ -2025,6 +2178,8 @@ static struct amdgpu_device_attr amdgpu_device_attrs[] = { AMDGPU_DEVICE_ATTR_RW(pp_dpm_mclk, ATTR_FLAG_BASIC|ATTR_FLAG_ONEVF), AMDGPU_DEVICE_ATTR_RW(pp_dpm_socclk, ATTR_FLAG_BASIC|ATTR_FLAG_ONEVF), AMDGPU_DEVICE_ATTR_RW(pp_dpm_fclk, ATTR_FLAG_BASIC|ATTR_FLAG_ONEVF), + AMDGPU_DEVICE_ATTR_RW(pp_dpm_vclk, ATTR_FLAG_BASIC|ATTR_FLAG_ONEVF), + AMDGPU_DEVICE_ATTR_RW(pp_dpm_dclk, ATTR_FLAG_BASIC|ATTR_FLAG_ONEVF), AMDGPU_DEVICE_ATTR_RW(pp_dpm_dcefclk, ATTR_FLAG_BASIC), AMDGPU_DEVICE_ATTR_RW(pp_dpm_pcie, ATTR_FLAG_BASIC), AMDGPU_DEVICE_ATTR_RW(pp_sclk_od, ATTR_FLAG_BASIC), @@ -2067,7 +2222,8 @@ static int default_attr_update(struct amdgpu_device *adev, struct amdgpu_device_ } else if (DEVICE_ATTR_IS(pp_od_clk_voltage)) { *states = ATTR_STATE_UNSUPPORTED; if ((is_support_sw_smu(adev) && adev->smu.od_enabled) || - (!is_support_sw_smu(adev) && hwmgr->od_enabled)) + (is_support_sw_smu(adev) && adev->smu.is_apu) || + (!is_support_sw_smu(adev) && hwmgr->od_enabled)) *states = ATTR_STATE_SUPPORTED; } else if (DEVICE_ATTR_IS(mem_busy_percent)) { if (adev->flags & AMD_IS_APU || asic_type == CHIP_VEGA10) @@ -2087,6 +2243,12 @@ static int default_attr_update(struct amdgpu_device *adev, struct amdgpu_device_ } else if (DEVICE_ATTR_IS(gpu_metrics)) { if (asic_type < CHIP_VEGA12) *states = ATTR_STATE_UNSUPPORTED; + } else if (DEVICE_ATTR_IS(pp_dpm_vclk)) { + if (!(asic_type == CHIP_VANGOGH)) + *states = ATTR_STATE_UNSUPPORTED; + } else if (DEVICE_ATTR_IS(pp_dpm_dclk)) { + if (!(asic_type == CHIP_VANGOGH)) + *states = ATTR_STATE_UNSUPPORTED; } if (asic_type == CHIP_ARCTURUS) { @@ -2897,7 +3059,8 @@ static ssize_t amdgpu_hwmon_show_power_cap_max(struct device *dev, char *buf) { struct amdgpu_device *adev = dev_get_drvdata(dev); - uint32_t limit = 0; + int limit_type = to_sensor_dev_attr(attr)->index; + uint32_t limit = limit_type << 24; ssize_t size; int r; @@ -2911,7 +3074,7 @@ static ssize_t amdgpu_hwmon_show_power_cap_max(struct device *dev, } if (is_support_sw_smu(adev)) { - smu_get_power_limit(&adev->smu, &limit, true); + smu_get_power_limit(&adev->smu, &limit, SMU_PPT_LIMIT_MAX); size = snprintf(buf, PAGE_SIZE, "%u\n", limit * 1000000); } else if (adev->powerplay.pp_funcs && adev->powerplay.pp_funcs->get_power_limit) { adev->powerplay.pp_funcs->get_power_limit(adev->powerplay.pp_handle, &limit, true); @@ -2931,7 +3094,8 @@ static ssize_t amdgpu_hwmon_show_power_cap(struct device *dev, char *buf) { struct amdgpu_device *adev = dev_get_drvdata(dev); - uint32_t limit = 0; + int limit_type = to_sensor_dev_attr(attr)->index; + uint32_t limit = limit_type << 24; ssize_t size; int r; @@ -2945,7 +3109,7 @@ static ssize_t amdgpu_hwmon_show_power_cap(struct device *dev, } if (is_support_sw_smu(adev)) { - smu_get_power_limit(&adev->smu, &limit, false); + smu_get_power_limit(&adev->smu, &limit, SMU_PPT_LIMIT_CURRENT); size = snprintf(buf, PAGE_SIZE, "%u\n", limit * 1000000); } else if (adev->powerplay.pp_funcs && adev->powerplay.pp_funcs->get_power_limit) { adev->powerplay.pp_funcs->get_power_limit(adev->powerplay.pp_handle, &limit, false); @@ -2960,6 +3124,15 @@ static ssize_t amdgpu_hwmon_show_power_cap(struct device *dev, return size; } +static ssize_t amdgpu_hwmon_show_power_label(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + int limit_type = to_sensor_dev_attr(attr)->index; + + return snprintf(buf, PAGE_SIZE, "%s\n", + limit_type == SMU_FAST_PPT_LIMIT ? "fastPPT" : "slowPPT"); +} static ssize_t amdgpu_hwmon_set_power_cap(struct device *dev, struct device_attribute *attr, @@ -2967,6 +3140,7 @@ static ssize_t amdgpu_hwmon_set_power_cap(struct device *dev, size_t count) { struct amdgpu_device *adev = dev_get_drvdata(dev); + int limit_type = to_sensor_dev_attr(attr)->index; int err; u32 value; @@ -2981,7 +3155,7 @@ static ssize_t amdgpu_hwmon_set_power_cap(struct device *dev, return err; value = value / 1000000; /* convert to Watt */ - + value |= limit_type << 24; err = pm_runtime_get_sync(adev_to_drm(adev)->dev); if (err < 0) { @@ -3193,6 +3367,12 @@ static SENSOR_DEVICE_ATTR(power1_average, S_IRUGO, amdgpu_hwmon_show_power_avg, static SENSOR_DEVICE_ATTR(power1_cap_max, S_IRUGO, amdgpu_hwmon_show_power_cap_max, NULL, 0); static SENSOR_DEVICE_ATTR(power1_cap_min, S_IRUGO, amdgpu_hwmon_show_power_cap_min, NULL, 0); static SENSOR_DEVICE_ATTR(power1_cap, S_IRUGO | S_IWUSR, amdgpu_hwmon_show_power_cap, amdgpu_hwmon_set_power_cap, 0); +static SENSOR_DEVICE_ATTR(power1_label, S_IRUGO, amdgpu_hwmon_show_power_label, NULL, 0); +static SENSOR_DEVICE_ATTR(power2_average, S_IRUGO, amdgpu_hwmon_show_power_avg, NULL, 1); +static SENSOR_DEVICE_ATTR(power2_cap_max, S_IRUGO, amdgpu_hwmon_show_power_cap_max, NULL, 1); +static SENSOR_DEVICE_ATTR(power2_cap_min, S_IRUGO, amdgpu_hwmon_show_power_cap_min, NULL, 1); +static SENSOR_DEVICE_ATTR(power2_cap, S_IRUGO | S_IWUSR, amdgpu_hwmon_show_power_cap, amdgpu_hwmon_set_power_cap, 1); +static SENSOR_DEVICE_ATTR(power2_label, S_IRUGO, amdgpu_hwmon_show_power_label, NULL, 1); static SENSOR_DEVICE_ATTR(freq1_input, S_IRUGO, amdgpu_hwmon_show_sclk, NULL, 0); static SENSOR_DEVICE_ATTR(freq1_label, S_IRUGO, amdgpu_hwmon_show_sclk_label, NULL, 0); static SENSOR_DEVICE_ATTR(freq2_input, S_IRUGO, amdgpu_hwmon_show_mclk, NULL, 0); @@ -3231,6 +3411,12 @@ static struct attribute *hwmon_attributes[] = { &sensor_dev_attr_power1_cap_max.dev_attr.attr, &sensor_dev_attr_power1_cap_min.dev_attr.attr, &sensor_dev_attr_power1_cap.dev_attr.attr, + &sensor_dev_attr_power1_label.dev_attr.attr, + &sensor_dev_attr_power2_average.dev_attr.attr, + &sensor_dev_attr_power2_cap_max.dev_attr.attr, + &sensor_dev_attr_power2_cap_min.dev_attr.attr, + &sensor_dev_attr_power2_cap.dev_attr.attr, + &sensor_dev_attr_power2_label.dev_attr.attr, &sensor_dev_attr_freq1_input.dev_attr.attr, &sensor_dev_attr_freq1_label.dev_attr.attr, &sensor_dev_attr_freq2_input.dev_attr.attr, @@ -3323,8 +3509,9 @@ static umode_t hwmon_attributes_visible(struct kobject *kobj, effective_mode &= ~S_IWUSR; } - if (((adev->flags & AMD_IS_APU) || - adev->family == AMDGPU_FAMILY_SI) && /* not implemented yet */ + if (((adev->family == AMDGPU_FAMILY_SI) || + ((adev->flags & AMD_IS_APU) && + (adev->asic_type != CHIP_VANGOGH))) && /* not implemented yet */ (attr == &sensor_dev_attr_power1_cap_max.dev_attr.attr || attr == &sensor_dev_attr_power1_cap_min.dev_attr.attr|| attr == &sensor_dev_attr_power1_cap.dev_attr.attr)) @@ -3387,6 +3574,16 @@ static umode_t hwmon_attributes_visible(struct kobject *kobj, attr == &sensor_dev_attr_temp3_label.dev_attr.attr)) return 0; + /* only Vangogh has fast PPT limit and power labels */ + if (!(adev->asic_type == CHIP_VANGOGH) && + (attr == &sensor_dev_attr_power2_average.dev_attr.attr || + attr == &sensor_dev_attr_power2_cap_max.dev_attr.attr || + attr == &sensor_dev_attr_power2_cap_min.dev_attr.attr || + attr == &sensor_dev_attr_power2_cap.dev_attr.attr || + attr == &sensor_dev_attr_power2_label.dev_attr.attr || + attr == &sensor_dev_attr_power1_label.dev_attr.attr)) + return 0; + return effective_mode; } @@ -3465,6 +3662,27 @@ void amdgpu_pm_sysfs_fini(struct amdgpu_device *adev) */ #if defined(CONFIG_DEBUG_FS) +static void amdgpu_debugfs_prints_cpu_info(struct seq_file *m, + struct amdgpu_device *adev) { + uint16_t *p_val; + uint32_t size; + int i; + + if (is_support_cclk_dpm(adev)) { + p_val = kcalloc(adev->smu.cpu_core_num, sizeof(uint16_t), + GFP_KERNEL); + + if (!amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_CPU_CLK, + (void *)p_val, &size)) { + for (i = 0; i < adev->smu.cpu_core_num; i++) + seq_printf(m, "\t%u MHz (CPU%d)\n", + *(p_val + i), i); + } + + kfree(p_val); + } +} + static int amdgpu_debugfs_pm_info_pp(struct seq_file *m, struct amdgpu_device *adev) { uint32_t value; @@ -3475,6 +3693,9 @@ static int amdgpu_debugfs_pm_info_pp(struct seq_file *m, struct amdgpu_device *a /* GPU Clocks */ size = sizeof(value); seq_printf(m, "GFX Clocks and Power:\n"); + + amdgpu_debugfs_prints_cpu_info(m, adev); + if (!amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_GFX_MCLK, (void *)&value, &size)) seq_printf(m, "\t%u MHz (MCLK)\n", value/100); if (!amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_GFX_SCLK, (void *)&value, &size)) diff --git a/drivers/gpu/drm/amd/pm/inc/amdgpu_smu.h b/drivers/gpu/drm/amd/pm/inc/amdgpu_smu.h index 0d797fa9f5cc69dd37314151d48d955a89e1e7d9..10b0624ade651eaa0d5ae30d4d8426cc7323f0c6 100644 --- a/drivers/gpu/drm/amd/pm/inc/amdgpu_smu.h +++ b/drivers/gpu/drm/amd/pm/inc/amdgpu_smu.h @@ -33,6 +33,8 @@ #define SMU_TEMPERATURE_UNITS_PER_CENTIGRADES 1000 #define SMU_FW_NAME_LEN 0x24 +#define SMU_DPM_USER_PROFILE_RESTORE (1 << 0) + struct smu_hw_power_state { unsigned int magic; }; @@ -159,6 +161,19 @@ enum smu_power_src_type SMU_POWER_SOURCE_COUNT, }; +enum smu_ppt_limit_type +{ + SMU_DEFAULT_PPT_LIMIT = 0, + SMU_FAST_PPT_LIMIT, +}; + +enum smu_ppt_limit_level +{ + SMU_PPT_LIMIT_MIN = -1, + SMU_PPT_LIMIT_CURRENT, + SMU_PPT_LIMIT_MAX, +}; + enum smu_memory_pool_size { SMU_MEMORY_POOL_SIZE_ZERO = 0, @@ -168,6 +183,17 @@ enum smu_memory_pool_size SMU_MEMORY_POOL_SIZE_2_GB = 0x80000000, }; +struct smu_user_dpm_profile { + uint32_t fan_mode; + uint32_t power_limit; + uint32_t fan_speed_percent; + uint32_t flags; + + /* user clock state information */ + uint32_t clk_mask[SMU_CLK_COUNT]; + uint32_t clk_dependency; +}; + #define SMU_TABLE_INIT(tables, table_id, s, a, d) \ do { \ tables[table_id].size = s; \ @@ -459,131 +485,672 @@ struct smu_context struct work_struct interrupt_work; unsigned fan_max_rpm; - unsigned manual_fan_speed_rpm; + unsigned manual_fan_speed_percent; uint32_t gfx_default_hard_min_freq; uint32_t gfx_default_soft_max_freq; uint32_t gfx_actual_hard_min_freq; uint32_t gfx_actual_soft_max_freq; + + /* APU only */ + uint32_t cpu_default_soft_min_freq; + uint32_t cpu_default_soft_max_freq; + uint32_t cpu_actual_soft_min_freq; + uint32_t cpu_actual_soft_max_freq; + uint32_t cpu_core_id_select; + uint16_t cpu_core_num; + + struct smu_user_dpm_profile user_dpm_profile; }; struct i2c_adapter; +/** + * struct pptable_funcs - Callbacks used to interact with the SMU. + */ struct pptable_funcs { + /** + * @run_btc: Calibrate voltage/frequency curve to fit the system's + * power delivery and voltage margins. Required for adaptive + * voltage frequency scaling (AVFS). + */ int (*run_btc)(struct smu_context *smu); + + /** + * @get_allowed_feature_mask: Get allowed feature mask. + * &feature_mask: Array to store feature mask. + * &num: Elements in &feature_mask. + */ int (*get_allowed_feature_mask)(struct smu_context *smu, uint32_t *feature_mask, uint32_t num); + + /** + * @get_current_power_state: Get the current power state. + * + * Return: Current power state on success, negative errno on failure. + */ enum amd_pm_state_type (*get_current_power_state)(struct smu_context *smu); + + /** + * @set_default_dpm_table: Retrieve the default overdrive settings from + * the SMU. + */ int (*set_default_dpm_table)(struct smu_context *smu); + int (*set_power_state)(struct smu_context *smu); + + /** + * @populate_umd_state_clk: Populate the UMD power state table with + * defaults. + */ int (*populate_umd_state_clk)(struct smu_context *smu); + + /** + * @print_clk_levels: Print DPM clock levels for a clock domain + * to buffer. Star current level. + * + * Used for sysfs interfaces. + */ int (*print_clk_levels)(struct smu_context *smu, enum smu_clk_type clk_type, char *buf); + + /** + * @force_clk_levels: Set a range of allowed DPM levels for a clock + * domain. + * &clk_type: Clock domain. + * &mask: Range of allowed DPM levels. + */ int (*force_clk_levels)(struct smu_context *smu, enum smu_clk_type clk_type, uint32_t mask); + + /** + * @od_edit_dpm_table: Edit the custom overdrive DPM table. + * &type: Type of edit. + * &input: Edit parameters. + * &size: Size of &input. + */ int (*od_edit_dpm_table)(struct smu_context *smu, enum PP_OD_DPM_TABLE_COMMAND type, long *input, uint32_t size); + + /** + * @get_clock_by_type_with_latency: Get the speed and latency of a clock + * domain. + */ int (*get_clock_by_type_with_latency)(struct smu_context *smu, enum smu_clk_type clk_type, struct pp_clock_levels_with_latency *clocks); + /** + * @get_clock_by_type_with_voltage: Get the speed and voltage of a clock + * domain. + */ + int (*get_clock_by_type_with_voltage)(struct smu_context *smu, + enum amd_pp_clock_type type, + struct + pp_clock_levels_with_voltage + *clocks); + + /** + * @get_power_profile_mode: Print all power profile modes to + * buffer. Star current mode. + */ int (*get_power_profile_mode)(struct smu_context *smu, char *buf); + + /** + * @set_power_profile_mode: Set a power profile mode. Also used to + * create/set custom power profile modes. + * &input: Power profile mode parameters. + * &size: Size of &input. + */ int (*set_power_profile_mode)(struct smu_context *smu, long *input, uint32_t size); + + /** + * @dpm_set_vcn_enable: Enable/disable VCN engine dynamic power + * management. + */ int (*dpm_set_vcn_enable)(struct smu_context *smu, bool enable); + + /** + * @dpm_set_jpeg_enable: Enable/disable JPEG engine dynamic power + * management. + */ int (*dpm_set_jpeg_enable)(struct smu_context *smu, bool enable); + + /** + * @read_sensor: Read data from a sensor. + * &sensor: Sensor to read data from. + * &data: Sensor reading. + * &size: Size of &data. + */ int (*read_sensor)(struct smu_context *smu, enum amd_pp_sensors sensor, void *data, uint32_t *size); + + /** + * @pre_display_config_changed: Prepare GPU for a display configuration + * change. + * + * Disable display tracking and pin memory clock speed to maximum. Used + * in display component synchronization. + */ int (*pre_display_config_changed)(struct smu_context *smu); + + /** + * @display_config_changed: Notify the SMU of the current display + * configuration. + * + * Allows SMU to properly track blanking periods for memory clock + * adjustment. Used in display component synchronization. + */ int (*display_config_changed)(struct smu_context *smu); + int (*apply_clocks_adjust_rules)(struct smu_context *smu); + + /** + * @notify_smc_display_config: Applies display requirements to the + * current power state. + * + * Optimize deep sleep DCEFclk and mclk for the current display + * configuration. Used in display component synchronization. + */ int (*notify_smc_display_config)(struct smu_context *smu); + + /** + * @is_dpm_running: Check if DPM is running. + * + * Return: True if DPM is running, false otherwise. + */ bool (*is_dpm_running)(struct smu_context *smu); - int (*get_fan_speed_rpm)(struct smu_context *smu, uint32_t *speed); + + /** + * @get_fan_speed_percent: Get the current fan speed in percent. + */ + int (*get_fan_speed_percent)(struct smu_context *smu, uint32_t *speed); + + /** + * @set_watermarks_table: Configure and upload the watermarks tables to + * the SMU. + */ int (*set_watermarks_table)(struct smu_context *smu, struct pp_smu_wm_range_sets *clock_ranges); + + /** + * @get_thermal_temperature_range: Get safe thermal limits in Celcius. + */ int (*get_thermal_temperature_range)(struct smu_context *smu, struct smu_temperature_range *range); + + /** + * @get_uclk_dpm_states: Get memory clock DPM levels in kHz. + * &clocks_in_khz: Array of DPM levels. + * &num_states: Elements in &clocks_in_khz. + */ int (*get_uclk_dpm_states)(struct smu_context *smu, uint32_t *clocks_in_khz, uint32_t *num_states); + + /** + * @set_default_od_settings: Set the overdrive tables to defaults. + */ int (*set_default_od_settings)(struct smu_context *smu); + + /** + * @set_performance_level: Set a performance level. + */ int (*set_performance_level)(struct smu_context *smu, enum amd_dpm_forced_level level); + + /** + * @display_disable_memory_clock_switch: Enable/disable dynamic memory + * clock switching. + * + * Disabling this feature forces memory clock speed to maximum. + * Enabling sets the minimum memory clock capable of driving the + * current display configuration. + */ int (*display_disable_memory_clock_switch)(struct smu_context *smu, bool disable_memory_clock_switch); + + /** + * @dump_pptable: Print the power play table to the system log. + */ void (*dump_pptable)(struct smu_context *smu); + + /** + * @get_power_limit: Get the device's power limits. + */ int (*get_power_limit)(struct smu_context *smu); + + /** + * @get_ppt_limit: Get the device's ppt limits. + */ + int (*get_ppt_limit)(struct smu_context *smu, uint32_t *ppt_limit, + enum smu_ppt_limit_type limit_type, enum smu_ppt_limit_level limit_level); + + /** + * @set_df_cstate: Set data fabric cstate. + */ int (*set_df_cstate)(struct smu_context *smu, enum pp_df_cstate state); + + /** + * @allow_xgmi_power_down: Enable/disable external global memory + * interconnect power down. + */ int (*allow_xgmi_power_down)(struct smu_context *smu, bool en); + + /** + * @update_pcie_parameters: Update and upload the system's PCIe + * capabilites to the SMU. + * &pcie_gen_cap: Maximum allowed PCIe generation. + * &pcie_width_cap: Maximum allowed PCIe width. + */ int (*update_pcie_parameters)(struct smu_context *smu, uint32_t pcie_gen_cap, uint32_t pcie_width_cap); + + /** + * @i2c_init: Initialize i2c. + * + * The i2c bus is used internally by the SMU voltage regulators and + * other devices. The i2c's EEPROM also stores bad page tables on boards + * with ECC. + */ int (*i2c_init)(struct smu_context *smu, struct i2c_adapter *control); + + /** + * @i2c_fini: Tear down i2c. + */ void (*i2c_fini)(struct smu_context *smu, struct i2c_adapter *control); + + /** + * @get_unique_id: Get the GPU's unique id. Used for asset tracking. + */ void (*get_unique_id)(struct smu_context *smu); + + /** + * @get_dpm_clock_table: Get a copy of the DPM clock table. + * + * Used by display component in bandwidth and watermark calculations. + */ int (*get_dpm_clock_table)(struct smu_context *smu, struct dpm_clocks *clock_table); + + /** + * @init_microcode: Request the SMU's firmware from the kernel. + */ int (*init_microcode)(struct smu_context *smu); + + /** + * @load_microcode: Load firmware onto the SMU. + */ int (*load_microcode)(struct smu_context *smu); + + /** + * @fini_microcode: Release the SMU's firmware. + */ void (*fini_microcode)(struct smu_context *smu); + + /** + * @init_smc_tables: Initialize the SMU tables. + */ int (*init_smc_tables)(struct smu_context *smu); + + /** + * @fini_smc_tables: Release the SMU tables. + */ int (*fini_smc_tables)(struct smu_context *smu); + + /** + * @init_power: Initialize the power gate table context. + */ int (*init_power)(struct smu_context *smu); + + /** + * @fini_power: Release the power gate table context. + */ int (*fini_power)(struct smu_context *smu); + + /** + * @check_fw_status: Check the SMU's firmware status. + * + * Return: Zero if check passes, negative errno on failure. + */ int (*check_fw_status)(struct smu_context *smu); + + /** + * @setup_pptable: Initialize the power play table and populate it with + * default values. + */ int (*setup_pptable)(struct smu_context *smu); + + /** + * @get_vbios_bootup_values: Get default boot values from the VBIOS. + */ int (*get_vbios_bootup_values)(struct smu_context *smu); + + /** + * @check_fw_version: Print driver and SMU interface versions to the + * system log. + * + * Interface mismatch is not a critical failure. + */ int (*check_fw_version)(struct smu_context *smu); + + /** + * @powergate_sdma: Power up/down system direct memory access. + */ int (*powergate_sdma)(struct smu_context *smu, bool gate); + + /** + * @set_gfx_cgpg: Enable/disable graphics engine course grain power + * gating. + */ int (*set_gfx_cgpg)(struct smu_context *smu, bool enable); + + /** + * @write_pptable: Write the power play table to the SMU. + */ int (*write_pptable)(struct smu_context *smu); + + /** + * @set_driver_table_location: Send the location of the driver table to + * the SMU. + */ int (*set_driver_table_location)(struct smu_context *smu); + + /** + * @set_tool_table_location: Send the location of the tool table to the + * SMU. + */ int (*set_tool_table_location)(struct smu_context *smu); + + /** + * @notify_memory_pool_location: Send the location of the memory pool to + * the SMU. + */ int (*notify_memory_pool_location)(struct smu_context *smu); + + /** + * @system_features_control: Enable/disable all SMU features. + */ int (*system_features_control)(struct smu_context *smu, bool en); + + /** + * @send_smc_msg_with_param: Send a message with a parameter to the SMU. + * &msg: Type of message. + * ¶m: Message parameter. + * &read_arg: SMU response (optional). + */ int (*send_smc_msg_with_param)(struct smu_context *smu, enum smu_message_type msg, uint32_t param, uint32_t *read_arg); + + /** + * @send_smc_msg: Send a message to the SMU. + * &msg: Type of message. + * &read_arg: SMU response (optional). + */ int (*send_smc_msg)(struct smu_context *smu, enum smu_message_type msg, uint32_t *read_arg); + + /** + * @init_display_count: Notify the SMU of the number of display + * components in current display configuration. + */ int (*init_display_count)(struct smu_context *smu, uint32_t count); + + /** + * @set_allowed_mask: Notify the SMU of the features currently allowed + * by the driver. + */ int (*set_allowed_mask)(struct smu_context *smu); + + /** + * @get_enabled_mask: Get a mask of features that are currently enabled + * on the SMU. + * &feature_mask: Array representing enabled feature mask. + * &num: Elements in &feature_mask. + */ int (*get_enabled_mask)(struct smu_context *smu, uint32_t *feature_mask, uint32_t num); + + /** + * @feature_is_enabled: Test if a feature is enabled. + * + * Return: One if enabled, zero if disabled. + */ int (*feature_is_enabled)(struct smu_context *smu, enum smu_feature_mask mask); + + /** + * @disable_all_features_with_exception: Disable all features with + * exception to those in &mask. + */ int (*disable_all_features_with_exception)(struct smu_context *smu, enum smu_feature_mask mask); + + /** + * @notify_display_change: Enable fast memory clock switching. + * + * Allows for fine grained memory clock switching but has more stringent + * timing requirements. + */ int (*notify_display_change)(struct smu_context *smu); + + /** + * @set_power_limit: Set power limit in watts. + */ int (*set_power_limit)(struct smu_context *smu, uint32_t n); + + /** + * @init_max_sustainable_clocks: Populate max sustainable clock speed + * table with values from the SMU. + */ int (*init_max_sustainable_clocks)(struct smu_context *smu); + + /** + * @enable_thermal_alert: Enable thermal alert interrupts. + */ int (*enable_thermal_alert)(struct smu_context *smu); + + /** + * @disable_thermal_alert: Disable thermal alert interrupts. + */ int (*disable_thermal_alert)(struct smu_context *smu); + + /** + * @set_min_dcef_deep_sleep: Set a minimum display fabric deep sleep + * clock speed in MHz. + */ int (*set_min_dcef_deep_sleep)(struct smu_context *smu, uint32_t clk); + + /** + * @display_clock_voltage_request: Set a hard minimum frequency + * for a clock domain. + */ int (*display_clock_voltage_request)(struct smu_context *smu, struct pp_display_clock_request *clock_req); + + /** + * @get_fan_control_mode: Get the current fan control mode. + */ uint32_t (*get_fan_control_mode)(struct smu_context *smu); + + /** + * @set_fan_control_mode: Set the fan control mode. + */ int (*set_fan_control_mode)(struct smu_context *smu, uint32_t mode); + + /** + * @set_fan_speed_percent: Set a static fan speed in percent. + */ int (*set_fan_speed_percent)(struct smu_context *smu, uint32_t speed); - int (*set_fan_speed_rpm)(struct smu_context *smu, uint32_t speed); + + /** + * @set_xgmi_pstate: Set inter-chip global memory interconnect pstate. + * &pstate: Pstate to set. D0 if Nonzero, D3 otherwise. + */ int (*set_xgmi_pstate)(struct smu_context *smu, uint32_t pstate); + + /** + * @gfx_off_control: Enable/disable graphics engine poweroff. + */ int (*gfx_off_control)(struct smu_context *smu, bool enable); + + + /** + * @get_gfx_off_status: Get graphics engine poweroff status. + * + * Return: + * 0 - GFXOFF(default). + * 1 - Transition out of GFX State. + * 2 - Not in GFXOFF. + * 3 - Transition into GFXOFF. + */ uint32_t (*get_gfx_off_status)(struct smu_context *smu); + + /** + * @register_irq_handler: Register interupt request handlers. + */ int (*register_irq_handler)(struct smu_context *smu); + + /** + * @set_azalia_d3_pme: Wake the audio decode engine from d3 sleep. + */ int (*set_azalia_d3_pme)(struct smu_context *smu); + + /** + * @get_max_sustainable_clocks_by_dc: Get a copy of the max sustainable + * clock speeds table. + * + * Provides a way for the display component (DC) to get the max + * sustainable clocks from the SMU. + */ int (*get_max_sustainable_clocks_by_dc)(struct smu_context *smu, struct pp_smu_nv_clock_table *max_clocks); + + /** + * @baco_is_support: Check if GPU supports BACO (Bus Active, Chip Off). + */ bool (*baco_is_support)(struct smu_context *smu); + + /** + * @baco_get_state: Get the current BACO state. + * + * Return: Current BACO state. + */ enum smu_baco_state (*baco_get_state)(struct smu_context *smu); + + /** + * @baco_set_state: Enter/exit BACO. + */ int (*baco_set_state)(struct smu_context *smu, enum smu_baco_state state); + + /** + * @baco_enter: Enter BACO. + */ int (*baco_enter)(struct smu_context *smu); + + /** + * @baco_exit: Exit Baco. + */ int (*baco_exit)(struct smu_context *smu); + + /** + * @mode1_reset_is_support: Check if GPU supports mode1 reset. + */ bool (*mode1_reset_is_support)(struct smu_context *smu); + + /** + * @mode1_reset: Perform mode1 reset. + * + * Complete GPU reset. + */ int (*mode1_reset)(struct smu_context *smu); + + /** + * @mode2_reset: Perform mode2 reset. + * + * Mode2 reset generally does not reset as many IPs as mode1 reset. The + * IPs reset varies by asic. + */ int (*mode2_reset)(struct smu_context *smu); + + /** + * @get_dpm_ultimate_freq: Get the hard frequency range of a clock + * domain in MHz. + */ int (*get_dpm_ultimate_freq)(struct smu_context *smu, enum smu_clk_type clk_type, uint32_t *min, uint32_t *max); + + /** + * @set_soft_freq_limited_range: Set the soft frequency range of a clock + * domain in MHz. + */ int (*set_soft_freq_limited_range)(struct smu_context *smu, enum smu_clk_type clk_type, uint32_t min, uint32_t max); + + /** + * @set_power_source: Notify the SMU of the current power source. + */ int (*set_power_source)(struct smu_context *smu, enum smu_power_src_type power_src); + + /** + * @log_thermal_throttling_event: Print a thermal throttling warning to + * the system's log. + */ void (*log_thermal_throttling_event)(struct smu_context *smu); + + /** + * @get_pp_feature_mask: Print a human readable table of enabled + * features to buffer. + */ size_t (*get_pp_feature_mask)(struct smu_context *smu, char *buf); + + /** + * @set_pp_feature_mask: Request the SMU enable/disable features to + * match those enabled in &new_mask. + */ int (*set_pp_feature_mask)(struct smu_context *smu, uint64_t new_mask); + + /** + * @get_gpu_metrics: Get a copy of the GPU metrics table from the SMU. + * + * Return: Size of &table + */ ssize_t (*get_gpu_metrics)(struct smu_context *smu, void **table); + + /** + * @enable_mgpu_fan_boost: Enable multi-GPU fan boost. + */ int (*enable_mgpu_fan_boost)(struct smu_context *smu); + + /** + * @gfx_ulv_control: Enable/disable ultra low voltage. + */ int (*gfx_ulv_control)(struct smu_context *smu, bool enablement); + + /** + * @deep_sleep_control: Enable/disable deep sleep. + */ int (*deep_sleep_control)(struct smu_context *smu, bool enablement); + + /** + * @get_fan_parameters: Get fan parameters. + * + * Get maximum fan speed from the power play table. + */ int (*get_fan_parameters)(struct smu_context *smu); + + /** + * @post_init: Helper function for asic specific workarounds. + */ int (*post_init)(struct smu_context *smu); + + /** + * @interrupt_work: Work task scheduled from SMU interrupt handler. + */ void (*interrupt_work)(struct smu_context *smu); + + /** + * @gpo_control: Enable/disable graphics power optimization if supported. + */ int (*gpo_control)(struct smu_context *smu, bool enablement); + + /** + * @gfx_state_change_set: Send the current graphics state to the SMU. + */ int (*gfx_state_change_set)(struct smu_context *smu, uint32_t state); + + /** + * @set_fine_grain_gfx_freq_parameters: Set fine grain graphics clock + * parameters to defaults. + */ int (*set_fine_grain_gfx_freq_parameters)(struct smu_context *smu); }; @@ -597,6 +1164,7 @@ typedef enum { METRICS_CURR_DCLK1, METRICS_CURR_FCLK, METRICS_CURR_DCEFCLK, + METRICS_AVERAGE_CPUCLK, METRICS_AVERAGE_GFXCLK, METRICS_AVERAGE_SOCCLK, METRICS_AVERAGE_FCLK, @@ -637,6 +1205,12 @@ enum smu_cmn2asic_mapping_type { #define FEA_MAP(fea) \ [SMU_FEATURE_##fea##_BIT] = {1, FEATURE_##fea##_BIT} +#define FEA_MAP_REVERSE(fea) \ + [SMU_FEATURE_DPM_##fea##_BIT] = {1, FEATURE_##fea##_DPM_BIT} + +#define FEA_MAP_HALF_REVERSE(fea) \ + [SMU_FEATURE_DPM_##fea##CLK_BIT] = {1, FEATURE_##fea##_DPM_BIT} + #define TAB_MAP(tab) \ [SMU_TABLE_##tab] = {1, TABLE_##tab} @@ -663,7 +1237,7 @@ int smu_set_fan_speed_rpm(struct smu_context *smu, uint32_t speed); int smu_get_power_limit(struct smu_context *smu, uint32_t *limit, - bool max_setting); + enum smu_ppt_limit_level limit_level); int smu_set_power_limit(struct smu_context *smu, uint32_t limit); int smu_print_clk_levels(struct smu_context *smu, enum smu_clk_type clk_type, char *buf); @@ -719,6 +1293,7 @@ extern const struct amdgpu_ip_block_version smu_v11_0_ip_block; extern const struct amdgpu_ip_block_version smu_v12_0_ip_block; bool is_support_sw_smu(struct amdgpu_device *adev); +bool is_support_cclk_dpm(struct amdgpu_device *adev); int smu_reset(struct smu_context *smu); int smu_sys_get_pp_table(struct smu_context *smu, void **table); int smu_sys_set_pp_table(struct smu_context *smu, void *buf, size_t size); diff --git a/drivers/gpu/drm/amd/pm/inc/smu11_driver_if_vangogh.h b/drivers/gpu/drm/amd/pm/inc/smu11_driver_if_vangogh.h index 1c19eae93ff1750aeefdb1ba3bc53743e6bed42f..6e23a3f803a7c53e7b49f0b76fd9a432e54245cc 100644 --- a/drivers/gpu/drm/amd/pm/inc/smu11_driver_if_vangogh.h +++ b/drivers/gpu/drm/amd/pm/inc/smu11_driver_if_vangogh.h @@ -141,7 +141,6 @@ typedef struct { uint32_t MaxGfxClk; uint8_t NumDfPstatesEnabled; - uint8_t NumDpmLevelsEnabled; uint8_t NumDcfclkLevelsEnabled; uint8_t NumDispClkLevelsEnabled; //applies to both dispclk and dppclk uint8_t NumSocClkLevelsEnabled; diff --git a/drivers/gpu/drm/amd/pm/inc/smu_types.h b/drivers/gpu/drm/amd/pm/inc/smu_types.h index 720d15612fe1ac45c1965d169be8069b198a8412..aa4822202587f327e06b96ad3d5a988cf528e0cc 100644 --- a/drivers/gpu/drm/amd/pm/inc/smu_types.h +++ b/drivers/gpu/drm/amd/pm/inc/smu_types.h @@ -133,8 +133,6 @@ __SMU_DUMMY_MAP(PowerUpSdma), \ __SMU_DUMMY_MAP(SetHardMinIspclkByFreq), \ __SMU_DUMMY_MAP(SetHardMinVcn), \ - __SMU_DUMMY_MAP(Spare1), \ - __SMU_DUMMY_MAP(Spare2), \ __SMU_DUMMY_MAP(SetAllowFclkSwitch), \ __SMU_DUMMY_MAP(SetMinVideoGfxclkFreq), \ __SMU_DUMMY_MAP(ActiveProcessNotify), \ @@ -211,6 +209,11 @@ __SMU_DUMMY_MAP(SetGpoFeaturePMask), \ __SMU_DUMMY_MAP(DisallowGpo), \ __SMU_DUMMY_MAP(Enable2ndUSB20Port), \ + __SMU_DUMMY_MAP(RequestActiveWgp), \ + __SMU_DUMMY_MAP(SetFastPPTLimit), \ + __SMU_DUMMY_MAP(SetSlowPPTLimit), \ + __SMU_DUMMY_MAP(GetFastPPTLimit), \ + __SMU_DUMMY_MAP(GetSlowPPTLimit), \ #undef __SMU_DUMMY_MAP #define __SMU_DUMMY_MAP(type) SMU_MSG_##type @@ -236,10 +239,12 @@ enum smu_clk_type { SMU_SCLK, SMU_MCLK, SMU_PCIE, + SMU_OD_CCLK, SMU_OD_SCLK, SMU_OD_MCLK, SMU_OD_VDDC_CURVE, SMU_OD_RANGE, + SMU_OD_VDDGFX_OFFSET, SMU_CLK_COUNT, }; diff --git a/drivers/gpu/drm/amd/pm/inc/smu_v11_0.h b/drivers/gpu/drm/amd/pm/inc/smu_v11_0.h index 5d0b29653ffa13ccf8b4ae265b69512331a5dc27..d4cddd2390a29d1d54f6003714321791f9bc516f 100644 --- a/drivers/gpu/drm/amd/pm/inc/smu_v11_0.h +++ b/drivers/gpu/drm/amd/pm/inc/smu_v11_0.h @@ -31,7 +31,7 @@ #define SMU11_DRIVER_IF_VERSION_NV12 0x36 #define SMU11_DRIVER_IF_VERSION_NV14 0x36 #define SMU11_DRIVER_IF_VERSION_Sienna_Cichlid 0x3D -#define SMU11_DRIVER_IF_VERSION_Navy_Flounder 0xC +#define SMU11_DRIVER_IF_VERSION_Navy_Flounder 0xE #define SMU11_DRIVER_IF_VERSION_VANGOGH 0x02 #define SMU11_DRIVER_IF_VERSION_Dimgrey_Cavefish 0xF @@ -129,6 +129,15 @@ struct smu_11_0_power_context { enum smu_11_0_power_state power_state; }; +struct smu_11_5_power_context { + uint32_t power_source; + uint8_t in_power_limit_boost_mode; + enum smu_11_0_power_state power_state; + + uint32_t current_fast_ppt_limit; + uint32_t max_fast_ppt_limit; +}; + enum smu_v11_0_baco_seq { BACO_SEQ_BACO = 0, BACO_SEQ_MSR, @@ -203,14 +212,8 @@ int smu_v11_0_set_fan_control_mode(struct smu_context *smu, uint32_t mode); -int -smu_v11_0_set_fan_speed_percent(struct smu_context *smu, uint32_t speed); - -int smu_v11_0_set_fan_speed_rpm(struct smu_context *smu, - uint32_t speed); - -int smu_v11_0_get_fan_speed_rpm(struct smu_context *smu, - uint32_t *speed); +int smu_v11_0_set_fan_speed_percent(struct smu_context *smu, + uint32_t speed); int smu_v11_0_set_xgmi_pstate(struct smu_context *smu, uint32_t pstate); @@ -278,10 +281,6 @@ int smu_v11_0_get_current_pcie_link_speed_level(struct smu_context *smu); int smu_v11_0_get_current_pcie_link_speed(struct smu_context *smu); -void smu_v11_0_init_gpu_metrics_v1_0(struct gpu_metrics_v1_0 *gpu_metrics); - -void smu_v11_0_init_gpu_metrics_v2_0(struct gpu_metrics_v2_0 *gpu_metrics); - int smu_v11_0_gfx_ulv_control(struct smu_context *smu, bool enablement); diff --git a/drivers/gpu/drm/amd/pm/inc/smu_v11_5_ppsmc.h b/drivers/gpu/drm/amd/pm/inc/smu_v11_5_ppsmc.h index 55d7892e4e0e52c3fe86f007c6178889968b1d5d..fe130a497d6c3b743142f20bf2a2090c19a72836 100644 --- a/drivers/gpu/drm/amd/pm/inc/smu_v11_5_ppsmc.h +++ b/drivers/gpu/drm/amd/pm/inc/smu_v11_5_ppsmc.h @@ -104,7 +104,11 @@ #define PPSMC_MSG_DramLogSetDramBufferSize 0x46 #define PPSMC_MSG_RequestActiveWgp 0x47 #define PPSMC_MSG_QueryActiveWgp 0x48 -#define PPSMC_Message_Count 0x49 +#define PPSMC_MSG_SetFastPPTLimit 0x49 +#define PPSMC_MSG_SetSlowPPTLimit 0x4A +#define PPSMC_MSG_GetFastPPTLimit 0x4B +#define PPSMC_MSG_GetSlowPPTLimit 0x4C +#define PPSMC_Message_Count 0x4D //Argument for PPSMC_MSG_GfxDeviceDriverReset enum { diff --git a/drivers/gpu/drm/amd/pm/inc/smu_v12_0.h b/drivers/gpu/drm/amd/pm/inc/smu_v12_0.h index fa2e8cb079670aa800d024c934d5767f23f6ad25..02de3b6199e5332afe1b7308f5dce9e12c27ed5e 100644 --- a/drivers/gpu/drm/amd/pm/inc/smu_v12_0.h +++ b/drivers/gpu/drm/amd/pm/inc/smu_v12_0.h @@ -60,7 +60,5 @@ int smu_v12_0_set_soft_freq_limited_range(struct smu_context *smu, enum smu_clk_ int smu_v12_0_set_driver_table_location(struct smu_context *smu); -void smu_v12_0_init_gpu_metrics_v2_0(struct gpu_metrics_v2_0 *gpu_metrics); - #endif #endif diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/hwmgr.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/hwmgr.c index 6a7de8b898fafeaaf217beb3e0c9a5421a36d3e0..f2cef0930aa96165b711827f40b94767c52324d3 100644 --- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/hwmgr.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/hwmgr.c @@ -33,6 +33,7 @@ #include "ppsmc.h" #include "amd_acpi.h" #include "pp_psm.h" +#include "vega10_hwmgr.h" extern const struct pp_smumgr_func ci_smu_funcs; extern const struct pp_smumgr_func smu8_smu_funcs; @@ -46,7 +47,6 @@ extern const struct pp_smumgr_func vega12_smu_funcs; extern const struct pp_smumgr_func smu10_smu_funcs; extern const struct pp_smumgr_func vega20_smu_funcs; -extern int vega10_hwmgr_init(struct pp_hwmgr *hwmgr); extern int smu10_init_function_pointers(struct pp_hwmgr *hwmgr); static int polaris_set_asic_special_caps(struct pp_hwmgr *hwmgr); diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/ppatomctrl.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/ppatomctrl.c index 83a6504e093cbba2fdb1cef05a02344b1315924f..b1038d30c8dcc558935d493f5567269532092093 100644 --- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/ppatomctrl.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/ppatomctrl.c @@ -279,7 +279,7 @@ static const ATOM_VOLTAGE_OBJECT_V3 *atomctrl_lookup_voltage_type_v3( * * @hwmgr: input parameter: pointer to HwMgr * @clock_value: input parameter: memory clock - * @dividers: output parameter: memory PLL dividers + * @mpll_param: output parameter: memory clock parameters * @strobe_mode: input parameter: 1 for strobe mode, 0 for performance mode */ int atomctrl_get_memory_pll_dividers_si( @@ -332,7 +332,7 @@ int atomctrl_get_memory_pll_dividers_si( * * @hwmgr: input parameter: pointer to HwMgr * @clock_value: input parameter: memory clock - * @dividers: output parameter: memory PLL dividers + * @mpll_param: output parameter: memory clock parameters */ int atomctrl_get_memory_pll_dividers_vi(struct pp_hwmgr *hwmgr, uint32_t clock_value, pp_atomctrl_memory_clock_param *mpll_param) diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/process_pptables_v1_0.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/process_pptables_v1_0.c index 741e03ad5311f115157496ca922f23668b45e175..f2a55c1413f597a4d643d1a2c99367517bdff17e 100644 --- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/process_pptables_v1_0.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/process_pptables_v1_0.c @@ -1362,6 +1362,7 @@ static int ppt_get_vce_state_table_entry_v1_0(struct pp_hwmgr *hwmgr, uint32_t i * @hwmgr: Pointer to the hardware manager. * @entry_index: The index of the entry to be extracted from the table. * @power_state: The address of the PowerState instance being created. + * @call_back_func: The function to call into to fill power state * Return: -1 if the entry cannot be retrieved. */ int get_powerplay_table_entry_v1_0(struct pp_hwmgr *hwmgr, diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu10_hwmgr.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu10_hwmgr.c index 88322781e447b5812896890b1a776fbae4010092..ed05a30d1139dc5488e3541c7420f33cce81f3c2 100644 --- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu10_hwmgr.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu10_hwmgr.c @@ -1487,7 +1487,7 @@ static int smu10_set_fine_grain_clk_vol(struct pp_hwmgr *hwmgr, } if (!smu10_data->fine_grain_enabled) { - pr_err("Fine grain not started\n"); + pr_err("pp_od_clk_voltage is not accessible if power_dpm_force_perfomance_level is not in manual mode!\n"); return -EINVAL; } diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_hwmgr.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_hwmgr.c index 82676c086ce464d9190b01097918df8c12a0036d..c57dc9ae81f2f4e7af3dc1dd3d9537cca3c18eb3 100644 --- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_hwmgr.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_hwmgr.c @@ -235,7 +235,7 @@ static int smu7_get_current_pcie_lane_number(struct pp_hwmgr *hwmgr) /** * smu7_enable_smc_voltage_controller - Enable voltage control * - * @hwmgr the address of the powerplay hardware manager. + * @hwmgr: the address of the powerplay hardware manager. * Return: always PP_Result_OK */ static int smu7_enable_smc_voltage_controller(struct pp_hwmgr *hwmgr) @@ -4501,7 +4501,7 @@ static int smu7_display_configuration_changed_task(struct pp_hwmgr *hwmgr) * smu7_set_max_fan_rpm_output - Set maximum target operating fan output RPM * * @hwmgr: the address of the powerplay hardware manager. - * @usMaxFanRpm: max operating fan RPM value. + * @us_max_fan_rpm: max operating fan RPM value. * Return: The response that came from the SMC. */ static int smu7_set_max_fan_rpm_output(struct pp_hwmgr *hwmgr, uint16_t us_max_fan_rpm) diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_hwmgr.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_hwmgr.c index 1b47f94e033178c118e0595bf669af7331e82a94..29c99642d22d407e5613aef1f1264efb9cc30f86 100644 --- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_hwmgr.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_hwmgr.c @@ -542,11 +542,11 @@ static int vega10_get_socclk_for_voltage_evv(struct pp_hwmgr *hwmgr, #define ATOM_VIRTUAL_VOLTAGE_ID0 0xff01 /** -* Get Leakage VDDC based on leakage ID. -* -* @param hwmgr the address of the powerplay hardware manager. -* @return always 0. -*/ + * Get Leakage VDDC based on leakage ID. + * + * @hwmgr: the address of the powerplay hardware manager. + * return: always 0. + */ static int vega10_get_evv_voltages(struct pp_hwmgr *hwmgr) { struct vega10_hwmgr *data = hwmgr->backend; @@ -600,9 +600,9 @@ static int vega10_get_evv_voltages(struct pp_hwmgr *hwmgr) /** * Change virtual leakage voltage to actual value. * - * @param hwmgr the address of the powerplay hardware manager. - * @param pointer to changing voltage - * @param pointer to leakage table + * @hwmgr: the address of the powerplay hardware manager. + * @voltage: pointer to changing voltage + * @leakage_table: pointer to leakage table */ static void vega10_patch_with_vdd_leakage(struct pp_hwmgr *hwmgr, uint16_t *voltage, struct vega10_leakage_voltage *leakage_table) @@ -624,13 +624,13 @@ static void vega10_patch_with_vdd_leakage(struct pp_hwmgr *hwmgr, } /** -* Patch voltage lookup table by EVV leakages. -* -* @param hwmgr the address of the powerplay hardware manager. -* @param pointer to voltage lookup table -* @param pointer to leakage table -* @return always 0 -*/ + * Patch voltage lookup table by EVV leakages. + * + * @hwmgr: the address of the powerplay hardware manager. + * @lookup_table: pointer to voltage lookup table + * @leakage_table: pointer to leakage table + * return: always 0 + */ static int vega10_patch_lookup_table_with_leakage(struct pp_hwmgr *hwmgr, phm_ppt_v1_voltage_lookup_table *lookup_table, struct vega10_leakage_voltage *leakage_table) @@ -1001,13 +1001,12 @@ static int vega10_setup_asic_task(struct pp_hwmgr *hwmgr) } /** -* Remove repeated voltage values and create table with unique values. -* -* @param hwmgr the address of the powerplay hardware manager. -* @param vol_table the pointer to changing voltage table -* @return 0 in success -*/ - + * Remove repeated voltage values and create table with unique values. + * + * @hwmgr: the address of the powerplay hardware manager. + * @vol_table: the pointer to changing voltage table + * return: 0 in success + */ static int vega10_trim_voltage_table(struct pp_hwmgr *hwmgr, struct pp_atomfwctrl_voltage_table *vol_table) { @@ -1151,11 +1150,11 @@ static void vega10_trim_voltage_table_to_fit_state_table( } /** -* Create Voltage Tables. -* -* @param hwmgr the address of the powerplay hardware manager. -* @return always 0 -*/ + * Create Voltage Tables. + * + * @hwmgr: the address of the powerplay hardware manager. + * return: always 0 + */ static int vega10_construct_voltage_tables(struct pp_hwmgr *hwmgr) { struct vega10_hwmgr *data = hwmgr->backend; @@ -1212,11 +1211,11 @@ static int vega10_construct_voltage_tables(struct pp_hwmgr *hwmgr) } /* - * @fn vega10_init_dpm_state - * @brief Function to initialize all Soft Min/Max and Hard Min/Max to 0xff. + * vega10_init_dpm_state + * Function to initialize all Soft Min/Max and Hard Min/Max to 0xff. * - * @param dpm_state - the address of the DPM Table to initiailize. - * @return None. + * @dpm_state: - the address of the DPM Table to initiailize. + * return: None. */ static void vega10_init_dpm_state(struct vega10_dpm_state *dpm_state) { @@ -1460,11 +1459,11 @@ static int vega10_setup_default_dpm_tables(struct pp_hwmgr *hwmgr) } /* - * @fn vega10_populate_ulv_state - * @brief Function to provide parameters for Utral Low Voltage state to SMC. + * vega10_populate_ulv_state + * Function to provide parameters for Utral Low Voltage state to SMC. * - * @param hwmgr - the address of the hardware manager. - * @return Always 0. + * @hwmgr: - the address of the hardware manager. + * return: Always 0. */ static int vega10_populate_ulv_state(struct pp_hwmgr *hwmgr) { @@ -1545,13 +1544,13 @@ static int vega10_populate_smc_link_levels(struct pp_hwmgr *hwmgr) } /** -* Populates single SMC GFXSCLK structure using the provided engine clock -* -* @param hwmgr the address of the hardware manager -* @param gfx_clock the GFX clock to use to populate the structure. -* @param current_gfxclk_level location in PPTable for the SMC GFXCLK structure. -*/ - + * Populates single SMC GFXSCLK structure using the provided engine clock + * + * @hwmgr: the address of the hardware manager + * @gfx_clock: the GFX clock to use to populate the structure. + * @current_gfxclk_level: location in PPTable for the SMC GFXCLK structure. + * @acg_freq: ACG frequenty to return (MHz) + */ static int vega10_populate_single_gfx_level(struct pp_hwmgr *hwmgr, uint32_t gfx_clock, PllSetting_t *current_gfxclk_level, uint32_t *acg_freq) @@ -1610,12 +1609,13 @@ static int vega10_populate_single_gfx_level(struct pp_hwmgr *hwmgr, } /** - * @brief Populates single SMC SOCCLK structure using the provided clock. + * Populates single SMC SOCCLK structure using the provided clock. * - * @param hwmgr - the address of the hardware manager. - * @param soc_clock - the SOC clock to use to populate the structure. - * @param current_socclk_level - location in PPTable for the SMC SOCCLK structure. - * @return 0 on success.. + * @hwmgr: the address of the hardware manager. + * @soc_clock: the SOC clock to use to populate the structure. + * @current_soc_did: DFS divider to pass back to caller + * @current_vol_index: index of current VDD to pass back to caller + * return: 0 on success */ static int vega10_populate_single_soc_level(struct pp_hwmgr *hwmgr, uint32_t soc_clock, uint8_t *current_soc_did, @@ -1659,10 +1659,10 @@ static int vega10_populate_single_soc_level(struct pp_hwmgr *hwmgr, } /** -* Populates all SMC SCLK levels' structure based on the trimmed allowed dpm engine clock states -* -* @param hwmgr the address of the hardware manager -*/ + * Populates all SMC SCLK levels' structure based on the trimmed allowed dpm engine clock states + * + * @hwmgr: the address of the hardware manager + */ static int vega10_populate_all_graphic_levels(struct pp_hwmgr *hwmgr) { struct vega10_hwmgr *data = hwmgr->backend; @@ -1746,12 +1746,12 @@ static void vega10_populate_vddc_soc_levels(struct pp_hwmgr *hwmgr) } } -/** - * @brief Populates single SMC GFXCLK structure using the provided clock. +/* + * Populates single SMC GFXCLK structure using the provided clock. * - * @param hwmgr - the address of the hardware manager. - * @param mem_clock - the memory clock to use to populate the structure. - * @return 0 on success.. + * @hwmgr: the address of the hardware manager. + * @mem_clock: the memory clock to use to populate the structure. + * return: 0 on success.. */ static int vega10_populate_single_memory_level(struct pp_hwmgr *hwmgr, uint32_t mem_clock, uint8_t *current_mem_vid, @@ -1808,10 +1808,10 @@ static int vega10_populate_single_memory_level(struct pp_hwmgr *hwmgr, } /** - * @brief Populates all SMC MCLK levels' structure based on the trimmed allowed dpm memory clock states. + * Populates all SMC MCLK levels' structure based on the trimmed allowed dpm memory clock states. * - * @param pHwMgr - the address of the hardware manager. - * @return PP_Result_OK on success. + * @hwmgr: the address of the hardware manager. + * return: PP_Result_OK on success. */ static int vega10_populate_all_memory_levels(struct pp_hwmgr *hwmgr) { @@ -2486,12 +2486,11 @@ static void vega10_check_dpm_table_updated(struct pp_hwmgr *hwmgr) } /** -* Initializes the SMC table and uploads it -* -* @param hwmgr the address of the powerplay hardware manager. -* @param pInput the pointer to input data (PowerState) -* @return always 0 -*/ + * Initializes the SMC table and uploads it + * + * @hwmgr: the address of the powerplay hardware manager. + * return: always 0 + */ static int vega10_init_smc_table(struct pp_hwmgr *hwmgr) { int result; @@ -2864,11 +2863,11 @@ static int vega10_stop_dpm(struct pp_hwmgr *hwmgr, uint32_t bitmap) } /** - * @brief Tell SMC to enabled the supported DPMs. + * Tell SMC to enabled the supported DPMs. * - * @param hwmgr - the address of the powerplay hardware manager. - * @Param bitmap - bitmap for the features to enabled. - * @return 0 on at least one DPM is successfully enabled. + * @hwmgr: the address of the powerplay hardware manager. + * @bitmap: bitmap for the features to enabled. + * return: 0 on at least one DPM is successfully enabled. */ static int vega10_start_dpm(struct pp_hwmgr *hwmgr, uint32_t bitmap) { diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_hwmgr.h b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_hwmgr.h index f752b4ad0c8ae76700996411ed78322444591459..07c06f8c90b093ed58577db19fb7c5537a9f75ab 100644 --- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_hwmgr.h +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_hwmgr.h @@ -442,5 +442,6 @@ int vega10_update_uvd_dpm(struct pp_hwmgr *hwmgr, bool bgate); int vega10_update_samu_dpm(struct pp_hwmgr *hwmgr, bool bgate); int vega10_update_acp_dpm(struct pp_hwmgr *hwmgr, bool bgate); int vega10_enable_disable_vce_dpm(struct pp_hwmgr *hwmgr, bool enable); +int vega10_hwmgr_init(struct pp_hwmgr *hwmgr); #endif /* _VEGA10_HWMGR_H_ */ diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega12_hwmgr.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega12_hwmgr.c index dc206fa88c5e539b7beb508b5240f77165fe5342..c0753029a8e2a9f6b1039ab906a2d354b047a463 100644 --- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega12_hwmgr.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega12_hwmgr.c @@ -718,12 +718,11 @@ static int vega12_save_default_power_profile(struct pp_hwmgr *hwmgr) #endif /** -* Initializes the SMC table and uploads it -* -* @param hwmgr the address of the powerplay hardware manager. -* @param pInput the pointer to input data (PowerState) -* @return always 0 -*/ + * Initializes the SMC table and uploads it + * + * @hwmgr: the address of the powerplay hardware manager. + * return: always 0 + */ static int vega12_init_smc_table(struct pp_hwmgr *hwmgr) { int result; diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega20_hwmgr.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega20_hwmgr.c index da84012b7fd5167c5883fb57ad164883c9b0f639..87811b005b85f314c3c9491533a6867fc3123791 100644 --- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega20_hwmgr.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega20_hwmgr.c @@ -771,12 +771,11 @@ static int vega20_setup_default_dpm_tables(struct pp_hwmgr *hwmgr) } /** -* Initializes the SMC table and uploads it -* -* @param hwmgr the address of the powerplay hardware manager. -* @param pInput the pointer to input data (PowerState) -* @return always 0 -*/ + * Initializes the SMC table and uploads it + * + * @hwmgr: the address of the powerplay hardware manager. + * return: always 0 + */ static int vega20_init_smc_table(struct pp_hwmgr *hwmgr) { int result; diff --git a/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c b/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c index e84c737e39673b35b429f85ea7076d320ebc1597..d143ef1b460bf6f887d0aaf49c97150f8a604445 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c +++ b/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c @@ -266,6 +266,119 @@ int smu_dpm_set_power_gate(struct smu_context *smu, uint32_t block_type, return ret; } +/** + * smu_set_user_clk_dependencies - set user profile clock dependencies + * + * @smu: smu_context pointer + * @clk: enum smu_clk_type type + * + * Enable/Disable the clock dependency for the @clk type. + */ +static void smu_set_user_clk_dependencies(struct smu_context *smu, enum smu_clk_type clk) +{ + if (smu->adev->in_suspend) + return; + + /* + * mclk, fclk and socclk are interdependent + * on each other + */ + if (clk == SMU_MCLK) { + /* reset clock dependency */ + smu->user_dpm_profile.clk_dependency = 0; + /* set mclk dependent clocks(fclk and socclk) */ + smu->user_dpm_profile.clk_dependency = BIT(SMU_FCLK) | BIT(SMU_SOCCLK); + } else if (clk == SMU_FCLK) { + /* give priority to mclk, if mclk dependent clocks are set */ + if (smu->user_dpm_profile.clk_dependency == (BIT(SMU_FCLK) | BIT(SMU_SOCCLK))) + return; + + /* reset clock dependency */ + smu->user_dpm_profile.clk_dependency = 0; + /* set fclk dependent clocks(mclk and socclk) */ + smu->user_dpm_profile.clk_dependency = BIT(SMU_MCLK) | BIT(SMU_SOCCLK); + } else if (clk == SMU_SOCCLK) { + /* give priority to mclk, if mclk dependent clocks are set */ + if (smu->user_dpm_profile.clk_dependency == (BIT(SMU_FCLK) | BIT(SMU_SOCCLK))) + return; + + /* reset clock dependency */ + smu->user_dpm_profile.clk_dependency = 0; + /* set socclk dependent clocks(mclk and fclk) */ + smu->user_dpm_profile.clk_dependency = BIT(SMU_MCLK) | BIT(SMU_FCLK); + } else + /* add clk dependencies here, if any */ + return; +} + +/** + * smu_restore_dpm_user_profile - reinstate user dpm profile + * + * @smu: smu_context pointer + * + * Restore the saved user power configurations include power limit, + * clock frequencies, fan control mode and fan speed. + */ +static void smu_restore_dpm_user_profile(struct smu_context *smu) +{ + struct smu_dpm_context *smu_dpm_ctx = &(smu->smu_dpm); + int ret = 0; + + if (!smu->adev->in_suspend) + return; + + if (!smu->pm_enabled || !smu->adev->pm.dpm_enabled) + return; + + /* Enable restore flag */ + smu->user_dpm_profile.flags = SMU_DPM_USER_PROFILE_RESTORE; + + /* set the user dpm power limit */ + if (smu->user_dpm_profile.power_limit) { + ret = smu_set_power_limit(smu, smu->user_dpm_profile.power_limit); + if (ret) + dev_err(smu->adev->dev, "Failed to set power limit value\n"); + } + + /* set the user dpm clock configurations */ + if (smu_dpm_ctx->dpm_level == AMD_DPM_FORCED_LEVEL_MANUAL) { + enum smu_clk_type clk_type; + + for (clk_type = 0; clk_type < SMU_CLK_COUNT; clk_type++) { + /* + * Iterate over smu clk type and force the saved user clk + * configs, skip if clock dependency is enabled + */ + if (!(smu->user_dpm_profile.clk_dependency & BIT(clk_type)) && + smu->user_dpm_profile.clk_mask[clk_type]) { + ret = smu_force_clk_levels(smu, clk_type, + smu->user_dpm_profile.clk_mask[clk_type]); + if (ret) + dev_err(smu->adev->dev, "Failed to set clock type = %d\n", + clk_type); + } + } + } + + /* set the user dpm fan configurations */ + if (smu->user_dpm_profile.fan_mode == AMD_FAN_CTRL_MANUAL) { + ret = smu_set_fan_control_mode(smu, smu->user_dpm_profile.fan_mode); + if (ret) { + dev_err(smu->adev->dev, "Failed to set manual fan control mode\n"); + return; + } + + if (!ret && smu->user_dpm_profile.fan_speed_percent) { + ret = smu_set_fan_speed_percent(smu, smu->user_dpm_profile.fan_speed_percent); + if (ret) + dev_err(smu->adev->dev, "Failed to set manual fan speed\n"); + } + } + + /* Disable restore flag */ + smu->user_dpm_profile.flags &= ~SMU_DPM_USER_PROFILE_RESTORE; +} + int smu_get_power_num_states(struct smu_context *smu, struct pp_states_info *state_info) { @@ -288,6 +401,20 @@ bool is_support_sw_smu(struct amdgpu_device *adev) return false; } +bool is_support_cclk_dpm(struct amdgpu_device *adev) +{ + struct smu_context *smu = &adev->smu; + + if (!is_support_sw_smu(adev)) + return false; + + if (!smu_feature_is_enabled(smu, SMU_FEATURE_CCLK_DPM_BIT)) + return false; + + return true; +} + + int smu_sys_get_pp_table(struct smu_context *smu, void **table) { struct smu_table_context *smu_table = &smu->smu_table; @@ -405,8 +532,6 @@ static int smu_set_funcs(struct amdgpu_device *adev) break; case CHIP_VANGOGH: vangogh_set_ppt_funcs(smu); - /* enable the OD by default to allow the fine grain tuning function */ - smu->od_enabled = true; break; default: return -EINVAL; @@ -478,9 +603,6 @@ static int smu_late_init(void *handle) smu_set_fine_grain_gfx_freq_parameters(smu); - if (adev->asic_type == CHIP_VANGOGH) - return 0; - if (!smu->pm_enabled) return 0; @@ -517,6 +639,8 @@ static int smu_late_init(void *handle) AMD_PP_TASK_COMPLETE_INIT, false); + smu_restore_dpm_user_profile(smu); + return 0; } @@ -1610,6 +1734,12 @@ int smu_force_performance_level(struct smu_context *smu, enum amd_dpm_forced_lev mutex_unlock(&smu->mutex); + /* reset user dpm clock state */ + if (!ret && smu_dpm_ctx->dpm_level != AMD_DPM_FORCED_LEVEL_MANUAL) { + memset(smu->user_dpm_profile.clk_mask, 0, sizeof(smu->user_dpm_profile.clk_mask)); + smu->user_dpm_profile.clk_dependency = 0; + } + return ret; } @@ -1644,8 +1774,13 @@ int smu_force_clk_levels(struct smu_context *smu, mutex_lock(&smu->mutex); - if (smu->ppt_funcs && smu->ppt_funcs->force_clk_levels) + if (smu->ppt_funcs && smu->ppt_funcs->force_clk_levels) { ret = smu->ppt_funcs->force_clk_levels(smu, clk_type, mask); + if (!ret && smu->user_dpm_profile.flags != SMU_DPM_USER_PROFILE_RESTORE) { + smu->user_dpm_profile.clk_mask[clk_type] = mask; + smu_set_user_clk_dependencies(smu, clk_type); + } + } mutex_unlock(&smu->mutex); @@ -1887,6 +2022,7 @@ int smu_set_gfx_cgpg(struct smu_context *smu, bool enabled) int smu_set_fan_speed_rpm(struct smu_context *smu, uint32_t speed) { + u32 percent; int ret = 0; if (!smu->pm_enabled || !smu->adev->pm.dpm_enabled) @@ -1894,8 +2030,12 @@ int smu_set_fan_speed_rpm(struct smu_context *smu, uint32_t speed) mutex_lock(&smu->mutex); - if (smu->ppt_funcs->set_fan_speed_rpm) - ret = smu->ppt_funcs->set_fan_speed_rpm(smu, speed); + if (smu->ppt_funcs->set_fan_speed_percent) { + percent = speed * 100 / smu->fan_max_rpm; + ret = smu->ppt_funcs->set_fan_speed_percent(smu, percent); + if (!ret && smu->user_dpm_profile.flags != SMU_DPM_USER_PROFILE_RESTORE) + smu->user_dpm_profile.fan_speed_percent = percent; + } mutex_unlock(&smu->mutex); @@ -1904,22 +2044,40 @@ int smu_set_fan_speed_rpm(struct smu_context *smu, uint32_t speed) int smu_get_power_limit(struct smu_context *smu, uint32_t *limit, - bool max_setting) + enum smu_ppt_limit_level limit_level) { + uint32_t limit_type = *limit >> 24; + int ret = 0; + if (!smu->pm_enabled || !smu->adev->pm.dpm_enabled) return -EOPNOTSUPP; mutex_lock(&smu->mutex); - *limit = (max_setting ? smu->max_power_limit : smu->current_power_limit); + if (limit_type != SMU_DEFAULT_PPT_LIMIT) { + if (smu->ppt_funcs->get_ppt_limit) + ret = smu->ppt_funcs->get_ppt_limit(smu, limit, limit_type, limit_level); + } else { + switch (limit_level) { + case SMU_PPT_LIMIT_CURRENT: + *limit = smu->current_power_limit; + break; + case SMU_PPT_LIMIT_MAX: + *limit = smu->max_power_limit; + break; + default: + break; + } + } mutex_unlock(&smu->mutex); - return 0; + return ret; } int smu_set_power_limit(struct smu_context *smu, uint32_t limit) { + uint32_t limit_type = limit >> 24; int ret = 0; if (!smu->pm_enabled || !smu->adev->pm.dpm_enabled) @@ -1927,6 +2085,12 @@ int smu_set_power_limit(struct smu_context *smu, uint32_t limit) mutex_lock(&smu->mutex); + if (limit_type != SMU_DEFAULT_PPT_LIMIT) + if (smu->ppt_funcs->set_power_limit) { + ret = smu->ppt_funcs->set_power_limit(smu, limit); + goto out; + } + if (limit > smu->max_power_limit) { dev_err(smu->adev->dev, "New power limit (%d) is over the max allowed %d\n", @@ -1937,8 +2101,11 @@ int smu_set_power_limit(struct smu_context *smu, uint32_t limit) if (!limit) limit = smu->current_power_limit; - if (smu->ppt_funcs->set_power_limit) + if (smu->ppt_funcs->set_power_limit) { ret = smu->ppt_funcs->set_power_limit(smu, limit); + if (!ret && smu->user_dpm_profile.flags != SMU_DPM_USER_PROFILE_RESTORE) + smu->user_dpm_profile.power_limit = limit; + } out: mutex_unlock(&smu->mutex); @@ -2115,11 +2282,19 @@ int smu_set_fan_control_mode(struct smu_context *smu, int value) mutex_lock(&smu->mutex); - if (smu->ppt_funcs->set_fan_control_mode) + if (smu->ppt_funcs->set_fan_control_mode) { ret = smu->ppt_funcs->set_fan_control_mode(smu, value); + if (!ret && smu->user_dpm_profile.flags != SMU_DPM_USER_PROFILE_RESTORE) + smu->user_dpm_profile.fan_mode = value; + } mutex_unlock(&smu->mutex); + /* reset user dpm fan speed */ + if (!ret && value != AMD_FAN_CTRL_MANUAL && + smu->user_dpm_profile.flags != SMU_DPM_USER_PROFILE_RESTORE) + smu->user_dpm_profile.fan_speed_percent = 0; + return ret; } @@ -2127,17 +2302,15 @@ int smu_get_fan_speed_percent(struct smu_context *smu, uint32_t *speed) { int ret = 0; uint32_t percent; - uint32_t current_rpm; if (!smu->pm_enabled || !smu->adev->pm.dpm_enabled) return -EOPNOTSUPP; mutex_lock(&smu->mutex); - if (smu->ppt_funcs->get_fan_speed_rpm) { - ret = smu->ppt_funcs->get_fan_speed_rpm(smu, ¤t_rpm); + if (smu->ppt_funcs->get_fan_speed_percent) { + ret = smu->ppt_funcs->get_fan_speed_percent(smu, &percent); if (!ret) { - percent = current_rpm * 100 / smu->fan_max_rpm; *speed = percent > 100 ? 100 : percent; } } @@ -2157,8 +2330,13 @@ int smu_set_fan_speed_percent(struct smu_context *smu, uint32_t speed) mutex_lock(&smu->mutex); - if (smu->ppt_funcs->set_fan_speed_percent) + if (smu->ppt_funcs->set_fan_speed_percent) { + if (speed > 100) + speed = 100; ret = smu->ppt_funcs->set_fan_speed_percent(smu, speed); + if (!ret && smu->user_dpm_profile.flags != SMU_DPM_USER_PROFILE_RESTORE) + smu->user_dpm_profile.fan_speed_percent = speed; + } mutex_unlock(&smu->mutex); @@ -2168,14 +2346,17 @@ int smu_set_fan_speed_percent(struct smu_context *smu, uint32_t speed) int smu_get_fan_speed_rpm(struct smu_context *smu, uint32_t *speed) { int ret = 0; + u32 percent; if (!smu->pm_enabled || !smu->adev->pm.dpm_enabled) return -EOPNOTSUPP; mutex_lock(&smu->mutex); - if (smu->ppt_funcs->get_fan_speed_rpm) - ret = smu->ppt_funcs->get_fan_speed_rpm(smu, speed); + if (smu->ppt_funcs->get_fan_speed_percent) { + ret = smu->ppt_funcs->get_fan_speed_percent(smu, &percent); + *speed = percent * smu->fan_max_rpm / 100; + } mutex_unlock(&smu->mutex); diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu11/arcturus_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu11/arcturus_ppt.c index 16db0b506b0de0a6ee3be4a80f385e861169e8c5..45564a776e9b2228d48ba299c1123a8ef343ae1c 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu11/arcturus_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu11/arcturus_ppt.c @@ -1080,15 +1080,27 @@ static int arcturus_read_sensor(struct smu_context *smu, return ret; } -static int arcturus_get_fan_speed_rpm(struct smu_context *smu, - uint32_t *speed) +static int arcturus_get_fan_speed_percent(struct smu_context *smu, + uint32_t *speed) { + int ret; + u32 rpm; + if (!speed) return -EINVAL; - return arcturus_get_smu_metrics_data(smu, - METRICS_CURR_FANSPEED, - speed); + switch (smu_v11_0_get_fan_control_mode(smu)) { + case AMD_FAN_CTRL_AUTO: + ret = arcturus_get_smu_metrics_data(smu, + METRICS_CURR_FANSPEED, + &rpm); + if (!ret && smu->fan_max_rpm) + *speed = rpm * 100 / smu->fan_max_rpm; + return ret; + default: + *speed = smu->user_dpm_profile.fan_speed_percent; + return 0; + } } static int arcturus_get_fan_parameters(struct smu_context *smu) @@ -2227,7 +2239,7 @@ static ssize_t arcturus_get_gpu_metrics(struct smu_context *smu, if (ret) return ret; - smu_v11_0_init_gpu_metrics_v1_0(gpu_metrics); + smu_cmn_init_soft_gpu_metrics(gpu_metrics, 1, 0); gpu_metrics->temperature_edge = metrics.TemperatureEdge; gpu_metrics->temperature_hotspot = metrics.TemperatureHotspot; @@ -2264,6 +2276,8 @@ static ssize_t arcturus_get_gpu_metrics(struct smu_context *smu, gpu_metrics->pcie_link_speed = arcturus_get_current_pcie_link_speed(smu); + gpu_metrics->system_clock_counter = ktime_get_boottime_ns(); + *table = (void *)gpu_metrics; return sizeof(struct gpu_metrics_v1_0); @@ -2281,7 +2295,7 @@ static const struct pptable_funcs arcturus_ppt_funcs = { .print_clk_levels = arcturus_print_clk_levels, .force_clk_levels = arcturus_force_clk_levels, .read_sensor = arcturus_read_sensor, - .get_fan_speed_rpm = arcturus_get_fan_speed_rpm, + .get_fan_speed_percent = arcturus_get_fan_speed_percent, .get_power_profile_mode = arcturus_get_power_profile_mode, .set_power_profile_mode = arcturus_set_power_profile_mode, .set_performance_level = arcturus_set_performance_level, @@ -2327,7 +2341,6 @@ static const struct pptable_funcs arcturus_ppt_funcs = { .get_fan_control_mode = smu_v11_0_get_fan_control_mode, .set_fan_control_mode = smu_v11_0_set_fan_control_mode, .set_fan_speed_percent = smu_v11_0_set_fan_speed_percent, - .set_fan_speed_rpm = smu_v11_0_set_fan_speed_rpm, .set_xgmi_pstate = smu_v11_0_set_xgmi_pstate, .gfx_off_control = smu_v11_0_gfx_off_control, .register_irq_handler = smu_v11_0_register_irq_handler, diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu11/navi10_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu11/navi10_ppt.c index cd7efa923195e3fa2a94414adf096fbfc1eccd0b..6e641f1513d80b0b48a2fce00dfb9485c79ae5a7 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu11/navi10_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu11/navi10_ppt.c @@ -1317,15 +1317,27 @@ static bool navi10_is_dpm_running(struct smu_context *smu) return !!(feature_enabled & SMC_DPM_FEATURE); } -static int navi10_get_fan_speed_rpm(struct smu_context *smu, - uint32_t *speed) +static int navi10_get_fan_speed_percent(struct smu_context *smu, + uint32_t *speed) { + int ret; + u32 rpm; + if (!speed) return -EINVAL; - return navi10_get_smu_metrics_data(smu, - METRICS_CURR_FANSPEED, - speed); + switch (smu_v11_0_get_fan_control_mode(smu)) { + case AMD_FAN_CTRL_AUTO: + ret = navi10_get_smu_metrics_data(smu, + METRICS_CURR_FANSPEED, + &rpm); + if (!ret && smu->fan_max_rpm) + *speed = rpm * 100 / smu->fan_max_rpm; + return ret; + default: + *speed = smu->user_dpm_profile.fan_speed_percent; + return 0; + } } static int navi10_get_fan_parameters(struct smu_context *smu) @@ -1673,7 +1685,7 @@ static int navi10_read_sensor(struct smu_context *smu, *size = 4; break; case AMDGPU_PP_SENSOR_GFX_SCLK: - ret = navi10_get_current_clk_freq_by_table(smu, SMU_GFXCLK, (uint32_t *)data); + ret = navi10_get_smu_metrics_data(smu, METRICS_AVERAGE_GFXCLK, (uint32_t *)data); *(uint32_t *)data *= 100; *size = 4; break; @@ -2302,7 +2314,7 @@ static ssize_t navi10_get_gpu_metrics(struct smu_context *smu, mutex_unlock(&smu->metrics_lock); - smu_v11_0_init_gpu_metrics_v1_0(gpu_metrics); + smu_cmn_init_soft_gpu_metrics(gpu_metrics, 1, 0); gpu_metrics->temperature_edge = metrics.TemperatureEdge; gpu_metrics->temperature_hotspot = metrics.TemperatureHotspot; @@ -2342,6 +2354,8 @@ static ssize_t navi10_get_gpu_metrics(struct smu_context *smu, gpu_metrics->pcie_link_speed = smu_v11_0_get_current_pcie_link_speed(smu); + gpu_metrics->system_clock_counter = ktime_get_boottime_ns(); + *table = (void *)gpu_metrics; return sizeof(struct gpu_metrics_v1_0); @@ -2413,7 +2427,7 @@ static const struct pptable_funcs navi10_ppt_funcs = { .display_config_changed = navi10_display_config_changed, .notify_smc_display_config = navi10_notify_smc_display_config, .is_dpm_running = navi10_is_dpm_running, - .get_fan_speed_rpm = navi10_get_fan_speed_rpm, + .get_fan_speed_percent = navi10_get_fan_speed_percent, .get_power_profile_mode = navi10_get_power_profile_mode, .set_power_profile_mode = navi10_set_power_profile_mode, .set_watermarks_table = navi10_set_watermarks_table, @@ -2457,7 +2471,6 @@ static const struct pptable_funcs navi10_ppt_funcs = { .get_fan_control_mode = smu_v11_0_get_fan_control_mode, .set_fan_control_mode = smu_v11_0_set_fan_control_mode, .set_fan_speed_percent = smu_v11_0_set_fan_speed_percent, - .set_fan_speed_rpm = smu_v11_0_set_fan_speed_rpm, .set_xgmi_pstate = smu_v11_0_set_xgmi_pstate, .gfx_off_control = smu_v11_0_gfx_off_control, .register_irq_handler = smu_v11_0_register_irq_handler, diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.c index d68d3dfee51d47c0c92c364c4495c332c7e37439..af73e1430af535beefff194d4140c8a3af4ca042 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.c @@ -261,6 +261,11 @@ sienna_cichlid_get_allowed_feature_mask(struct smu_context *smu, *(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_DPM_GFX_GPO_BIT); } + if ((adev->pm.pp_feature & PP_GFX_DCS_MASK) && + (adev->asic_type > CHIP_SIENNA_CICHLID) && + !(adev->flags & AMD_IS_APU)) + *(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_GFX_DCS_BIT); + if (adev->pm.pp_feature & PP_MCLK_DPM_MASK) *(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_DPM_UCLK_BIT) | FEATURE_MASK(FEATURE_MEM_VDDCI_SCALING_BIT) @@ -294,6 +299,12 @@ sienna_cichlid_get_allowed_feature_mask(struct smu_context *smu, smu->adev->pg_flags & AMD_PG_SUPPORT_JPEG) *(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_MM_DPM_PG_BIT); + if (smu->dc_controlled_by_gpio) + *(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_ACDC_BIT); + + if (amdgpu_aspm == 1) + *(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_DS_LCLK_BIT); + return 0; } @@ -314,6 +325,12 @@ static int sienna_cichlid_check_powerplay_table(struct smu_context *smu) table_context->thermal_controller_type = powerplay_table->thermal_controller_type; + /* + * Instead of having its own buffer space and get overdrive_table copied, + * smu->od_settings just points to the actual overdrive_table + */ + smu->od_settings = &powerplay_table->overdrive_table; + return 0; } @@ -907,6 +924,22 @@ static bool sienna_cichlid_is_support_fine_grained_dpm(struct smu_context *smu, return dpm_desc->SnapToDiscrete == 0 ? true : false; } +static bool sienna_cichlid_is_od_feature_supported(struct smu_11_0_7_overdrive_table *od_table, + enum SMU_11_0_7_ODFEATURE_CAP cap) +{ + return od_table->cap[cap]; +} + +static void sienna_cichlid_get_od_setting_range(struct smu_11_0_7_overdrive_table *od_table, + enum SMU_11_0_7_ODSETTING_ID setting, + uint32_t *min, uint32_t *max) +{ + if (min) + *min = od_table->min[setting]; + if (max) + *max = od_table->max[setting]; +} + static int sienna_cichlid_print_clk_levels(struct smu_context *smu, enum smu_clk_type clk_type, char *buf) { @@ -915,11 +948,16 @@ static int sienna_cichlid_print_clk_levels(struct smu_context *smu, struct smu_dpm_context *smu_dpm = &smu->smu_dpm; struct smu_11_0_dpm_context *dpm_context = smu_dpm->dpm_context; PPTable_t *pptable = (PPTable_t *)table_context->driver_pptable; + struct smu_11_0_7_overdrive_table *od_settings = smu->od_settings; + OverDriveTable_t *od_table = + (OverDriveTable_t *)table_context->overdrive_table; int i, size = 0, ret = 0; uint32_t cur_value = 0, value = 0, count = 0; uint32_t freq_values[3] = {0}; uint32_t mark_index = 0; uint32_t gen_speed, lane_width; + uint32_t min_value, max_value; + uint32_t smu_version; switch (clk_type) { case SMU_GFXCLK: @@ -995,6 +1033,70 @@ static int sienna_cichlid_print_clk_levels(struct smu_context *smu, (lane_width == dpm_context->dpm_tables.pcie_table.pcie_lane[i]) ? "*" : ""); break; + case SMU_OD_SCLK: + if (!smu->od_enabled || !od_table || !od_settings) + break; + + if (!sienna_cichlid_is_od_feature_supported(od_settings, SMU_11_0_7_ODCAP_GFXCLK_LIMITS)) + break; + + size += sprintf(buf + size, "OD_SCLK:\n"); + size += sprintf(buf + size, "0: %uMhz\n1: %uMhz\n", od_table->GfxclkFmin, od_table->GfxclkFmax); + break; + + case SMU_OD_MCLK: + if (!smu->od_enabled || !od_table || !od_settings) + break; + + if (!sienna_cichlid_is_od_feature_supported(od_settings, SMU_11_0_7_ODCAP_UCLK_LIMITS)) + break; + + size += sprintf(buf + size, "OD_MCLK:\n"); + size += sprintf(buf + size, "0: %uMhz\n1: %uMHz\n", od_table->UclkFmin, od_table->UclkFmax); + break; + + case SMU_OD_VDDGFX_OFFSET: + if (!smu->od_enabled || !od_table || !od_settings) + break; + + /* + * OD GFX Voltage Offset functionality is supported only by 58.41.0 + * and onwards SMU firmwares. + */ + smu_cmn_get_smc_version(smu, NULL, &smu_version); + if ((adev->asic_type == CHIP_SIENNA_CICHLID) && + (smu_version < 0x003a2900)) + break; + + size += sprintf(buf + size, "OD_VDDGFX_OFFSET:\n"); + size += sprintf(buf + size, "%dmV\n", od_table->VddGfxOffset); + break; + + case SMU_OD_RANGE: + if (!smu->od_enabled || !od_table || !od_settings) + break; + + size = sprintf(buf, "%s:\n", "OD_RANGE"); + + if (sienna_cichlid_is_od_feature_supported(od_settings, SMU_11_0_7_ODCAP_GFXCLK_LIMITS)) { + sienna_cichlid_get_od_setting_range(od_settings, SMU_11_0_7_ODSETTING_GFXCLKFMIN, + &min_value, NULL); + sienna_cichlid_get_od_setting_range(od_settings, SMU_11_0_7_ODSETTING_GFXCLKFMAX, + NULL, &max_value); + size += sprintf(buf + size, "SCLK: %7uMhz %10uMhz\n", + min_value, max_value); + } + + if (sienna_cichlid_is_od_feature_supported(od_settings, SMU_11_0_7_ODCAP_UCLK_LIMITS)) { + sienna_cichlid_get_od_setting_range(od_settings, SMU_11_0_7_ODSETTING_UCLKFMIN, + &min_value, NULL); + sienna_cichlid_get_od_setting_range(od_settings, SMU_11_0_7_ODSETTING_UCLKFMAX, + NULL, &max_value); + size += sprintf(buf + size, "MCLK: %7uMhz %10uMhz\n", + min_value, max_value); + } + break; + default: break; } @@ -1146,15 +1248,27 @@ static bool sienna_cichlid_is_dpm_running(struct smu_context *smu) return !!(feature_enabled & SMC_DPM_FEATURE); } -static int sienna_cichlid_get_fan_speed_rpm(struct smu_context *smu, - uint32_t *speed) +static int sienna_cichlid_get_fan_speed_percent(struct smu_context *smu, + uint32_t *speed) { + int ret; + u32 rpm; + if (!speed) return -EINVAL; - return sienna_cichlid_get_smu_metrics_data(smu, - METRICS_CURR_FANSPEED, - speed); + switch (smu_v11_0_get_fan_control_mode(smu)) { + case AMD_FAN_CTRL_AUTO: + ret = sienna_cichlid_get_smu_metrics_data(smu, + METRICS_CURR_FANSPEED, + &rpm); + if (!ret && smu->fan_max_rpm) + *speed = rpm * 100 / smu->fan_max_rpm; + return ret; + default: + *speed = smu->user_dpm_profile.fan_speed_percent; + return 0; + } } static int sienna_cichlid_get_fan_parameters(struct smu_context *smu) @@ -1694,6 +1808,243 @@ static int sienna_cichlid_get_dpm_ultimate_freq(struct smu_context *smu, return ret; } +static void sienna_cichlid_dump_od_table(struct smu_context *smu, + OverDriveTable_t *od_table) +{ + struct amdgpu_device *adev = smu->adev; + uint32_t smu_version; + + dev_dbg(smu->adev->dev, "OD: Gfxclk: (%d, %d)\n", od_table->GfxclkFmin, + od_table->GfxclkFmax); + dev_dbg(smu->adev->dev, "OD: Uclk: (%d, %d)\n", od_table->UclkFmin, + od_table->UclkFmax); + + smu_cmn_get_smc_version(smu, NULL, &smu_version); + if (!((adev->asic_type == CHIP_SIENNA_CICHLID) && + (smu_version < 0x003a2900))) + dev_dbg(smu->adev->dev, "OD: VddGfxOffset: %d\n", od_table->VddGfxOffset); +} + +static int sienna_cichlid_set_default_od_settings(struct smu_context *smu) +{ + OverDriveTable_t *od_table = + (OverDriveTable_t *)smu->smu_table.overdrive_table; + OverDriveTable_t *boot_od_table = + (OverDriveTable_t *)smu->smu_table.boot_overdrive_table; + int ret = 0; + + ret = smu_cmn_update_table(smu, SMU_TABLE_OVERDRIVE, + 0, (void *)od_table, false); + if (ret) { + dev_err(smu->adev->dev, "Failed to get overdrive table!\n"); + return ret; + } + + memcpy(boot_od_table, od_table, sizeof(OverDriveTable_t)); + + sienna_cichlid_dump_od_table(smu, od_table); + + return 0; +} + +static int sienna_cichlid_od_setting_check_range(struct smu_context *smu, + struct smu_11_0_7_overdrive_table *od_table, + enum SMU_11_0_7_ODSETTING_ID setting, + uint32_t value) +{ + if (value < od_table->min[setting]) { + dev_warn(smu->adev->dev, "OD setting (%d, %d) is less than the minimum allowed (%d)\n", + setting, value, od_table->min[setting]); + return -EINVAL; + } + if (value > od_table->max[setting]) { + dev_warn(smu->adev->dev, "OD setting (%d, %d) is greater than the maximum allowed (%d)\n", + setting, value, od_table->max[setting]); + return -EINVAL; + } + + return 0; +} + +static int sienna_cichlid_od_edit_dpm_table(struct smu_context *smu, + enum PP_OD_DPM_TABLE_COMMAND type, + long input[], uint32_t size) +{ + struct smu_table_context *table_context = &smu->smu_table; + OverDriveTable_t *od_table = + (OverDriveTable_t *)table_context->overdrive_table; + struct smu_11_0_7_overdrive_table *od_settings = + (struct smu_11_0_7_overdrive_table *)smu->od_settings; + struct amdgpu_device *adev = smu->adev; + enum SMU_11_0_7_ODSETTING_ID freq_setting; + uint16_t *freq_ptr; + int i, ret = 0; + uint32_t smu_version; + + if (!smu->od_enabled) { + dev_warn(smu->adev->dev, "OverDrive is not enabled!\n"); + return -EINVAL; + } + + if (!smu->od_settings) { + dev_err(smu->adev->dev, "OD board limits are not set!\n"); + return -ENOENT; + } + + if (!(table_context->overdrive_table && table_context->boot_overdrive_table)) { + dev_err(smu->adev->dev, "Overdrive table was not initialized!\n"); + return -EINVAL; + } + + switch (type) { + case PP_OD_EDIT_SCLK_VDDC_TABLE: + if (!sienna_cichlid_is_od_feature_supported(od_settings, + SMU_11_0_7_ODCAP_GFXCLK_LIMITS)) { + dev_warn(smu->adev->dev, "GFXCLK_LIMITS not supported!\n"); + return -ENOTSUPP; + } + + for (i = 0; i < size; i += 2) { + if (i + 2 > size) { + dev_info(smu->adev->dev, "invalid number of input parameters %d\n", size); + return -EINVAL; + } + + switch (input[i]) { + case 0: + if (input[i + 1] > od_table->GfxclkFmax) { + dev_info(smu->adev->dev, "GfxclkFmin (%ld) must be <= GfxclkFmax (%u)!\n", + input[i + 1], od_table->GfxclkFmax); + return -EINVAL; + } + + freq_setting = SMU_11_0_7_ODSETTING_GFXCLKFMIN; + freq_ptr = &od_table->GfxclkFmin; + break; + + case 1: + if (input[i + 1] < od_table->GfxclkFmin) { + dev_info(smu->adev->dev, "GfxclkFmax (%ld) must be >= GfxclkFmin (%u)!\n", + input[i + 1], od_table->GfxclkFmin); + return -EINVAL; + } + + freq_setting = SMU_11_0_7_ODSETTING_GFXCLKFMAX; + freq_ptr = &od_table->GfxclkFmax; + break; + + default: + dev_info(smu->adev->dev, "Invalid SCLK_VDDC_TABLE index: %ld\n", input[i]); + dev_info(smu->adev->dev, "Supported indices: [0:min,1:max]\n"); + return -EINVAL; + } + + ret = sienna_cichlid_od_setting_check_range(smu, od_settings, + freq_setting, input[i + 1]); + if (ret) + return ret; + + *freq_ptr = (uint16_t)input[i + 1]; + } + break; + + case PP_OD_EDIT_MCLK_VDDC_TABLE: + if (!sienna_cichlid_is_od_feature_supported(od_settings, SMU_11_0_7_ODCAP_UCLK_LIMITS)) { + dev_warn(smu->adev->dev, "UCLK_LIMITS not supported!\n"); + return -ENOTSUPP; + } + + for (i = 0; i < size; i += 2) { + if (i + 2 > size) { + dev_info(smu->adev->dev, "invalid number of input parameters %d\n", size); + return -EINVAL; + } + + switch (input[i]) { + case 0: + if (input[i + 1] > od_table->UclkFmax) { + dev_info(smu->adev->dev, "UclkFmin (%ld) must be <= UclkFmax (%u)!\n", + input[i + 1], od_table->UclkFmax); + return -EINVAL; + } + + freq_setting = SMU_11_0_7_ODSETTING_UCLKFMIN; + freq_ptr = &od_table->UclkFmin; + break; + + case 1: + if (input[i + 1] < od_table->UclkFmin) { + dev_info(smu->adev->dev, "UclkFmax (%ld) must be >= UclkFmin (%u)!\n", + input[i + 1], od_table->UclkFmin); + return -EINVAL; + } + + freq_setting = SMU_11_0_7_ODSETTING_UCLKFMAX; + freq_ptr = &od_table->UclkFmax; + break; + + default: + dev_info(smu->adev->dev, "Invalid MCLK_VDDC_TABLE index: %ld\n", input[i]); + dev_info(smu->adev->dev, "Supported indices: [0:min,1:max]\n"); + return -EINVAL; + } + + ret = sienna_cichlid_od_setting_check_range(smu, od_settings, + freq_setting, input[i + 1]); + if (ret) + return ret; + + *freq_ptr = (uint16_t)input[i + 1]; + } + break; + + case PP_OD_RESTORE_DEFAULT_TABLE: + memcpy(table_context->overdrive_table, + table_context->boot_overdrive_table, + sizeof(OverDriveTable_t)); + fallthrough; + + case PP_OD_COMMIT_DPM_TABLE: + sienna_cichlid_dump_od_table(smu, od_table); + + ret = smu_cmn_update_table(smu, SMU_TABLE_OVERDRIVE, + 0, (void *)od_table, true); + if (ret) { + dev_err(smu->adev->dev, "Failed to import overdrive table!\n"); + return ret; + } + break; + + case PP_OD_EDIT_VDDGFX_OFFSET: + if (size != 1) { + dev_info(smu->adev->dev, "invalid number of parameters: %d\n", size); + return -EINVAL; + } + + /* + * OD GFX Voltage Offset functionality is supported only by 58.41.0 + * and onwards SMU firmwares. + */ + smu_cmn_get_smc_version(smu, NULL, &smu_version); + if ((adev->asic_type == CHIP_SIENNA_CICHLID) && + (smu_version < 0x003a2900)) { + dev_err(smu->adev->dev, "OD GFX Voltage offset functionality is supported " + "only by 58.41.0 and onwards SMU firmwares!\n"); + return -EOPNOTSUPP; + } + + od_table->VddGfxOffset = (int16_t)input[0]; + + sienna_cichlid_dump_od_table(smu, od_table); + break; + + default: + return -ENOSYS; + } + + return ret; +} + static int sienna_cichlid_run_btc(struct smu_context *smu) { return smu_cmn_send_smc_msg(smu, SMU_MSG_RunDcBtc, NULL); @@ -2610,7 +2961,7 @@ static ssize_t sienna_cichlid_get_gpu_metrics(struct smu_context *smu, if (ret) return ret; - smu_v11_0_init_gpu_metrics_v1_0(gpu_metrics); + smu_cmn_init_soft_gpu_metrics(gpu_metrics, 1, 0); gpu_metrics->temperature_edge = metrics->TemperatureEdge; gpu_metrics->temperature_hotspot = metrics->TemperatureHotspot; @@ -2653,6 +3004,8 @@ static ssize_t sienna_cichlid_get_gpu_metrics(struct smu_context *smu, gpu_metrics->pcie_link_speed = smu_v11_0_get_current_pcie_link_speed(smu); + gpu_metrics->system_clock_counter = ktime_get_boottime_ns(); + *table = (void *)gpu_metrics; return sizeof(struct gpu_metrics_v1_0); @@ -2759,7 +3112,7 @@ static const struct pptable_funcs sienna_cichlid_ppt_funcs = { .display_config_changed = sienna_cichlid_display_config_changed, .notify_smc_display_config = sienna_cichlid_notify_smc_display_config, .is_dpm_running = sienna_cichlid_is_dpm_running, - .get_fan_speed_rpm = sienna_cichlid_get_fan_speed_rpm, + .get_fan_speed_percent = sienna_cichlid_get_fan_speed_percent, .get_power_profile_mode = sienna_cichlid_get_power_profile_mode, .set_power_profile_mode = sienna_cichlid_set_power_profile_mode, .set_watermarks_table = sienna_cichlid_set_watermarks_table, @@ -2803,7 +3156,6 @@ static const struct pptable_funcs sienna_cichlid_ppt_funcs = { .get_fan_control_mode = smu_v11_0_get_fan_control_mode, .set_fan_control_mode = smu_v11_0_set_fan_control_mode, .set_fan_speed_percent = smu_v11_0_set_fan_speed_percent, - .set_fan_speed_rpm = smu_v11_0_set_fan_speed_rpm, .set_xgmi_pstate = smu_v11_0_set_xgmi_pstate, .gfx_off_control = smu_v11_0_gfx_off_control, .register_irq_handler = smu_v11_0_register_irq_handler, @@ -2818,6 +3170,8 @@ static const struct pptable_funcs sienna_cichlid_ppt_funcs = { .mode1_reset = smu_v11_0_mode1_reset, .get_dpm_ultimate_freq = sienna_cichlid_get_dpm_ultimate_freq, .set_soft_freq_limited_range = smu_v11_0_set_soft_freq_limited_range, + .set_default_od_settings = sienna_cichlid_set_default_od_settings, + .od_edit_dpm_table = sienna_cichlid_od_edit_dpm_table, .run_btc = sienna_cichlid_run_btc, .set_power_source = smu_v11_0_set_power_source, .get_pp_feature_mask = smu_cmn_get_pp_feature_mask, diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c b/drivers/gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c index 5aeb5f5a04478a04f75dc94bfcee685a672c3ee5..90585461a56e9c17cf6ccb873bdfcb06868be694 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c @@ -474,12 +474,14 @@ int smu_v11_0_fini_smc_tables(struct smu_context *smu) int smu_v11_0_init_power(struct smu_context *smu) { struct smu_power_context *smu_power = &smu->smu_power; + size_t size = smu->adev->asic_type == CHIP_VANGOGH ? + sizeof(struct smu_11_5_power_context) : + sizeof(struct smu_11_0_power_context); - smu_power->power_context = kzalloc(sizeof(struct smu_11_0_power_context), - GFP_KERNEL); + smu_power->power_context = kzalloc(size, GFP_KERNEL); if (!smu_power->power_context) return -ENOMEM; - smu_power->power_context_size = sizeof(struct smu_11_0_power_context); + smu_power->power_context_size = size; return 0; } @@ -1119,6 +1121,7 @@ int smu_v11_0_gfx_off_control(struct smu_context *smu, bool enable) case CHIP_SIENNA_CICHLID: case CHIP_NAVY_FLOUNDER: case CHIP_DIMGREY_CAVEFISH: + case CHIP_VANGOGH: if (!(adev->pm.pp_feature & PP_GFXOFF_MASK)) return 0; if (enable) @@ -1136,10 +1139,10 @@ int smu_v11_0_gfx_off_control(struct smu_context *smu, bool enable) uint32_t smu_v11_0_get_fan_control_mode(struct smu_context *smu) { - if (!smu_cmn_feature_is_enabled(smu, SMU_FEATURE_FAN_CONTROL_BIT)) - return AMD_FAN_CTRL_MANUAL; - else + if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_FAN_CONTROL_BIT)) return AMD_FAN_CTRL_AUTO; + else + return smu->user_dpm_profile.fan_mode; } static int @@ -1230,58 +1233,6 @@ smu_v11_0_set_fan_control_mode(struct smu_context *smu, return ret; } -int smu_v11_0_set_fan_speed_rpm(struct smu_context *smu, - uint32_t speed) -{ - struct amdgpu_device *adev = smu->adev; - int ret; - uint32_t tach_period, crystal_clock_freq; - - if (!speed) - return -EINVAL; - - ret = smu_v11_0_auto_fan_control(smu, 0); - if (ret) - return ret; - - /* - * crystal_clock_freq div by 4 is required since the fan control - * module refers to 25MHz - */ - - crystal_clock_freq = amdgpu_asic_get_xclk(adev) / 4; - tach_period = 60 * crystal_clock_freq * 10000 / (8 * speed); - WREG32_SOC15(THM, 0, mmCG_TACH_CTRL, - REG_SET_FIELD(RREG32_SOC15(THM, 0, mmCG_TACH_CTRL), - CG_TACH_CTRL, TARGET_PERIOD, - tach_period)); - - ret = smu_v11_0_set_fan_static_mode(smu, FDO_PWM_MODE_STATIC_RPM); - - return ret; -} - -int smu_v11_0_get_fan_speed_rpm(struct smu_context *smu, - uint32_t *speed) -{ - struct amdgpu_device *adev = smu->adev; - uint32_t tach_period, crystal_clock_freq; - uint64_t tmp64; - - tach_period = REG_GET_FIELD(RREG32_SOC15(THM, 0, mmCG_TACH_CTRL), - CG_TACH_CTRL, TARGET_PERIOD); - if (!tach_period) - return -EINVAL; - - crystal_clock_freq = amdgpu_asic_get_xclk(adev); - - tmp64 = (uint64_t)crystal_clock_freq * 60 * 10000; - do_div(tmp64, (tach_period * 8)); - *speed = (uint32_t)tmp64; - - return 0; -} - int smu_v11_0_set_xgmi_pstate(struct smu_context *smu, uint32_t pstate) { @@ -2072,30 +2023,6 @@ int smu_v11_0_get_current_pcie_link_speed(struct smu_context *smu) return link_speed[speed_level]; } -void smu_v11_0_init_gpu_metrics_v1_0(struct gpu_metrics_v1_0 *gpu_metrics) -{ - memset(gpu_metrics, 0xFF, sizeof(struct gpu_metrics_v1_0)); - - gpu_metrics->common_header.structure_size = - sizeof(struct gpu_metrics_v1_0); - gpu_metrics->common_header.format_revision = 1; - gpu_metrics->common_header.content_revision = 0; - - gpu_metrics->system_clock_counter = ktime_get_boottime_ns(); -} - -void smu_v11_0_init_gpu_metrics_v2_0(struct gpu_metrics_v2_0 *gpu_metrics) -{ - memset(gpu_metrics, 0xFF, sizeof(struct gpu_metrics_v2_0)); - - gpu_metrics->common_header.structure_size = - sizeof(struct gpu_metrics_v2_0); - gpu_metrics->common_header.format_revision = 2; - gpu_metrics->common_header.content_revision = 0; - - gpu_metrics->system_clock_counter = ktime_get_boottime_ns(); -} - int smu_v11_0_gfx_ulv_control(struct smu_context *smu, bool enablement) { diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu11/vangogh_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu11/vangogh_ppt.c index 92ad2cdbae107c18aef29ac906e3ea4a14e08ba4..093b01159408d7661d04606664b8189bad017dca 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu11/vangogh_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu11/vangogh_ppt.c @@ -31,6 +31,10 @@ #include "smu_v11_5_ppsmc.h" #include "smu_v11_5_pmfw.h" #include "smu_cmn.h" +#include "soc15_common.h" +#include "asic_reg/gc/gc_10_3_0_offset.h" +#include "asic_reg/gc/gc_10_3_0_sh_mask.h" +#include /* * DO NOT use these for err/warn/info/debug messages. @@ -59,7 +63,8 @@ static struct cmn2asic_msg_mapping vangogh_message_map[SMU_MSG_MAX_COUNT] = { MSG_MAP(GetSmuVersion, PPSMC_MSG_GetSmuVersion, 0), MSG_MAP(GetDriverIfVersion, PPSMC_MSG_GetDriverIfVersion, 0), MSG_MAP(EnableGfxOff, PPSMC_MSG_EnableGfxOff, 0), - MSG_MAP(DisallowGfxOff, PPSMC_MSG_DisableGfxOff, 0), + MSG_MAP(AllowGfxOff, PPSMC_MSG_AllowGfxOff, 0), + MSG_MAP(DisallowGfxOff, PPSMC_MSG_DisallowGfxOff, 0), MSG_MAP(PowerDownIspByTile, PPSMC_MSG_PowerDownIspByTile, 0), MSG_MAP(PowerUpIspByTile, PPSMC_MSG_PowerUpIspByTile, 0), MSG_MAP(PowerDownVcn, PPSMC_MSG_PowerDownVcn, 0), @@ -76,7 +81,6 @@ static struct cmn2asic_msg_mapping vangogh_message_map[SMU_MSG_MAX_COUNT] = { MSG_MAP(TransferTableDram2Smu, PPSMC_MSG_TransferTableDram2Smu, 0), MSG_MAP(GfxDeviceDriverReset, PPSMC_MSG_GfxDeviceDriverReset, 0), MSG_MAP(GetEnabledSmuFeatures, PPSMC_MSG_GetEnabledSmuFeatures, 0), - MSG_MAP(Spare1, PPSMC_MSG_spare1, 0), MSG_MAP(SetHardMinSocclkByFreq, PPSMC_MSG_SetHardMinSocclkByFreq, 0), MSG_MAP(SetSoftMinFclk, PPSMC_MSG_SetSoftMinFclk, 0), MSG_MAP(SetSoftMinVcn, PPSMC_MSG_SetSoftMinVcn, 0), @@ -88,7 +92,6 @@ static struct cmn2asic_msg_mapping vangogh_message_map[SMU_MSG_MAX_COUNT] = { MSG_MAP(SetSoftMaxSocclkByFreq, PPSMC_MSG_SetSoftMaxSocclkByFreq, 0), MSG_MAP(SetSoftMaxFclkByFreq, PPSMC_MSG_SetSoftMaxFclkByFreq, 0), MSG_MAP(SetSoftMaxVcn, PPSMC_MSG_SetSoftMaxVcn, 0), - MSG_MAP(Spare2, PPSMC_MSG_spare2, 0), MSG_MAP(SetPowerLimitPercentage, PPSMC_MSG_SetPowerLimitPercentage, 0), MSG_MAP(PowerDownJpeg, PPSMC_MSG_PowerDownJpeg, 0), MSG_MAP(PowerUpJpeg, PPSMC_MSG_PowerUpJpeg, 0), @@ -118,6 +121,11 @@ static struct cmn2asic_msg_mapping vangogh_message_map[SMU_MSG_MAX_COUNT] = { MSG_MAP(StopDramLogging, PPSMC_MSG_StopDramLogging, 0), MSG_MAP(SetSoftMinCclk, PPSMC_MSG_SetSoftMinCclk, 0), MSG_MAP(SetSoftMaxCclk, PPSMC_MSG_SetSoftMaxCclk, 0), + MSG_MAP(RequestActiveWgp, PPSMC_MSG_RequestActiveWgp, 0), + MSG_MAP(SetFastPPTLimit, PPSMC_MSG_SetFastPPTLimit, 0), + MSG_MAP(SetSlowPPTLimit, PPSMC_MSG_SetSlowPPTLimit, 0), + MSG_MAP(GetFastPPTLimit, PPSMC_MSG_GetFastPPTLimit, 0), + MSG_MAP(GetSlowPPTLimit, PPSMC_MSG_GetSlowPPTLimit, 0), }; static struct cmn2asic_mapping vangogh_feature_mask_map[SMU_FEATURE_COUNT] = { @@ -162,6 +170,9 @@ static struct cmn2asic_mapping vangogh_feature_mask_map[SMU_FEATURE_COUNT] = { FEA_MAP(A55_DPM), FEA_MAP(CVIP_DSP_DPM), FEA_MAP(MSMU_LOW_POWER), + FEA_MAP_REVERSE(SOCCLK), + FEA_MAP_REVERSE(FCLK), + FEA_MAP_HALF_REVERSE(GFX), }; static struct cmn2asic_mapping vangogh_table_map[SMU_TABLE_COUNT] = { @@ -171,6 +182,14 @@ static struct cmn2asic_mapping vangogh_table_map[SMU_TABLE_COUNT] = { TAB_MAP_VALID(DPMCLOCKS), }; +static struct cmn2asic_mapping vangogh_workload_map[PP_SMC_POWER_PROFILE_COUNT] = { + WORKLOAD_MAP(PP_SMC_POWER_PROFILE_FULLSCREEN3D, WORKLOAD_PPLIB_FULL_SCREEN_3D_BIT), + WORKLOAD_MAP(PP_SMC_POWER_PROFILE_VIDEO, WORKLOAD_PPLIB_VIDEO_BIT), + WORKLOAD_MAP(PP_SMC_POWER_PROFILE_VR, WORKLOAD_PPLIB_VR_BIT), + WORKLOAD_MAP(PP_SMC_POWER_PROFILE_COMPUTE, WORKLOAD_PPLIB_COMPUTE_BIT), + WORKLOAD_MAP(PP_SMC_POWER_PROFILE_CUSTOM, WORKLOAD_PPLIB_CUSTOM_BIT), +}; + static int vangogh_tables_init(struct smu_context *smu) { struct smu_table_context *smu_table = &smu->smu_table; @@ -242,6 +261,12 @@ static int vangogh_get_smu_metrics_data(struct smu_context *smu, case METRICS_AVERAGE_SOCCLK: *value = metrics->SocclkFrequency; break; + case METRICS_AVERAGE_VCLK: + *value = metrics->VclkFrequency; + break; + case METRICS_AVERAGE_DCLK: + *value = metrics->DclkFrequency; + break; case METRICS_AVERAGE_UCLK: *value = metrics->MemclkFrequency; break; @@ -272,6 +297,10 @@ static int vangogh_get_smu_metrics_data(struct smu_context *smu, case METRICS_VOLTAGE_VDDSOC: *value = metrics->Voltage[1]; break; + case METRICS_AVERAGE_CPUCLK: + memcpy(value, &metrics->CoreFrequency[0], + smu->cpu_core_num * sizeof(uint16_t)); + break; default: *value = UINT_MAX; break; @@ -308,6 +337,13 @@ static int vangogh_init_smc_tables(struct smu_context *smu) if (ret) return ret; +#ifdef CONFIG_X86 + /* AMD x86 APU only */ + smu->cpu_core_num = boot_cpu_data.x86_max_cores; +#else + smu->cpu_core_num = 4; +#endif + return smu_v11_0_init_smc_tables(smu); } @@ -317,17 +353,13 @@ static int vangogh_dpm_set_vcn_enable(struct smu_context *smu, bool enable) if (enable) { /* vcn dpm on is a prerequisite for vcn power gate messages */ - if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_VCN_PG_BIT)) { - ret = smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_PowerUpVcn, 0, NULL); - if (ret) - return ret; - } + ret = smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_PowerUpVcn, 0, NULL); + if (ret) + return ret; } else { - if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_VCN_PG_BIT)) { - ret = smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_PowerDownVcn, 0, NULL); - if (ret) - return ret; - } + ret = smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_PowerDownVcn, 0, NULL); + if (ret) + return ret; } return ret; @@ -338,54 +370,18 @@ static int vangogh_dpm_set_jpeg_enable(struct smu_context *smu, bool enable) int ret = 0; if (enable) { - if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_JPEG_PG_BIT)) { - ret = smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_PowerUpJpeg, 0, NULL); - if (ret) - return ret; - } + ret = smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_PowerUpJpeg, 0, NULL); + if (ret) + return ret; } else { - if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_JPEG_PG_BIT)) { - ret = smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_PowerDownJpeg, 0, NULL); - if (ret) - return ret; - } + ret = smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_PowerDownJpeg, 0, NULL); + if (ret) + return ret; } return ret; } -static int vangogh_get_allowed_feature_mask(struct smu_context *smu, - uint32_t *feature_mask, - uint32_t num) -{ - struct amdgpu_device *adev = smu->adev; - - if (num > 2) - return -EINVAL; - - memset(feature_mask, 0, sizeof(uint32_t) * num); - - *(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_GFX_DPM_BIT) - | FEATURE_MASK(FEATURE_MP0CLK_DPM_BIT) - | FEATURE_MASK(FEATURE_DS_SOCCLK_BIT) - | FEATURE_MASK(FEATURE_PPT_BIT) - | FEATURE_MASK(FEATURE_TDC_BIT) - | FEATURE_MASK(FEATURE_FAN_CONTROLLER_BIT) - | FEATURE_MASK(FEATURE_DS_LCLK_BIT) - | FEATURE_MASK(FEATURE_DS_DCFCLK_BIT); - - if (adev->pm.pp_feature & PP_SOCCLK_DPM_MASK) - *(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_SOCCLK_DPM_BIT); - - if (adev->pm.pp_feature & PP_DCEFCLK_DPM_MASK) - *(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_DCFCLK_DPM_BIT); - - if (smu->adev->pg_flags & AMD_PG_SUPPORT_ATHUB) - *(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_ATHUB_PG_BIT); - - return 0; -} - static bool vangogh_is_dpm_running(struct smu_context *smu) { int ret = 0; @@ -403,14 +399,68 @@ static bool vangogh_is_dpm_running(struct smu_context *smu) return !!(feature_enabled & SMC_DPM_FEATURE); } +static int vangogh_get_dpm_clk_limited(struct smu_context *smu, enum smu_clk_type clk_type, + uint32_t dpm_level, uint32_t *freq) +{ + DpmClocks_t *clk_table = smu->smu_table.clocks_table; + + if (!clk_table || clk_type >= SMU_CLK_COUNT) + return -EINVAL; + + switch (clk_type) { + case SMU_SOCCLK: + if (dpm_level >= clk_table->NumSocClkLevelsEnabled) + return -EINVAL; + *freq = clk_table->SocClocks[dpm_level]; + break; + case SMU_VCLK: + if (dpm_level >= clk_table->VcnClkLevelsEnabled) + return -EINVAL; + *freq = clk_table->VcnClocks[dpm_level].vclk; + break; + case SMU_DCLK: + if (dpm_level >= clk_table->VcnClkLevelsEnabled) + return -EINVAL; + *freq = clk_table->VcnClocks[dpm_level].dclk; + break; + case SMU_UCLK: + case SMU_MCLK: + if (dpm_level >= clk_table->NumDfPstatesEnabled) + return -EINVAL; + *freq = clk_table->DfPstateTable[dpm_level].memclk; + + break; + case SMU_FCLK: + if (dpm_level >= clk_table->NumDfPstatesEnabled) + return -EINVAL; + *freq = clk_table->DfPstateTable[dpm_level].fclk; + break; + default: + return -EINVAL; + } + + return 0; +} + static int vangogh_print_fine_grain_clk(struct smu_context *smu, enum smu_clk_type clk_type, char *buf) { - int size = 0; + DpmClocks_t *clk_table = smu->smu_table.clocks_table; + SmuMetrics_t metrics; + struct smu_dpm_context *smu_dpm_ctx = &(smu->smu_dpm); + int i, size = 0, ret = 0; + uint32_t cur_value = 0, value = 0, count = 0; + bool cur_value_match_level = false; + + memset(&metrics, 0, sizeof(metrics)); + + ret = smu_cmn_get_metrics_table(smu, &metrics, false); + if (ret) + return ret; switch (clk_type) { case SMU_OD_SCLK: - if (smu->od_enabled) { + if (smu_dpm_ctx->dpm_level == AMD_DPM_FORCED_LEVEL_MANUAL) { size = sprintf(buf, "%s:\n", "OD_SCLK"); size += sprintf(buf + size, "0: %10uMhz\n", (smu->gfx_actual_hard_min_freq > 0) ? smu->gfx_actual_hard_min_freq : smu->gfx_default_hard_min_freq); @@ -418,12 +468,71 @@ static int vangogh_print_fine_grain_clk(struct smu_context *smu, (smu->gfx_actual_soft_max_freq > 0) ? smu->gfx_actual_soft_max_freq : smu->gfx_default_soft_max_freq); } break; + case SMU_OD_CCLK: + if (smu_dpm_ctx->dpm_level == AMD_DPM_FORCED_LEVEL_MANUAL) { + size = sprintf(buf, "CCLK_RANGE in Core%d:\n", smu->cpu_core_id_select); + size += sprintf(buf + size, "0: %10uMhz\n", + (smu->cpu_actual_soft_min_freq > 0) ? smu->cpu_actual_soft_min_freq : smu->cpu_default_soft_min_freq); + size += sprintf(buf + size, "1: %10uMhz\n", + (smu->cpu_actual_soft_max_freq > 0) ? smu->cpu_actual_soft_max_freq : smu->cpu_default_soft_max_freq); + } + break; case SMU_OD_RANGE: - if (smu->od_enabled) { + if (smu_dpm_ctx->dpm_level == AMD_DPM_FORCED_LEVEL_MANUAL) { size = sprintf(buf, "%s:\n", "OD_RANGE"); size += sprintf(buf + size, "SCLK: %7uMhz %10uMhz\n", smu->gfx_default_hard_min_freq, smu->gfx_default_soft_max_freq); + size += sprintf(buf + size, "CCLK: %7uMhz %10uMhz\n", + smu->cpu_default_soft_min_freq, smu->cpu_default_soft_max_freq); + } + break; + case SMU_SOCCLK: + /* the level 3 ~ 6 of socclk use the same frequency for vangogh */ + count = clk_table->NumSocClkLevelsEnabled; + cur_value = metrics.SocclkFrequency; + break; + case SMU_VCLK: + count = clk_table->VcnClkLevelsEnabled; + cur_value = metrics.VclkFrequency; + break; + case SMU_DCLK: + count = clk_table->VcnClkLevelsEnabled; + cur_value = metrics.DclkFrequency; + break; + case SMU_MCLK: + count = clk_table->NumDfPstatesEnabled; + cur_value = metrics.MemclkFrequency; + break; + case SMU_FCLK: + count = clk_table->NumDfPstatesEnabled; + ret = smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_GetFclkFrequency, 0, &cur_value); + if (ret) + return ret; + break; + default: + break; + } + + switch (clk_type) { + case SMU_SOCCLK: + case SMU_VCLK: + case SMU_DCLK: + case SMU_MCLK: + case SMU_FCLK: + for (i = 0; i < count; i++) { + ret = vangogh_get_dpm_clk_limited(smu, clk_type, i, &value); + if (ret) + return ret; + if (!value) + continue; + size += sprintf(buf + size, "%d: %uMhz %s\n", i, value, + cur_value == value ? "*" : ""); + if (cur_value == value) + cur_value_match_level = true; } + + if (!cur_value_match_level) + size += sprintf(buf + size, " %uMhz *\n", cur_value); break; default: break; @@ -432,6 +541,726 @@ static int vangogh_print_fine_grain_clk(struct smu_context *smu, return size; } +static int vangogh_get_profiling_clk_mask(struct smu_context *smu, + enum amd_dpm_forced_level level, + uint32_t *vclk_mask, + uint32_t *dclk_mask, + uint32_t *mclk_mask, + uint32_t *fclk_mask, + uint32_t *soc_mask) +{ + DpmClocks_t *clk_table = smu->smu_table.clocks_table; + + if (level == AMD_DPM_FORCED_LEVEL_PROFILE_MIN_MCLK) { + if (mclk_mask) + *mclk_mask = clk_table->NumDfPstatesEnabled - 1; + + if (fclk_mask) + *fclk_mask = clk_table->NumDfPstatesEnabled - 1; + + if (soc_mask) + *soc_mask = 0; + } else if (level == AMD_DPM_FORCED_LEVEL_PROFILE_PEAK) { + if (mclk_mask) + *mclk_mask = 0; + + if (fclk_mask) + *fclk_mask = 0; + + if (soc_mask) + *soc_mask = 1; + + if (vclk_mask) + *vclk_mask = 1; + + if (dclk_mask) + *dclk_mask = 1; + } else if (level == AMD_DPM_FORCED_LEVEL_PROFILE_STANDARD) { + if (mclk_mask) + *mclk_mask = 0; + + if (fclk_mask) + *fclk_mask = 0; + + if (soc_mask) + *soc_mask = 1; + + if (vclk_mask) + *vclk_mask = 1; + + if (dclk_mask) + *dclk_mask = 1; + } + + return 0; +} + +static bool vangogh_clk_dpm_is_enabled(struct smu_context *smu, + enum smu_clk_type clk_type) +{ + enum smu_feature_mask feature_id = 0; + + switch (clk_type) { + case SMU_MCLK: + case SMU_UCLK: + case SMU_FCLK: + feature_id = SMU_FEATURE_DPM_FCLK_BIT; + break; + case SMU_GFXCLK: + case SMU_SCLK: + feature_id = SMU_FEATURE_DPM_GFXCLK_BIT; + break; + case SMU_SOCCLK: + feature_id = SMU_FEATURE_DPM_SOCCLK_BIT; + break; + case SMU_VCLK: + case SMU_DCLK: + feature_id = SMU_FEATURE_VCN_DPM_BIT; + break; + default: + return true; + } + + if (!smu_cmn_feature_is_enabled(smu, feature_id)) + return false; + + return true; +} + +static int vangogh_get_dpm_ultimate_freq(struct smu_context *smu, + enum smu_clk_type clk_type, + uint32_t *min, + uint32_t *max) +{ + int ret = 0; + uint32_t soc_mask; + uint32_t vclk_mask; + uint32_t dclk_mask; + uint32_t mclk_mask; + uint32_t fclk_mask; + uint32_t clock_limit; + + if (!vangogh_clk_dpm_is_enabled(smu, clk_type)) { + switch (clk_type) { + case SMU_MCLK: + case SMU_UCLK: + clock_limit = smu->smu_table.boot_values.uclk; + break; + case SMU_FCLK: + clock_limit = smu->smu_table.boot_values.fclk; + break; + case SMU_GFXCLK: + case SMU_SCLK: + clock_limit = smu->smu_table.boot_values.gfxclk; + break; + case SMU_SOCCLK: + clock_limit = smu->smu_table.boot_values.socclk; + break; + case SMU_VCLK: + clock_limit = smu->smu_table.boot_values.vclk; + break; + case SMU_DCLK: + clock_limit = smu->smu_table.boot_values.dclk; + break; + default: + clock_limit = 0; + break; + } + + /* clock in Mhz unit */ + if (min) + *min = clock_limit / 100; + if (max) + *max = clock_limit / 100; + + return 0; + } + if (max) { + ret = vangogh_get_profiling_clk_mask(smu, + AMD_DPM_FORCED_LEVEL_PROFILE_PEAK, + &vclk_mask, + &dclk_mask, + &mclk_mask, + &fclk_mask, + &soc_mask); + if (ret) + goto failed; + + switch (clk_type) { + case SMU_UCLK: + case SMU_MCLK: + ret = vangogh_get_dpm_clk_limited(smu, clk_type, mclk_mask, max); + if (ret) + goto failed; + break; + case SMU_SOCCLK: + ret = vangogh_get_dpm_clk_limited(smu, clk_type, soc_mask, max); + if (ret) + goto failed; + break; + case SMU_FCLK: + ret = vangogh_get_dpm_clk_limited(smu, clk_type, fclk_mask, max); + if (ret) + goto failed; + break; + case SMU_VCLK: + ret = vangogh_get_dpm_clk_limited(smu, clk_type, vclk_mask, max); + if (ret) + goto failed; + break; + case SMU_DCLK: + ret = vangogh_get_dpm_clk_limited(smu, clk_type, dclk_mask, max); + if (ret) + goto failed; + break; + default: + ret = -EINVAL; + goto failed; + } + } + if (min) { + switch (clk_type) { + case SMU_UCLK: + case SMU_MCLK: + ret = vangogh_get_dpm_clk_limited(smu, clk_type, mclk_mask, min); + if (ret) + goto failed; + break; + case SMU_SOCCLK: + ret = vangogh_get_dpm_clk_limited(smu, clk_type, soc_mask, min); + if (ret) + goto failed; + break; + case SMU_FCLK: + ret = vangogh_get_dpm_clk_limited(smu, clk_type, fclk_mask, min); + if (ret) + goto failed; + break; + case SMU_VCLK: + ret = vangogh_get_dpm_clk_limited(smu, clk_type, vclk_mask, min); + if (ret) + goto failed; + break; + case SMU_DCLK: + ret = vangogh_get_dpm_clk_limited(smu, clk_type, dclk_mask, min); + if (ret) + goto failed; + break; + default: + ret = -EINVAL; + goto failed; + } + } +failed: + return ret; +} + +static int vangogh_get_power_profile_mode(struct smu_context *smu, + char *buf) +{ + static const char *profile_name[] = { + "BOOTUP_DEFAULT", + "3D_FULL_SCREEN", + "POWER_SAVING", + "VIDEO", + "VR", + "COMPUTE", + "CUSTOM"}; + uint32_t i, size = 0; + int16_t workload_type = 0; + + if (!buf) + return -EINVAL; + + for (i = 0; i <= PP_SMC_POWER_PROFILE_CUSTOM; i++) { + /* + * Conv PP_SMC_POWER_PROFILE* to WORKLOAD_PPLIB_*_BIT + * Not all profile modes are supported on vangogh. + */ + workload_type = smu_cmn_to_asic_specific_index(smu, + CMN2ASIC_MAPPING_WORKLOAD, + i); + + if (workload_type < 0) + continue; + + size += sprintf(buf + size, "%2d %14s%s\n", + i, profile_name[i], (i == smu->power_profile_mode) ? "*" : " "); + } + + return size; +} + +static int vangogh_set_power_profile_mode(struct smu_context *smu, long *input, uint32_t size) +{ + int workload_type, ret; + uint32_t profile_mode = input[size]; + + if (profile_mode > PP_SMC_POWER_PROFILE_CUSTOM) { + dev_err(smu->adev->dev, "Invalid power profile mode %d\n", profile_mode); + return -EINVAL; + } + + if (profile_mode == PP_SMC_POWER_PROFILE_BOOTUP_DEFAULT || + profile_mode == PP_SMC_POWER_PROFILE_POWERSAVING) + return 0; + + /* conv PP_SMC_POWER_PROFILE* to WORKLOAD_PPLIB_*_BIT */ + workload_type = smu_cmn_to_asic_specific_index(smu, + CMN2ASIC_MAPPING_WORKLOAD, + profile_mode); + if (workload_type < 0) { + dev_err_once(smu->adev->dev, "Unsupported power profile mode %d on VANGOGH\n", + profile_mode); + return -EINVAL; + } + + ret = smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_ActiveProcessNotify, + 1 << workload_type, + NULL); + if (ret) { + dev_err_once(smu->adev->dev, "Fail to set workload type %d\n", + workload_type); + return ret; + } + + smu->power_profile_mode = profile_mode; + + return 0; +} + +static int vangogh_set_soft_freq_limited_range(struct smu_context *smu, + enum smu_clk_type clk_type, + uint32_t min, + uint32_t max) +{ + int ret = 0; + + if (!vangogh_clk_dpm_is_enabled(smu, clk_type)) + return 0; + + switch (clk_type) { + case SMU_GFXCLK: + case SMU_SCLK: + ret = smu_cmn_send_smc_msg_with_param(smu, + SMU_MSG_SetHardMinGfxClk, + min, NULL); + if (ret) + return ret; + + ret = smu_cmn_send_smc_msg_with_param(smu, + SMU_MSG_SetSoftMaxGfxClk, + max, NULL); + if (ret) + return ret; + break; + case SMU_FCLK: + case SMU_MCLK: + ret = smu_cmn_send_smc_msg_with_param(smu, + SMU_MSG_SetHardMinFclkByFreq, + min, NULL); + if (ret) + return ret; + + ret = smu_cmn_send_smc_msg_with_param(smu, + SMU_MSG_SetSoftMaxFclkByFreq, + max, NULL); + if (ret) + return ret; + break; + case SMU_SOCCLK: + ret = smu_cmn_send_smc_msg_with_param(smu, + SMU_MSG_SetHardMinSocclkByFreq, + min, NULL); + if (ret) + return ret; + + ret = smu_cmn_send_smc_msg_with_param(smu, + SMU_MSG_SetSoftMaxSocclkByFreq, + max, NULL); + if (ret) + return ret; + break; + case SMU_VCLK: + ret = smu_cmn_send_smc_msg_with_param(smu, + SMU_MSG_SetHardMinVcn, + min << 16, NULL); + if (ret) + return ret; + ret = smu_cmn_send_smc_msg_with_param(smu, + SMU_MSG_SetSoftMaxVcn, + max << 16, NULL); + if (ret) + return ret; + break; + case SMU_DCLK: + ret = smu_cmn_send_smc_msg_with_param(smu, + SMU_MSG_SetHardMinVcn, + min, NULL); + if (ret) + return ret; + ret = smu_cmn_send_smc_msg_with_param(smu, + SMU_MSG_SetSoftMaxVcn, + max, NULL); + if (ret) + return ret; + break; + default: + return -EINVAL; + } + + return ret; +} + +static int vangogh_force_clk_levels(struct smu_context *smu, + enum smu_clk_type clk_type, uint32_t mask) +{ + uint32_t soft_min_level = 0, soft_max_level = 0; + uint32_t min_freq = 0, max_freq = 0; + int ret = 0 ; + + soft_min_level = mask ? (ffs(mask) - 1) : 0; + soft_max_level = mask ? (fls(mask) - 1) : 0; + + switch (clk_type) { + case SMU_SOCCLK: + ret = vangogh_get_dpm_clk_limited(smu, clk_type, + soft_min_level, &min_freq); + if (ret) + return ret; + ret = vangogh_get_dpm_clk_limited(smu, clk_type, + soft_max_level, &max_freq); + if (ret) + return ret; + ret = smu_cmn_send_smc_msg_with_param(smu, + SMU_MSG_SetSoftMaxSocclkByFreq, + max_freq, NULL); + if (ret) + return ret; + ret = smu_cmn_send_smc_msg_with_param(smu, + SMU_MSG_SetHardMinSocclkByFreq, + min_freq, NULL); + if (ret) + return ret; + break; + case SMU_MCLK: + case SMU_FCLK: + ret = vangogh_get_dpm_clk_limited(smu, + clk_type, soft_min_level, &min_freq); + if (ret) + return ret; + ret = vangogh_get_dpm_clk_limited(smu, + clk_type, soft_max_level, &max_freq); + if (ret) + return ret; + ret = smu_cmn_send_smc_msg_with_param(smu, + SMU_MSG_SetSoftMaxFclkByFreq, + max_freq, NULL); + if (ret) + return ret; + ret = smu_cmn_send_smc_msg_with_param(smu, + SMU_MSG_SetHardMinFclkByFreq, + min_freq, NULL); + if (ret) + return ret; + break; + case SMU_VCLK: + ret = vangogh_get_dpm_clk_limited(smu, + clk_type, soft_min_level, &min_freq); + if (ret) + return ret; + + ret = vangogh_get_dpm_clk_limited(smu, + clk_type, soft_max_level, &max_freq); + if (ret) + return ret; + + + ret = smu_cmn_send_smc_msg_with_param(smu, + SMU_MSG_SetHardMinVcn, + min_freq << 16, NULL); + if (ret) + return ret; + + ret = smu_cmn_send_smc_msg_with_param(smu, + SMU_MSG_SetSoftMaxVcn, + max_freq << 16, NULL); + if (ret) + return ret; + + break; + case SMU_DCLK: + ret = vangogh_get_dpm_clk_limited(smu, + clk_type, soft_min_level, &min_freq); + if (ret) + return ret; + + ret = vangogh_get_dpm_clk_limited(smu, + clk_type, soft_max_level, &max_freq); + if (ret) + return ret; + + ret = smu_cmn_send_smc_msg_with_param(smu, + SMU_MSG_SetHardMinVcn, + min_freq, NULL); + if (ret) + return ret; + + ret = smu_cmn_send_smc_msg_with_param(smu, + SMU_MSG_SetSoftMaxVcn, + max_freq, NULL); + if (ret) + return ret; + + break; + default: + break; + } + + return ret; +} + +static int vangogh_force_dpm_limit_value(struct smu_context *smu, bool highest) +{ + int ret = 0, i = 0; + uint32_t min_freq, max_freq, force_freq; + enum smu_clk_type clk_type; + + enum smu_clk_type clks[] = { + SMU_SOCCLK, + SMU_VCLK, + SMU_DCLK, + SMU_MCLK, + SMU_FCLK, + }; + + for (i = 0; i < ARRAY_SIZE(clks); i++) { + clk_type = clks[i]; + ret = vangogh_get_dpm_ultimate_freq(smu, clk_type, &min_freq, &max_freq); + if (ret) + return ret; + + force_freq = highest ? max_freq : min_freq; + ret = vangogh_set_soft_freq_limited_range(smu, clk_type, force_freq, force_freq); + if (ret) + return ret; + } + + return ret; +} + +static int vangogh_unforce_dpm_levels(struct smu_context *smu) +{ + int ret = 0, i = 0; + uint32_t min_freq, max_freq; + enum smu_clk_type clk_type; + + struct clk_feature_map { + enum smu_clk_type clk_type; + uint32_t feature; + } clk_feature_map[] = { + {SMU_MCLK, SMU_FEATURE_DPM_FCLK_BIT}, + {SMU_FCLK, SMU_FEATURE_DPM_FCLK_BIT}, + {SMU_SOCCLK, SMU_FEATURE_DPM_SOCCLK_BIT}, + {SMU_VCLK, SMU_FEATURE_VCN_DPM_BIT}, + {SMU_DCLK, SMU_FEATURE_VCN_DPM_BIT}, + }; + + for (i = 0; i < ARRAY_SIZE(clk_feature_map); i++) { + + if (!smu_cmn_feature_is_enabled(smu, clk_feature_map[i].feature)) + continue; + + clk_type = clk_feature_map[i].clk_type; + + ret = vangogh_get_dpm_ultimate_freq(smu, clk_type, &min_freq, &max_freq); + + if (ret) + return ret; + + ret = vangogh_set_soft_freq_limited_range(smu, clk_type, min_freq, max_freq); + + if (ret) + return ret; + } + + return ret; +} + +static int vangogh_set_peak_clock_by_device(struct smu_context *smu) +{ + int ret = 0; + uint32_t socclk_freq = 0, fclk_freq = 0; + uint32_t vclk_freq = 0, dclk_freq = 0; + + ret = vangogh_get_dpm_ultimate_freq(smu, SMU_FCLK, NULL, &fclk_freq); + if (ret) + return ret; + + ret = vangogh_set_soft_freq_limited_range(smu, SMU_FCLK, fclk_freq, fclk_freq); + if (ret) + return ret; + + ret = vangogh_get_dpm_ultimate_freq(smu, SMU_SOCCLK, NULL, &socclk_freq); + if (ret) + return ret; + + ret = vangogh_set_soft_freq_limited_range(smu, SMU_SOCCLK, socclk_freq, socclk_freq); + if (ret) + return ret; + + ret = vangogh_get_dpm_ultimate_freq(smu, SMU_VCLK, NULL, &vclk_freq); + if (ret) + return ret; + + ret = vangogh_set_soft_freq_limited_range(smu, SMU_VCLK, vclk_freq, vclk_freq); + if (ret) + return ret; + + ret = vangogh_get_dpm_ultimate_freq(smu, SMU_DCLK, NULL, &dclk_freq); + if (ret) + return ret; + + ret = vangogh_set_soft_freq_limited_range(smu, SMU_DCLK, dclk_freq, dclk_freq); + if (ret) + return ret; + + return ret; +} + +static int vangogh_set_performance_level(struct smu_context *smu, + enum amd_dpm_forced_level level) +{ + int ret = 0; + uint32_t soc_mask, mclk_mask, fclk_mask; + uint32_t vclk_mask = 0, dclk_mask = 0; + + switch (level) { + case AMD_DPM_FORCED_LEVEL_HIGH: + smu->gfx_actual_hard_min_freq = smu->gfx_default_hard_min_freq; + smu->gfx_actual_soft_max_freq = smu->gfx_default_soft_max_freq; + + smu->cpu_actual_soft_min_freq = smu->cpu_default_soft_min_freq; + smu->cpu_actual_soft_max_freq = smu->cpu_default_soft_max_freq; + + ret = vangogh_force_dpm_limit_value(smu, true); + break; + case AMD_DPM_FORCED_LEVEL_LOW: + smu->gfx_actual_hard_min_freq = smu->gfx_default_hard_min_freq; + smu->gfx_actual_soft_max_freq = smu->gfx_default_soft_max_freq; + + smu->cpu_actual_soft_min_freq = smu->cpu_default_soft_min_freq; + smu->cpu_actual_soft_max_freq = smu->cpu_default_soft_max_freq; + + ret = vangogh_force_dpm_limit_value(smu, false); + break; + case AMD_DPM_FORCED_LEVEL_AUTO: + smu->gfx_actual_hard_min_freq = smu->gfx_default_hard_min_freq; + smu->gfx_actual_soft_max_freq = smu->gfx_default_soft_max_freq; + + smu->cpu_actual_soft_min_freq = smu->cpu_default_soft_min_freq; + smu->cpu_actual_soft_max_freq = smu->cpu_default_soft_max_freq; + + ret = vangogh_unforce_dpm_levels(smu); + break; + case AMD_DPM_FORCED_LEVEL_PROFILE_STANDARD: + smu->gfx_actual_hard_min_freq = smu->gfx_default_hard_min_freq; + smu->gfx_actual_soft_max_freq = smu->gfx_default_soft_max_freq; + + smu->cpu_actual_soft_min_freq = smu->cpu_default_soft_min_freq; + smu->cpu_actual_soft_max_freq = smu->cpu_default_soft_max_freq; + + ret = smu_cmn_send_smc_msg_with_param(smu, + SMU_MSG_SetHardMinGfxClk, + VANGOGH_UMD_PSTATE_STANDARD_GFXCLK, NULL); + if (ret) + return ret; + + ret = smu_cmn_send_smc_msg_with_param(smu, + SMU_MSG_SetSoftMaxGfxClk, + VANGOGH_UMD_PSTATE_STANDARD_GFXCLK, NULL); + if (ret) + return ret; + + ret = vangogh_get_profiling_clk_mask(smu, level, + &vclk_mask, + &dclk_mask, + &mclk_mask, + &fclk_mask, + &soc_mask); + if (ret) + return ret; + + vangogh_force_clk_levels(smu, SMU_MCLK, 1 << mclk_mask); + vangogh_force_clk_levels(smu, SMU_FCLK, 1 << fclk_mask); + vangogh_force_clk_levels(smu, SMU_SOCCLK, 1 << soc_mask); + vangogh_force_clk_levels(smu, SMU_VCLK, 1 << vclk_mask); + vangogh_force_clk_levels(smu, SMU_DCLK, 1 << dclk_mask); + + break; + case AMD_DPM_FORCED_LEVEL_PROFILE_MIN_SCLK: + smu->gfx_actual_hard_min_freq = smu->gfx_default_hard_min_freq; + smu->gfx_actual_soft_max_freq = smu->gfx_default_soft_max_freq; + + smu->cpu_actual_soft_min_freq = smu->cpu_default_soft_min_freq; + smu->cpu_actual_soft_max_freq = smu->cpu_default_soft_max_freq; + + ret = smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_SetHardMinVcn, + VANGOGH_UMD_PSTATE_PEAK_DCLK, NULL); + if (ret) + return ret; + + ret = smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_SetSoftMaxVcn, + VANGOGH_UMD_PSTATE_PEAK_DCLK, NULL); + if (ret) + return ret; + break; + case AMD_DPM_FORCED_LEVEL_PROFILE_MIN_MCLK: + smu->gfx_actual_hard_min_freq = smu->gfx_default_hard_min_freq; + smu->gfx_actual_soft_max_freq = smu->gfx_default_soft_max_freq; + + smu->cpu_actual_soft_min_freq = smu->cpu_default_soft_min_freq; + smu->cpu_actual_soft_max_freq = smu->cpu_default_soft_max_freq; + + ret = vangogh_get_profiling_clk_mask(smu, level, + NULL, + NULL, + &mclk_mask, + &fclk_mask, + NULL); + if (ret) + return ret; + + vangogh_force_clk_levels(smu, SMU_MCLK, 1 << mclk_mask); + vangogh_force_clk_levels(smu, SMU_FCLK, 1 << fclk_mask); + break; + case AMD_DPM_FORCED_LEVEL_PROFILE_PEAK: + smu->gfx_actual_hard_min_freq = smu->gfx_default_hard_min_freq; + smu->gfx_actual_soft_max_freq = smu->gfx_default_soft_max_freq; + + smu->cpu_actual_soft_min_freq = smu->cpu_default_soft_min_freq; + smu->cpu_actual_soft_max_freq = smu->cpu_default_soft_max_freq; + + ret = smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_SetHardMinGfxClk, + VANGOGH_UMD_PSTATE_PEAK_GFXCLK, NULL); + if (ret) + return ret; + + ret = smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_SetSoftMaxGfxClk, + VANGOGH_UMD_PSTATE_PEAK_GFXCLK, NULL); + if (ret) + return ret; + + ret = vangogh_set_peak_clock_by_device(smu); + break; + case AMD_DPM_FORCED_LEVEL_MANUAL: + case AMD_DPM_FORCED_LEVEL_PROFILE_EXIT: + default: + break; + } + return ret; +} + static int vangogh_read_sensor(struct smu_context *smu, enum amd_pp_sensors sensor, void *data, uint32_t *size) @@ -493,6 +1322,12 @@ static int vangogh_read_sensor(struct smu_context *smu, (uint32_t *)data); *size = 4; break; + case AMDGPU_PP_SENSOR_CPU_CLK: + ret = vangogh_get_smu_metrics_data(smu, + METRICS_AVERAGE_CPUCLK, + (uint32_t *)data); + *size = smu->cpu_core_num * sizeof(uint16_t); + break; default: ret = -EOPNOTSUPP; break; @@ -514,7 +1349,7 @@ static int vangogh_set_watermarks_table(struct smu_context *smu, if (clock_ranges) { if (clock_ranges->num_reader_wm_sets > NUM_WM_RANGES || - clock_ranges->num_writer_wm_sets > NUM_WM_RANGES) + clock_ranges->num_writer_wm_sets > NUM_WM_RANGES) return -EINVAL; for (i = 0; i < clock_ranges->num_reader_wm_sets; i++) { @@ -575,7 +1410,7 @@ static ssize_t vangogh_get_gpu_metrics(struct smu_context *smu, if (ret) return ret; - smu_v11_0_init_gpu_metrics_v2_0(gpu_metrics); + smu_cmn_init_soft_gpu_metrics(gpu_metrics, 2, 0); gpu_metrics->temperature_gfx = metrics.GfxTemperature; gpu_metrics->temperature_soc = metrics.SocTemperature; @@ -611,22 +1446,55 @@ static ssize_t vangogh_get_gpu_metrics(struct smu_context *smu, gpu_metrics->throttle_status = metrics.ThrottlerStatus; + gpu_metrics->system_clock_counter = ktime_get_boottime_ns(); + *table = (void *)gpu_metrics; return sizeof(struct gpu_metrics_v2_0); } static int vangogh_od_edit_dpm_table(struct smu_context *smu, enum PP_OD_DPM_TABLE_COMMAND type, - long input[], uint32_t size) + long input[], uint32_t size) { int ret = 0; + int i; + struct smu_dpm_context *smu_dpm_ctx = &(smu->smu_dpm); - if (!smu->od_enabled) { - dev_warn(smu->adev->dev, "Fine grain is not enabled!\n"); + if (!(smu_dpm_ctx->dpm_level == AMD_DPM_FORCED_LEVEL_MANUAL)) { + dev_warn(smu->adev->dev, + "pp_od_clk_voltage is not accessible if power_dpm_force_perfomance_level is not in manual mode!\n"); return -EINVAL; } switch (type) { + case PP_OD_EDIT_CCLK_VDDC_TABLE: + if (size != 3) { + dev_err(smu->adev->dev, "Input parameter number not correct (should be 4 for processor)\n"); + return -EINVAL; + } + if (input[0] >= smu->cpu_core_num) { + dev_err(smu->adev->dev, "core index is overflow, should be less than %d\n", + smu->cpu_core_num); + } + smu->cpu_core_id_select = input[0]; + if (input[1] == 0) { + if (input[2] < smu->cpu_default_soft_min_freq) { + dev_warn(smu->adev->dev, "Fine grain setting minimum cclk (%ld) MHz is less than the minimum allowed (%d) MHz\n", + input[2], smu->cpu_default_soft_min_freq); + return -EINVAL; + } + smu->cpu_actual_soft_min_freq = input[2]; + } else if (input[1] == 1) { + if (input[2] > smu->cpu_default_soft_max_freq) { + dev_warn(smu->adev->dev, "Fine grain setting maximum cclk (%ld) MHz is greater than the maximum allowed (%d) MHz\n", + input[2], smu->cpu_default_soft_max_freq); + return -EINVAL; + } + smu->cpu_actual_soft_max_freq = input[2]; + } else { + return -EINVAL; + } + break; case PP_OD_EDIT_SCLK_VDDC_TABLE: if (size != 2) { dev_err(smu->adev->dev, "Input parameter number not correct\n"); @@ -635,14 +1503,16 @@ static int vangogh_od_edit_dpm_table(struct smu_context *smu, enum PP_OD_DPM_TAB if (input[0] == 0) { if (input[1] < smu->gfx_default_hard_min_freq) { - dev_warn(smu->adev->dev, "Fine grain setting minimum sclk (%ld) MHz is less than the minimum allowed (%d) MHz\n", + dev_warn(smu->adev->dev, + "Fine grain setting minimum sclk (%ld) MHz is less than the minimum allowed (%d) MHz\n", input[1], smu->gfx_default_hard_min_freq); return -EINVAL; } smu->gfx_actual_hard_min_freq = input[1]; } else if (input[0] == 1) { if (input[1] > smu->gfx_default_soft_max_freq) { - dev_warn(smu->adev->dev, "Fine grain setting maximum sclk (%ld) MHz is greater than the maximum allowed (%d) MHz\n", + dev_warn(smu->adev->dev, + "Fine grain setting maximum sclk (%ld) MHz is greater than the maximum allowed (%d) MHz\n", input[1], smu->gfx_default_soft_max_freq); return -EINVAL; } @@ -658,6 +1528,8 @@ static int vangogh_od_edit_dpm_table(struct smu_context *smu, enum PP_OD_DPM_TAB } else { smu->gfx_actual_hard_min_freq = smu->gfx_default_hard_min_freq; smu->gfx_actual_soft_max_freq = smu->gfx_default_soft_max_freq; + smu->cpu_actual_soft_min_freq = smu->cpu_default_soft_min_freq; + smu->cpu_actual_soft_max_freq = smu->cpu_default_soft_max_freq; ret = smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_SetHardMinGfxClk, smu->gfx_actual_hard_min_freq, NULL); @@ -672,6 +1544,29 @@ static int vangogh_od_edit_dpm_table(struct smu_context *smu, enum PP_OD_DPM_TAB dev_err(smu->adev->dev, "Restore the default soft max sclk failed!"); return ret; } + + if (smu->adev->pm.fw_version < 0x43f1b00) { + dev_warn(smu->adev->dev, "CPUSoftMax/CPUSoftMin are not supported, please update SBIOS!\n"); + break; + } + + for (i = 0; i < smu->cpu_core_num; i++) { + ret = smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_SetSoftMinCclk, + (i << 20) | smu->cpu_actual_soft_min_freq, + NULL); + if (ret) { + dev_err(smu->adev->dev, "Set hard min cclk failed!"); + return ret; + } + + ret = smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_SetSoftMaxCclk, + (i << 20) | smu->cpu_actual_soft_max_freq, + NULL); + if (ret) { + dev_err(smu->adev->dev, "Set soft max cclk failed!"); + return ret; + } + } } break; case PP_OD_COMMIT_DPM_TABLE: @@ -680,8 +1575,10 @@ static int vangogh_od_edit_dpm_table(struct smu_context *smu, enum PP_OD_DPM_TAB return -EINVAL; } else { if (smu->gfx_actual_hard_min_freq > smu->gfx_actual_soft_max_freq) { - dev_err(smu->adev->dev, "The setting minimun sclk (%d) MHz is greater than the setting maximum sclk (%d) MHz\n", - smu->gfx_actual_hard_min_freq, smu->gfx_actual_soft_max_freq); + dev_err(smu->adev->dev, + "The setting minimun sclk (%d) MHz is greater than the setting maximum sclk (%d) MHz\n", + smu->gfx_actual_hard_min_freq, + smu->gfx_actual_soft_max_freq); return -EINVAL; } @@ -698,6 +1595,29 @@ static int vangogh_od_edit_dpm_table(struct smu_context *smu, enum PP_OD_DPM_TAB dev_err(smu->adev->dev, "Set soft max sclk failed!"); return ret; } + + if (smu->adev->pm.fw_version < 0x43f1b00) { + dev_warn(smu->adev->dev, "CPUSoftMax/CPUSoftMin are not supported, please update SBIOS!\n"); + break; + } + + ret = smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_SetSoftMinCclk, + ((smu->cpu_core_id_select << 20) + | smu->cpu_actual_soft_min_freq), + NULL); + if (ret) { + dev_err(smu->adev->dev, "Set hard min cclk failed!"); + return ret; + } + + ret = smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_SetSoftMaxCclk, + ((smu->cpu_core_id_select << 20) + | smu->cpu_actual_soft_max_freq), + NULL); + if (ret) { + dev_err(smu->adev->dev, "Set soft max cclk failed!"); + return ret; + } } break; default: @@ -723,18 +1643,245 @@ static int vangogh_set_fine_grain_gfx_freq_parameters(struct smu_context *smu) smu->gfx_actual_hard_min_freq = 0; smu->gfx_actual_soft_max_freq = 0; + smu->cpu_default_soft_min_freq = 1400; + smu->cpu_default_soft_max_freq = 3500; + smu->cpu_actual_soft_min_freq = 0; + smu->cpu_actual_soft_max_freq = 0; + + return 0; +} + +static int vangogh_get_dpm_clock_table(struct smu_context *smu, struct dpm_clocks *clock_table) +{ + DpmClocks_t *table = smu->smu_table.clocks_table; + int i; + + if (!clock_table || !table) + return -EINVAL; + + for (i = 0; i < NUM_SOCCLK_DPM_LEVELS; i++) { + clock_table->SocClocks[i].Freq = table->SocClocks[i]; + clock_table->SocClocks[i].Vol = table->SocVoltage[i]; + } + + for (i = 0; i < NUM_FCLK_DPM_LEVELS; i++) { + clock_table->FClocks[i].Freq = table->DfPstateTable[i].fclk; + clock_table->FClocks[i].Vol = table->DfPstateTable[i].voltage; + } + + for (i = 0; i < NUM_FCLK_DPM_LEVELS; i++) { + clock_table->MemClocks[i].Freq = table->DfPstateTable[i].memclk; + clock_table->MemClocks[i].Vol = table->DfPstateTable[i].voltage; + } + return 0; } + static int vangogh_system_features_control(struct smu_context *smu, bool en) { struct amdgpu_device *adev = smu->adev; + struct smu_feature *feature = &smu->smu_feature; + uint32_t feature_mask[2]; + int ret = 0; if (adev->pm.fw_version >= 0x43f1700) - return smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_RlcPowerNotify, - en ? RLC_STATUS_NORMAL : RLC_STATUS_OFF, NULL); - else + ret = smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_RlcPowerNotify, + en ? RLC_STATUS_NORMAL : RLC_STATUS_OFF, NULL); + + bitmap_zero(feature->enabled, feature->feature_num); + bitmap_zero(feature->supported, feature->feature_num); + + if (!en) + return ret; + + ret = smu_cmn_get_enabled_32_bits_mask(smu, feature_mask, 2); + if (ret) + return ret; + + bitmap_copy(feature->enabled, (unsigned long *)&feature_mask, + feature->feature_num); + bitmap_copy(feature->supported, (unsigned long *)&feature_mask, + feature->feature_num); + + return 0; +} + +static int vangogh_post_smu_init(struct smu_context *smu) +{ + struct amdgpu_device *adev = smu->adev; + uint32_t tmp; + int ret = 0; + uint8_t aon_bits = 0; + /* Two CUs in one WGP */ + uint32_t req_active_wgps = adev->gfx.cu_info.number/2; + uint32_t total_cu = adev->gfx.config.max_cu_per_sh * + adev->gfx.config.max_sh_per_se * adev->gfx.config.max_shader_engines; + + /* allow message will be sent after enable message on Vangogh*/ + if (smu_cmn_feature_is_supported(smu, SMU_FEATURE_DPM_GFXCLK_BIT) && + (adev->pg_flags & AMD_PG_SUPPORT_GFX_PG)) { + ret = smu_cmn_send_smc_msg(smu, SMU_MSG_EnableGfxOff, NULL); + if (ret) { + dev_err(adev->dev, "Failed to Enable GfxOff!\n"); + return ret; + } + } else { + adev->pm.pp_feature &= ~PP_GFXOFF_MASK; + dev_info(adev->dev, "If GFX DPM or power gate disabled, disable GFXOFF\n"); + } + + /* if all CUs are active, no need to power off any WGPs */ + if (total_cu == adev->gfx.cu_info.number) return 0; + + /* + * Calculate the total bits number of always on WGPs for all SA/SEs in + * RLC_PG_ALWAYS_ON_WGP_MASK. + */ + tmp = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_ALWAYS_ON_WGP_MASK)); + tmp &= RLC_PG_ALWAYS_ON_WGP_MASK__AON_WGP_MASK_MASK; + + aon_bits = hweight32(tmp) * adev->gfx.config.max_sh_per_se * adev->gfx.config.max_shader_engines; + + /* Do not request any WGPs less than set in the AON_WGP_MASK */ + if (aon_bits > req_active_wgps) { + dev_info(adev->dev, "Number of always on WGPs greater than active WGPs: WGP power save not requested.\n"); + return 0; + } else { + return smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_RequestActiveWgp, req_active_wgps, NULL); + } +} + +static int vangogh_mode_reset(struct smu_context *smu, int type) +{ + int ret = 0, index = 0; + + index = smu_cmn_to_asic_specific_index(smu, CMN2ASIC_MAPPING_MSG, + SMU_MSG_GfxDeviceDriverReset); + if (index < 0) + return index == -EACCES ? 0 : index; + + mutex_lock(&smu->message_lock); + + ret = smu_cmn_send_msg_without_waiting(smu, (uint16_t)index, type); + + mutex_unlock(&smu->message_lock); + + mdelay(10); + + return ret; +} + +static int vangogh_mode2_reset(struct smu_context *smu) +{ + return vangogh_mode_reset(smu, SMU_RESET_MODE_2); +} + +static int vangogh_get_power_limit(struct smu_context *smu) +{ + struct smu_11_5_power_context *power_context = + smu->smu_power.power_context; + uint32_t ppt_limit; + int ret = 0; + + if (smu->adev->pm.fw_version < 0x43f1e00) + return ret; + + ret = smu_cmn_send_smc_msg(smu, SMU_MSG_GetSlowPPTLimit, &ppt_limit); + if (ret) { + dev_err(smu->adev->dev, "Get slow PPT limit failed!\n"); + return ret; + } + /* convert from milliwatt to watt */ + smu->current_power_limit = ppt_limit / 1000; + smu->max_power_limit = 29; + + ret = smu_cmn_send_smc_msg(smu, SMU_MSG_GetFastPPTLimit, &ppt_limit); + if (ret) { + dev_err(smu->adev->dev, "Get fast PPT limit failed!\n"); + return ret; + } + /* convert from milliwatt to watt */ + power_context->current_fast_ppt_limit = ppt_limit / 1000; + power_context->max_fast_ppt_limit = 30; + + return ret; +} + +static int vangogh_get_ppt_limit(struct smu_context *smu, + uint32_t *ppt_limit, + enum smu_ppt_limit_type type, + enum smu_ppt_limit_level level) +{ + struct smu_11_5_power_context *power_context = + smu->smu_power.power_context; + + if (!power_context) + return -EOPNOTSUPP; + + if (type == SMU_FAST_PPT_LIMIT) { + switch (level) { + case SMU_PPT_LIMIT_MAX: + *ppt_limit = power_context->max_fast_ppt_limit; + break; + case SMU_PPT_LIMIT_CURRENT: + *ppt_limit = power_context->current_fast_ppt_limit; + break; + default: + break; + } + } + + return 0; +} + +static int vangogh_set_power_limit(struct smu_context *smu, uint32_t ppt_limit) +{ + struct smu_11_5_power_context *power_context = + smu->smu_power.power_context; + uint32_t limit_type = ppt_limit >> 24; + int ret = 0; + + if (!smu_cmn_feature_is_enabled(smu, SMU_FEATURE_PPT_BIT)) { + dev_err(smu->adev->dev, "Setting new power limit is not supported!\n"); + return -EOPNOTSUPP; + } + + switch (limit_type) { + case SMU_DEFAULT_PPT_LIMIT: + ret = smu_cmn_send_smc_msg_with_param(smu, + SMU_MSG_SetSlowPPTLimit, + ppt_limit * 1000, /* convert from watt to milliwatt */ + NULL); + if (ret) + return ret; + + smu->current_power_limit = ppt_limit; + break; + case SMU_FAST_PPT_LIMIT: + ppt_limit &= ~(SMU_FAST_PPT_LIMIT << 24); + if (ppt_limit > power_context->max_fast_ppt_limit) { + dev_err(smu->adev->dev, + "New power limit (%d) is over the max allowed %d\n", + ppt_limit, power_context->max_fast_ppt_limit); + return ret; + } + + ret = smu_cmn_send_smc_msg_with_param(smu, + SMU_MSG_SetFastPPTLimit, + ppt_limit * 1000, /* convert from watt to milliwatt */ + NULL); + if (ret) + return ret; + + power_context->current_fast_ppt_limit = ppt_limit; + break; + default: + return -EINVAL; + } + + return ret; } static const struct pptable_funcs vangogh_ppt_funcs = { @@ -746,7 +1893,6 @@ static const struct pptable_funcs vangogh_ppt_funcs = { .init_power = smu_v11_0_init_power, .fini_power = smu_v11_0_fini_power, .register_irq_handler = smu_v11_0_register_irq_handler, - .get_allowed_feature_mask = vangogh_get_allowed_feature_mask, .notify_memory_pool_location = smu_v11_0_notify_memory_pool_location, .send_smc_msg_with_param = smu_cmn_send_smc_msg_with_param, .send_smc_msg = smu_cmn_send_smc_msg, @@ -765,6 +1911,18 @@ static const struct pptable_funcs vangogh_ppt_funcs = { .set_default_dpm_table = vangogh_set_default_dpm_tables, .set_fine_grain_gfx_freq_parameters = vangogh_set_fine_grain_gfx_freq_parameters, .system_features_control = vangogh_system_features_control, + .feature_is_enabled = smu_cmn_feature_is_enabled, + .set_power_profile_mode = vangogh_set_power_profile_mode, + .get_power_profile_mode = vangogh_get_power_profile_mode, + .get_dpm_clock_table = vangogh_get_dpm_clock_table, + .force_clk_levels = vangogh_force_clk_levels, + .set_performance_level = vangogh_set_performance_level, + .post_init = vangogh_post_smu_init, + .mode2_reset = vangogh_mode2_reset, + .gfx_off_control = smu_v11_0_gfx_off_control, + .get_ppt_limit = vangogh_get_ppt_limit, + .get_power_limit = vangogh_get_power_limit, + .set_power_limit = vangogh_set_power_limit, }; void vangogh_set_ppt_funcs(struct smu_context *smu) @@ -773,5 +1931,6 @@ void vangogh_set_ppt_funcs(struct smu_context *smu) smu->message_map = vangogh_message_map; smu->feature_map = vangogh_feature_mask_map; smu->table_map = vangogh_table_map; + smu->workload_map = vangogh_workload_map; smu->is_apu = true; } diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu11/vangogh_ppt.h b/drivers/gpu/drm/amd/pm/swsmu/smu11/vangogh_ppt.h index eab45549307646c1a3665d44d41d2b91034e005c..c56d4583dc7233aa4ab4a6f6dfbe2e96dea74ff2 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu11/vangogh_ppt.h +++ b/drivers/gpu/drm/amd/pm/swsmu/smu11/vangogh_ppt.h @@ -28,9 +28,29 @@ extern void vangogh_set_ppt_funcs(struct smu_context *smu); /* UMD PState Vangogh Msg Parameters in MHz */ -#define VANGOGH_UMD_PSTATE_GFXCLK 700 -#define VANGOGH_UMD_PSTATE_SOCCLK 678 -#define VANGOGH_UMD_PSTATE_FCLK 800 +#define VANGOGH_UMD_PSTATE_STANDARD_GFXCLK 1100 +#define VANGOGH_UMD_PSTATE_STANDARD_SOCCLK 600 +#define VANGOGH_UMD_PSTATE_STANDARD_FCLK 800 +#define VANGOGH_UMD_PSTATE_STANDARD_VCLK 705 +#define VANGOGH_UMD_PSTATE_STANDARD_DCLK 600 + +#define VANGOGH_UMD_PSTATE_PEAK_GFXCLK 1300 +#define VANGOGH_UMD_PSTATE_PEAK_SOCCLK 600 +#define VANGOGH_UMD_PSTATE_PEAK_FCLK 800 +#define VANGOGH_UMD_PSTATE_PEAK_VCLK 705 +#define VANGOGH_UMD_PSTATE_PEAK_DCLK 600 + +#define VANGOGH_UMD_PSTATE_MIN_SCLK_GFXCLK 400 +#define VANGOGH_UMD_PSTATE_MIN_SCLK_SOCCLK 1000 +#define VANGOGH_UMD_PSTATE_MIN_SCLK_FCLK 800 +#define VANGOGH_UMD_PSTATE_MIN_SCLK_VCLK 1000 +#define VANGOGH_UMD_PSTATE_MIN_SCLK_DCLK 800 + +#define VANGOGH_UMD_PSTATE_MIN_MCLK_GFXCLK 1100 +#define VANGOGH_UMD_PSTATE_MIN_MCLK_SOCCLK 1000 +#define VANGOGH_UMD_PSTATE_MIN_MCLK_FCLK 400 +#define VANGOGH_UMD_PSTATE_MIN_MCLK_VCLK 1000 +#define VANGOGH_UMD_PSTATE_MIN_MCLK_DCLK 800 /* RLC Power Status */ #define RLC_STATUS_OFF 0 diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu12/renoir_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu12/renoir_ppt.c index 9a9697038016027d84c2cc2696a86922df5a8eda..5faa509f0dba9fb84c1a4699cd826c25218ba88d 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu12/renoir_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu12/renoir_ppt.c @@ -56,8 +56,6 @@ static struct cmn2asic_msg_mapping renoir_message_map[SMU_MSG_MAX_COUNT] = { MSG_MAP(PowerUpSdma, PPSMC_MSG_PowerUpSdma, 1), MSG_MAP(SetHardMinIspclkByFreq, PPSMC_MSG_SetHardMinIspclkByFreq, 1), MSG_MAP(SetHardMinVcn, PPSMC_MSG_SetHardMinVcn, 1), - MSG_MAP(Spare1, PPSMC_MSG_spare1, 1), - MSG_MAP(Spare2, PPSMC_MSG_spare2, 1), MSG_MAP(SetAllowFclkSwitch, PPSMC_MSG_SetAllowFclkSwitch, 1), MSG_MAP(SetMinVideoGfxclkFreq, PPSMC_MSG_SetMinVideoGfxclkFreq, 1), MSG_MAP(ActiveProcessNotify, PPSMC_MSG_ActiveProcessNotify, 1), @@ -344,12 +342,142 @@ static int renoir_get_dpm_ultimate_freq(struct smu_context *smu, return ret; } +static int renoir_od_edit_dpm_table(struct smu_context *smu, + enum PP_OD_DPM_TABLE_COMMAND type, + long input[], uint32_t size) +{ + int ret = 0; + struct smu_dpm_context *smu_dpm_ctx = &(smu->smu_dpm); + + if (!(smu_dpm_ctx->dpm_level == AMD_DPM_FORCED_LEVEL_MANUAL)) { + dev_warn(smu->adev->dev, + "pp_od_clk_voltage is not accessible if power_dpm_force_perfomance_level is not in manual mode!\n"); + return -EINVAL; + } + + switch (type) { + case PP_OD_EDIT_SCLK_VDDC_TABLE: + if (size != 2) { + dev_err(smu->adev->dev, "Input parameter number not correct\n"); + return -EINVAL; + } + + if (input[0] == 0) { + if (input[1] < smu->gfx_default_hard_min_freq) { + dev_warn(smu->adev->dev, + "Fine grain setting minimum sclk (%ld) MHz is less than the minimum allowed (%d) MHz\n", + input[1], smu->gfx_default_hard_min_freq); + return -EINVAL; + } + smu->gfx_actual_hard_min_freq = input[1]; + } else if (input[0] == 1) { + if (input[1] > smu->gfx_default_soft_max_freq) { + dev_warn(smu->adev->dev, + "Fine grain setting maximum sclk (%ld) MHz is greater than the maximum allowed (%d) MHz\n", + input[1], smu->gfx_default_soft_max_freq); + return -EINVAL; + } + smu->gfx_actual_soft_max_freq = input[1]; + } else { + return -EINVAL; + } + break; + case PP_OD_RESTORE_DEFAULT_TABLE: + if (size != 0) { + dev_err(smu->adev->dev, "Input parameter number not correct\n"); + return -EINVAL; + } + smu->gfx_actual_hard_min_freq = smu->gfx_default_hard_min_freq; + smu->gfx_actual_soft_max_freq = smu->gfx_default_soft_max_freq; + + ret = smu_cmn_send_smc_msg_with_param(smu, + SMU_MSG_SetHardMinGfxClk, + smu->gfx_actual_hard_min_freq, + NULL); + if (ret) { + dev_err(smu->adev->dev, "Restore the default hard min sclk failed!"); + return ret; + } + + ret = smu_cmn_send_smc_msg_with_param(smu, + SMU_MSG_SetSoftMaxGfxClk, + smu->gfx_actual_soft_max_freq, + NULL); + if (ret) { + dev_err(smu->adev->dev, "Restore the default soft max sclk failed!"); + return ret; + } + break; + case PP_OD_COMMIT_DPM_TABLE: + if (size != 0) { + dev_err(smu->adev->dev, "Input parameter number not correct\n"); + return -EINVAL; + } else { + if (smu->gfx_actual_hard_min_freq > smu->gfx_actual_soft_max_freq) { + dev_err(smu->adev->dev, + "The setting minimun sclk (%d) MHz is greater than the setting maximum sclk (%d) MHz\n", + smu->gfx_actual_hard_min_freq, + smu->gfx_actual_soft_max_freq); + return -EINVAL; + } + + ret = smu_cmn_send_smc_msg_with_param(smu, + SMU_MSG_SetHardMinGfxClk, + smu->gfx_actual_hard_min_freq, + NULL); + if (ret) { + dev_err(smu->adev->dev, "Set hard min sclk failed!"); + return ret; + } + + ret = smu_cmn_send_smc_msg_with_param(smu, + SMU_MSG_SetSoftMaxGfxClk, + smu->gfx_actual_soft_max_freq, + NULL); + if (ret) { + dev_err(smu->adev->dev, "Set soft max sclk failed!"); + return ret; + } + } + break; + default: + return -ENOSYS; + } + + return ret; +} + +static int renoir_set_fine_grain_gfx_freq_parameters(struct smu_context *smu) +{ + uint32_t min = 0, max = 0; + uint32_t ret = 0; + + ret = smu_cmn_send_smc_msg_with_param(smu, + SMU_MSG_GetMinGfxclkFrequency, + 0, &min); + if (ret) + return ret; + ret = smu_cmn_send_smc_msg_with_param(smu, + SMU_MSG_GetMaxGfxclkFrequency, + 0, &max); + if (ret) + return ret; + + smu->gfx_default_hard_min_freq = min; + smu->gfx_default_soft_max_freq = max; + smu->gfx_actual_hard_min_freq = 0; + smu->gfx_actual_soft_max_freq = 0; + + return 0; +} + static int renoir_print_clk_levels(struct smu_context *smu, enum smu_clk_type clk_type, char *buf) { int i, size = 0, ret = 0; uint32_t cur_value = 0, value = 0, count = 0, min = 0, max = 0; SmuMetrics_t metrics; + struct smu_dpm_context *smu_dpm_ctx = &(smu->smu_dpm); bool cur_value_match_level = false; memset(&metrics, 0, sizeof(metrics)); @@ -359,6 +487,30 @@ static int renoir_print_clk_levels(struct smu_context *smu, return ret; switch (clk_type) { + case SMU_OD_RANGE: + if (smu_dpm_ctx->dpm_level == AMD_DPM_FORCED_LEVEL_MANUAL) { + ret = smu_cmn_send_smc_msg_with_param(smu, + SMU_MSG_GetMinGfxclkFrequency, + 0, &min); + if (ret) + return ret; + ret = smu_cmn_send_smc_msg_with_param(smu, + SMU_MSG_GetMaxGfxclkFrequency, + 0, &max); + if (ret) + return ret; + size += sprintf(buf + size, "OD_RANGE\nSCLK: %10uMhz %10uMhz\n", min, max); + } + break; + case SMU_OD_SCLK: + if (smu_dpm_ctx->dpm_level == AMD_DPM_FORCED_LEVEL_MANUAL) { + min = (smu->gfx_actual_hard_min_freq > 0) ? smu->gfx_actual_hard_min_freq : smu->gfx_default_hard_min_freq; + max = (smu->gfx_actual_soft_max_freq > 0) ? smu->gfx_actual_soft_max_freq : smu->gfx_default_soft_max_freq; + size += sprintf(buf + size, "OD_SCLK\n"); + size += sprintf(buf + size, "0:%10uMhz\n", min); + size += sprintf(buf + size, "1:%10uMhz\n", max); + } + break; case SMU_GFXCLK: case SMU_SCLK: /* retirve table returned paramters unit is MHz */ @@ -399,23 +551,35 @@ static int renoir_print_clk_levels(struct smu_context *smu, cur_value = metrics.ClockFrequency[CLOCK_FCLK]; break; default: - return -EINVAL; + break; } - for (i = 0; i < count; i++) { - ret = renoir_get_dpm_clk_limited(smu, clk_type, i, &value); - if (ret) - return ret; - if (!value) - continue; - size += sprintf(buf + size, "%d: %uMhz %s\n", i, value, - cur_value == value ? "*" : ""); - if (cur_value == value) - cur_value_match_level = true; - } + switch (clk_type) { + case SMU_GFXCLK: + case SMU_SCLK: + case SMU_SOCCLK: + case SMU_MCLK: + case SMU_DCEFCLK: + case SMU_FCLK: + for (i = 0; i < count; i++) { + ret = renoir_get_dpm_clk_limited(smu, clk_type, i, &value); + if (ret) + return ret; + if (!value) + continue; + size += sprintf(buf + size, "%d: %uMhz %s\n", i, value, + cur_value == value ? "*" : ""); + if (cur_value == value) + cur_value_match_level = true; + } + + if (!cur_value_match_level) + size += sprintf(buf + size, " %uMhz *\n", cur_value); - if (!cur_value_match_level) - size += sprintf(buf + size, " %uMhz *\n", cur_value); + break; + default: + break; + } return size; } @@ -667,6 +831,10 @@ static int renoir_set_power_profile_mode(struct smu_context *smu, long *input, u return -EINVAL; } + if (profile_mode == PP_SMC_POWER_PROFILE_BOOTUP_DEFAULT || + profile_mode == PP_SMC_POWER_PROFILE_POWERSAVING) + return 0; + /* conv PP_SMC_POWER_PROFILE* to WORKLOAD_PPLIB_*_BIT */ workload_type = smu_cmn_to_asic_specific_index(smu, CMN2ASIC_MAPPING_WORKLOAD, @@ -725,15 +893,27 @@ static int renoir_set_performance_level(struct smu_context *smu, switch (level) { case AMD_DPM_FORCED_LEVEL_HIGH: + smu->gfx_actual_hard_min_freq = smu->gfx_default_hard_min_freq; + smu->gfx_actual_soft_max_freq = smu->gfx_default_soft_max_freq; + ret = renoir_force_dpm_limit_value(smu, true); break; case AMD_DPM_FORCED_LEVEL_LOW: + smu->gfx_actual_hard_min_freq = smu->gfx_default_hard_min_freq; + smu->gfx_actual_soft_max_freq = smu->gfx_default_soft_max_freq; + ret = renoir_force_dpm_limit_value(smu, false); break; case AMD_DPM_FORCED_LEVEL_AUTO: + smu->gfx_actual_hard_min_freq = smu->gfx_default_hard_min_freq; + smu->gfx_actual_soft_max_freq = smu->gfx_default_soft_max_freq; + ret = renoir_unforce_dpm_levels(smu); break; case AMD_DPM_FORCED_LEVEL_PROFILE_STANDARD: + smu->gfx_actual_hard_min_freq = smu->gfx_default_hard_min_freq; + smu->gfx_actual_soft_max_freq = smu->gfx_default_soft_max_freq; + ret = smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_SetHardMinGfxClk, RENOIR_UMD_PSTATE_GFXCLK, @@ -786,6 +966,9 @@ static int renoir_set_performance_level(struct smu_context *smu, break; case AMD_DPM_FORCED_LEVEL_PROFILE_MIN_SCLK: case AMD_DPM_FORCED_LEVEL_PROFILE_MIN_MCLK: + smu->gfx_actual_hard_min_freq = smu->gfx_default_hard_min_freq; + smu->gfx_actual_soft_max_freq = smu->gfx_default_soft_max_freq; + ret = renoir_get_profiling_clk_mask(smu, level, &sclk_mask, &mclk_mask, @@ -797,6 +980,9 @@ static int renoir_set_performance_level(struct smu_context *smu, renoir_force_clk_levels(smu, SMU_SOCCLK, 1 << soc_mask); break; case AMD_DPM_FORCED_LEVEL_PROFILE_PEAK: + smu->gfx_actual_hard_min_freq = smu->gfx_default_hard_min_freq; + smu->gfx_actual_soft_max_freq = smu->gfx_default_soft_max_freq; + ret = renoir_set_peak_clock_by_device(smu); break; case AMD_DPM_FORCED_LEVEL_MANUAL: @@ -944,7 +1130,7 @@ static int renoir_get_smu_metrics_data(struct smu_context *smu, *value = metrics->AverageUvdActivity / 100; break; case METRICS_AVERAGE_SOCKETPOWER: - *value = metrics->CurrentSocketPower << 8; + *value = (metrics->CurrentSocketPower << 8) / 1000; break; case METRICS_TEMPERATURE_EDGE: *value = (metrics->GfxTemperature / 100) * @@ -1072,7 +1258,7 @@ static ssize_t renoir_get_gpu_metrics(struct smu_context *smu, if (ret) return ret; - smu_v12_0_init_gpu_metrics_v2_0(gpu_metrics); + smu_cmn_init_soft_gpu_metrics(gpu_metrics, 2, 0); gpu_metrics->temperature_gfx = metrics.GfxTemperature; gpu_metrics->temperature_soc = metrics.SocTemperature; @@ -1113,6 +1299,8 @@ static ssize_t renoir_get_gpu_metrics(struct smu_context *smu, gpu_metrics->fan_pwm = metrics.FanPwm; + gpu_metrics->system_clock_counter = ktime_get_boottime_ns(); + *table = (void *)gpu_metrics; return sizeof(struct gpu_metrics_v2_0); @@ -1160,6 +1348,8 @@ static const struct pptable_funcs renoir_ppt_funcs = { .set_pp_feature_mask = smu_cmn_set_pp_feature_mask, .get_gpu_metrics = renoir_get_gpu_metrics, .gfx_state_change_set = renoir_gfx_state_change_set, + .set_fine_grain_gfx_freq_parameters = renoir_set_fine_grain_gfx_freq_parameters, + .od_edit_dpm_table = renoir_od_edit_dpm_table, }; void renoir_set_ppt_funcs(struct smu_context *smu) diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu12/smu_v12_0.c b/drivers/gpu/drm/amd/pm/swsmu/smu12/smu_v12_0.c index 06abf2a7ce9e9e0f7f116f5c36223f5264d838b3..6cc4855c8a37148431716dac2152178001f3542a 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu12/smu_v12_0.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu12/smu_v12_0.c @@ -278,15 +278,3 @@ int smu_v12_0_set_driver_table_location(struct smu_context *smu) return ret; } - -void smu_v12_0_init_gpu_metrics_v2_0(struct gpu_metrics_v2_0 *gpu_metrics) -{ - memset(gpu_metrics, 0xFF, sizeof(struct gpu_metrics_v2_0)); - - gpu_metrics->common_header.structure_size = - sizeof(struct gpu_metrics_v2_0); - gpu_metrics->common_header.format_revision = 2; - gpu_metrics->common_header.content_revision = 0; - - gpu_metrics->system_clock_counter = ktime_get_boottime_ns(); -} diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c b/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c index f8260769061c538e630b60f1857d161806d9ba48..bcedd4d92e356c172fa1d6c00d37b8fe4ce8b97f 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c @@ -68,14 +68,6 @@ static const char *smu_get_message_name(struct smu_context *smu, return __smu_message_names[type]; } -static void smu_cmn_send_msg_without_waiting(struct smu_context *smu, - uint16_t msg) -{ - struct amdgpu_device *adev = smu->adev; - - WREG32_SOC15_NO_KIQ(MP1, 0, mmMP1_SMN_C2PMSG_66, msg); -} - static void smu_cmn_read_arg(struct smu_context *smu, uint32_t *arg) { @@ -92,7 +84,7 @@ static int smu_cmn_wait_for_response(struct smu_context *smu) for (i = 0; i < timeout; i++) { cur_value = RREG32_SOC15_NO_KIQ(MP1, 0, mmMP1_SMN_C2PMSG_90); if ((cur_value & MP1_C2PMSG_90__CONTENT_MASK) != 0) - return cur_value == 0x1 ? 0 : -EIO; + return cur_value; udelay(1); } @@ -101,7 +93,29 @@ static int smu_cmn_wait_for_response(struct smu_context *smu) if (i == timeout) return -ETIME; - return RREG32_SOC15_NO_KIQ(MP1, 0, mmMP1_SMN_C2PMSG_90) == 0x1 ? 0 : -EIO; + return RREG32_SOC15_NO_KIQ(MP1, 0, mmMP1_SMN_C2PMSG_90); +} + +int smu_cmn_send_msg_without_waiting(struct smu_context *smu, + uint16_t msg, uint32_t param) +{ + struct amdgpu_device *adev = smu->adev; + int ret; + + ret = smu_cmn_wait_for_response(smu); + if (ret != 0x1) { + dev_err(adev->dev, "Msg issuing pre-check failed and " + "SMU may be not in the right state!\n"); + if (ret != -ETIME) + ret = -EIO; + return ret; + } + + WREG32_SOC15_NO_KIQ(MP1, 0, mmMP1_SMN_C2PMSG_90, 0); + WREG32_SOC15_NO_KIQ(MP1, 0, mmMP1_SMN_C2PMSG_82, param); + WREG32_SOC15_NO_KIQ(MP1, 0, mmMP1_SMN_C2PMSG_66, msg); + + return 0; } int smu_cmn_send_smc_msg_with_param(struct smu_context *smu, @@ -122,29 +136,28 @@ int smu_cmn_send_smc_msg_with_param(struct smu_context *smu, return index == -EACCES ? 0 : index; mutex_lock(&smu->message_lock); - ret = smu_cmn_wait_for_response(smu); - if (ret) { - dev_err(adev->dev, "Msg issuing pre-check failed and " - "SMU may be not in the right state!\n"); + ret = smu_cmn_send_msg_without_waiting(smu, (uint16_t)index, param); + if (ret) goto out; - } - - WREG32_SOC15_NO_KIQ(MP1, 0, mmMP1_SMN_C2PMSG_90, 0); - - WREG32_SOC15_NO_KIQ(MP1, 0, mmMP1_SMN_C2PMSG_82, param); - - smu_cmn_send_msg_without_waiting(smu, (uint16_t)index); ret = smu_cmn_wait_for_response(smu); - if (ret) { - dev_err(adev->dev, "failed send message: %10s (%d) \tparam: 0x%08x response %#x\n", - smu_get_message_name(smu, msg), index, param, ret); + if (ret != 0x1) { + if (ret == -ETIME) { + dev_err(adev->dev, "message: %15s (%d) \tparam: 0x%08x is timeout (no response)\n", + smu_get_message_name(smu, msg), index, param); + } else { + dev_err(adev->dev, "failed send message: %15s (%d) \tparam: 0x%08x response %#x\n", + smu_get_message_name(smu, msg), index, param, + ret); + ret = -EIO; + } goto out; } if (read_arg) smu_cmn_read_arg(smu, read_arg); + ret = 0; /* 0 as driver return value */ out: mutex_unlock(&smu->message_lock); return ret; @@ -269,11 +282,13 @@ int smu_cmn_feature_is_enabled(struct smu_context *smu, enum smu_feature_mask mask) { struct smu_feature *feature = &smu->smu_feature; + struct amdgpu_device *adev = smu->adev; int feature_id; int ret = 0; - if (smu->is_apu) + if (smu->is_apu && adev->family < AMDGPU_FAMILY_VGH) return 1; + feature_id = smu_cmn_to_asic_specific_index(smu, CMN2ASIC_MAPPING_FEATURE, mask); @@ -731,3 +746,31 @@ int smu_cmn_get_metrics_table(struct smu_context *smu, return ret; } + +void smu_cmn_init_soft_gpu_metrics(void *table, uint8_t frev, uint8_t crev) +{ + struct metrics_table_header *header = (struct metrics_table_header *)table; + uint16_t structure_size; + +#define METRICS_VERSION(a, b) ((a << 16) | b ) + + switch (METRICS_VERSION(frev, crev)) { + case METRICS_VERSION(1, 0): + structure_size = sizeof(struct gpu_metrics_v1_0); + break; + case METRICS_VERSION(2, 0): + structure_size = sizeof(struct gpu_metrics_v2_0); + break; + default: + return; + } + +#undef METRICS_VERSION + + memset(header, 0xFF, structure_size); + + header->format_revision = frev; + header->content_revision = crev; + header->structure_size = structure_size; + +} diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.h b/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.h index 01e825d83d8df87c59b1ef39afa1eae6d15b0a1f..c6925018557555c91e4efc8ab4ee3263f07b52b8 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.h +++ b/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.h @@ -26,6 +26,8 @@ #include "amdgpu_smu.h" #if defined(SWSMU_CODE_LAYER_L2) || defined(SWSMU_CODE_LAYER_L3) || defined(SWSMU_CODE_LAYER_L4) +int smu_cmn_send_msg_without_waiting(struct smu_context *smu, + uint16_t msg, uint32_t param); int smu_cmn_send_smc_msg_with_param(struct smu_context *smu, enum smu_message_type msg, uint32_t param, @@ -95,5 +97,7 @@ int smu_cmn_get_metrics_table(struct smu_context *smu, void *metrics_table, bool bypass_cache); +void smu_cmn_init_soft_gpu_metrics(void *table, uint8_t frev, uint8_t crev); + #endif #endif diff --git a/drivers/gpu/drm/arc/arcpgu_crtc.c b/drivers/gpu/drm/arc/arcpgu_crtc.c index 042d7b54a6deae4264620c2d10d934b9982e3210..895cdd991af688b6aa5482adadfc3f05387efd4f 100644 --- a/drivers/gpu/drm/arc/arcpgu_crtc.c +++ b/drivers/gpu/drm/arc/arcpgu_crtc.c @@ -162,15 +162,10 @@ static const struct drm_plane_helper_funcs arc_pgu_plane_helper_funcs = { .atomic_update = arc_pgu_plane_atomic_update, }; -static void arc_pgu_plane_destroy(struct drm_plane *plane) -{ - drm_plane_cleanup(plane); -} - static const struct drm_plane_funcs arc_pgu_plane_funcs = { .update_plane = drm_atomic_helper_update_plane, .disable_plane = drm_atomic_helper_disable_plane, - .destroy = arc_pgu_plane_destroy, + .destroy = drm_plane_cleanup, .reset = drm_atomic_helper_plane_reset, .atomic_duplicate_state = drm_atomic_helper_plane_duplicate_state, .atomic_destroy_state = drm_atomic_helper_plane_destroy_state, @@ -213,7 +208,7 @@ int arc_pgu_setup_crtc(struct drm_device *drm) ret = drm_crtc_init_with_planes(drm, &arcpgu->crtc, primary, NULL, &arc_pgu_crtc_funcs, NULL); if (ret) { - arc_pgu_plane_destroy(primary); + drm_plane_cleanup(primary); return ret; } diff --git a/drivers/gpu/drm/arc/arcpgu_drv.c b/drivers/gpu/drm/arc/arcpgu_drv.c index f164818ec477a5dfbf694231d400ed0b6c4e3d03..077d006b1fbf50f9488203969d0fab5c8a9498b5 100644 --- a/drivers/gpu/drm/arc/arcpgu_drv.c +++ b/drivers/gpu/drm/arc/arcpgu_drv.c @@ -145,7 +145,7 @@ static void arcpgu_debugfs_init(struct drm_minor *minor) } #endif -static struct drm_driver arcpgu_drm_driver = { +static const struct drm_driver arcpgu_drm_driver = { .driver_features = DRIVER_MODESET | DRIVER_GEM | DRIVER_ATOMIC, .name = "arcpgu", .desc = "ARC PGU Controller", diff --git a/drivers/gpu/drm/arm/display/komeda/komeda_crtc.c b/drivers/gpu/drm/arm/display/komeda/komeda_crtc.c index 4b485eb512e2e3686b2025aac69b0beb5f0a36eb..59172acb973803d29c853ea4f2ad9a5538770ae3 100644 --- a/drivers/gpu/drm/arm/display/komeda/komeda_crtc.c +++ b/drivers/gpu/drm/arm/display/komeda/komeda_crtc.c @@ -550,7 +550,6 @@ static void komeda_crtc_vblank_disable(struct drm_crtc *crtc) } static const struct drm_crtc_funcs komeda_crtc_funcs = { - .gamma_set = drm_atomic_helper_legacy_gamma_set, .destroy = drm_crtc_cleanup, .set_config = drm_atomic_helper_set_config, .page_flip = drm_atomic_helper_page_flip, diff --git a/drivers/gpu/drm/arm/malidp_crtc.c b/drivers/gpu/drm/arm/malidp_crtc.c index 108e7a31bd26bed69a4349b3a8c534befb7ce38d..494075ddbef68374aaca8927a1141379925adb38 100644 --- a/drivers/gpu/drm/arm/malidp_crtc.c +++ b/drivers/gpu/drm/arm/malidp_crtc.c @@ -510,7 +510,6 @@ static void malidp_crtc_disable_vblank(struct drm_crtc *crtc) } static const struct drm_crtc_funcs malidp_crtc_funcs = { - .gamma_set = drm_atomic_helper_legacy_gamma_set, .destroy = drm_crtc_cleanup, .set_config = drm_atomic_helper_set_config, .page_flip = drm_atomic_helper_page_flip, diff --git a/drivers/gpu/drm/armada/armada_crtc.c b/drivers/gpu/drm/armada/armada_crtc.c index 3ebcf5a52c8bc42421e99a90638dc0a1cdc25a3b..b7bb90ae787f7cb209aedff1caf3fa383f340f50 100644 --- a/drivers/gpu/drm/armada/armada_crtc.c +++ b/drivers/gpu/drm/armada/armada_crtc.c @@ -820,7 +820,6 @@ static const struct drm_crtc_funcs armada_crtc_funcs = { .cursor_set = armada_drm_crtc_cursor_set, .cursor_move = armada_drm_crtc_cursor_move, .destroy = armada_drm_crtc_destroy, - .gamma_set = drm_atomic_helper_legacy_gamma_set, .set_config = drm_atomic_helper_set_config, .page_flip = drm_atomic_helper_page_flip, .atomic_duplicate_state = drm_atomic_helper_crtc_duplicate_state, diff --git a/drivers/gpu/drm/ast/ast_cursor.c b/drivers/gpu/drm/ast/ast_cursor.c index 742d43a7edf433f6d6631f44bd72a3146c44aa06..fac1ee79c372e65af6be7d2d1cc7479f111a27a5 100644 --- a/drivers/gpu/drm/ast/ast_cursor.c +++ b/drivers/gpu/drm/ast/ast_cursor.c @@ -39,7 +39,6 @@ static void ast_cursor_fini(struct ast_private *ast) for (i = 0; i < ARRAY_SIZE(ast->cursor.gbo); ++i) { gbo = ast->cursor.gbo[i]; - drm_gem_vram_vunmap(gbo, &ast->cursor.map[i]); drm_gem_vram_unpin(gbo); drm_gem_vram_put(gbo); } @@ -53,14 +52,13 @@ static void ast_cursor_release(struct drm_device *dev, void *ptr) } /* - * Allocate cursor BOs and pins them at the end of VRAM. + * Allocate cursor BOs and pin them at the end of VRAM. */ int ast_cursor_init(struct ast_private *ast) { struct drm_device *dev = &ast->base; size_t size, i; struct drm_gem_vram_object *gbo; - struct dma_buf_map map; int ret; size = roundup(AST_HWC_SIZE + AST_HWC_SIGNATURE_SIZE, PAGE_SIZE); @@ -77,15 +75,7 @@ int ast_cursor_init(struct ast_private *ast) drm_gem_vram_put(gbo); goto err_drm_gem_vram_put; } - ret = drm_gem_vram_vmap(gbo, &map); - if (ret) { - drm_gem_vram_unpin(gbo); - drm_gem_vram_put(gbo); - goto err_drm_gem_vram_put; - } - ast->cursor.gbo[i] = gbo; - ast->cursor.map[i] = map; } return drmm_add_action_or_reset(dev, ast_cursor_release, NULL); @@ -94,7 +84,6 @@ int ast_cursor_init(struct ast_private *ast) while (i) { --i; gbo = ast->cursor.gbo[i]; - drm_gem_vram_vunmap(gbo, &ast->cursor.map[i]); drm_gem_vram_unpin(gbo); drm_gem_vram_put(gbo); } @@ -168,38 +157,37 @@ static void update_cursor_image(u8 __iomem *dst, const u8 *src, int width, int h int ast_cursor_blit(struct ast_private *ast, struct drm_framebuffer *fb) { struct drm_device *dev = &ast->base; - struct drm_gem_vram_object *gbo; - struct dma_buf_map map; - int ret; - void *src; + struct drm_gem_vram_object *dst_gbo = ast->cursor.gbo[ast->cursor.next_index]; + struct drm_gem_vram_object *src_gbo = drm_gem_vram_of_gem(fb->obj[0]); + struct dma_buf_map src_map, dst_map; void __iomem *dst; + void *src; + int ret; if (drm_WARN_ON_ONCE(dev, fb->width > AST_MAX_HWC_WIDTH) || drm_WARN_ON_ONCE(dev, fb->height > AST_MAX_HWC_HEIGHT)) return -EINVAL; - gbo = drm_gem_vram_of_gem(fb->obj[0]); - - ret = drm_gem_vram_pin(gbo, 0); + ret = drm_gem_vram_vmap(src_gbo, &src_map); if (ret) return ret; - ret = drm_gem_vram_vmap(gbo, &map); - if (ret) - goto err_drm_gem_vram_unpin; - src = map.vaddr; /* TODO: Use mapping abstraction properly */ + src = src_map.vaddr; /* TODO: Use mapping abstraction properly */ - dst = ast->cursor.map[ast->cursor.next_index].vaddr_iomem; + ret = drm_gem_vram_vmap(dst_gbo, &dst_map); + if (ret) + goto err_drm_gem_vram_vunmap; + dst = dst_map.vaddr_iomem; /* TODO: Use mapping abstraction properly */ /* do data transfer to cursor BO */ update_cursor_image(dst, src, fb->width, fb->height); - drm_gem_vram_vunmap(gbo, &map); - drm_gem_vram_unpin(gbo); + drm_gem_vram_vunmap(dst_gbo, &dst_map); + drm_gem_vram_vunmap(src_gbo, &src_map); return 0; -err_drm_gem_vram_unpin: - drm_gem_vram_unpin(gbo); +err_drm_gem_vram_vunmap: + drm_gem_vram_vunmap(src_gbo, &src_map); return ret; } @@ -251,17 +239,26 @@ static void ast_cursor_set_location(struct ast_private *ast, u16 x, u16 y, void ast_cursor_show(struct ast_private *ast, int x, int y, unsigned int offset_x, unsigned int offset_y) { + struct drm_device *dev = &ast->base; + struct drm_gem_vram_object *gbo = ast->cursor.gbo[ast->cursor.next_index]; + struct dma_buf_map map; u8 x_offset, y_offset; u8 __iomem *dst; u8 __iomem *sig; u8 jreg; + int ret; - dst = ast->cursor.map[ast->cursor.next_index].vaddr; + ret = drm_gem_vram_vmap(gbo, &map); + if (drm_WARN_ONCE(dev, ret, "drm_gem_vram_vmap() failed, ret=%d\n", ret)) + return; + dst = map.vaddr_iomem; /* TODO: Use mapping abstraction properly */ sig = dst + AST_HWC_SIZE; writel(x, sig + AST_HWC_SIGNATURE_X); writel(y, sig + AST_HWC_SIGNATURE_Y); + drm_gem_vram_vunmap(gbo, &map); + if (x < 0) { x_offset = (-x) + offset_x; x = 0; diff --git a/drivers/gpu/drm/ast/ast_drv.c b/drivers/gpu/drm/ast/ast_drv.c index 667b450606ef8a0284bc3808dd9992f76235a2de..ea8164e7a6dc6f81bb6db7a26baa16101e3299f2 100644 --- a/drivers/gpu/drm/ast/ast_drv.c +++ b/drivers/gpu/drm/ast/ast_drv.c @@ -147,7 +147,7 @@ static int ast_drm_freeze(struct drm_device *dev) error = drm_mode_config_helper_suspend(dev); if (error) return error; - pci_save_state(dev->pdev); + pci_save_state(to_pci_dev(dev->dev)); return 0; } @@ -162,7 +162,7 @@ static int ast_drm_resume(struct drm_device *dev) { int ret; - if (pci_enable_device(dev->pdev)) + if (pci_enable_device(to_pci_dev(dev->dev))) return -EIO; ret = ast_drm_thaw(dev); diff --git a/drivers/gpu/drm/ast/ast_drv.h b/drivers/gpu/drm/ast/ast_drv.h index ccaff81924ee00faee43c830b35c6ec34d8b864a..f871fc36c2f7aaa320de71b2bc60b516ae2616f5 100644 --- a/drivers/gpu/drm/ast/ast_drv.h +++ b/drivers/gpu/drm/ast/ast_drv.h @@ -28,7 +28,6 @@ #ifndef __AST_DRV_H__ #define __AST_DRV_H__ -#include #include #include #include @@ -133,7 +132,6 @@ struct ast_private { struct { struct drm_gem_vram_object *gbo[AST_DEFAULT_HWC_NUM]; - struct dma_buf_map map[AST_DEFAULT_HWC_NUM]; unsigned int next_index; } cursor; diff --git a/drivers/gpu/drm/ast/ast_main.c b/drivers/gpu/drm/ast/ast_main.c index 1b13199858cbafa2b80be32c1cfe306009c66905..0ac3c2039c4b19ec0f2e5b7adfbd43cb50397a92 100644 --- a/drivers/gpu/drm/ast/ast_main.c +++ b/drivers/gpu/drm/ast/ast_main.c @@ -67,8 +67,9 @@ uint8_t ast_get_index_reg_mask(struct ast_private *ast, static void ast_detect_config_mode(struct drm_device *dev, u32 *scu_rev) { - struct device_node *np = dev->pdev->dev.of_node; + struct device_node *np = dev->dev->of_node; struct ast_private *ast = to_ast_private(dev); + struct pci_dev *pdev = to_pci_dev(dev->dev); uint32_t data, jregd0, jregd1; /* Defaults */ @@ -85,7 +86,7 @@ static void ast_detect_config_mode(struct drm_device *dev, u32 *scu_rev) } /* Not all families have a P2A bridge */ - if (dev->pdev->device != PCI_CHIP_AST2000) + if (pdev->device != PCI_CHIP_AST2000) return; /* @@ -119,6 +120,7 @@ static void ast_detect_config_mode(struct drm_device *dev, u32 *scu_rev) static int ast_detect_chip(struct drm_device *dev, bool *need_post) { struct ast_private *ast = to_ast_private(dev); + struct pci_dev *pdev = to_pci_dev(dev->dev); uint32_t jreg, scu_rev; /* @@ -143,19 +145,19 @@ static int ast_detect_chip(struct drm_device *dev, bool *need_post) ast_detect_config_mode(dev, &scu_rev); /* Identify chipset */ - if (dev->pdev->revision >= 0x50) { + if (pdev->revision >= 0x50) { ast->chip = AST2600; drm_info(dev, "AST 2600 detected\n"); - } else if (dev->pdev->revision >= 0x40) { + } else if (pdev->revision >= 0x40) { ast->chip = AST2500; drm_info(dev, "AST 2500 detected\n"); - } else if (dev->pdev->revision >= 0x30) { + } else if (pdev->revision >= 0x30) { ast->chip = AST2400; drm_info(dev, "AST 2400 detected\n"); - } else if (dev->pdev->revision >= 0x20) { + } else if (pdev->revision >= 0x20) { ast->chip = AST2300; drm_info(dev, "AST 2300 detected\n"); - } else if (dev->pdev->revision >= 0x10) { + } else if (pdev->revision >= 0x10) { switch (scu_rev & 0x0300) { case 0x0200: ast->chip = AST1100; @@ -265,7 +267,7 @@ static int ast_detect_chip(struct drm_device *dev, bool *need_post) static int ast_get_dram_info(struct drm_device *dev) { - struct device_node *np = dev->pdev->dev.of_node; + struct device_node *np = dev->dev->of_node; struct ast_private *ast = to_ast_private(dev); uint32_t mcr_cfg, mcr_scu_mpll, mcr_scu_strap; uint32_t denum, num, div, ref_pll, dsel; @@ -409,10 +411,9 @@ struct ast_private *ast_device_create(const struct drm_driver *drv, return ast; dev = &ast->base; - dev->pdev = pdev; pci_set_drvdata(pdev, dev); - ast->regs = pci_iomap(dev->pdev, 1, 0); + ast->regs = pci_iomap(pdev, 1, 0); if (!ast->regs) return ERR_PTR(-EIO); @@ -421,14 +422,14 @@ struct ast_private *ast_device_create(const struct drm_driver *drv, * assume the chip has MMIO enabled by default (rev 0x20 * and higher). */ - if (!(pci_resource_flags(dev->pdev, 2) & IORESOURCE_IO)) { + if (!(pci_resource_flags(pdev, 2) & IORESOURCE_IO)) { drm_info(dev, "platform has no IO space, trying MMIO\n"); ast->ioregs = ast->regs + AST_IO_MM_OFFSET; } /* "map" IO regs if the above hasn't done so already */ if (!ast->ioregs) { - ast->ioregs = pci_iomap(dev->pdev, 2, 0); + ast->ioregs = pci_iomap(pdev, 2, 0); if (!ast->ioregs) return ERR_PTR(-EIO); } diff --git a/drivers/gpu/drm/ast/ast_mm.c b/drivers/gpu/drm/ast/ast_mm.c index 8392ebde504b33b226c8f71c718ccc12b9f15cb1..7592f1b9e1f1973107aca1a52b38c02a3a1f2700 100644 --- a/drivers/gpu/drm/ast/ast_mm.c +++ b/drivers/gpu/drm/ast/ast_mm.c @@ -77,31 +77,32 @@ static u32 ast_get_vram_size(struct ast_private *ast) static void ast_mm_release(struct drm_device *dev, void *ptr) { struct ast_private *ast = to_ast_private(dev); + struct pci_dev *pdev = to_pci_dev(dev->dev); arch_phys_wc_del(ast->fb_mtrr); - arch_io_free_memtype_wc(pci_resource_start(dev->pdev, 0), - pci_resource_len(dev->pdev, 0)); + arch_io_free_memtype_wc(pci_resource_start(pdev, 0), + pci_resource_len(pdev, 0)); } int ast_mm_init(struct ast_private *ast) { struct drm_device *dev = &ast->base; + struct pci_dev *pdev = to_pci_dev(dev->dev); u32 vram_size; int ret; vram_size = ast_get_vram_size(ast); - ret = drmm_vram_helper_init(dev, pci_resource_start(dev->pdev, 0), - vram_size); + ret = drmm_vram_helper_init(dev, pci_resource_start(pdev, 0), vram_size); if (ret) { drm_err(dev, "Error initializing VRAM MM; %d\n", ret); return ret; } - arch_io_reserve_memtype_wc(pci_resource_start(dev->pdev, 0), - pci_resource_len(dev->pdev, 0)); - ast->fb_mtrr = arch_phys_wc_add(pci_resource_start(dev->pdev, 0), - pci_resource_len(dev->pdev, 0)); + arch_io_reserve_memtype_wc(pci_resource_start(pdev, 0), + pci_resource_len(pdev, 0)); + ast->fb_mtrr = arch_phys_wc_add(pci_resource_start(pdev, 0), + pci_resource_len(pdev, 0)); return drmm_add_action_or_reset(dev, ast_mm_release, NULL); } diff --git a/drivers/gpu/drm/ast/ast_mode.c b/drivers/gpu/drm/ast/ast_mode.c index 9db371f4054f3c3996e569602d2f37d06863a5a6..988b270fea5ebe9cfd682e5f6fb3003d8f2c4372 100644 --- a/drivers/gpu/drm/ast/ast_mode.c +++ b/drivers/gpu/drm/ast/ast_mode.c @@ -903,7 +903,6 @@ static void ast_crtc_atomic_destroy_state(struct drm_crtc *crtc, static const struct drm_crtc_funcs ast_crtc_funcs = { .reset = ast_crtc_reset, - .gamma_set = drm_atomic_helper_legacy_gamma_set, .destroy = drm_crtc_cleanup, .set_config = drm_atomic_helper_set_config, .page_flip = drm_atomic_helper_page_flip, @@ -1107,6 +1106,7 @@ static const struct drm_mode_config_funcs ast_mode_config_funcs = { int ast_mode_config_init(struct ast_private *ast) { struct drm_device *dev = &ast->base; + struct pci_dev *pdev = to_pci_dev(dev->dev); int ret; ret = ast_cursor_init(ast); @@ -1122,7 +1122,7 @@ int ast_mode_config_init(struct ast_private *ast) dev->mode_config.min_height = 0; dev->mode_config.preferred_depth = 24; dev->mode_config.prefer_shadow = 1; - dev->mode_config.fb_base = pci_resource_start(dev->pdev, 0); + dev->mode_config.fb_base = pci_resource_start(pdev, 0); if (ast->chip == AST2100 || ast->chip == AST2200 || @@ -1259,7 +1259,7 @@ static struct ast_i2c_chan *ast_i2c_create(struct drm_device *dev) i2c->adapter.owner = THIS_MODULE; i2c->adapter.class = I2C_CLASS_DDC; - i2c->adapter.dev.parent = &dev->pdev->dev; + i2c->adapter.dev.parent = dev->dev; i2c->dev = dev; i2c_set_adapdata(&i2c->adapter, i2c); snprintf(i2c->adapter.name, sizeof(i2c->adapter.name), diff --git a/drivers/gpu/drm/ast/ast_post.c b/drivers/gpu/drm/ast/ast_post.c index 8902c2f84bf99ab0def4b95870cfcc1816d66b22..0607658dde51b70518c27fb28ab396c5e6708de2 100644 --- a/drivers/gpu/drm/ast/ast_post.c +++ b/drivers/gpu/drm/ast/ast_post.c @@ -71,6 +71,7 @@ static void ast_set_def_ext_reg(struct drm_device *dev) { struct ast_private *ast = to_ast_private(dev); + struct pci_dev *pdev = to_pci_dev(dev->dev); u8 i, index, reg; const u8 *ext_reg_info; @@ -80,7 +81,7 @@ ast_set_def_ext_reg(struct drm_device *dev) if (ast->chip == AST2300 || ast->chip == AST2400 || ast->chip == AST2500) { - if (dev->pdev->revision >= 0x20) + if (pdev->revision >= 0x20) ext_reg_info = extreginfo_ast2300; else ext_reg_info = extreginfo_ast2300a0; @@ -366,11 +367,12 @@ static void ast_init_dram_reg(struct drm_device *dev) void ast_post_gpu(struct drm_device *dev) { struct ast_private *ast = to_ast_private(dev); + struct pci_dev *pdev = to_pci_dev(dev->dev); u32 reg; - pci_read_config_dword(dev->pdev, 0x04, ®); + pci_read_config_dword(pdev, 0x04, ®); reg |= 0x3; - pci_write_config_dword(dev->pdev, 0x04, reg); + pci_write_config_dword(pdev, 0x04, reg); ast_enable_vga(dev); ast_open_key(ast); diff --git a/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_crtc.c b/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_crtc.c index c58fa00b4848c45b9d8acb0d33d31f46b902dada..05ad75d155e8407f18a3b6af72a835a958cb8d32 100644 --- a/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_crtc.c +++ b/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_crtc.c @@ -473,7 +473,6 @@ static const struct drm_crtc_funcs atmel_hlcdc_crtc_funcs = { .atomic_destroy_state = atmel_hlcdc_crtc_destroy_state, .enable_vblank = atmel_hlcdc_crtc_enable_vblank, .disable_vblank = atmel_hlcdc_crtc_disable_vblank, - .gamma_set = drm_atomic_helper_legacy_gamma_set, }; int atmel_hlcdc_crtc_create(struct drm_device *dev) diff --git a/drivers/gpu/drm/bochs/bochs_drv.c b/drivers/gpu/drm/bochs/bochs_drv.c index fd454225fd196983173183c9cb1924b023fd0148..b469624fe40dee985d67a76ba3a44a0dbf153bfb 100644 --- a/drivers/gpu/drm/bochs/bochs_drv.c +++ b/drivers/gpu/drm/bochs/bochs_drv.c @@ -121,7 +121,6 @@ static int bochs_pci_probe(struct pci_dev *pdev, if (ret) goto err_free_dev; - dev->pdev = pdev; pci_set_drvdata(pdev, dev); ret = bochs_load(dev); diff --git a/drivers/gpu/drm/bochs/bochs_hw.c b/drivers/gpu/drm/bochs/bochs_hw.c index dce4672e3fc8d15529fde53e3e66984cfc6797d6..2d7380a9890ed1ef4ebb42549a9f36e94f3cdc1e 100644 --- a/drivers/gpu/drm/bochs/bochs_hw.c +++ b/drivers/gpu/drm/bochs/bochs_hw.c @@ -110,7 +110,7 @@ int bochs_hw_load_edid(struct bochs_device *bochs) int bochs_hw_init(struct drm_device *dev) { struct bochs_device *bochs = dev->dev_private; - struct pci_dev *pdev = dev->pdev; + struct pci_dev *pdev = to_pci_dev(dev->dev); unsigned long addr, size, mem, ioaddr, iosize; u16 id; @@ -201,7 +201,7 @@ void bochs_hw_fini(struct drm_device *dev) release_region(VBE_DISPI_IOPORT_INDEX, 2); if (bochs->fb_map) iounmap(bochs->fb_map); - pci_release_regions(dev->pdev); + pci_release_regions(to_pci_dev(dev->dev)); kfree(bochs->edid); } diff --git a/drivers/gpu/drm/bridge/adv7511/adv7511_drv.c b/drivers/gpu/drm/bridge/adv7511/adv7511_drv.c index a0d392c338da5241744835e0ed3d89ba14440e8b..76555ae64e9ce1272eed82a32d66e1bfd89aa604 100644 --- a/drivers/gpu/drm/bridge/adv7511/adv7511_drv.c +++ b/drivers/gpu/drm/bridge/adv7511/adv7511_drv.c @@ -1292,8 +1292,7 @@ static int adv7511_probe(struct i2c_client *i2c, const struct i2c_device_id *id) err_unregister_cec: i2c_unregister_device(adv7511->i2c_cec); - if (adv7511->cec_clk) - clk_disable_unprepare(adv7511->cec_clk); + clk_disable_unprepare(adv7511->cec_clk); err_i2c_unregister_packet: i2c_unregister_device(adv7511->i2c_packet); err_i2c_unregister_edid: @@ -1311,8 +1310,7 @@ static int adv7511_remove(struct i2c_client *i2c) if (adv7511->type == ADV7533 || adv7511->type == ADV7535) adv7533_detach_dsi(adv7511); i2c_unregister_device(adv7511->i2c_cec); - if (adv7511->cec_clk) - clk_disable_unprepare(adv7511->cec_clk); + clk_disable_unprepare(adv7511->cec_clk); adv7511_uninit_regulators(adv7511); diff --git a/drivers/gpu/drm/bridge/display-connector.c b/drivers/gpu/drm/bridge/display-connector.c index 4d278573cdb99817bab4e58705fedcd9f43f8949..05eb759da6fc62395707b98023ada082c0fe1626 100644 --- a/drivers/gpu/drm/bridge/display-connector.c +++ b/drivers/gpu/drm/bridge/display-connector.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -20,6 +21,8 @@ struct display_connector { struct gpio_desc *hpd_gpio; int hpd_irq; + + struct regulator *dp_pwr; }; static inline struct display_connector * @@ -172,11 +175,12 @@ static int display_connector_probe(struct platform_device *pdev) of_property_read_string(pdev->dev.of_node, "label", &label); /* - * Get the HPD GPIO for DVI and HDMI connectors. If the GPIO can provide + * Get the HPD GPIO for DVI, HDMI and DP connectors. If the GPIO can provide * edge interrupts, register an interrupt handler. */ if (type == DRM_MODE_CONNECTOR_DVII || - type == DRM_MODE_CONNECTOR_HDMIA) { + type == DRM_MODE_CONNECTOR_HDMIA || + type == DRM_MODE_CONNECTOR_DisplayPort) { conn->hpd_gpio = devm_gpiod_get_optional(&pdev->dev, "hpd", GPIOD_IN); if (IS_ERR(conn->hpd_gpio)) { @@ -223,6 +227,38 @@ static int display_connector_probe(struct platform_device *pdev) } } + /* Get the DP PWR for DP connector. */ + if (type == DRM_MODE_CONNECTOR_DisplayPort) { + int ret; + + conn->dp_pwr = devm_regulator_get_optional(&pdev->dev, "dp-pwr"); + + if (IS_ERR(conn->dp_pwr)) { + ret = PTR_ERR(conn->dp_pwr); + + switch (ret) { + case -ENODEV: + conn->dp_pwr = NULL; + break; + + case -EPROBE_DEFER: + return -EPROBE_DEFER; + + default: + dev_err(&pdev->dev, "failed to get DP PWR regulator: %d\n", ret); + return ret; + } + } + + if (conn->dp_pwr) { + ret = regulator_enable(conn->dp_pwr); + if (ret) { + dev_err(&pdev->dev, "failed to enable DP PWR regulator: %d\n", ret); + return ret; + } + } + } + conn->bridge.funcs = &display_connector_bridge_funcs; conn->bridge.of_node = pdev->dev.of_node; @@ -251,6 +287,9 @@ static int display_connector_remove(struct platform_device *pdev) { struct display_connector *conn = platform_get_drvdata(pdev); + if (conn->dp_pwr) + regulator_disable(conn->dp_pwr); + drm_bridge_remove(&conn->bridge); if (!IS_ERR(conn->bridge.ddc)) @@ -275,6 +314,9 @@ static const struct of_device_id display_connector_match[] = { }, { .compatible = "vga-connector", .data = (void *)DRM_MODE_CONNECTOR_VGA, + }, { + .compatible = "dp-connector", + .data = (void *)DRM_MODE_CONNECTOR_DisplayPort, }, {}, }; diff --git a/drivers/gpu/drm/bridge/synopsys/dw-hdmi.c b/drivers/gpu/drm/bridge/synopsys/dw-hdmi.c index 0c79a9ba48bb6407d7a001b2e03cca139d4ed28d..dda4fa9a1a08d8ef792b7cd2959360bb79e6b79d 100644 --- a/drivers/gpu/drm/bridge/synopsys/dw-hdmi.c +++ b/drivers/gpu/drm/bridge/synopsys/dw-hdmi.c @@ -3440,8 +3440,7 @@ struct dw_hdmi *dw_hdmi_probe(struct platform_device *pdev, err_iahb: clk_disable_unprepare(hdmi->iahb_clk); - if (hdmi->cec_clk) - clk_disable_unprepare(hdmi->cec_clk); + clk_disable_unprepare(hdmi->cec_clk); err_isfr: clk_disable_unprepare(hdmi->isfr_clk); err_res: @@ -3465,8 +3464,7 @@ void dw_hdmi_remove(struct dw_hdmi *hdmi) clk_disable_unprepare(hdmi->iahb_clk); clk_disable_unprepare(hdmi->isfr_clk); - if (hdmi->cec_clk) - clk_disable_unprepare(hdmi->cec_clk); + clk_disable_unprepare(hdmi->cec_clk); if (hdmi->i2c) i2c_del_adapter(&hdmi->i2c->adap); diff --git a/drivers/gpu/drm/bridge/thc63lvd1024.c b/drivers/gpu/drm/bridge/thc63lvd1024.c index 86b06975bfdd43e1892b0c744ba1720cf1937eac..e21078b2f8b5b998f5a26af90119c96531e8881f 100644 --- a/drivers/gpu/drm/bridge/thc63lvd1024.c +++ b/drivers/gpu/drm/bridge/thc63lvd1024.c @@ -202,7 +202,7 @@ static int thc63_probe(struct platform_device *pdev) thc63->dev = &pdev->dev; platform_set_drvdata(pdev, thc63); - thc63->vcc = devm_regulator_get_optional(thc63->dev, "vcc"); + thc63->vcc = devm_regulator_get(thc63->dev, "vcc"); if (IS_ERR(thc63->vcc)) { if (PTR_ERR(thc63->vcc) == -EPROBE_DEFER) return -EPROBE_DEFER; diff --git a/drivers/gpu/drm/drm_agpsupport.c b/drivers/gpu/drm/drm_agpsupport.c index 4c7ad46fdd219f59c2830d2d52428763d5e606d2..5311d03d49cc093f76b04cd3d743a79d9a6d3a6d 100644 --- a/drivers/gpu/drm/drm_agpsupport.c +++ b/drivers/gpu/drm/drm_agpsupport.c @@ -45,13 +45,9 @@ #include "drm_legacy.h" -/** +/* * Get AGP information. * - * \param inode device inode. - * \param file_priv DRM file private. - * \param cmd command. - * \param arg pointer to a (output) drm_agp_info structure. * \return zero on success or a negative number on failure. * * Verifies the AGP device has been initialized and acquired and fills in the @@ -92,7 +88,7 @@ int drm_agp_info_ioctl(struct drm_device *dev, void *data, return 0; } -/** +/* * Acquire the AGP device. * * \param dev DRM device that is to acquire AGP. @@ -103,11 +99,13 @@ int drm_agp_info_ioctl(struct drm_device *dev, void *data, */ int drm_agp_acquire(struct drm_device *dev) { + struct pci_dev *pdev = to_pci_dev(dev->dev); + if (!dev->agp) return -ENODEV; if (dev->agp->acquired) return -EBUSY; - dev->agp->bridge = agp_backend_acquire(dev->pdev); + dev->agp->bridge = agp_backend_acquire(pdev); if (!dev->agp->bridge) return -ENODEV; dev->agp->acquired = 1; @@ -115,13 +113,9 @@ int drm_agp_acquire(struct drm_device *dev) } EXPORT_SYMBOL(drm_agp_acquire); -/** +/* * Acquire the AGP device (ioctl). * - * \param inode device inode. - * \param file_priv DRM file private. - * \param cmd command. - * \param arg user argument. * \return zero on success or a negative number on failure. * * Verifies the AGP device hasn't been acquired before and calls @@ -133,7 +127,7 @@ int drm_agp_acquire_ioctl(struct drm_device *dev, void *data, return drm_agp_acquire((struct drm_device *) file_priv->minor->dev); } -/** +/* * Release the AGP device. * * \param dev DRM device that is to release AGP. @@ -157,7 +151,7 @@ int drm_agp_release_ioctl(struct drm_device *dev, void *data, return drm_agp_release(dev); } -/** +/* * Enable the AGP bus. * * \param dev DRM device that has previously acquired AGP. @@ -187,13 +181,9 @@ int drm_agp_enable_ioctl(struct drm_device *dev, void *data, return drm_agp_enable(dev, *mode); } -/** +/* * Allocate AGP memory. * - * \param inode device inode. - * \param file_priv file private pointer. - * \param cmd command. - * \param arg pointer to a drm_agp_buffer structure. * \return zero on success or a negative number on failure. * * Verifies the AGP device is present and has been acquired, allocates the @@ -242,7 +232,7 @@ int drm_agp_alloc_ioctl(struct drm_device *dev, void *data, return drm_agp_alloc(dev, request); } -/** +/* * Search for the AGP memory entry associated with a handle. * * \param dev DRM device structure. @@ -263,13 +253,9 @@ static struct drm_agp_mem *drm_agp_lookup_entry(struct drm_device *dev, return NULL; } -/** +/* * Unbind AGP memory from the GATT (ioctl). * - * \param inode device inode. - * \param file_priv DRM file private. - * \param cmd command. - * \param arg pointer to a drm_agp_binding structure. * \return zero on success or a negative number on failure. * * Verifies the AGP device is present and acquired, looks-up the AGP memory @@ -285,7 +271,7 @@ int drm_agp_unbind(struct drm_device *dev, struct drm_agp_binding *request) entry = drm_agp_lookup_entry(dev, request->handle); if (!entry || !entry->bound) return -EINVAL; - ret = drm_unbind_agp(entry->memory); + ret = agp_unbind_memory(entry->memory); if (ret == 0) entry->bound = 0; return ret; @@ -301,13 +287,9 @@ int drm_agp_unbind_ioctl(struct drm_device *dev, void *data, return drm_agp_unbind(dev, request); } -/** +/* * Bind AGP memory into the GATT (ioctl) * - * \param inode device inode. - * \param file_priv DRM file private. - * \param cmd command. - * \param arg pointer to a drm_agp_binding structure. * \return zero on success or a negative number on failure. * * Verifies the AGP device is present and has been acquired and that no memory @@ -326,7 +308,7 @@ int drm_agp_bind(struct drm_device *dev, struct drm_agp_binding *request) if (!entry || entry->bound) return -EINVAL; page = DIV_ROUND_UP(request->offset, PAGE_SIZE); - retcode = drm_bind_agp(entry->memory, page); + retcode = agp_bind_memory(entry->memory, page); if (retcode) return retcode; entry->bound = dev->agp->base + (page << PAGE_SHIFT); @@ -345,13 +327,9 @@ int drm_agp_bind_ioctl(struct drm_device *dev, void *data, return drm_agp_bind(dev, request); } -/** +/* * Free AGP memory (ioctl). * - * \param inode device inode. - * \param file_priv DRM file private. - * \param cmd command. - * \param arg pointer to a drm_agp_buffer structure. * \return zero on success or a negative number on failure. * * Verifies the AGP device is present and has been acquired and looks up the @@ -369,11 +347,11 @@ int drm_agp_free(struct drm_device *dev, struct drm_agp_buffer *request) if (!entry) return -EINVAL; if (entry->bound) - drm_unbind_agp(entry->memory); + agp_unbind_memory(entry->memory); list_del(&entry->head); - drm_free_agp(entry->memory, entry->pages); + agp_free_memory(entry->memory); kfree(entry); return 0; } @@ -388,7 +366,7 @@ int drm_agp_free_ioctl(struct drm_device *dev, void *data, return drm_agp_free(dev, request); } -/** +/* * Initialize the AGP resources. * * \return pointer to a drm_agp_head structure. @@ -402,14 +380,15 @@ int drm_agp_free_ioctl(struct drm_device *dev, void *data, */ struct drm_agp_head *drm_agp_init(struct drm_device *dev) { + struct pci_dev *pdev = to_pci_dev(dev->dev); struct drm_agp_head *head = NULL; head = kzalloc(sizeof(*head), GFP_KERNEL); if (!head) return NULL; - head->bridge = agp_find_bridge(dev->pdev); + head->bridge = agp_find_bridge(pdev); if (!head->bridge) { - head->bridge = agp_backend_acquire(dev->pdev); + head->bridge = agp_backend_acquire(pdev); if (!head->bridge) { kfree(head); return NULL; @@ -453,8 +432,8 @@ void drm_legacy_agp_clear(struct drm_device *dev) list_for_each_entry_safe(entry, tempe, &dev->agp->memory, head) { if (entry->bound) - drm_unbind_agp(entry->memory); - drm_free_agp(entry->memory, entry->pages); + agp_unbind_memory(entry->memory); + agp_free_memory(entry->memory); kfree(entry); } INIT_LIST_HEAD(&dev->agp->memory); diff --git a/drivers/gpu/drm/drm_atomic_helper.c b/drivers/gpu/drm/drm_atomic_helper.c index 4a8cbec832bc46051a8d500fa1ab3df67d23929d..560aaecba31bc3c5b50b978749fd353568a53939 100644 --- a/drivers/gpu/drm/drm_atomic_helper.c +++ b/drivers/gpu/drm/drm_atomic_helper.c @@ -2040,6 +2040,9 @@ crtc_or_fake_commit(struct drm_atomic_state *state, struct drm_crtc *crtc) * should always call this function from their * &drm_mode_config_funcs.atomic_commit hook. * + * Drivers that need to extend the commit setup to private objects can use the + * &drm_mode_config_helper_funcs.atomic_commit_setup hook. + * * To be able to use this support drivers need to use a few more helper * functions. drm_atomic_helper_wait_for_dependencies() must be called before * actually committing the hardware state, and for nonblocking commits this call @@ -2083,8 +2086,11 @@ int drm_atomic_helper_setup_commit(struct drm_atomic_state *state, struct drm_plane *plane; struct drm_plane_state *old_plane_state, *new_plane_state; struct drm_crtc_commit *commit; + const struct drm_mode_config_helper_funcs *funcs; int i, ret; + funcs = state->dev->mode_config.helper_private; + for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) { commit = kzalloc(sizeof(*commit), GFP_KERNEL); if (!commit) @@ -2169,6 +2175,9 @@ int drm_atomic_helper_setup_commit(struct drm_atomic_state *state, new_plane_state->commit = drm_crtc_commit_get(commit); } + if (funcs && funcs->atomic_commit_setup) + return funcs->atomic_commit_setup(state); + return 0; } EXPORT_SYMBOL(drm_atomic_helper_setup_commit); @@ -3499,76 +3508,6 @@ int drm_atomic_helper_page_flip_target(struct drm_crtc *crtc, } EXPORT_SYMBOL(drm_atomic_helper_page_flip_target); -/** - * drm_atomic_helper_legacy_gamma_set - set the legacy gamma correction table - * @crtc: CRTC object - * @red: red correction table - * @green: green correction table - * @blue: green correction table - * @size: size of the tables - * @ctx: lock acquire context - * - * Implements support for legacy gamma correction table for drivers - * that support color management through the DEGAMMA_LUT/GAMMA_LUT - * properties. See drm_crtc_enable_color_mgmt() and the containing chapter for - * how the atomic color management and gamma tables work. - */ -int drm_atomic_helper_legacy_gamma_set(struct drm_crtc *crtc, - u16 *red, u16 *green, u16 *blue, - uint32_t size, - struct drm_modeset_acquire_ctx *ctx) -{ - struct drm_device *dev = crtc->dev; - struct drm_atomic_state *state; - struct drm_crtc_state *crtc_state; - struct drm_property_blob *blob = NULL; - struct drm_color_lut *blob_data; - int i, ret = 0; - bool replaced; - - state = drm_atomic_state_alloc(crtc->dev); - if (!state) - return -ENOMEM; - - blob = drm_property_create_blob(dev, - sizeof(struct drm_color_lut) * size, - NULL); - if (IS_ERR(blob)) { - ret = PTR_ERR(blob); - blob = NULL; - goto fail; - } - - /* Prepare GAMMA_LUT with the legacy values. */ - blob_data = blob->data; - for (i = 0; i < size; i++) { - blob_data[i].red = red[i]; - blob_data[i].green = green[i]; - blob_data[i].blue = blue[i]; - } - - state->acquire_ctx = ctx; - crtc_state = drm_atomic_get_crtc_state(state, crtc); - if (IS_ERR(crtc_state)) { - ret = PTR_ERR(crtc_state); - goto fail; - } - - /* Reset DEGAMMA_LUT and CTM properties. */ - replaced = drm_property_replace_blob(&crtc_state->degamma_lut, NULL); - replaced |= drm_property_replace_blob(&crtc_state->ctm, NULL); - replaced |= drm_property_replace_blob(&crtc_state->gamma_lut, blob); - crtc_state->color_mgmt_changed |= replaced; - - ret = drm_atomic_commit(state); - -fail: - drm_atomic_state_put(state); - drm_property_blob_put(blob); - return ret; -} -EXPORT_SYMBOL(drm_atomic_helper_legacy_gamma_set); - /** * drm_atomic_helper_bridge_propagate_bus_fmt() - Propagate output format to * the input end of a bridge diff --git a/drivers/gpu/drm/drm_blend.c b/drivers/gpu/drm/drm_blend.c index 5c2141e9a9f483a3472b0ec96a3918761ee1b920..26e2f2ffd255b3b6631e2340330abbfbaa90e59a 100644 --- a/drivers/gpu/drm/drm_blend.c +++ b/drivers/gpu/drm/drm_blend.c @@ -185,12 +185,6 @@ * plane does not expose the "alpha" property, then this is * assumed to be 1.0 * - * IN_FORMATS: - * Blob property which contains the set of buffer format and modifier - * pairs supported by this plane. The blob is a drm_format_modifier_blob - * struct. Without this property the plane doesn't support buffers with - * modifiers. Userspace cannot change this property. - * * Note that all the property extensions described here apply either to the * plane or the CRTC (e.g. for the background color, which currently is not * exposed and assumed to be black). diff --git a/drivers/gpu/drm/drm_bufs.c b/drivers/gpu/drm/drm_bufs.c index aeb1327e3077cf8484ddacfe003a69d9e93c6dcf..e3d77dfefb0afe4b13059f19119a274dc9521731 100644 --- a/drivers/gpu/drm/drm_bufs.c +++ b/drivers/gpu/drm/drm_bufs.c @@ -326,7 +326,7 @@ static int drm_addmap_core(struct drm_device *dev, resource_size_t offset, * As we're limiting the address to 2^32-1 (or less), * casting it down to 32 bits is no problem, but we * need to point to a 64bit variable first. */ - map->handle = dma_alloc_coherent(&dev->pdev->dev, + map->handle = dma_alloc_coherent(dev->dev, map->size, &map->offset, GFP_KERNEL); @@ -556,7 +556,7 @@ int drm_legacy_rmmap_locked(struct drm_device *dev, struct drm_local_map *map) case _DRM_SCATTER_GATHER: break; case _DRM_CONSISTENT: - dma_free_coherent(&dev->pdev->dev, + dma_free_coherent(dev->dev, map->size, map->handle, map->offset); diff --git a/drivers/gpu/drm/drm_cache.c b/drivers/gpu/drm/drm_cache.c index 0fe3c496002a904aa9bd95e19593812dcd16c833..79a50ef1250fd1d8370547e13fae21cb193ec209 100644 --- a/drivers/gpu/drm/drm_cache.c +++ b/drivers/gpu/drm/drm_cache.c @@ -30,6 +30,8 @@ #include #include +#include +#include #include @@ -176,3 +178,34 @@ drm_clflush_virt_range(void *addr, unsigned long length) #endif } EXPORT_SYMBOL(drm_clflush_virt_range); + +bool drm_need_swiotlb(int dma_bits) +{ + struct resource *tmp; + resource_size_t max_iomem = 0; + + /* + * Xen paravirtual hosts require swiotlb regardless of requested dma + * transfer size. + * + * NOTE: Really, what it requires is use of the dma_alloc_coherent + * allocator used in ttm_dma_populate() instead of + * ttm_populate_and_map_pages(), which bounce buffers so much in + * Xen it leads to swiotlb buffer exhaustion. + */ + if (xen_pv_domain()) + return true; + + /* + * Enforce dma_alloc_coherent when memory encryption is active as well + * for the same reasons as for Xen paravirtual hosts. + */ + if (mem_encrypt_active()) + return true; + + for (tmp = iomem_resource.child; tmp; tmp = tmp->sibling) + max_iomem = max(max_iomem, tmp->end); + + return max_iomem > ((u64)1 << dma_bits); +} +EXPORT_SYMBOL(drm_need_swiotlb); diff --git a/drivers/gpu/drm/drm_client_modeset.c b/drivers/gpu/drm/drm_client_modeset.c index b7e9e1c2564cd4b88a9ace3ae708d91010bc31e3..ced09c7c06f90f34bde86dc1ac9ac1fb5df78d5d 100644 --- a/drivers/gpu/drm/drm_client_modeset.c +++ b/drivers/gpu/drm/drm_client_modeset.c @@ -7,6 +7,7 @@ * Copyright (c) 2007 Dave Airlie */ +#include "drm/drm_modeset_lock.h" #include #include #include @@ -1181,9 +1182,11 @@ static void drm_client_modeset_dpms_legacy(struct drm_client_dev *client, int dp struct drm_device *dev = client->dev; struct drm_connector *connector; struct drm_mode_set *modeset; + struct drm_modeset_acquire_ctx ctx; int j; + int ret; - drm_modeset_lock_all(dev); + DRM_MODESET_LOCK_ALL_BEGIN(dev, ctx, 0, ret); drm_client_for_each_modeset(modeset, client) { if (!modeset->crtc->enabled) continue; @@ -1195,7 +1198,7 @@ static void drm_client_modeset_dpms_legacy(struct drm_client_dev *client, int dp dev->mode_config.dpms_property, dpms_mode); } } - drm_modeset_unlock_all(dev); + DRM_MODESET_LOCK_ALL_END(dev, ctx, ret); } /** diff --git a/drivers/gpu/drm/drm_color_mgmt.c b/drivers/gpu/drm/drm_color_mgmt.c index 3bcabc2f6e0e47bfb7606e7298f9c7f5d288d69b..bb14f488c8f6c7f05ec80fb5300c0d1bbdf049c4 100644 --- a/drivers/gpu/drm/drm_color_mgmt.c +++ b/drivers/gpu/drm/drm_color_mgmt.c @@ -22,6 +22,7 @@ #include +#include #include #include #include @@ -89,9 +90,8 @@ * modes) appropriately. * * There is also support for a legacy gamma table, which is set up by calling - * drm_mode_crtc_set_gamma_size(). Drivers which support both should use - * drm_atomic_helper_legacy_gamma_set() to alias the legacy gamma ramp with the - * "GAMMA_LUT" property above. + * drm_mode_crtc_set_gamma_size(). The DRM core will then alias the legacy gamma + * ramp with "GAMMA_LUT" or, if that is unavailable, "DEGAMMA_LUT". * * Support for different non RGB color encodings is controlled through * &drm_plane specific COLOR_ENCODING and COLOR_RANGE properties. They @@ -156,9 +156,6 @@ EXPORT_SYMBOL(drm_color_ctm_s31_32_to_qm_n); * optional. The gamma and degamma properties are only attached if * their size is not 0 and ctm_property is only attached if has_ctm is * true. - * - * Drivers should use drm_atomic_helper_legacy_gamma_set() to implement the - * legacy &drm_crtc_funcs.gamma_set callback. */ void drm_crtc_enable_color_mgmt(struct drm_crtc *crtc, uint degamma_lut_size, @@ -231,6 +228,116 @@ int drm_mode_crtc_set_gamma_size(struct drm_crtc *crtc, } EXPORT_SYMBOL(drm_mode_crtc_set_gamma_size); +/** + * drm_crtc_supports_legacy_gamma - does the crtc support legacy gamma correction table + * @crtc: CRTC object + * + * Returns true/false if the given crtc supports setting the legacy gamma + * correction table. + */ +static bool drm_crtc_supports_legacy_gamma(struct drm_crtc *crtc) +{ + u32 gamma_id = crtc->dev->mode_config.gamma_lut_property->base.id; + u32 degamma_id = crtc->dev->mode_config.degamma_lut_property->base.id; + + if (!crtc->gamma_size) + return false; + + if (crtc->funcs->gamma_set) + return true; + + return !!(drm_mode_obj_find_prop_id(&crtc->base, gamma_id) || + drm_mode_obj_find_prop_id(&crtc->base, degamma_id)); +} + +/** + * drm_crtc_legacy_gamma_set - set the legacy gamma correction table + * @crtc: CRTC object + * @red: red correction table + * @green: green correction table + * @blue: green correction table + * @size: size of the tables + * @ctx: lock acquire context + * + * Implements support for legacy gamma correction table for drivers + * that have set drm_crtc_funcs.gamma_set or that support color management + * through the DEGAMMA_LUT/GAMMA_LUT properties. See + * drm_crtc_enable_color_mgmt() and the containing chapter for + * how the atomic color management and gamma tables work. + * + * This function sets the gamma using drm_crtc_funcs.gamma_set if set, or + * alternatively using crtc color management properties. + */ +static int drm_crtc_legacy_gamma_set(struct drm_crtc *crtc, + u16 *red, u16 *green, u16 *blue, + u32 size, + struct drm_modeset_acquire_ctx *ctx) +{ + struct drm_device *dev = crtc->dev; + struct drm_atomic_state *state; + struct drm_crtc_state *crtc_state; + struct drm_property_blob *blob; + struct drm_color_lut *blob_data; + u32 gamma_id = dev->mode_config.gamma_lut_property->base.id; + u32 degamma_id = dev->mode_config.degamma_lut_property->base.id; + bool use_gamma_lut; + int i, ret = 0; + bool replaced; + + if (crtc->funcs->gamma_set) + return crtc->funcs->gamma_set(crtc, red, green, blue, size, ctx); + + if (drm_mode_obj_find_prop_id(&crtc->base, gamma_id)) + use_gamma_lut = true; + else if (drm_mode_obj_find_prop_id(&crtc->base, degamma_id)) + use_gamma_lut = false; + else + return -ENODEV; + + state = drm_atomic_state_alloc(crtc->dev); + if (!state) + return -ENOMEM; + + blob = drm_property_create_blob(dev, + sizeof(struct drm_color_lut) * size, + NULL); + if (IS_ERR(blob)) { + ret = PTR_ERR(blob); + blob = NULL; + goto fail; + } + + /* Prepare GAMMA_LUT with the legacy values. */ + blob_data = blob->data; + for (i = 0; i < size; i++) { + blob_data[i].red = red[i]; + blob_data[i].green = green[i]; + blob_data[i].blue = blue[i]; + } + + state->acquire_ctx = ctx; + crtc_state = drm_atomic_get_crtc_state(state, crtc); + if (IS_ERR(crtc_state)) { + ret = PTR_ERR(crtc_state); + goto fail; + } + + /* Set GAMMA_LUT and reset DEGAMMA_LUT and CTM */ + replaced = drm_property_replace_blob(&crtc_state->degamma_lut, + use_gamma_lut ? NULL : blob); + replaced |= drm_property_replace_blob(&crtc_state->ctm, NULL); + replaced |= drm_property_replace_blob(&crtc_state->gamma_lut, + use_gamma_lut ? blob : NULL); + crtc_state->color_mgmt_changed |= replaced; + + ret = drm_atomic_commit(state); + +fail: + drm_atomic_state_put(state); + drm_property_blob_put(blob); + return ret; +} + /** * drm_mode_gamma_set_ioctl - set the gamma table * @dev: DRM device @@ -262,7 +369,7 @@ int drm_mode_gamma_set_ioctl(struct drm_device *dev, if (!crtc) return -ENOENT; - if (crtc->funcs->gamma_set == NULL) + if (!drm_crtc_supports_legacy_gamma(crtc)) return -ENOSYS; /* memcpy into gamma store */ @@ -290,8 +397,8 @@ int drm_mode_gamma_set_ioctl(struct drm_device *dev, goto out; } - ret = crtc->funcs->gamma_set(crtc, r_base, g_base, b_base, - crtc->gamma_size, &ctx); + ret = drm_crtc_legacy_gamma_set(crtc, r_base, g_base, b_base, + crtc->gamma_size, &ctx); out: DRM_MODESET_LOCK_ALL_END(dev, ctx, ret); diff --git a/drivers/gpu/drm/drm_crtc.c b/drivers/gpu/drm/drm_crtc.c index 74090fc3aa552a54d047ddfff2af5e4483e3e009..9c4f9947b194fcaf954ada858b73a41fd680ea15 100644 --- a/drivers/gpu/drm/drm_crtc.c +++ b/drivers/gpu/drm/drm_crtc.c @@ -38,6 +38,7 @@ #include #include #include +#include #include #include #include @@ -67,7 +68,7 @@ * &drm_crtc_funcs.page_flip and &drm_crtc_funcs.cursor_set2, and other legacy * operations like &drm_crtc_funcs.gamma_set. For atomic drivers all these * features are controlled through &drm_property and - * &drm_mode_config_funcs.atomic_check and &drm_mode_config_funcs.atomic_check. + * &drm_mode_config_funcs.atomic_check. */ /** @@ -240,30 +241,12 @@ struct dma_fence *drm_crtc_create_fence(struct drm_crtc *crtc) * Nearest Neighbor scaling filter */ -/** - * drm_crtc_init_with_planes - Initialise a new CRTC object with - * specified primary and cursor planes. - * @dev: DRM device - * @crtc: CRTC object to init - * @primary: Primary plane for CRTC - * @cursor: Cursor plane for CRTC - * @funcs: callbacks for the new CRTC - * @name: printf style format string for the CRTC name, or NULL for default name - * - * Inits a new object created as base part of a driver crtc object. Drivers - * should use this function instead of drm_crtc_init(), which is only provided - * for backwards compatibility with drivers which do not yet support universal - * planes). For really simple hardware which has only 1 plane look at - * drm_simple_display_pipe_init() instead. - * - * Returns: - * Zero on success, error code on failure. - */ -int drm_crtc_init_with_planes(struct drm_device *dev, struct drm_crtc *crtc, - struct drm_plane *primary, - struct drm_plane *cursor, - const struct drm_crtc_funcs *funcs, - const char *name, ...) +__printf(6, 0) +static int __drm_crtc_init_with_planes(struct drm_device *dev, struct drm_crtc *crtc, + struct drm_plane *primary, + struct drm_plane *cursor, + const struct drm_crtc_funcs *funcs, + const char *name, va_list ap) { struct drm_mode_config *config = &dev->mode_config; int ret; @@ -291,11 +274,7 @@ int drm_crtc_init_with_planes(struct drm_device *dev, struct drm_crtc *crtc, return ret; if (name) { - va_list ap; - - va_start(ap, name); crtc->name = kvasprintf(GFP_KERNEL, name, ap); - va_end(ap); } else { crtc->name = kasprintf(GFP_KERNEL, "crtc-%d", drm_num_crtcs(dev)); @@ -339,8 +318,101 @@ int drm_crtc_init_with_planes(struct drm_device *dev, struct drm_crtc *crtc, return 0; } + +/** + * drm_crtc_init_with_planes - Initialise a new CRTC object with + * specified primary and cursor planes. + * @dev: DRM device + * @crtc: CRTC object to init + * @primary: Primary plane for CRTC + * @cursor: Cursor plane for CRTC + * @funcs: callbacks for the new CRTC + * @name: printf style format string for the CRTC name, or NULL for default name + * + * Inits a new object created as base part of a driver crtc object. Drivers + * should use this function instead of drm_crtc_init(), which is only provided + * for backwards compatibility with drivers which do not yet support universal + * planes). For really simple hardware which has only 1 plane look at + * drm_simple_display_pipe_init() instead. + * The &drm_crtc_funcs.destroy hook should call drm_crtc_cleanup() and kfree() + * the crtc structure. The crtc structure should not be allocated with + * devm_kzalloc(). + * + * The @primary and @cursor planes are only relevant for legacy uAPI, see + * &drm_crtc.primary and &drm_crtc.cursor. + * + * Note: consider using drmm_crtc_alloc_with_planes() instead of + * drm_crtc_init_with_planes() to let the DRM managed resource infrastructure + * take care of cleanup and deallocation. + * + * Returns: + * Zero on success, error code on failure. + */ +int drm_crtc_init_with_planes(struct drm_device *dev, struct drm_crtc *crtc, + struct drm_plane *primary, + struct drm_plane *cursor, + const struct drm_crtc_funcs *funcs, + const char *name, ...) +{ + va_list ap; + int ret; + + WARN_ON(!funcs->destroy); + + va_start(ap, name); + ret = __drm_crtc_init_with_planes(dev, crtc, primary, cursor, funcs, + name, ap); + va_end(ap); + + return ret; +} EXPORT_SYMBOL(drm_crtc_init_with_planes); +static void drmm_crtc_alloc_with_planes_cleanup(struct drm_device *dev, + void *ptr) +{ + struct drm_crtc *crtc = ptr; + + drm_crtc_cleanup(crtc); +} + +void *__drmm_crtc_alloc_with_planes(struct drm_device *dev, + size_t size, size_t offset, + struct drm_plane *primary, + struct drm_plane *cursor, + const struct drm_crtc_funcs *funcs, + const char *name, ...) +{ + void *container; + struct drm_crtc *crtc; + va_list ap; + int ret; + + if (WARN_ON(!funcs || funcs->destroy)) + return ERR_PTR(-EINVAL); + + container = drmm_kzalloc(dev, size, GFP_KERNEL); + if (!container) + return ERR_PTR(-ENOMEM); + + crtc = container + offset; + + va_start(ap, name); + ret = __drm_crtc_init_with_planes(dev, crtc, primary, cursor, funcs, + name, ap); + va_end(ap); + if (ret) + return ERR_PTR(ret); + + ret = drmm_add_action_or_reset(dev, drmm_crtc_alloc_with_planes_cleanup, + crtc); + if (ret) + return ERR_PTR(ret); + + return container; +} +EXPORT_SYMBOL(__drmm_crtc_alloc_with_planes); + /** * drm_crtc_cleanup - Clean up the core crtc usage * @crtc: CRTC to cleanup diff --git a/drivers/gpu/drm/drm_crtc_helper_internal.h b/drivers/gpu/drm/drm_crtc_helper_internal.h index 25ce42e799952c6776d6d76bec0b81de446c3cbb..61e09f8a8d0ff07d15da3e7ea378602208adc972 100644 --- a/drivers/gpu/drm/drm_crtc_helper_internal.h +++ b/drivers/gpu/drm/drm_crtc_helper_internal.h @@ -32,16 +32,6 @@ #include #include -/* drm_fb_helper.c */ -#ifdef CONFIG_DRM_FBDEV_EMULATION -int drm_fb_helper_modinit(void); -#else -static inline int drm_fb_helper_modinit(void) -{ - return 0; -} -#endif - /* drm_dp_aux_dev.c */ #ifdef CONFIG_DRM_DP_AUX_CHARDEV int drm_dp_aux_dev_init(void); diff --git a/drivers/gpu/drm/drm_dp_helper.c b/drivers/gpu/drm/drm_dp_helper.c index 5bd0934004e30c1f4757528efacb0c2bf8205e17..eedbb48815b7ba3b7c7962e063d6c580b756438e 100644 --- a/drivers/gpu/drm/drm_dp_helper.c +++ b/drivers/gpu/drm/drm_dp_helper.c @@ -949,6 +949,38 @@ bool drm_dp_downstream_444_to_420_conversion(const u8 dpcd[DP_RECEIVER_CAP_SIZE] } EXPORT_SYMBOL(drm_dp_downstream_444_to_420_conversion); +/** + * drm_dp_downstream_rgb_to_ycbcr_conversion() - determine downstream facing port + * RGB->YCbCr conversion capability + * @dpcd: DisplayPort configuration data + * @port_cap: downstream facing port capabilities + * @color_spc: Colorspace for which conversion cap is sought + * + * Returns: whether the downstream facing port can convert RGB->YCbCr for a given + * colorspace. + */ +bool drm_dp_downstream_rgb_to_ycbcr_conversion(const u8 dpcd[DP_RECEIVER_CAP_SIZE], + const u8 port_cap[4], + u8 color_spc) +{ + if (!drm_dp_is_branch(dpcd)) + return false; + + if (dpcd[DP_DPCD_REV] < 0x13) + return false; + + switch (port_cap[0] & DP_DS_PORT_TYPE_MASK) { + case DP_DS_PORT_TYPE_HDMI: + if ((dpcd[DP_DOWNSTREAMPORT_PRESENT] & DP_DETAILED_CAP_INFO_AVAILABLE) == 0) + return false; + + return port_cap[3] & color_spc; + default: + return false; + } +} +EXPORT_SYMBOL(drm_dp_downstream_rgb_to_ycbcr_conversion); + /** * drm_dp_downstream_mode() - return a mode for downstream facing port * @dev: DRM device @@ -1204,7 +1236,7 @@ bool drm_dp_read_sink_count_cap(struct drm_connector *connector, return connector->connector_type != DRM_MODE_CONNECTOR_eDP && dpcd[DP_DPCD_REV] >= DP_DPCD_REV_11 && dpcd[DP_DOWNSTREAMPORT_PRESENT] & DP_DWN_STRM_PORT_PRESENT && - !drm_dp_has_quirk(desc, 0, DP_DPCD_QUIRK_NO_SINK_COUNT); + !drm_dp_has_quirk(desc, DP_DPCD_QUIRK_NO_SINK_COUNT); } EXPORT_SYMBOL(drm_dp_read_sink_count_cap); @@ -1925,87 +1957,6 @@ drm_dp_get_quirks(const struct drm_dp_dpcd_ident *ident, bool is_branch) #undef DEVICE_ID_ANY #undef DEVICE_ID -struct edid_quirk { - u8 mfg_id[2]; - u8 prod_id[2]; - u32 quirks; -}; - -#define MFG(first, second) { (first), (second) } -#define PROD_ID(first, second) { (first), (second) } - -/* - * Some devices have unreliable OUIDs where they don't set the device ID - * correctly, and as a result we need to use the EDID for finding additional - * DP quirks in such cases. - */ -static const struct edid_quirk edid_quirk_list[] = { - /* Optional 4K AMOLED panel in the ThinkPad X1 Extreme 2nd Generation - * only supports DPCD backlight controls - */ - { MFG(0x4c, 0x83), PROD_ID(0x41, 0x41), BIT(DP_QUIRK_FORCE_DPCD_BACKLIGHT) }, - /* - * Some Dell CML 2020 systems have panels support both AUX and PWM - * backlight control, and some only support AUX backlight control. All - * said panels start up in AUX mode by default, and we don't have any - * support for disabling HDR mode on these panels which would be - * required to switch to PWM backlight control mode (plus, I'm not - * even sure we want PWM backlight controls over DPCD backlight - * controls anyway...). Until we have a better way of detecting these, - * force DPCD backlight mode on all of them. - */ - { MFG(0x06, 0xaf), PROD_ID(0x9b, 0x32), BIT(DP_QUIRK_FORCE_DPCD_BACKLIGHT) }, - { MFG(0x06, 0xaf), PROD_ID(0xeb, 0x41), BIT(DP_QUIRK_FORCE_DPCD_BACKLIGHT) }, - { MFG(0x4d, 0x10), PROD_ID(0xc7, 0x14), BIT(DP_QUIRK_FORCE_DPCD_BACKLIGHT) }, - { MFG(0x4d, 0x10), PROD_ID(0xe6, 0x14), BIT(DP_QUIRK_FORCE_DPCD_BACKLIGHT) }, - { MFG(0x4c, 0x83), PROD_ID(0x47, 0x41), BIT(DP_QUIRK_FORCE_DPCD_BACKLIGHT) }, - { MFG(0x09, 0xe5), PROD_ID(0xde, 0x08), BIT(DP_QUIRK_FORCE_DPCD_BACKLIGHT) }, -}; - -#undef MFG -#undef PROD_ID - -/** - * drm_dp_get_edid_quirks() - Check the EDID of a DP device to find additional - * DP-specific quirks - * @edid: The EDID to check - * - * While OUIDs are meant to be used to recognize a DisplayPort device, a lot - * of manufacturers don't seem to like following standards and neglect to fill - * the dev-ID in, making it impossible to only use OUIDs for determining - * quirks in some cases. This function can be used to check the EDID and look - * up any additional DP quirks. The bits returned by this function correspond - * to the quirk bits in &drm_dp_quirk. - * - * Returns: a bitmask of quirks, if any. The driver can check this using - * drm_dp_has_quirk(). - */ -u32 drm_dp_get_edid_quirks(const struct edid *edid) -{ - const struct edid_quirk *quirk; - u32 quirks = 0; - int i; - - if (!edid) - return 0; - - for (i = 0; i < ARRAY_SIZE(edid_quirk_list); i++) { - quirk = &edid_quirk_list[i]; - if (memcmp(quirk->mfg_id, edid->mfg_id, - sizeof(edid->mfg_id)) == 0 && - memcmp(quirk->prod_id, edid->prod_code, - sizeof(edid->prod_code)) == 0) - quirks |= quirk->quirks; - } - - DRM_DEBUG_KMS("DP sink: EDID mfg %*phD prod-ID %*phD quirks: 0x%04x\n", - (int)sizeof(edid->mfg_id), edid->mfg_id, - (int)sizeof(edid->prod_code), edid->prod_code, quirks); - - return quirks; -} -EXPORT_SYMBOL(drm_dp_get_edid_quirks); - /** * drm_dp_read_desc - read sink/branch descriptor from DPCD * @aux: DisplayPort AUX channel @@ -2596,3 +2547,538 @@ void drm_dp_vsc_sdp_log(const char *level, struct device *dev, #undef DP_SDP_LOG } EXPORT_SYMBOL(drm_dp_vsc_sdp_log); + +/** + * drm_dp_get_pcon_max_frl_bw() - maximum frl supported by PCON + * @dpcd: DisplayPort configuration data + * @port_cap: port capabilities + * + * Returns maximum frl bandwidth supported by PCON in GBPS, + * returns 0 if not supported. + */ +int drm_dp_get_pcon_max_frl_bw(const u8 dpcd[DP_RECEIVER_CAP_SIZE], + const u8 port_cap[4]) +{ + int bw; + u8 buf; + + buf = port_cap[2]; + bw = buf & DP_PCON_MAX_FRL_BW; + + switch (bw) { + case DP_PCON_MAX_9GBPS: + return 9; + case DP_PCON_MAX_18GBPS: + return 18; + case DP_PCON_MAX_24GBPS: + return 24; + case DP_PCON_MAX_32GBPS: + return 32; + case DP_PCON_MAX_40GBPS: + return 40; + case DP_PCON_MAX_48GBPS: + return 48; + case DP_PCON_MAX_0GBPS: + default: + return 0; + } + + return 0; +} +EXPORT_SYMBOL(drm_dp_get_pcon_max_frl_bw); + +/** + * drm_dp_pcon_frl_prepare() - Prepare PCON for FRL. + * @aux: DisplayPort AUX channel + * @enable_frl_ready_hpd: Configure DP_PCON_ENABLE_HPD_READY. + * + * Returns 0 if success, else returns negative error code. + */ +int drm_dp_pcon_frl_prepare(struct drm_dp_aux *aux, bool enable_frl_ready_hpd) +{ + int ret; + u8 buf = DP_PCON_ENABLE_SOURCE_CTL_MODE | + DP_PCON_ENABLE_LINK_FRL_MODE; + + if (enable_frl_ready_hpd) + buf |= DP_PCON_ENABLE_HPD_READY; + + ret = drm_dp_dpcd_writeb(aux, DP_PCON_HDMI_LINK_CONFIG_1, buf); + + return ret; +} +EXPORT_SYMBOL(drm_dp_pcon_frl_prepare); + +/** + * drm_dp_pcon_is_frl_ready() - Is PCON ready for FRL + * @aux: DisplayPort AUX channel + * + * Returns true if success, else returns false. + */ +bool drm_dp_pcon_is_frl_ready(struct drm_dp_aux *aux) +{ + int ret; + u8 buf; + + ret = drm_dp_dpcd_readb(aux, DP_PCON_HDMI_TX_LINK_STATUS, &buf); + if (ret < 0) + return false; + + if (buf & DP_PCON_FRL_READY) + return true; + + return false; +} +EXPORT_SYMBOL(drm_dp_pcon_is_frl_ready); + +/** + * drm_dp_pcon_frl_configure_1() - Set HDMI LINK Configuration-Step1 + * @aux: DisplayPort AUX channel + * @max_frl_gbps: maximum frl bw to be configured between PCON and HDMI sink + * @concurrent_mode: true if concurrent mode or operation is required, + * false otherwise. + * + * Returns 0 if success, else returns negative error code. + */ + +int drm_dp_pcon_frl_configure_1(struct drm_dp_aux *aux, int max_frl_gbps, + bool concurrent_mode) +{ + int ret; + u8 buf; + + ret = drm_dp_dpcd_readb(aux, DP_PCON_HDMI_LINK_CONFIG_1, &buf); + if (ret < 0) + return ret; + + if (concurrent_mode) + buf |= DP_PCON_ENABLE_CONCURRENT_LINK; + else + buf &= ~DP_PCON_ENABLE_CONCURRENT_LINK; + + switch (max_frl_gbps) { + case 9: + buf |= DP_PCON_ENABLE_MAX_BW_9GBPS; + break; + case 18: + buf |= DP_PCON_ENABLE_MAX_BW_18GBPS; + break; + case 24: + buf |= DP_PCON_ENABLE_MAX_BW_24GBPS; + break; + case 32: + buf |= DP_PCON_ENABLE_MAX_BW_32GBPS; + break; + case 40: + buf |= DP_PCON_ENABLE_MAX_BW_40GBPS; + break; + case 48: + buf |= DP_PCON_ENABLE_MAX_BW_48GBPS; + break; + case 0: + buf |= DP_PCON_ENABLE_MAX_BW_0GBPS; + break; + default: + return -EINVAL; + } + + ret = drm_dp_dpcd_writeb(aux, DP_PCON_HDMI_LINK_CONFIG_1, buf); + if (ret < 0) + return ret; + + return 0; +} +EXPORT_SYMBOL(drm_dp_pcon_frl_configure_1); + +/** + * drm_dp_pcon_frl_configure_2() - Set HDMI Link configuration Step-2 + * @aux: DisplayPort AUX channel + * @max_frl_mask : Max FRL BW to be tried by the PCON with HDMI Sink + * @extended_train_mode : true for Extended Mode, false for Normal Mode. + * In Normal mode, the PCON tries each frl bw from the max_frl_mask starting + * from min, and stops when link training is successful. In Extended mode, all + * frl bw selected in the mask are trained by the PCON. + * + * Returns 0 if success, else returns negative error code. + */ +int drm_dp_pcon_frl_configure_2(struct drm_dp_aux *aux, int max_frl_mask, + bool extended_train_mode) +{ + int ret; + u8 buf = max_frl_mask; + + if (extended_train_mode) + buf |= DP_PCON_FRL_LINK_TRAIN_EXTENDED; + + ret = drm_dp_dpcd_writeb(aux, DP_PCON_HDMI_LINK_CONFIG_2, buf); + if (ret < 0) + return ret; + + return 0; +} +EXPORT_SYMBOL(drm_dp_pcon_frl_configure_2); + +/** + * drm_dp_pcon_reset_frl_config() - Re-Set HDMI Link configuration. + * @aux: DisplayPort AUX channel + * + * Returns 0 if success, else returns negative error code. + */ +int drm_dp_pcon_reset_frl_config(struct drm_dp_aux *aux) +{ + int ret; + + ret = drm_dp_dpcd_writeb(aux, DP_PCON_HDMI_LINK_CONFIG_1, 0x0); + if (ret < 0) + return ret; + + return 0; +} +EXPORT_SYMBOL(drm_dp_pcon_reset_frl_config); + +/** + * drm_dp_pcon_frl_enable() - Enable HDMI link through FRL + * @aux: DisplayPort AUX channel + * + * Returns 0 if success, else returns negative error code. + */ +int drm_dp_pcon_frl_enable(struct drm_dp_aux *aux) +{ + int ret; + u8 buf = 0; + + ret = drm_dp_dpcd_readb(aux, DP_PCON_HDMI_LINK_CONFIG_1, &buf); + if (ret < 0) + return ret; + if (!(buf & DP_PCON_ENABLE_SOURCE_CTL_MODE)) { + DRM_DEBUG_KMS("PCON in Autonomous mode, can't enable FRL\n"); + return -EINVAL; + } + buf |= DP_PCON_ENABLE_HDMI_LINK; + ret = drm_dp_dpcd_writeb(aux, DP_PCON_HDMI_LINK_CONFIG_1, buf); + if (ret < 0) + return ret; + + return 0; +} +EXPORT_SYMBOL(drm_dp_pcon_frl_enable); + +/** + * drm_dp_pcon_hdmi_link_active() - check if the PCON HDMI LINK status is active. + * @aux: DisplayPort AUX channel + * + * Returns true if link is active else returns false. + */ +bool drm_dp_pcon_hdmi_link_active(struct drm_dp_aux *aux) +{ + u8 buf; + int ret; + + ret = drm_dp_dpcd_readb(aux, DP_PCON_HDMI_TX_LINK_STATUS, &buf); + if (ret < 0) + return false; + + return buf & DP_PCON_HDMI_TX_LINK_ACTIVE; +} +EXPORT_SYMBOL(drm_dp_pcon_hdmi_link_active); + +/** + * drm_dp_pcon_hdmi_link_mode() - get the PCON HDMI LINK MODE + * @aux: DisplayPort AUX channel + * @frl_trained_mask: pointer to store bitmask of the trained bw configuration. + * Valid only if the MODE returned is FRL. For Normal Link training mode + * only 1 of the bits will be set, but in case of Extended mode, more than + * one bits can be set. + * + * Returns the link mode : TMDS or FRL on success, else returns negative error + * code. + */ +int drm_dp_pcon_hdmi_link_mode(struct drm_dp_aux *aux, u8 *frl_trained_mask) +{ + u8 buf; + int mode; + int ret; + + ret = drm_dp_dpcd_readb(aux, DP_PCON_HDMI_POST_FRL_STATUS, &buf); + if (ret < 0) + return ret; + + mode = buf & DP_PCON_HDMI_LINK_MODE; + + if (frl_trained_mask && DP_PCON_HDMI_MODE_FRL == mode) + *frl_trained_mask = (buf & DP_PCON_HDMI_FRL_TRAINED_BW) >> 1; + + return mode; +} +EXPORT_SYMBOL(drm_dp_pcon_hdmi_link_mode); + +/** + * drm_dp_pcon_hdmi_frl_link_error_count() - print the error count per lane + * during link failure between PCON and HDMI sink + * @aux: DisplayPort AUX channel + * @connector: DRM connector + * code. + **/ + +void drm_dp_pcon_hdmi_frl_link_error_count(struct drm_dp_aux *aux, + struct drm_connector *connector) +{ + u8 buf, error_count; + int i, num_error; + struct drm_hdmi_info *hdmi = &connector->display_info.hdmi; + + for (i = 0; i < hdmi->max_lanes; i++) { + if (drm_dp_dpcd_readb(aux, DP_PCON_HDMI_ERROR_STATUS_LN0 + i, &buf) < 0) + return; + + error_count = buf & DP_PCON_HDMI_ERROR_COUNT_MASK; + switch (error_count) { + case DP_PCON_HDMI_ERROR_COUNT_HUNDRED_PLUS: + num_error = 100; + break; + case DP_PCON_HDMI_ERROR_COUNT_TEN_PLUS: + num_error = 10; + break; + case DP_PCON_HDMI_ERROR_COUNT_THREE_PLUS: + num_error = 3; + break; + default: + num_error = 0; + } + + DRM_ERROR("More than %d errors since the last read for lane %d", num_error, i); + } +} +EXPORT_SYMBOL(drm_dp_pcon_hdmi_frl_link_error_count); + +/* + * drm_dp_pcon_enc_is_dsc_1_2 - Does PCON Encoder supports DSC 1.2 + * @pcon_dsc_dpcd: DSC capabilities of the PCON DSC Encoder + * + * Returns true is PCON encoder is DSC 1.2 else returns false. + */ +bool drm_dp_pcon_enc_is_dsc_1_2(const u8 pcon_dsc_dpcd[DP_PCON_DSC_ENCODER_CAP_SIZE]) +{ + u8 buf; + u8 major_v, minor_v; + + buf = pcon_dsc_dpcd[DP_PCON_DSC_VERSION - DP_PCON_DSC_ENCODER]; + major_v = (buf & DP_PCON_DSC_MAJOR_MASK) >> DP_PCON_DSC_MAJOR_SHIFT; + minor_v = (buf & DP_PCON_DSC_MINOR_MASK) >> DP_PCON_DSC_MINOR_SHIFT; + + if (major_v == 1 && minor_v == 2) + return true; + + return false; +} +EXPORT_SYMBOL(drm_dp_pcon_enc_is_dsc_1_2); + +/* + * drm_dp_pcon_dsc_max_slices - Get max slices supported by PCON DSC Encoder + * @pcon_dsc_dpcd: DSC capabilities of the PCON DSC Encoder + * + * Returns maximum no. of slices supported by the PCON DSC Encoder. + */ +int drm_dp_pcon_dsc_max_slices(const u8 pcon_dsc_dpcd[DP_PCON_DSC_ENCODER_CAP_SIZE]) +{ + u8 slice_cap1, slice_cap2; + + slice_cap1 = pcon_dsc_dpcd[DP_PCON_DSC_SLICE_CAP_1 - DP_PCON_DSC_ENCODER]; + slice_cap2 = pcon_dsc_dpcd[DP_PCON_DSC_SLICE_CAP_2 - DP_PCON_DSC_ENCODER]; + + if (slice_cap2 & DP_PCON_DSC_24_PER_DSC_ENC) + return 24; + if (slice_cap2 & DP_PCON_DSC_20_PER_DSC_ENC) + return 20; + if (slice_cap2 & DP_PCON_DSC_16_PER_DSC_ENC) + return 16; + if (slice_cap1 & DP_PCON_DSC_12_PER_DSC_ENC) + return 12; + if (slice_cap1 & DP_PCON_DSC_10_PER_DSC_ENC) + return 10; + if (slice_cap1 & DP_PCON_DSC_8_PER_DSC_ENC) + return 8; + if (slice_cap1 & DP_PCON_DSC_6_PER_DSC_ENC) + return 6; + if (slice_cap1 & DP_PCON_DSC_4_PER_DSC_ENC) + return 4; + if (slice_cap1 & DP_PCON_DSC_2_PER_DSC_ENC) + return 2; + if (slice_cap1 & DP_PCON_DSC_1_PER_DSC_ENC) + return 1; + + return 0; +} +EXPORT_SYMBOL(drm_dp_pcon_dsc_max_slices); + +/* + * drm_dp_pcon_dsc_max_slice_width() - Get max slice width for Pcon DSC encoder + * @pcon_dsc_dpcd: DSC capabilities of the PCON DSC Encoder + * + * Returns maximum width of the slices in pixel width i.e. no. of pixels x 320. + */ +int drm_dp_pcon_dsc_max_slice_width(const u8 pcon_dsc_dpcd[DP_PCON_DSC_ENCODER_CAP_SIZE]) +{ + u8 buf; + + buf = pcon_dsc_dpcd[DP_PCON_DSC_MAX_SLICE_WIDTH - DP_PCON_DSC_ENCODER]; + + return buf * DP_DSC_SLICE_WIDTH_MULTIPLIER; +} +EXPORT_SYMBOL(drm_dp_pcon_dsc_max_slice_width); + +/* + * drm_dp_pcon_dsc_bpp_incr() - Get bits per pixel increment for PCON DSC encoder + * @pcon_dsc_dpcd: DSC capabilities of the PCON DSC Encoder + * + * Returns the bpp precision supported by the PCON encoder. + */ +int drm_dp_pcon_dsc_bpp_incr(const u8 pcon_dsc_dpcd[DP_PCON_DSC_ENCODER_CAP_SIZE]) +{ + u8 buf; + + buf = pcon_dsc_dpcd[DP_PCON_DSC_BPP_INCR - DP_PCON_DSC_ENCODER]; + + switch (buf & DP_PCON_DSC_BPP_INCR_MASK) { + case DP_PCON_DSC_ONE_16TH_BPP: + return 16; + case DP_PCON_DSC_ONE_8TH_BPP: + return 8; + case DP_PCON_DSC_ONE_4TH_BPP: + return 4; + case DP_PCON_DSC_ONE_HALF_BPP: + return 2; + case DP_PCON_DSC_ONE_BPP: + return 1; + } + + return 0; +} +EXPORT_SYMBOL(drm_dp_pcon_dsc_bpp_incr); + +static +int drm_dp_pcon_configure_dsc_enc(struct drm_dp_aux *aux, u8 pps_buf_config) +{ + u8 buf; + int ret; + + ret = drm_dp_dpcd_readb(aux, DP_PROTOCOL_CONVERTER_CONTROL_2, &buf); + if (ret < 0) + return ret; + + buf |= DP_PCON_ENABLE_DSC_ENCODER; + + if (pps_buf_config <= DP_PCON_ENC_PPS_OVERRIDE_EN_BUFFER) { + buf &= ~DP_PCON_ENCODER_PPS_OVERRIDE_MASK; + buf |= pps_buf_config << 2; + } + + ret = drm_dp_dpcd_writeb(aux, DP_PROTOCOL_CONVERTER_CONTROL_2, buf); + if (ret < 0) + return ret; + + return 0; +} + +/** + * drm_dp_pcon_pps_default() - Let PCON fill the default pps parameters + * for DSC1.2 between PCON & HDMI2.1 sink + * @aux: DisplayPort AUX channel + * + * Returns 0 on success, else returns negative error code. + */ +int drm_dp_pcon_pps_default(struct drm_dp_aux *aux) +{ + int ret; + + ret = drm_dp_pcon_configure_dsc_enc(aux, DP_PCON_ENC_PPS_OVERRIDE_DISABLED); + if (ret < 0) + return ret; + + return 0; +} +EXPORT_SYMBOL(drm_dp_pcon_pps_default); + +/** + * drm_dp_pcon_pps_override_buf() - Configure PPS encoder override buffer for + * HDMI sink + * @aux: DisplayPort AUX channel + * @pps_buf: 128 bytes to be written into PPS buffer for HDMI sink by PCON. + * + * Returns 0 on success, else returns negative error code. + */ +int drm_dp_pcon_pps_override_buf(struct drm_dp_aux *aux, u8 pps_buf[128]) +{ + int ret; + + ret = drm_dp_dpcd_write(aux, DP_PCON_HDMI_PPS_OVERRIDE_BASE, &pps_buf, 128); + if (ret < 0) + return ret; + + ret = drm_dp_pcon_configure_dsc_enc(aux, DP_PCON_ENC_PPS_OVERRIDE_EN_BUFFER); + if (ret < 0) + return ret; + + return 0; +} +EXPORT_SYMBOL(drm_dp_pcon_pps_override_buf); + +/* + * drm_dp_pcon_pps_override_param() - Write PPS parameters to DSC encoder + * override registers + * @aux: DisplayPort AUX channel + * @pps_param: 3 Parameters (2 Bytes each) : Slice Width, Slice Height, + * bits_per_pixel. + * + * Returns 0 on success, else returns negative error code. + */ +int drm_dp_pcon_pps_override_param(struct drm_dp_aux *aux, u8 pps_param[6]) +{ + int ret; + + ret = drm_dp_dpcd_write(aux, DP_PCON_HDMI_PPS_OVRD_SLICE_HEIGHT, &pps_param[0], 2); + if (ret < 0) + return ret; + ret = drm_dp_dpcd_write(aux, DP_PCON_HDMI_PPS_OVRD_SLICE_WIDTH, &pps_param[2], 2); + if (ret < 0) + return ret; + ret = drm_dp_dpcd_write(aux, DP_PCON_HDMI_PPS_OVRD_BPP, &pps_param[4], 2); + if (ret < 0) + return ret; + + ret = drm_dp_pcon_configure_dsc_enc(aux, DP_PCON_ENC_PPS_OVERRIDE_EN_BUFFER); + if (ret < 0) + return ret; + + return 0; +} +EXPORT_SYMBOL(drm_dp_pcon_pps_override_param); + +/* + * drm_dp_pcon_convert_rgb_to_ycbcr() - Configure the PCon to convert RGB to Ycbcr + * @aux: displayPort AUX channel + * @color_spc: Color-space/s for which conversion is to be enabled, 0 for disable. + * + * Returns 0 on success, else returns negative error code. + */ +int drm_dp_pcon_convert_rgb_to_ycbcr(struct drm_dp_aux *aux, u8 color_spc) +{ + int ret; + u8 buf; + + ret = drm_dp_dpcd_readb(aux, DP_PROTOCOL_CONVERTER_CONTROL_2, &buf); + if (ret < 0) + return ret; + + if (color_spc & DP_CONVERSION_RGB_YCBCR_MASK) + buf |= (color_spc & DP_CONVERSION_RGB_YCBCR_MASK); + else + buf &= ~DP_CONVERSION_RGB_YCBCR_MASK; + + ret = drm_dp_dpcd_writeb(aux, DP_PROTOCOL_CONVERTER_CONTROL_2, buf); + if (ret < 0) + return ret; + + return 0; +} +EXPORT_SYMBOL(drm_dp_pcon_convert_rgb_to_ycbcr); diff --git a/drivers/gpu/drm/drm_dp_mst_topology.c b/drivers/gpu/drm/drm_dp_mst_topology.c index b11c0522a4410676c8ef6180a98ea984ac0687e2..309afe61afdd55616e8f943b4c81de8d34188e50 100644 --- a/drivers/gpu/drm/drm_dp_mst_topology.c +++ b/drivers/gpu/drm/drm_dp_mst_topology.c @@ -2302,7 +2302,8 @@ drm_dp_mst_port_add_connector(struct drm_dp_mst_branch *mstb, } if (port->pdt != DP_PEER_DEVICE_NONE && - drm_dp_mst_is_end_device(port->pdt, port->mcs)) { + drm_dp_mst_is_end_device(port->pdt, port->mcs) && + port->port_num >= DP_MST_LOGICAL_PORT_0) { port->cached_edid = drm_get_edid(port->connector, &port->aux.ddc); drm_connector_set_tile_property(port->connector); @@ -2751,7 +2752,7 @@ static void drm_dp_mst_link_probe_work(struct work_struct *work) drm_dp_mst_topology_put_mstb(mstb); mutex_unlock(&mgr->probe_lock); - if (ret) + if (ret > 0) drm_kms_helper_hotplug_event(dev); } @@ -5837,8 +5838,7 @@ struct drm_dp_aux *drm_dp_mst_dsc_aux_for_port(struct drm_dp_mst_port *port) if (drm_dp_read_desc(port->mgr->aux, &desc, true)) return NULL; - if (drm_dp_has_quirk(&desc, 0, - DP_DPCD_QUIRK_DSC_WITHOUT_VIRTUAL_DPCD) && + if (drm_dp_has_quirk(&desc, DP_DPCD_QUIRK_DSC_WITHOUT_VIRTUAL_DPCD) && port->mgr->dpcd[DP_DPCD_REV] >= DP_DPCD_REV_14 && port->parent == port->mgr->mst_primary) { u8 downstreamport; diff --git a/drivers/gpu/drm/drm_drv.c b/drivers/gpu/drm/drm_drv.c index 734303802bc3989ac7f3b88f7111d74bf7c72881..20d22e41d7ce74d7f7b0698d271dabd64f7ce488 100644 --- a/drivers/gpu/drm/drm_drv.c +++ b/drivers/gpu/drm/drm_drv.c @@ -469,6 +469,9 @@ void drm_dev_unplug(struct drm_device *dev) synchronize_srcu(&drm_unplug_srcu); drm_dev_unregister(dev); + + /* Clear all CPU mappings pointing to this device */ + unmap_mapping_range(dev->anon_inode->i_mapping, 0, 0, 1); } EXPORT_SYMBOL(drm_dev_unplug); @@ -589,11 +592,7 @@ static int drm_dev_init(struct drm_device *dev, kref_init(&dev->ref); dev->dev = get_device(parent); -#ifdef CONFIG_DRM_LEGACY - dev->driver = (struct drm_driver *)driver; -#else dev->driver = driver; -#endif INIT_LIST_HEAD(&dev->managed.resources); spin_lock_init(&dev->managed.lock); @@ -675,11 +674,8 @@ static int devm_drm_dev_init(struct device *parent, if (ret) return ret; - ret = devm_add_action(parent, devm_drm_dev_init_release, dev); - if (ret) - devm_drm_dev_init_release(dev); - - return ret; + return devm_add_action_or_reset(parent, + devm_drm_dev_init_release, dev); } void *__devm_drm_dev_alloc(struct device *parent, @@ -897,8 +893,6 @@ int drm_dev_register(struct drm_device *dev, unsigned long flags) if (drm_core_check_feature(dev, DRIVER_MODESET)) drm_modeset_register_all(dev); - ret = 0; - DRM_INFO("Initialized %s %d.%d.%d %s for %s on minor %d\n", driver->name, driver->major, driver->minor, driver->patchlevel, driver->date, diff --git a/drivers/gpu/drm/drm_dsc.c b/drivers/gpu/drm/drm_dsc.c index 4a475d9696ff39dae1ace772c0af1f27897a07cc..ff602f7ec65b1e722458a562abf99a138b59b755 100644 --- a/drivers/gpu/drm/drm_dsc.c +++ b/drivers/gpu/drm/drm_dsc.c @@ -49,6 +49,33 @@ void drm_dsc_dp_pps_header_init(struct dp_sdp_header *pps_header) } EXPORT_SYMBOL(drm_dsc_dp_pps_header_init); +/** + * drm_dsc_dp_rc_buffer_size - get rc buffer size in bytes + * @rc_buffer_block_size: block size code, according to DPCD offset 62h + * @rc_buffer_size: number of blocks - 1, according to DPCD offset 63h + * + * return: + * buffer size in bytes, or 0 on invalid input + */ +int drm_dsc_dp_rc_buffer_size(u8 rc_buffer_block_size, u8 rc_buffer_size) +{ + int size = 1024 * (rc_buffer_size + 1); + + switch (rc_buffer_block_size) { + case DP_DSC_RC_BUF_BLK_SIZE_1: + return 1 * size; + case DP_DSC_RC_BUF_BLK_SIZE_4: + return 4 * size; + case DP_DSC_RC_BUF_BLK_SIZE_16: + return 16 * size; + case DP_DSC_RC_BUF_BLK_SIZE_64: + return 64 * size; + default: + return 0; + } +} +EXPORT_SYMBOL(drm_dsc_dp_rc_buffer_size); + /** * drm_dsc_pps_payload_pack() - Populates the DSC PPS * @@ -186,8 +213,7 @@ void drm_dsc_pps_payload_pack(struct drm_dsc_picture_parameter_set *pps_payload, pps_payload->flatness_max_qp = dsc_cfg->flatness_max_qp; /* PPS 38, 39 */ - pps_payload->rc_model_size = - cpu_to_be16(DSC_RC_MODEL_SIZE_CONST); + pps_payload->rc_model_size = cpu_to_be16(dsc_cfg->rc_model_size); /* PPS 40 */ pps_payload->rc_edge_factor = DSC_RC_EDGE_FACTOR_CONST; diff --git a/drivers/gpu/drm/drm_dumb_buffers.c b/drivers/gpu/drm/drm_dumb_buffers.c index d18a740fe0f181f076f33360d321bf7c10ed8df2..ad17fa21cebb514b16212bd092a44df5497e5c3a 100644 --- a/drivers/gpu/drm/drm_dumb_buffers.c +++ b/drivers/gpu/drm/drm_dumb_buffers.c @@ -29,6 +29,7 @@ #include #include "drm_crtc_internal.h" +#include "drm_internal.h" /** * DOC: overview @@ -46,9 +47,10 @@ * KMS frame buffers. * * To support dumb objects drivers must implement the &drm_driver.dumb_create - * operation. &drm_driver.dumb_destroy defaults to drm_gem_dumb_destroy() if - * not set and &drm_driver.dumb_map_offset defaults to - * drm_gem_dumb_map_offset(). See the callbacks for further details. + * and &drm_driver.dumb_map_offset operations (the latter defaults to + * drm_gem_dumb_map_offset() if not set). Drivers that don't use GEM handles + * additionally need to implement the &drm_driver.dumb_destroy operation. See + * the callbacks for further details. * * Note that dumb objects may not be used for gpu acceleration, as has been * attempted on some ARM embedded platforms. Such drivers really must have diff --git a/drivers/gpu/drm/drm_edid.c b/drivers/gpu/drm/drm_edid.c index e95cce8e736dc7090a6091046aae53871e92a27b..c2bbe7bee7b658a911decaab5e0d6c23ca9565cc 100644 --- a/drivers/gpu/drm/drm_edid.c +++ b/drivers/gpu/drm/drm_edid.c @@ -32,6 +32,7 @@ #include #include #include +#include #include #include @@ -2075,9 +2076,13 @@ EXPORT_SYMBOL(drm_get_edid); struct edid *drm_get_edid_switcheroo(struct drm_connector *connector, struct i2c_adapter *adapter) { - struct pci_dev *pdev = connector->dev->pdev; + struct drm_device *dev = connector->dev; + struct pci_dev *pdev = to_pci_dev(dev->dev); struct edid *edid; + if (drm_WARN_ON_ONCE(dev, !dev_is_pci(dev->dev))) + return NULL; + vga_switcheroo_lock_ddc(pdev); edid = drm_get_edid(connector, adapter); vga_switcheroo_unlock_ddc(pdev); @@ -4851,6 +4856,41 @@ static void drm_parse_vcdb(struct drm_connector *connector, const u8 *db) info->rgb_quant_range_selectable = true; } +static +void drm_get_max_frl_rate(int max_frl_rate, u8 *max_lanes, u8 *max_rate_per_lane) +{ + switch (max_frl_rate) { + case 1: + *max_lanes = 3; + *max_rate_per_lane = 3; + break; + case 2: + *max_lanes = 3; + *max_rate_per_lane = 6; + break; + case 3: + *max_lanes = 4; + *max_rate_per_lane = 6; + break; + case 4: + *max_lanes = 4; + *max_rate_per_lane = 8; + break; + case 5: + *max_lanes = 4; + *max_rate_per_lane = 10; + break; + case 6: + *max_lanes = 4; + *max_rate_per_lane = 12; + break; + case 0: + default: + *max_lanes = 0; + *max_rate_per_lane = 0; + } +} + static void drm_parse_ycbcr420_deep_color_info(struct drm_connector *connector, const u8 *db) { @@ -4904,6 +4944,74 @@ static void drm_parse_hdmi_forum_vsdb(struct drm_connector *connector, } } + if (hf_vsdb[7]) { + u8 max_frl_rate; + u8 dsc_max_frl_rate; + u8 dsc_max_slices; + struct drm_hdmi_dsc_cap *hdmi_dsc = &hdmi->dsc_cap; + + DRM_DEBUG_KMS("hdmi_21 sink detected. parsing edid\n"); + max_frl_rate = (hf_vsdb[7] & DRM_EDID_MAX_FRL_RATE_MASK) >> 4; + drm_get_max_frl_rate(max_frl_rate, &hdmi->max_lanes, + &hdmi->max_frl_rate_per_lane); + hdmi_dsc->v_1p2 = hf_vsdb[11] & DRM_EDID_DSC_1P2; + + if (hdmi_dsc->v_1p2) { + hdmi_dsc->native_420 = hf_vsdb[11] & DRM_EDID_DSC_NATIVE_420; + hdmi_dsc->all_bpp = hf_vsdb[11] & DRM_EDID_DSC_ALL_BPP; + + if (hf_vsdb[11] & DRM_EDID_DSC_16BPC) + hdmi_dsc->bpc_supported = 16; + else if (hf_vsdb[11] & DRM_EDID_DSC_12BPC) + hdmi_dsc->bpc_supported = 12; + else if (hf_vsdb[11] & DRM_EDID_DSC_10BPC) + hdmi_dsc->bpc_supported = 10; + else + hdmi_dsc->bpc_supported = 0; + + dsc_max_frl_rate = (hf_vsdb[12] & DRM_EDID_DSC_MAX_FRL_RATE_MASK) >> 4; + drm_get_max_frl_rate(dsc_max_frl_rate, &hdmi_dsc->max_lanes, + &hdmi_dsc->max_frl_rate_per_lane); + hdmi_dsc->total_chunk_kbytes = hf_vsdb[13] & DRM_EDID_DSC_TOTAL_CHUNK_KBYTES; + + dsc_max_slices = hf_vsdb[12] & DRM_EDID_DSC_MAX_SLICES; + switch (dsc_max_slices) { + case 1: + hdmi_dsc->max_slices = 1; + hdmi_dsc->clk_per_slice = 340; + break; + case 2: + hdmi_dsc->max_slices = 2; + hdmi_dsc->clk_per_slice = 340; + break; + case 3: + hdmi_dsc->max_slices = 4; + hdmi_dsc->clk_per_slice = 340; + break; + case 4: + hdmi_dsc->max_slices = 8; + hdmi_dsc->clk_per_slice = 340; + break; + case 5: + hdmi_dsc->max_slices = 8; + hdmi_dsc->clk_per_slice = 400; + break; + case 6: + hdmi_dsc->max_slices = 12; + hdmi_dsc->clk_per_slice = 400; + break; + case 7: + hdmi_dsc->max_slices = 16; + hdmi_dsc->clk_per_slice = 400; + break; + case 0: + default: + hdmi_dsc->max_slices = 0; + hdmi_dsc->clk_per_slice = 0; + } + } + } + drm_parse_ycbcr420_deep_color_info(connector, hf_vsdb); } diff --git a/drivers/gpu/drm/drm_encoder.c b/drivers/gpu/drm/drm_encoder.c index e555281f43d4b4daf5919e61ccf1549fca90f1d6..72e982323a5e5b02081a6ba396e421d06bc2c20d 100644 --- a/drivers/gpu/drm/drm_encoder.c +++ b/drivers/gpu/drm/drm_encoder.c @@ -26,6 +26,7 @@ #include #include #include +#include #include "drm_crtc_internal.h" @@ -72,7 +73,7 @@ int drm_encoder_register_all(struct drm_device *dev) int ret = 0; drm_for_each_encoder(encoder, dev) { - if (encoder->funcs->late_register) + if (encoder->funcs && encoder->funcs->late_register) ret = encoder->funcs->late_register(encoder); if (ret) return ret; @@ -86,30 +87,16 @@ void drm_encoder_unregister_all(struct drm_device *dev) struct drm_encoder *encoder; drm_for_each_encoder(encoder, dev) { - if (encoder->funcs->early_unregister) + if (encoder->funcs && encoder->funcs->early_unregister) encoder->funcs->early_unregister(encoder); } } -/** - * drm_encoder_init - Init a preallocated encoder - * @dev: drm device - * @encoder: the encoder to init - * @funcs: callbacks for this encoder - * @encoder_type: user visible type of the encoder - * @name: printf style format string for the encoder name, or NULL for default name - * - * Initialises a preallocated encoder. Encoder should be subclassed as part of - * driver encoder objects. At driver unload time drm_encoder_cleanup() should be - * called from the driver's &drm_encoder_funcs.destroy hook. - * - * Returns: - * Zero on success, error code on failure. - */ -int drm_encoder_init(struct drm_device *dev, - struct drm_encoder *encoder, - const struct drm_encoder_funcs *funcs, - int encoder_type, const char *name, ...) +__printf(5, 0) +static int __drm_encoder_init(struct drm_device *dev, + struct drm_encoder *encoder, + const struct drm_encoder_funcs *funcs, + int encoder_type, const char *name, va_list ap) { int ret; @@ -125,11 +112,7 @@ int drm_encoder_init(struct drm_device *dev, encoder->encoder_type = encoder_type; encoder->funcs = funcs; if (name) { - va_list ap; - - va_start(ap, name); encoder->name = kvasprintf(GFP_KERNEL, name, ap); - va_end(ap); } else { encoder->name = kasprintf(GFP_KERNEL, "%s-%d", drm_encoder_enum_list[encoder_type].name, @@ -150,6 +133,44 @@ int drm_encoder_init(struct drm_device *dev, return ret; } + +/** + * drm_encoder_init - Init a preallocated encoder + * @dev: drm device + * @encoder: the encoder to init + * @funcs: callbacks for this encoder + * @encoder_type: user visible type of the encoder + * @name: printf style format string for the encoder name, or NULL for default name + * + * Initializes a preallocated encoder. Encoder should be subclassed as part of + * driver encoder objects. At driver unload time the driver's + * &drm_encoder_funcs.destroy hook should call drm_encoder_cleanup() and kfree() + * the encoder structure. The encoder structure should not be allocated with + * devm_kzalloc(). + * + * Note: consider using drmm_encoder_alloc() instead of drm_encoder_init() to + * let the DRM managed resource infrastructure take care of cleanup and + * deallocation. + * + * Returns: + * Zero on success, error code on failure. + */ +int drm_encoder_init(struct drm_device *dev, + struct drm_encoder *encoder, + const struct drm_encoder_funcs *funcs, + int encoder_type, const char *name, ...) +{ + va_list ap; + int ret; + + WARN_ON(!funcs->destroy); + + va_start(ap, name); + ret = __drm_encoder_init(dev, encoder, funcs, encoder_type, name, ap); + va_end(ap); + + return ret; +} EXPORT_SYMBOL(drm_encoder_init); /** @@ -181,6 +202,48 @@ void drm_encoder_cleanup(struct drm_encoder *encoder) } EXPORT_SYMBOL(drm_encoder_cleanup); +static void drmm_encoder_alloc_release(struct drm_device *dev, void *ptr) +{ + struct drm_encoder *encoder = ptr; + + if (WARN_ON(!encoder->dev)) + return; + + drm_encoder_cleanup(encoder); +} + +void *__drmm_encoder_alloc(struct drm_device *dev, size_t size, size_t offset, + const struct drm_encoder_funcs *funcs, + int encoder_type, const char *name, ...) +{ + void *container; + struct drm_encoder *encoder; + va_list ap; + int ret; + + if (WARN_ON(funcs && funcs->destroy)) + return ERR_PTR(-EINVAL); + + container = drmm_kzalloc(dev, size, GFP_KERNEL); + if (!container) + return ERR_PTR(-EINVAL); + + encoder = container + offset; + + va_start(ap, name); + ret = __drm_encoder_init(dev, encoder, funcs, encoder_type, name, ap); + va_end(ap); + if (ret) + return ERR_PTR(ret); + + ret = drmm_add_action_or_reset(dev, drmm_encoder_alloc_release, encoder); + if (ret) + return ERR_PTR(ret); + + return container; +} +EXPORT_SYMBOL(__drmm_encoder_alloc); + static struct drm_crtc *drm_encoder_get_crtc(struct drm_encoder *encoder) { struct drm_connector *connector; diff --git a/drivers/gpu/drm/drm_fb_helper.c b/drivers/gpu/drm/drm_fb_helper.c index 4b81195106875d97abf38310ab4f8b4105ebfcd1..b9a616737c0ee2e90b0cfc3bb3674f1004e7b3e2 100644 --- a/drivers/gpu/drm/drm_fb_helper.c +++ b/drivers/gpu/drm/drm_fb_helper.c @@ -946,11 +946,15 @@ static int setcmap_legacy(struct fb_cmap *cmap, struct fb_info *info) drm_modeset_lock_all(fb_helper->dev); drm_client_for_each_modeset(modeset, &fb_helper->client) { crtc = modeset->crtc; - if (!crtc->funcs->gamma_set || !crtc->gamma_size) - return -EINVAL; + if (!crtc->funcs->gamma_set || !crtc->gamma_size) { + ret = -EINVAL; + goto out; + } - if (cmap->start + cmap->len > crtc->gamma_size) - return -EINVAL; + if (cmap->start + cmap->len > crtc->gamma_size) { + ret = -EINVAL; + goto out; + } r = crtc->gamma_store; g = r + crtc->gamma_size; @@ -963,8 +967,9 @@ static int setcmap_legacy(struct fb_cmap *cmap, struct fb_info *info) ret = crtc->funcs->gamma_set(crtc, r, g, b, crtc->gamma_size, NULL); if (ret) - return ret; + goto out; } +out: drm_modeset_unlock_all(fb_helper->dev); return ret; @@ -1054,6 +1059,11 @@ static int setcmap_atomic(struct fb_cmap *cmap, struct fb_info *info) goto out_state; } + /* + * FIXME: This always uses gamma_lut. Some HW have only + * degamma_lut, in which case we should reset gamma_lut and set + * degamma_lut. See drm_crtc_legacy_gamma_set(). + */ replaced = drm_property_replace_blob(&crtc_state->degamma_lut, NULL); replaced |= drm_property_replace_blob(&crtc_state->ctm, NULL); @@ -2486,6 +2496,11 @@ void drm_fbdev_generic_setup(struct drm_device *dev, return; } + /* + * FIXME: This mixes up depth with bpp, which results in a glorious + * mess, resulting in some drivers picking wrong fbdev defaults and + * others wrong preferred_depth defaults. + */ if (!preferred_bpp) preferred_bpp = dev->mode_config.preferred_depth; if (!preferred_bpp) @@ -2499,24 +2514,3 @@ void drm_fbdev_generic_setup(struct drm_device *dev, drm_client_register(&fb_helper->client); } EXPORT_SYMBOL(drm_fbdev_generic_setup); - -/* The Kconfig DRM_KMS_HELPER selects FRAMEBUFFER_CONSOLE (if !EXPERT) - * but the module doesn't depend on any fb console symbols. At least - * attempt to load fbcon to avoid leaving the system without a usable console. - */ -int __init drm_fb_helper_modinit(void) -{ -#if defined(CONFIG_FRAMEBUFFER_CONSOLE_MODULE) && !defined(CONFIG_EXPERT) - const char name[] = "fbcon"; - struct module *fbcon; - - mutex_lock(&module_mutex); - fbcon = find_module(name); - mutex_unlock(&module_mutex); - - if (!fbcon) - request_module_nowait(name); -#endif - return 0; -} -EXPORT_SYMBOL(drm_fb_helper_modinit); diff --git a/drivers/gpu/drm/drm_file.c b/drivers/gpu/drm/drm_file.c index b50380fa80ce83389172cf4f3a053299aaaf46cc..7efbccffc2eaf7f86cae0278af50f61867c91db6 100644 --- a/drivers/gpu/drm/drm_file.c +++ b/drivers/gpu/drm/drm_file.c @@ -113,8 +113,7 @@ bool drm_dev_needs_global_mutex(struct drm_device *dev) * The memory mapping implementation will vary depending on how the driver * manages memory. Legacy drivers will use the deprecated drm_legacy_mmap() * function, modern drivers should use one of the provided memory-manager - * specific implementations. For GEM-based drivers this is drm_gem_mmap(), and - * for drivers which use the CMA GEM helpers it's drm_gem_cma_mmap(). + * specific implementations. For GEM-based drivers this is drm_gem_mmap(). * * No other file operations are supported by the DRM userspace API. Overall the * following is an example &file_operations structure:: @@ -240,9 +239,6 @@ static void drm_events_release(struct drm_file *file_priv) * before calling this. * * If NULL is passed, this is a no-op. - * - * RETURNS: - * 0 on success, or error code on failure. */ void drm_file_free(struct drm_file *file) { @@ -371,6 +367,7 @@ static int drm_open_helper(struct file *filp, struct drm_minor *minor) list_add(&priv->lhead, &dev->filelist); mutex_unlock(&dev->filelist_mutex); +#ifdef CONFIG_DRM_LEGACY #ifdef __alpha__ /* * Default the hose @@ -390,6 +387,7 @@ static int drm_open_helper(struct file *filp, struct drm_minor *minor) dev->hose = b->sysdata; } } +#endif #endif return 0; @@ -777,20 +775,19 @@ void drm_event_cancel_free(struct drm_device *dev, EXPORT_SYMBOL(drm_event_cancel_free); /** - * drm_send_event_locked - send DRM event to file descriptor + * drm_send_event_helper - send DRM event to file descriptor * @dev: DRM device * @e: DRM event to deliver + * @timestamp: timestamp to set for the fence event in kernel's CLOCK_MONOTONIC + * time domain * - * This function sends the event @e, initialized with drm_event_reserve_init(), - * to its associated userspace DRM file. Callers must already hold - * &drm_device.event_lock, see drm_send_event() for the unlocked version. - * - * Note that the core will take care of unlinking and disarming events when the - * corresponding DRM file is closed. Drivers need not worry about whether the - * DRM file for this event still exists and can call this function upon - * completion of the asynchronous work unconditionally. + * This helper function sends the event @e, initialized with + * drm_event_reserve_init(), to its associated userspace DRM file. + * The timestamp variant of dma_fence_signal is used when the caller + * sends a valid timestamp. */ -void drm_send_event_locked(struct drm_device *dev, struct drm_pending_event *e) +void drm_send_event_helper(struct drm_device *dev, + struct drm_pending_event *e, ktime_t timestamp) { assert_spin_locked(&dev->event_lock); @@ -801,7 +798,10 @@ void drm_send_event_locked(struct drm_device *dev, struct drm_pending_event *e) } if (e->fence) { - dma_fence_signal(e->fence); + if (timestamp) + dma_fence_signal_timestamp(e->fence, timestamp); + else + dma_fence_signal(e->fence); dma_fence_put(e->fence); } @@ -816,6 +816,48 @@ void drm_send_event_locked(struct drm_device *dev, struct drm_pending_event *e) wake_up_interruptible_poll(&e->file_priv->event_wait, EPOLLIN | EPOLLRDNORM); } + +/** + * drm_send_event_timestamp_locked - send DRM event to file descriptor + * @dev: DRM device + * @e: DRM event to deliver + * @timestamp: timestamp to set for the fence event in kernel's CLOCK_MONOTONIC + * time domain + * + * This function sends the event @e, initialized with drm_event_reserve_init(), + * to its associated userspace DRM file. Callers must already hold + * &drm_device.event_lock. + * + * Note that the core will take care of unlinking and disarming events when the + * corresponding DRM file is closed. Drivers need not worry about whether the + * DRM file for this event still exists and can call this function upon + * completion of the asynchronous work unconditionally. + */ +void drm_send_event_timestamp_locked(struct drm_device *dev, + struct drm_pending_event *e, ktime_t timestamp) +{ + drm_send_event_helper(dev, e, timestamp); +} +EXPORT_SYMBOL(drm_send_event_timestamp_locked); + +/** + * drm_send_event_locked - send DRM event to file descriptor + * @dev: DRM device + * @e: DRM event to deliver + * + * This function sends the event @e, initialized with drm_event_reserve_init(), + * to its associated userspace DRM file. Callers must already hold + * &drm_device.event_lock, see drm_send_event() for the unlocked version. + * + * Note that the core will take care of unlinking and disarming events when the + * corresponding DRM file is closed. Drivers need not worry about whether the + * DRM file for this event still exists and can call this function upon + * completion of the asynchronous work unconditionally. + */ +void drm_send_event_locked(struct drm_device *dev, struct drm_pending_event *e) +{ + drm_send_event_helper(dev, e, 0); +} EXPORT_SYMBOL(drm_send_event_locked); /** @@ -838,7 +880,7 @@ void drm_send_event(struct drm_device *dev, struct drm_pending_event *e) unsigned long irqflags; spin_lock_irqsave(&dev->event_lock, irqflags); - drm_send_event_locked(dev, e); + drm_send_event_helper(dev, e, 0); spin_unlock_irqrestore(&dev->event_lock, irqflags); } EXPORT_SYMBOL(drm_send_event); diff --git a/drivers/gpu/drm/drm_gem.c b/drivers/gpu/drm/drm_gem.c index 92f89cee213e8290a9a703dc0919d6424d227526..c2ce78c4edc3ee287e346c088e8bfbc1c440392d 100644 --- a/drivers/gpu/drm/drm_gem.c +++ b/drivers/gpu/drm/drm_gem.c @@ -335,22 +335,12 @@ int drm_gem_dumb_map_offset(struct drm_file *file, struct drm_device *dev, } EXPORT_SYMBOL_GPL(drm_gem_dumb_map_offset); -/** - * drm_gem_dumb_destroy - dumb fb callback helper for gem based drivers - * @file: drm file-private structure to remove the dumb handle from - * @dev: corresponding drm_device - * @handle: the dumb handle to remove - * - * This implements the &drm_driver.dumb_destroy kms driver callback for drivers - * which use gem to manage their backing storage. - */ int drm_gem_dumb_destroy(struct drm_file *file, struct drm_device *dev, - uint32_t handle) + u32 handle) { return drm_gem_handle_delete(file, handle); } -EXPORT_SYMBOL(drm_gem_dumb_destroy); /** * drm_gem_handle_create_tail - internal functions to create a handle @@ -1078,20 +1068,17 @@ int drm_gem_mmap_obj(struct drm_gem_object *obj, unsigned long obj_size, drm_gem_object_get(obj); vma->vm_private_data = obj; + vma->vm_ops = obj->funcs->vm_ops; if (obj->funcs->mmap) { ret = obj->funcs->mmap(obj, vma); - if (ret) { - drm_gem_object_put(obj); - return ret; - } + if (ret) + goto err_drm_gem_object_put; WARN_ON(!(vma->vm_flags & VM_DONTEXPAND)); } else { - if (obj->funcs->vm_ops) - vma->vm_ops = obj->funcs->vm_ops; - else { - drm_gem_object_put(obj); - return -EINVAL; + if (!vma->vm_ops) { + ret = -EINVAL; + goto err_drm_gem_object_put; } vma->vm_flags |= VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP; @@ -1100,6 +1087,10 @@ int drm_gem_mmap_obj(struct drm_gem_object *obj, unsigned long obj_size, } return 0; + +err_drm_gem_object_put: + drm_gem_object_put(obj); + return ret; } EXPORT_SYMBOL(drm_gem_mmap_obj); diff --git a/drivers/gpu/drm/drm_gem_cma_helper.c b/drivers/gpu/drm/drm_gem_cma_helper.c index 4d5c1d86b0222d2c42e77342454f8f016a10e5aa..7942cf05cd93825d6778d98f22ebb959eb9977fe 100644 --- a/drivers/gpu/drm/drm_gem_cma_helper.c +++ b/drivers/gpu/drm/drm_gem_cma_helper.c @@ -36,8 +36,9 @@ static const struct drm_gem_object_funcs drm_gem_cma_default_funcs = { .free = drm_gem_cma_free_object, .print_info = drm_gem_cma_print_info, - .get_sg_table = drm_gem_cma_prime_get_sg_table, - .vmap = drm_gem_cma_prime_vmap, + .get_sg_table = drm_gem_cma_get_sg_table, + .vmap = drm_gem_cma_vmap, + .mmap = drm_gem_cma_mmap, .vm_ops = &drm_gem_cma_vm_ops, }; @@ -277,62 +278,6 @@ const struct vm_operations_struct drm_gem_cma_vm_ops = { }; EXPORT_SYMBOL_GPL(drm_gem_cma_vm_ops); -static int drm_gem_cma_mmap_obj(struct drm_gem_cma_object *cma_obj, - struct vm_area_struct *vma) -{ - int ret; - - /* - * Clear the VM_PFNMAP flag that was set by drm_gem_mmap(), and set the - * vm_pgoff (used as a fake buffer offset by DRM) to 0 as we want to map - * the whole buffer. - */ - vma->vm_flags &= ~VM_PFNMAP; - vma->vm_pgoff = 0; - - ret = dma_mmap_wc(cma_obj->base.dev->dev, vma, cma_obj->vaddr, - cma_obj->paddr, vma->vm_end - vma->vm_start); - if (ret) - drm_gem_vm_close(vma); - - return ret; -} - -/** - * drm_gem_cma_mmap - memory-map a CMA GEM object - * @filp: file object - * @vma: VMA for the area to be mapped - * - * This function implements an augmented version of the GEM DRM file mmap - * operation for CMA objects: In addition to the usual GEM VMA setup it - * immediately faults in the entire object instead of using on-demaind - * faulting. Drivers which employ the CMA helpers should use this function - * as their ->mmap() handler in the DRM device file's file_operations - * structure. - * - * Instead of directly referencing this function, drivers should use the - * DEFINE_DRM_GEM_CMA_FOPS().macro. - * - * Returns: - * 0 on success or a negative error code on failure. - */ -int drm_gem_cma_mmap(struct file *filp, struct vm_area_struct *vma) -{ - struct drm_gem_cma_object *cma_obj; - struct drm_gem_object *gem_obj; - int ret; - - ret = drm_gem_mmap(filp, vma); - if (ret) - return ret; - - gem_obj = vma->vm_private_data; - cma_obj = to_drm_gem_cma_obj(gem_obj); - - return drm_gem_cma_mmap_obj(cma_obj, vma); -} -EXPORT_SYMBOL_GPL(drm_gem_cma_mmap); - #ifndef CONFIG_MMU /** * drm_gem_cma_get_unmapped_area - propose address for mapping in noMMU cases @@ -424,18 +369,18 @@ void drm_gem_cma_print_info(struct drm_printer *p, unsigned int indent, EXPORT_SYMBOL(drm_gem_cma_print_info); /** - * drm_gem_cma_prime_get_sg_table - provide a scatter/gather table of pinned + * drm_gem_cma_get_sg_table - provide a scatter/gather table of pinned * pages for a CMA GEM object * @obj: GEM object * - * This function exports a scatter/gather table suitable for PRIME usage by + * This function exports a scatter/gather table by * calling the standard DMA mapping API. Drivers using the CMA helpers should * set this as their &drm_gem_object_funcs.get_sg_table callback. * * Returns: * A pointer to the scatter/gather table of pinned pages or NULL on failure. */ -struct sg_table *drm_gem_cma_prime_get_sg_table(struct drm_gem_object *obj) +struct sg_table *drm_gem_cma_get_sg_table(struct drm_gem_object *obj) { struct drm_gem_cma_object *cma_obj = to_drm_gem_cma_obj(obj); struct sg_table *sgt; @@ -456,7 +401,7 @@ struct sg_table *drm_gem_cma_prime_get_sg_table(struct drm_gem_object *obj) kfree(sgt); return ERR_PTR(ret); } -EXPORT_SYMBOL_GPL(drm_gem_cma_prime_get_sg_table); +EXPORT_SYMBOL_GPL(drm_gem_cma_get_sg_table); /** * drm_gem_cma_prime_import_sg_table - produce a CMA GEM object from another @@ -501,40 +446,13 @@ drm_gem_cma_prime_import_sg_table(struct drm_device *dev, EXPORT_SYMBOL_GPL(drm_gem_cma_prime_import_sg_table); /** - * drm_gem_cma_prime_mmap - memory-map an exported CMA GEM object - * @obj: GEM object - * @vma: VMA for the area to be mapped - * - * This function maps a buffer imported via DRM PRIME into a userspace - * process's address space. Drivers that use the CMA helpers should set this - * as their &drm_driver.gem_prime_mmap callback. - * - * Returns: - * 0 on success or a negative error code on failure. - */ -int drm_gem_cma_prime_mmap(struct drm_gem_object *obj, - struct vm_area_struct *vma) -{ - struct drm_gem_cma_object *cma_obj; - int ret; - - ret = drm_gem_mmap_obj(obj, obj->size, vma); - if (ret < 0) - return ret; - - cma_obj = to_drm_gem_cma_obj(obj); - return drm_gem_cma_mmap_obj(cma_obj, vma); -} -EXPORT_SYMBOL_GPL(drm_gem_cma_prime_mmap); - -/** - * drm_gem_cma_prime_vmap - map a CMA GEM object into the kernel's virtual + * drm_gem_cma_vmap - map a CMA GEM object into the kernel's virtual * address space * @obj: GEM object * @map: Returns the kernel virtual address of the CMA GEM object's backing * store. * - * This function maps a buffer exported via DRM PRIME into the kernel's + * This function maps a buffer into the kernel's * virtual address space. Since the CMA buffers are already mapped into the * kernel virtual address space this simply returns the cached virtual * address. Drivers using the CMA helpers should set this as their DRM @@ -543,7 +461,7 @@ EXPORT_SYMBOL_GPL(drm_gem_cma_prime_mmap); * Returns: * 0 on success, or a negative error code otherwise. */ -int drm_gem_cma_prime_vmap(struct drm_gem_object *obj, struct dma_buf_map *map) +int drm_gem_cma_vmap(struct drm_gem_object *obj, struct dma_buf_map *map) { struct drm_gem_cma_object *cma_obj = to_drm_gem_cma_obj(obj); @@ -551,7 +469,44 @@ int drm_gem_cma_prime_vmap(struct drm_gem_object *obj, struct dma_buf_map *map) return 0; } -EXPORT_SYMBOL_GPL(drm_gem_cma_prime_vmap); +EXPORT_SYMBOL_GPL(drm_gem_cma_vmap); + +/** + * drm_gem_cma_mmap - memory-map an exported CMA GEM object + * @obj: GEM object + * @vma: VMA for the area to be mapped + * + * This function maps a buffer into a userspace process's address space. + * In addition to the usual GEM VMA setup it immediately faults in the entire + * object instead of using on-demand faulting. Drivers that use the CMA + * helpers should set this as their &drm_gem_object_funcs.mmap callback. + * + * Returns: + * 0 on success or a negative error code on failure. + */ +int drm_gem_cma_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma) +{ + struct drm_gem_cma_object *cma_obj; + int ret; + + /* + * Clear the VM_PFNMAP flag that was set by drm_gem_mmap(), and set the + * vm_pgoff (used as a fake buffer offset by DRM) to 0 as we want to map + * the whole buffer. + */ + vma->vm_pgoff -= drm_vma_node_start(&obj->vma_node); + vma->vm_flags &= ~VM_PFNMAP; + + cma_obj = to_drm_gem_cma_obj(obj); + + ret = dma_mmap_wc(cma_obj->base.dev->dev, vma, cma_obj->vaddr, + cma_obj->paddr, vma->vm_end - vma->vm_start); + if (ret) + drm_gem_vm_close(vma); + + return ret; +} +EXPORT_SYMBOL_GPL(drm_gem_cma_mmap); /** * drm_gem_cma_prime_import_sg_table_vmap - PRIME import another driver's diff --git a/drivers/gpu/drm/drm_internal.h b/drivers/gpu/drm/drm_internal.h index 81d386b5b92a03902e13de829a687e513d6ed1ab..fad2249ee67b2810d9e12cd0de3933c7ab7245ad 100644 --- a/drivers/gpu/drm/drm_internal.h +++ b/drivers/gpu/drm/drm_internal.h @@ -191,6 +191,9 @@ void drm_gem_unpin(struct drm_gem_object *obj); int drm_gem_vmap(struct drm_gem_object *obj, struct dma_buf_map *map); void drm_gem_vunmap(struct drm_gem_object *obj, struct dma_buf_map *map); +int drm_gem_dumb_destroy(struct drm_file *file, struct drm_device *dev, + u32 handle); + /* drm_debugfs.c drm_debugfs_crc.c */ #if defined(CONFIG_DEBUG_FS) int drm_debugfs_init(struct drm_minor *minor, int minor_id, diff --git a/drivers/gpu/drm/drm_irq.c b/drivers/gpu/drm/drm_irq.c index 09d6e9e2e07515240a230097bd3b1f6b0cf2a02c..c3bd664ea733fe3311be5de77f95de2816392148 100644 --- a/drivers/gpu/drm/drm_irq.c +++ b/drivers/gpu/drm/drm_irq.c @@ -122,7 +122,7 @@ int drm_irq_install(struct drm_device *dev, int irq) dev->driver->irq_preinstall(dev); /* PCI devices require shared interrupts. */ - if (dev->pdev) + if (dev_is_pci(dev->dev)) sh_flags = IRQF_SHARED; ret = request_irq(irq, dev->driver->irq_handler, @@ -140,7 +140,7 @@ int drm_irq_install(struct drm_device *dev, int irq) if (ret < 0) { dev->irq_enabled = false; if (drm_core_check_feature(dev, DRIVER_LEGACY)) - vga_client_register(dev->pdev, NULL, NULL, NULL); + vga_client_register(to_pci_dev(dev->dev), NULL, NULL, NULL); free_irq(irq, dev); } else { dev->irq = irq; @@ -203,7 +203,7 @@ int drm_irq_uninstall(struct drm_device *dev) DRM_DEBUG("irq=%d\n", dev->irq); if (drm_core_check_feature(dev, DRIVER_LEGACY)) - vga_client_register(dev->pdev, NULL, NULL, NULL); + vga_client_register(to_pci_dev(dev->dev), NULL, NULL, NULL); if (dev->driver->irq_uninstall) dev->driver->irq_uninstall(dev); @@ -214,12 +214,45 @@ int drm_irq_uninstall(struct drm_device *dev) } EXPORT_SYMBOL(drm_irq_uninstall); +static void devm_drm_irq_uninstall(void *data) +{ + drm_irq_uninstall(data); +} + +/** + * devm_drm_irq_install - install IRQ handler + * @dev: DRM device + * @irq: IRQ number to install the handler for + * + * devm_drm_irq_install is a help function of drm_irq_install. + * + * if the driver uses devm_drm_irq_install, there is no need + * to call drm_irq_uninstall when the drm module get unloaded, + * as this will done automagically. + * + * Returns: + * Zero on success or a negative error code on failure. + */ +int devm_drm_irq_install(struct drm_device *dev, int irq) +{ + int ret; + + ret = drm_irq_install(dev, irq); + if (ret) + return ret; + + return devm_add_action_or_reset(dev->dev, + devm_drm_irq_uninstall, dev); +} +EXPORT_SYMBOL(devm_drm_irq_install); + #if IS_ENABLED(CONFIG_DRM_LEGACY) int drm_legacy_irq_control(struct drm_device *dev, void *data, struct drm_file *file_priv) { struct drm_control *ctl = data; int ret = 0, irq; + struct pci_dev *pdev; /* if we haven't irq we fallback for compatibility reasons - * this used to be a separate function in drm_dma.h @@ -230,12 +263,13 @@ int drm_legacy_irq_control(struct drm_device *dev, void *data, if (!drm_core_check_feature(dev, DRIVER_LEGACY)) return 0; /* UMS was only ever supported on pci devices. */ - if (WARN_ON(!dev->pdev)) + if (WARN_ON(!dev_is_pci(dev->dev))) return -EINVAL; switch (ctl->func) { case DRM_INST_HANDLER: - irq = dev->pdev->irq; + pdev = to_pci_dev(dev->dev); + irq = pdev->irq; if (dev->if_version < DRM_IF_VERSION(1, 2) && ctl->irq != irq) diff --git a/drivers/gpu/drm/drm_kms_helper_common.c b/drivers/gpu/drm/drm_kms_helper_common.c index 221a8528c9937a7b01ba0d8d5b6fdf778f1dfcf3..f933da1656eb52f23b379e9637b8ea16ac77eef9 100644 --- a/drivers/gpu/drm/drm_kms_helper_common.c +++ b/drivers/gpu/drm/drm_kms_helper_common.c @@ -64,19 +64,18 @@ MODULE_PARM_DESC(edid_firmware, static int __init drm_kms_helper_init(void) { - int ret; - - /* Call init functions from specific kms helpers here */ - ret = drm_fb_helper_modinit(); - if (ret < 0) - goto out; - - ret = drm_dp_aux_dev_init(); - if (ret < 0) - goto out; - -out: - return ret; + /* + * The Kconfig DRM_KMS_HELPER selects FRAMEBUFFER_CONSOLE (if !EXPERT) + * but the module doesn't depend on any fb console symbols. At least + * attempt to load fbcon to avoid leaving the system without a usable + * console. + */ + if (IS_ENABLED(CONFIG_DRM_FBDEV_EMULATION) && + IS_MODULE(CONFIG_FRAMEBUFFER_CONSOLE) && + !IS_ENABLED(CONFIG_EXPERT)) + request_module_nowait("fbcon"); + + return drm_dp_aux_dev_init(); } static void __exit drm_kms_helper_exit(void) diff --git a/drivers/gpu/drm/drm_legacy.h b/drivers/gpu/drm/drm_legacy.h index 1be3ea32047495379f2468854972c968136699a4..f71358f9eac94295317396053e37a7f10c48af8b 100644 --- a/drivers/gpu/drm/drm_legacy.h +++ b/drivers/gpu/drm/drm_legacy.h @@ -127,7 +127,7 @@ static inline void drm_legacy_master_rmmaps(struct drm_device *dev, static inline void drm_legacy_rmmaps(struct drm_device *dev) {} #endif -#if IS_ENABLED(CONFIG_DRM_VM) && IS_ENABLED(CONFIG_DRM_LEGACY) +#if IS_ENABLED(CONFIG_DRM_LEGACY) void drm_legacy_vma_flush(struct drm_device *d); #else static inline void drm_legacy_vma_flush(struct drm_device *d) diff --git a/drivers/gpu/drm/drm_memory.c b/drivers/gpu/drm/drm_memory.c index fbea69d6f90902d93f910a4d93e50ba1ed9e3c61..e4f20a2eb6e793163756a22c1666b5ee894f9f78 100644 --- a/drivers/gpu/drm/drm_memory.c +++ b/drivers/gpu/drm/drm_memory.c @@ -37,7 +37,6 @@ #include #include #include -#include #include #include @@ -100,24 +99,6 @@ static void *agp_remap(unsigned long offset, unsigned long size, return addr; } -/** Wrapper around agp_free_memory() */ -void drm_free_agp(struct agp_memory *handle, int pages) -{ - agp_free_memory(handle); -} - -/** Wrapper around agp_bind_memory() */ -int drm_bind_agp(struct agp_memory *handle, unsigned int start) -{ - return agp_bind_memory(handle, start); -} - -/** Wrapper around agp_unbind_memory() */ -int drm_unbind_agp(struct agp_memory *handle) -{ - return agp_unbind_memory(handle); -} - #else /* CONFIG_AGP */ static inline void *agp_remap(unsigned long offset, unsigned long size, struct drm_device *dev) @@ -156,35 +137,3 @@ void drm_legacy_ioremapfree(struct drm_local_map *map, struct drm_device *dev) iounmap(map->handle); } EXPORT_SYMBOL(drm_legacy_ioremapfree); - -bool drm_need_swiotlb(int dma_bits) -{ - struct resource *tmp; - resource_size_t max_iomem = 0; - - /* - * Xen paravirtual hosts require swiotlb regardless of requested dma - * transfer size. - * - * NOTE: Really, what it requires is use of the dma_alloc_coherent - * allocator used in ttm_dma_populate() instead of - * ttm_populate_and_map_pages(), which bounce buffers so much in - * Xen it leads to swiotlb buffer exhaustion. - */ - if (xen_pv_domain()) - return true; - - /* - * Enforce dma_alloc_coherent when memory encryption is active as well - * for the same reasons as for Xen paravirtual hosts. - */ - if (mem_encrypt_active()) - return true; - - for (tmp = iomem_resource.child; tmp; tmp = tmp->sibling) { - max_iomem = max(max_iomem, tmp->end); - } - - return max_iomem > ((u64)1 << dma_bits); -} -EXPORT_SYMBOL(drm_need_swiotlb); diff --git a/drivers/gpu/drm/drm_mode_config.c b/drivers/gpu/drm/drm_mode_config.c index f1affc1bb6799fdce233dbc27b4dd62754465953..37b4b9f0e468ad9a750ebd078637da71dd251db5 100644 --- a/drivers/gpu/drm/drm_mode_config.c +++ b/drivers/gpu/drm/drm_mode_config.c @@ -195,7 +195,7 @@ void drm_mode_config_reset(struct drm_device *dev) crtc->funcs->reset(crtc); drm_for_each_encoder(encoder, dev) - if (encoder->funcs->reset) + if (encoder->funcs && encoder->funcs->reset) encoder->funcs->reset(encoder); drm_connector_list_iter_begin(dev, &conn_iter); @@ -625,6 +625,10 @@ static void validate_encoder_possible_crtcs(struct drm_encoder *encoder) void drm_mode_config_validate(struct drm_device *dev) { struct drm_encoder *encoder; + struct drm_crtc *crtc; + struct drm_plane *plane; + u32 primary_with_crtc = 0, cursor_with_crtc = 0; + unsigned int num_primary = 0; if (!drm_core_check_feature(dev, DRIVER_MODESET)) return; @@ -636,4 +640,49 @@ void drm_mode_config_validate(struct drm_device *dev) validate_encoder_possible_clones(encoder); validate_encoder_possible_crtcs(encoder); } + + drm_for_each_crtc(crtc, dev) { + WARN(!crtc->primary, "Missing primary plane on [CRTC:%d:%s]\n", + crtc->base.id, crtc->name); + + WARN(crtc->cursor && crtc->funcs->cursor_set, + "[CRTC:%d:%s] must not have both a cursor plane and a cursor_set func", + crtc->base.id, crtc->name); + WARN(crtc->cursor && crtc->funcs->cursor_set2, + "[CRTC:%d:%s] must not have both a cursor plane and a cursor_set2 func", + crtc->base.id, crtc->name); + WARN(crtc->cursor && crtc->funcs->cursor_move, + "[CRTC:%d:%s] must not have both a cursor plane and a cursor_move func", + crtc->base.id, crtc->name); + + if (crtc->primary) { + WARN(!(crtc->primary->possible_crtcs & drm_crtc_mask(crtc)), + "Bogus primary plane possible_crtcs: [PLANE:%d:%s] must be compatible with [CRTC:%d:%s]\n", + crtc->primary->base.id, crtc->primary->name, + crtc->base.id, crtc->name); + WARN(primary_with_crtc & drm_plane_mask(crtc->primary), + "Primary plane [PLANE:%d:%s] used for multiple CRTCs", + crtc->primary->base.id, crtc->primary->name); + primary_with_crtc |= drm_plane_mask(crtc->primary); + } + if (crtc->cursor) { + WARN(!(crtc->cursor->possible_crtcs & drm_crtc_mask(crtc)), + "Bogus cursor plane possible_crtcs: [PLANE:%d:%s] must be compatible with [CRTC:%d:%s]\n", + crtc->cursor->base.id, crtc->cursor->name, + crtc->base.id, crtc->name); + WARN(cursor_with_crtc & drm_plane_mask(crtc->cursor), + "Cursor plane [PLANE:%d:%s] used for multiple CRTCs", + crtc->cursor->base.id, crtc->cursor->name); + cursor_with_crtc |= drm_plane_mask(crtc->cursor); + } + } + + drm_for_each_plane(plane, dev) { + if (plane->type == DRM_PLANE_TYPE_PRIMARY) + num_primary++; + } + + WARN(num_primary != dev->mode_config.num_crtc, + "Must have as many primary planes as there are CRTCs, but have %u primary planes and %u CRTCs", + num_primary, dev->mode_config.num_crtc); } diff --git a/drivers/gpu/drm/drm_modes.c b/drivers/gpu/drm/drm_modes.c index 33fb2f05ce66291c2de8ad9b1b0d8ab7d43132df..1ac67d4505e073933dd79d9e2ad21cf96f78844c 100644 --- a/drivers/gpu/drm/drm_modes.c +++ b/drivers/gpu/drm/drm_modes.c @@ -762,7 +762,7 @@ int drm_mode_vrefresh(const struct drm_display_mode *mode) if (mode->htotal == 0 || mode->vtotal == 0) return 0; - num = mode->clock * 1000; + num = mode->clock; den = mode->htotal * mode->vtotal; if (mode->flags & DRM_MODE_FLAG_INTERLACE) @@ -772,7 +772,7 @@ int drm_mode_vrefresh(const struct drm_display_mode *mode) if (mode->vscan > 1) den *= mode->vscan; - return DIV_ROUND_CLOSEST(num, den); + return DIV_ROUND_CLOSEST_ULL(mul_u32_u32(num, 1000), den); } EXPORT_SYMBOL(drm_mode_vrefresh); diff --git a/drivers/gpu/drm/drm_pci.c b/drivers/gpu/drm/drm_pci.c index 6dba4b8ce4fe2fd438f4d09ff870151a5e7255ec..2294a1580d359c5fcce0164ea011052e223092de 100644 --- a/drivers/gpu/drm/drm_pci.c +++ b/drivers/gpu/drm/drm_pci.c @@ -24,6 +24,8 @@ #include #include +#include +#include #include #include @@ -36,6 +38,9 @@ #include "drm_legacy.h" #ifdef CONFIG_DRM_LEGACY +/* List of devices hanging off drivers with stealth attach. */ +static LIST_HEAD(legacy_dev_list); +static DEFINE_MUTEX(legacy_dev_list_lock); /** * drm_pci_alloc - Allocate a PCI consistent memory block, for DMA. @@ -65,7 +70,7 @@ drm_dma_handle_t *drm_pci_alloc(struct drm_device * dev, size_t size, size_t ali return NULL; dmah->size = size; - dmah->vaddr = dma_alloc_coherent(&dev->pdev->dev, size, + dmah->vaddr = dma_alloc_coherent(dev->dev, size, &dmah->busaddr, GFP_KERNEL); @@ -88,7 +93,7 @@ EXPORT_SYMBOL(drm_pci_alloc); */ void drm_pci_free(struct drm_device * dev, drm_dma_handle_t * dmah) { - dma_free_coherent(&dev->pdev->dev, dmah->size, dmah->vaddr, + dma_free_coherent(dev->dev, dmah->size, dmah->vaddr, dmah->busaddr); kfree(dmah); } @@ -107,16 +112,18 @@ static int drm_get_pci_domain(struct drm_device *dev) return 0; #endif /* __alpha__ */ - return pci_domain_nr(dev->pdev->bus); + return pci_domain_nr(to_pci_dev(dev->dev)->bus); } int drm_pci_set_busid(struct drm_device *dev, struct drm_master *master) { + struct pci_dev *pdev = to_pci_dev(dev->dev); + master->unique = kasprintf(GFP_KERNEL, "pci:%04x:%02x:%02x.%d", drm_get_pci_domain(dev), - dev->pdev->bus->number, - PCI_SLOT(dev->pdev->devfn), - PCI_FUNC(dev->pdev->devfn)); + pdev->bus->number, + PCI_SLOT(pdev->devfn), + PCI_FUNC(pdev->devfn)); if (!master->unique) return -ENOMEM; @@ -126,12 +133,14 @@ int drm_pci_set_busid(struct drm_device *dev, struct drm_master *master) static int drm_pci_irq_by_busid(struct drm_device *dev, struct drm_irq_busid *p) { + struct pci_dev *pdev = to_pci_dev(dev->dev); + if ((p->busnum >> 8) != drm_get_pci_domain(dev) || - (p->busnum & 0xff) != dev->pdev->bus->number || - p->devnum != PCI_SLOT(dev->pdev->devfn) || p->funcnum != PCI_FUNC(dev->pdev->devfn)) + (p->busnum & 0xff) != pdev->bus->number || + p->devnum != PCI_SLOT(pdev->devfn) || p->funcnum != PCI_FUNC(pdev->devfn)) return -EINVAL; - p->irq = dev->pdev->irq; + p->irq = pdev->irq; DRM_DEBUG("%d:%d:%d => IRQ %d\n", p->busnum, p->devnum, p->funcnum, p->irq); @@ -159,7 +168,7 @@ int drm_legacy_irq_by_busid(struct drm_device *dev, void *data, return -EOPNOTSUPP; /* UMS was only ever support on PCI devices. */ - if (WARN_ON(!dev->pdev)) + if (WARN_ON(!dev_is_pci(dev->dev))) return -EINVAL; if (!drm_core_check_feature(dev, DRIVER_HAVE_IRQ)) @@ -183,7 +192,7 @@ void drm_pci_agp_destroy(struct drm_device *dev) static void drm_pci_agp_init(struct drm_device *dev) { if (drm_core_check_feature(dev, DRIVER_USE_AGP)) { - if (pci_find_capability(dev->pdev, PCI_CAP_ID_AGP)) + if (pci_find_capability(to_pci_dev(dev->dev), PCI_CAP_ID_AGP)) dev->agp = drm_agp_init(dev); if (dev->agp) { dev->agp->agp_mtrr = arch_phys_wc_add( @@ -196,7 +205,7 @@ static void drm_pci_agp_init(struct drm_device *dev) static int drm_get_pci_dev(struct pci_dev *pdev, const struct pci_device_id *ent, - struct drm_driver *driver) + const struct drm_driver *driver) { struct drm_device *dev; int ret; @@ -225,10 +234,11 @@ static int drm_get_pci_dev(struct pci_dev *pdev, if (ret) goto err_agp; - /* No locking needed since shadow-attach is single-threaded since it may - * only be called from the per-driver module init hook. */ - if (drm_core_check_feature(dev, DRIVER_LEGACY)) - list_add_tail(&dev->legacy_dev_list, &driver->legacy_dev_list); + if (drm_core_check_feature(dev, DRIVER_LEGACY)) { + mutex_lock(&legacy_dev_list_lock); + list_add_tail(&dev->legacy_dev_list, &legacy_dev_list); + mutex_unlock(&legacy_dev_list_lock); + } return 0; @@ -249,7 +259,8 @@ static int drm_get_pci_dev(struct pci_dev *pdev, * * Return: 0 on success or a negative error code on failure. */ -int drm_legacy_pci_init(struct drm_driver *driver, struct pci_driver *pdriver) +int drm_legacy_pci_init(const struct drm_driver *driver, + struct pci_driver *pdriver) { struct pci_dev *pdev = NULL; const struct pci_device_id *pid; @@ -261,7 +272,6 @@ int drm_legacy_pci_init(struct drm_driver *driver, struct pci_driver *pdriver) return -EINVAL; /* If not using KMS, fall back to stealth mode manual scanning. */ - INIT_LIST_HEAD(&driver->legacy_dev_list); for (i = 0; pdriver->id_table[i].vendor != 0; i++) { pid = &pdriver->id_table[i]; @@ -295,7 +305,8 @@ EXPORT_SYMBOL(drm_legacy_pci_init); * Unregister a DRM driver shadow-attached through drm_legacy_pci_init(). This * is deprecated and only used by dri1 drivers. */ -void drm_legacy_pci_exit(struct drm_driver *driver, struct pci_driver *pdriver) +void drm_legacy_pci_exit(const struct drm_driver *driver, + struct pci_driver *pdriver) { struct drm_device *dev, *tmp; @@ -304,11 +315,15 @@ void drm_legacy_pci_exit(struct drm_driver *driver, struct pci_driver *pdriver) if (!(driver->driver_features & DRIVER_LEGACY)) { WARN_ON(1); } else { - list_for_each_entry_safe(dev, tmp, &driver->legacy_dev_list, + mutex_lock(&legacy_dev_list_lock); + list_for_each_entry_safe(dev, tmp, &legacy_dev_list, legacy_dev_list) { - list_del(&dev->legacy_dev_list); - drm_put_dev(dev); + if (dev->driver == driver) { + list_del(&dev->legacy_dev_list); + drm_put_dev(dev); + } } + mutex_unlock(&legacy_dev_list_lock); } DRM_INFO("Module unloaded\n"); } diff --git a/drivers/gpu/drm/drm_plane.c b/drivers/gpu/drm/drm_plane.c index a0cb746bcb0a9a1344e0cd23fa3dcf7a228a9f3b..338650abd267a9ab97c045a8ba1bdc20bef6fe2d 100644 --- a/drivers/gpu/drm/drm_plane.c +++ b/drivers/gpu/drm/drm_plane.c @@ -30,6 +30,7 @@ #include #include #include +#include #include #include "drm_crtc_internal.h" @@ -40,7 +41,7 @@ * A plane represents an image source that can be blended with or overlayed on * top of a CRTC during the scanout process. Planes take their input data from a * &drm_framebuffer object. The plane itself specifies the cropping and scaling - * of that image, and where it is placed on the visible are of a display + * of that image, and where it is placed on the visible area of a display * pipeline, represented by &drm_crtc. A plane can also have additional * properties that specify how the pixels are positioned and blended, like * rotation or Z-position. All these properties are stored in &drm_plane_state. @@ -49,14 +50,34 @@ * &struct drm_plane (possibly as part of a larger structure) and registers it * with a call to drm_universal_plane_init(). * - * Cursor and overlay planes are optional. All drivers should provide one - * primary plane per CRTC to avoid surprising userspace too much. See enum - * drm_plane_type for a more in-depth discussion of these special uapi-relevant - * plane types. Special planes are associated with their CRTC by calling - * drm_crtc_init_with_planes(). - * * The type of a plane is exposed in the immutable "type" enumeration property, - * which has one of the following values: "Overlay", "Primary", "Cursor". + * which has one of the following values: "Overlay", "Primary", "Cursor" (see + * enum drm_plane_type). A plane can be compatible with multiple CRTCs, see + * &drm_plane.possible_crtcs. + * + * Each CRTC must have a unique primary plane userspace can attach to enable + * the CRTC. In other words, userspace must be able to attach a different + * primary plane to each CRTC at the same time. Primary planes can still be + * compatible with multiple CRTCs. There must be exactly as many primary planes + * as there are CRTCs. + * + * Legacy uAPI doesn't expose the primary and cursor planes directly. DRM core + * relies on the driver to set the primary and optionally the cursor plane used + * for legacy IOCTLs. This is done by calling drm_crtc_init_with_planes(). All + * drivers must provide one primary plane per CRTC to avoid surprising legacy + * userspace too much. + */ + +/** + * DOC: standard plane properties + * + * DRM planes have a few standardized properties: + * + * IN_FORMATS: + * Blob property which contains the set of buffer format and modifier + * pairs supported by this plane. The blob is a struct + * drm_format_modifier_blob. Without this property the plane doesn't + * support buffers with modifiers. Userspace cannot change this property. */ static unsigned int drm_num_planes(struct drm_device *dev) @@ -152,31 +173,16 @@ static int create_in_format_blob(struct drm_device *dev, struct drm_plane *plane return 0; } -/** - * drm_universal_plane_init - Initialize a new universal plane object - * @dev: DRM device - * @plane: plane object to init - * @possible_crtcs: bitmask of possible CRTCs - * @funcs: callbacks for the new plane - * @formats: array of supported formats (DRM_FORMAT\_\*) - * @format_count: number of elements in @formats - * @format_modifiers: array of struct drm_format modifiers terminated by - * DRM_FORMAT_MOD_INVALID - * @type: type of plane (overlay, primary, cursor) - * @name: printf style format string for the plane name, or NULL for default name - * - * Initializes a plane object of type @type. - * - * Returns: - * Zero on success, error code on failure. - */ -int drm_universal_plane_init(struct drm_device *dev, struct drm_plane *plane, - uint32_t possible_crtcs, - const struct drm_plane_funcs *funcs, - const uint32_t *formats, unsigned int format_count, - const uint64_t *format_modifiers, - enum drm_plane_type type, - const char *name, ...) +__printf(9, 0) +static int __drm_universal_plane_init(struct drm_device *dev, + struct drm_plane *plane, + uint32_t possible_crtcs, + const struct drm_plane_funcs *funcs, + const uint32_t *formats, + unsigned int format_count, + const uint64_t *format_modifiers, + enum drm_plane_type type, + const char *name, va_list ap) { struct drm_mode_config *config = &dev->mode_config; unsigned int format_modifier_count = 0; @@ -237,11 +243,7 @@ int drm_universal_plane_init(struct drm_device *dev, struct drm_plane *plane, } if (name) { - va_list ap; - - va_start(ap, name); plane->name = kvasprintf(GFP_KERNEL, name, ap); - va_end(ap); } else { plane->name = kasprintf(GFP_KERNEL, "plane-%d", drm_num_planes(dev)); @@ -286,8 +288,102 @@ int drm_universal_plane_init(struct drm_device *dev, struct drm_plane *plane, return 0; } + +/** + * drm_universal_plane_init - Initialize a new universal plane object + * @dev: DRM device + * @plane: plane object to init + * @possible_crtcs: bitmask of possible CRTCs + * @funcs: callbacks for the new plane + * @formats: array of supported formats (DRM_FORMAT\_\*) + * @format_count: number of elements in @formats + * @format_modifiers: array of struct drm_format modifiers terminated by + * DRM_FORMAT_MOD_INVALID + * @type: type of plane (overlay, primary, cursor) + * @name: printf style format string for the plane name, or NULL for default name + * + * Initializes a plane object of type @type. The &drm_plane_funcs.destroy hook + * should call drm_plane_cleanup() and kfree() the plane structure. The plane + * structure should not be allocated with devm_kzalloc(). + * + * Note: consider using drmm_universal_plane_alloc() instead of + * drm_universal_plane_init() to let the DRM managed resource infrastructure + * take care of cleanup and deallocation. + * + * Returns: + * Zero on success, error code on failure. + */ +int drm_universal_plane_init(struct drm_device *dev, struct drm_plane *plane, + uint32_t possible_crtcs, + const struct drm_plane_funcs *funcs, + const uint32_t *formats, unsigned int format_count, + const uint64_t *format_modifiers, + enum drm_plane_type type, + const char *name, ...) +{ + va_list ap; + int ret; + + WARN_ON(!funcs->destroy); + + va_start(ap, name); + ret = __drm_universal_plane_init(dev, plane, possible_crtcs, funcs, + formats, format_count, format_modifiers, + type, name, ap); + va_end(ap); + return ret; +} EXPORT_SYMBOL(drm_universal_plane_init); +static void drmm_universal_plane_alloc_release(struct drm_device *dev, void *ptr) +{ + struct drm_plane *plane = ptr; + + if (WARN_ON(!plane->dev)) + return; + + drm_plane_cleanup(plane); +} + +void *__drmm_universal_plane_alloc(struct drm_device *dev, size_t size, + size_t offset, uint32_t possible_crtcs, + const struct drm_plane_funcs *funcs, + const uint32_t *formats, unsigned int format_count, + const uint64_t *format_modifiers, + enum drm_plane_type type, + const char *name, ...) +{ + void *container; + struct drm_plane *plane; + va_list ap; + int ret; + + if (WARN_ON(!funcs || funcs->destroy)) + return ERR_PTR(-EINVAL); + + container = drmm_kzalloc(dev, size, GFP_KERNEL); + if (!container) + return ERR_PTR(-ENOMEM); + + plane = container + offset; + + va_start(ap, name); + ret = __drm_universal_plane_init(dev, plane, possible_crtcs, funcs, + formats, format_count, format_modifiers, + type, name, ap); + va_end(ap); + if (ret) + return ERR_PTR(ret); + + ret = drmm_add_action_or_reset(dev, drmm_universal_plane_alloc_release, + plane); + if (ret) + return ERR_PTR(ret); + + return container; +} +EXPORT_SYMBOL(__drmm_universal_plane_alloc); + int drm_plane_register_all(struct drm_device *dev) { unsigned int num_planes = 0; diff --git a/drivers/gpu/drm/drm_prime.c b/drivers/gpu/drm/drm_prime.c index 7db55fce35d837153f427148b9044cd607df3bfc..2a54f86856afef25ebaeff9654926b23d42e579f 100644 --- a/drivers/gpu/drm/drm_prime.c +++ b/drivers/gpu/drm/drm_prime.c @@ -717,6 +717,8 @@ int drm_gem_prime_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma) vma->vm_pgoff += drm_vma_node_start(&obj->vma_node); if (obj->funcs && obj->funcs->mmap) { + vma->vm_ops = obj->funcs->vm_ops; + ret = obj->funcs->mmap(obj, vma); if (ret) return ret; @@ -978,44 +980,58 @@ struct drm_gem_object *drm_gem_prime_import(struct drm_device *dev, EXPORT_SYMBOL(drm_gem_prime_import); /** - * drm_prime_sg_to_page_addr_arrays - convert an sg table into a page array + * drm_prime_sg_to_page_array - convert an sg table into a page array + * @sgt: scatter-gather table to convert + * @pages: array of page pointers to store the pages in + * @max_entries: size of the passed-in array + * + * Exports an sg table into an array of pages. + * + * This function is deprecated and strongly discouraged to be used. + * The page array is only useful for page faults and those can corrupt fields + * in the struct page if they are not handled by the exporting driver. + */ +int __deprecated drm_prime_sg_to_page_array(struct sg_table *sgt, + struct page **pages, + int max_entries) +{ + struct sg_page_iter page_iter; + struct page **p = pages; + + for_each_sgtable_page(sgt, &page_iter, 0) { + if (WARN_ON(p - pages >= max_entries)) + return -1; + *p++ = sg_page_iter_page(&page_iter); + } + return 0; +} +EXPORT_SYMBOL(drm_prime_sg_to_page_array); + +/** + * drm_prime_sg_to_dma_addr_array - convert an sg table into a dma addr array * @sgt: scatter-gather table to convert - * @pages: optional array of page pointers to store the page array in - * @addrs: optional array to store the dma bus address of each page + * @addrs: array to store the dma bus address of each page * @max_entries: size of both the passed-in arrays * - * Exports an sg table into an array of pages and addresses. This is currently - * required by the TTM driver in order to do correct fault handling. + * Exports an sg table into an array of addresses. * - * Drivers can use this in their &drm_driver.gem_prime_import_sg_table + * Drivers should use this in their &drm_driver.gem_prime_import_sg_table * implementation. */ -int drm_prime_sg_to_page_addr_arrays(struct sg_table *sgt, struct page **pages, - dma_addr_t *addrs, int max_entries) +int drm_prime_sg_to_dma_addr_array(struct sg_table *sgt, dma_addr_t *addrs, + int max_entries) { struct sg_dma_page_iter dma_iter; - struct sg_page_iter page_iter; - struct page **p = pages; dma_addr_t *a = addrs; - if (pages) { - for_each_sgtable_page(sgt, &page_iter, 0) { - if (WARN_ON(p - pages >= max_entries)) - return -1; - *p++ = sg_page_iter_page(&page_iter); - } + for_each_sgtable_dma_page(sgt, &dma_iter, 0) { + if (WARN_ON(a - addrs >= max_entries)) + return -1; + *a++ = sg_page_iter_dma_address(&dma_iter); } - if (addrs) { - for_each_sgtable_dma_page(sgt, &dma_iter, 0) { - if (WARN_ON(a - addrs >= max_entries)) - return -1; - *a++ = sg_page_iter_dma_address(&dma_iter); - } - } - return 0; } -EXPORT_SYMBOL(drm_prime_sg_to_page_addr_arrays); +EXPORT_SYMBOL(drm_prime_sg_to_dma_addr_array); /** * drm_prime_gem_destroy - helper to clean up a PRIME-imported GEM object diff --git a/drivers/gpu/drm/drm_probe_helper.c b/drivers/gpu/drm/drm_probe_helper.c index d6017726cc2a0c84005997ed577d599e7f0e2160..ad59a51eab6d897f863ce0ba8dffded5f7266c6d 100644 --- a/drivers/gpu/drm/drm_probe_helper.c +++ b/drivers/gpu/drm/drm_probe_helper.c @@ -515,7 +515,8 @@ int drm_helper_probe_single_connector_modes(struct drm_connector *connector, if (count == 0 && connector->status == connector_status_connected) count = drm_add_override_edid_modes(connector); - if (count == 0 && connector->status == connector_status_connected) + if (count == 0 && (connector->status == connector_status_connected || + connector->status == connector_status_unknown)) count = drm_add_modes_noedid(connector, 1024, 768); count += drm_helper_probe_add_cmdline_mode(connector); if (count == 0) diff --git a/drivers/gpu/drm/drm_simple_kms_helper.c b/drivers/gpu/drm/drm_simple_kms_helper.c index 743e57c1b44f3eeecbbd475caba851efe48f5206..6ce8f5cd1eb59c886a1af7014b1e5ba10250bbcb 100644 --- a/drivers/gpu/drm/drm_simple_kms_helper.c +++ b/drivers/gpu/drm/drm_simple_kms_helper.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -55,8 +56,9 @@ static const struct drm_encoder_funcs drm_simple_encoder_funcs_cleanup = { * stored in the device structure. Free the encoder's memory as part of * the device release function. * - * FIXME: Later improvements to DRM's resource management may allow for - * an automated kfree() of the encoder's memory. + * Note: consider using drmm_simple_encoder_alloc() instead of + * drm_simple_encoder_init() to let the DRM managed resource infrastructure + * take care of cleanup and deallocation. * * Returns: * Zero on success, error code on failure. @@ -71,6 +73,14 @@ int drm_simple_encoder_init(struct drm_device *dev, } EXPORT_SYMBOL(drm_simple_encoder_init); +void *__drmm_simple_encoder_alloc(struct drm_device *dev, size_t size, + size_t offset, int encoder_type) +{ + return __drmm_encoder_alloc(dev, size, offset, NULL, encoder_type, + NULL); +} +EXPORT_SYMBOL(__drmm_simple_encoder_alloc); + static enum drm_mode_status drm_simple_kms_crtc_mode_valid(struct drm_crtc *crtc, const struct drm_display_mode *mode) diff --git a/drivers/gpu/drm/drm_vblank.c b/drivers/gpu/drm/drm_vblank.c index d30e2f2b8f3c47bf13d9e7bb531f676537ca2806..893165eeddf3f4d56b5d6773107f325b29c9496a 100644 --- a/drivers/gpu/drm/drm_vblank.c +++ b/drivers/gpu/drm/drm_vblank.c @@ -74,7 +74,7 @@ * |↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓| updates the * | | frame as it * | | travels down - * | | ("sacn out") + * | | ("scan out") * | Old frame | * | | * | | @@ -1006,7 +1006,14 @@ static void send_vblank_event(struct drm_device *dev, break; } trace_drm_vblank_event_delivered(e->base.file_priv, e->pipe, seq); - drm_send_event_locked(dev, &e->base); + /* + * Use the same timestamp for any associated fence signal to avoid + * mismatch in timestamps for vsync & fence events triggered by the + * same HW event. Frameworks like SurfaceFlinger in Android expects the + * retire-fence timestamp to match exactly with HW vsync as it uses it + * for its software vsync modeling. + */ + drm_send_event_timestamp_locked(dev, &e->base, now); } /** diff --git a/drivers/gpu/drm/drm_vm.c b/drivers/gpu/drm/drm_vm.c index 6d5a03b32238007ad5a71263c068360291c39bb6..9b3b989d7cad12e0101661472cb2f058cb4464e4 100644 --- a/drivers/gpu/drm/drm_vm.c +++ b/drivers/gpu/drm/drm_vm.c @@ -278,7 +278,7 @@ static void drm_vm_shm_close(struct vm_area_struct *vma) case _DRM_SCATTER_GATHER: break; case _DRM_CONSISTENT: - dma_free_coherent(&dev->pdev->dev, + dma_free_coherent(dev->dev, map->size, map->handle, map->offset); diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c b/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c index d9bd83203a15e5d4cb10f0875e1067a6afbe4c49..b390dd4d60b76d929fc2847484f1a8067126d2b9 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_gem_prime.c @@ -135,8 +135,7 @@ struct drm_gem_object *etnaviv_gem_prime_import_sg_table(struct drm_device *dev, goto fail; } - ret = drm_prime_sg_to_page_addr_arrays(sgt, etnaviv_obj->pages, - NULL, npages); + ret = drm_prime_sg_to_page_array(sgt, etnaviv_obj->pages, npages); if (ret) goto fail; diff --git a/drivers/gpu/drm/exynos/Kconfig b/drivers/gpu/drm/exynos/Kconfig index 951d5f708e92b0b9600eb8501bc2390c6b85246b..6a251e3aa779ed5a401f4f99e0b6c579c713519a 100644 --- a/drivers/gpu/drm/exynos/Kconfig +++ b/drivers/gpu/drm/exynos/Kconfig @@ -89,7 +89,6 @@ comment "Sub-drivers" config DRM_EXYNOS_G2D bool "G2D" depends on VIDEO_SAMSUNG_S5P_G2D=n || COMPILE_TEST - select FRAME_VECTOR help Choose this option if you want to use Exynos G2D for DRM. diff --git a/drivers/gpu/drm/exynos/exynos_drm_g2d.c b/drivers/gpu/drm/exynos/exynos_drm_g2d.c index 967a5cdc120e33d83a751177207ef2732f81c444..1e0c5a7f206e2fbbcaaa512423bd89dca229585c 100644 --- a/drivers/gpu/drm/exynos/exynos_drm_g2d.c +++ b/drivers/gpu/drm/exynos/exynos_drm_g2d.c @@ -205,7 +205,8 @@ struct g2d_cmdlist_userptr { dma_addr_t dma_addr; unsigned long userptr; unsigned long size; - struct frame_vector *vec; + struct page **pages; + unsigned int npages; struct sg_table *sgt; atomic_t refcount; bool in_pool; @@ -378,7 +379,6 @@ static void g2d_userptr_put_dma_addr(struct g2d_data *g2d, bool force) { struct g2d_cmdlist_userptr *g2d_userptr = obj; - struct page **pages; if (!obj) return; @@ -398,15 +398,9 @@ static void g2d_userptr_put_dma_addr(struct g2d_data *g2d, dma_unmap_sgtable(to_dma_dev(g2d->drm_dev), g2d_userptr->sgt, DMA_BIDIRECTIONAL, 0); - pages = frame_vector_pages(g2d_userptr->vec); - if (!IS_ERR(pages)) { - int i; - - for (i = 0; i < frame_vector_count(g2d_userptr->vec); i++) - set_page_dirty_lock(pages[i]); - } - put_vaddr_frames(g2d_userptr->vec); - frame_vector_destroy(g2d_userptr->vec); + unpin_user_pages_dirty_lock(g2d_userptr->pages, g2d_userptr->npages, + true); + kvfree(g2d_userptr->pages); if (!g2d_userptr->out_of_list) list_del_init(&g2d_userptr->list); @@ -474,35 +468,35 @@ static dma_addr_t *g2d_userptr_get_dma_addr(struct g2d_data *g2d, offset = userptr & ~PAGE_MASK; end = PAGE_ALIGN(userptr + size); npages = (end - start) >> PAGE_SHIFT; - g2d_userptr->vec = frame_vector_create(npages); - if (!g2d_userptr->vec) { + g2d_userptr->pages = kvmalloc_array(npages, sizeof(*g2d_userptr->pages), + GFP_KERNEL); + if (!g2d_userptr->pages) { ret = -ENOMEM; goto err_free; } - ret = get_vaddr_frames(start, npages, FOLL_FORCE | FOLL_WRITE, - g2d_userptr->vec); + ret = pin_user_pages_fast(start, npages, + FOLL_FORCE | FOLL_WRITE | FOLL_LONGTERM, + g2d_userptr->pages); if (ret != npages) { DRM_DEV_ERROR(g2d->dev, "failed to get user pages from userptr.\n"); if (ret < 0) - goto err_destroy_framevec; - ret = -EFAULT; - goto err_put_framevec; - } - if (frame_vector_to_pages(g2d_userptr->vec) < 0) { + goto err_destroy_pages; + npages = ret; ret = -EFAULT; - goto err_put_framevec; + goto err_unpin_pages; } + g2d_userptr->npages = npages; sgt = kzalloc(sizeof(*sgt), GFP_KERNEL); if (!sgt) { ret = -ENOMEM; - goto err_put_framevec; + goto err_unpin_pages; } ret = sg_alloc_table_from_pages(sgt, - frame_vector_pages(g2d_userptr->vec), + g2d_userptr->pages, npages, offset, size, GFP_KERNEL); if (ret < 0) { DRM_DEV_ERROR(g2d->dev, "failed to get sgt from pages.\n"); @@ -538,11 +532,11 @@ static dma_addr_t *g2d_userptr_get_dma_addr(struct g2d_data *g2d, err_free_sgt: kfree(sgt); -err_put_framevec: - put_vaddr_frames(g2d_userptr->vec); +err_unpin_pages: + unpin_user_pages(g2d_userptr->pages, npages); -err_destroy_framevec: - frame_vector_destroy(g2d_userptr->vec); +err_destroy_pages: + kvfree(g2d_userptr->pages); err_free: kfree(g2d_userptr); diff --git a/drivers/gpu/drm/gma500/Kconfig b/drivers/gpu/drm/gma500/Kconfig index 0e23c93a1094cdc9e7ec0664edd8d87b040a4221..ec395658a43f1bf154e12af3cedecedffcab7603 100644 --- a/drivers/gpu/drm/gma500/Kconfig +++ b/drivers/gpu/drm/gma500/Kconfig @@ -1,9 +1,8 @@ # SPDX-License-Identifier: GPL-2.0-only config DRM_GMA500 - tristate "Intel GMA5/600 KMS Framebuffer" + tristate "Intel GMA500/600/3600/3650 KMS Framebuffer" depends on DRM && PCI && X86 && MMU select DRM_KMS_HELPER - select DRM_TTM # GMA500 depends on ACPI_VIDEO when ACPI is enabled, just like i915 select ACPI_VIDEO if ACPI select BACKLIGHT_CLASS_DEVICE if ACPI @@ -19,17 +18,3 @@ config DRM_GMA600 help Say yes to include support for GMA600 (Intel Moorestown/Oaktrail) platforms with LVDS ports. MIPI is not currently supported. - -config DRM_GMA3600 - bool "Intel GMA3600/3650 support (Experimental)" - depends on DRM_GMA500 - help - Say yes to include basic support for Intel GMA3600/3650 (Intel - Cedar Trail) platforms. - -config DRM_MEDFIELD - bool "Intel Medfield support (Experimental)" - depends on DRM_GMA500 && X86_INTEL_MID - help - Say yes to include support for the Intel Medfield platform. - diff --git a/drivers/gpu/drm/gma500/Makefile b/drivers/gpu/drm/gma500/Makefile index c8f2c89be99db49e45b574250fa37873e8783a7d..884ab1f9063efdcf7eedb7b2ac6c02b1103ce40e 100644 --- a/drivers/gpu/drm/gma500/Makefile +++ b/drivers/gpu/drm/gma500/Makefile @@ -6,36 +6,35 @@ gma500_gfx-y += \ accel_2d.o \ backlight.o \ + blitter.o \ + cdv_device.o \ + cdv_intel_crt.o \ + cdv_intel_display.o \ + cdv_intel_dp.o \ + cdv_intel_hdmi.o \ + cdv_intel_lvds.o \ framebuffer.o \ gem.o \ + gma_device.o \ + gma_display.o \ gtt.o \ intel_bios.o \ - intel_i2c.o \ intel_gmbus.o \ + intel_i2c.o \ + mid_bios.o \ mmu.o \ - blitter.o \ power.o \ + psb_device.o \ psb_drv.o \ - gma_display.o \ - gma_device.o \ psb_intel_display.o \ psb_intel_lvds.o \ psb_intel_modes.o \ psb_intel_sdvo.o \ psb_lid.o \ - psb_irq.o \ - psb_device.o \ - mid_bios.o + psb_irq.o gma500_gfx-$(CONFIG_ACPI) += opregion.o \ -gma500_gfx-$(CONFIG_DRM_GMA3600) += cdv_device.o \ - cdv_intel_crt.o \ - cdv_intel_display.o \ - cdv_intel_hdmi.o \ - cdv_intel_lvds.o \ - cdv_intel_dp.o - gma500_gfx-$(CONFIG_DRM_GMA600) += oaktrail_device.o \ oaktrail_crtc.o \ oaktrail_lvds.o \ @@ -43,14 +42,4 @@ gma500_gfx-$(CONFIG_DRM_GMA600) += oaktrail_device.o \ oaktrail_hdmi.o \ oaktrail_hdmi_i2c.o -gma500_gfx-$(CONFIG_DRM_MEDFIELD) += mdfld_device.o \ - mdfld_output.o \ - mdfld_intel_display.o \ - mdfld_dsi_output.o \ - mdfld_dsi_dpi.o \ - mdfld_dsi_pkg_sender.o \ - mdfld_tpo_vid.o \ - mdfld_tmd_vid.o \ - tc35876x-dsi-lvds.o - obj-$(CONFIG_DRM_GMA500) += gma500_gfx.o diff --git a/drivers/gpu/drm/gma500/cdv_device.c b/drivers/gpu/drm/gma500/cdv_device.c index e75293e4a52f1262f3119853fec39f7f58857414..19e055dbd4c2c07a5dc70a35ccef4226119b09be 100644 --- a/drivers/gpu/drm/gma500/cdv_device.c +++ b/drivers/gpu/drm/gma500/cdv_device.c @@ -95,13 +95,14 @@ static u32 cdv_get_max_backlight(struct drm_device *dev) static int cdv_get_brightness(struct backlight_device *bd) { struct drm_device *dev = bl_get_data(bd); + struct pci_dev *pdev = to_pci_dev(dev->dev); u32 val = REG_READ(BLC_PWM_CTL) & BACKLIGHT_DUTY_CYCLE_MASK; if (cdv_backlight_combination_mode(dev)) { u8 lbpc; val &= ~1; - pci_read_config_byte(dev->pdev, 0xF4, &lbpc); + pci_read_config_byte(pdev, 0xF4, &lbpc); val *= lbpc; } return (val * 100)/cdv_get_max_backlight(dev); @@ -111,6 +112,7 @@ static int cdv_get_brightness(struct backlight_device *bd) static int cdv_set_brightness(struct backlight_device *bd) { struct drm_device *dev = bl_get_data(bd); + struct pci_dev *pdev = to_pci_dev(dev->dev); int level = bd->props.brightness; u32 blc_pwm_ctl; @@ -128,7 +130,7 @@ static int cdv_set_brightness(struct backlight_device *bd) lbpc = level * 0xfe / max + 1; level /= lbpc; - pci_write_config_byte(dev->pdev, 0xF4, lbpc); + pci_write_config_byte(pdev, 0xF4, lbpc); } blc_pwm_ctl = REG_READ(BLC_PWM_CTL) & ~BACKLIGHT_DUTY_CYCLE_MASK; @@ -205,8 +207,9 @@ static inline void CDV_MSG_WRITE32(int domain, uint port, uint offset, static void cdv_init_pm(struct drm_device *dev) { struct drm_psb_private *dev_priv = dev->dev_private; + struct pci_dev *pdev = to_pci_dev(dev->dev); u32 pwr_cnt; - int domain = pci_domain_nr(dev->pdev->bus); + int domain = pci_domain_nr(pdev->bus); int i; dev_priv->apm_base = CDV_MSG_READ32(domain, PSB_PUNIT_PORT, @@ -234,6 +237,8 @@ static void cdv_init_pm(struct drm_device *dev) static void cdv_errata(struct drm_device *dev) { + struct pci_dev *pdev = to_pci_dev(dev->dev); + /* Disable bonus launch. * CPU and GPU competes for memory and display misses updates and * flickers. Worst with dual core, dual displays. @@ -242,7 +247,7 @@ static void cdv_errata(struct drm_device *dev) * Bonus Launch to work around the issue, by degrading * performance. */ - CDV_MSG_WRITE32(pci_domain_nr(dev->pdev->bus), 3, 0x30, 0x08027108); + CDV_MSG_WRITE32(pci_domain_nr(pdev->bus), 3, 0x30, 0x08027108); } /** @@ -255,12 +260,13 @@ static void cdv_errata(struct drm_device *dev) static int cdv_save_display_registers(struct drm_device *dev) { struct drm_psb_private *dev_priv = dev->dev_private; + struct pci_dev *pdev = to_pci_dev(dev->dev); struct psb_save_area *regs = &dev_priv->regs; struct drm_connector *connector; dev_dbg(dev->dev, "Saving GPU registers.\n"); - pci_read_config_byte(dev->pdev, 0xF4, ®s->cdv.saveLBB); + pci_read_config_byte(pdev, 0xF4, ®s->cdv.saveLBB); regs->cdv.saveDSPCLK_GATE_D = REG_READ(DSPCLK_GATE_D); regs->cdv.saveRAMCLK_GATE_D = REG_READ(RAMCLK_GATE_D); @@ -309,11 +315,12 @@ static int cdv_save_display_registers(struct drm_device *dev) static int cdv_restore_display_registers(struct drm_device *dev) { struct drm_psb_private *dev_priv = dev->dev_private; + struct pci_dev *pdev = to_pci_dev(dev->dev); struct psb_save_area *regs = &dev_priv->regs; struct drm_connector *connector; u32 temp; - pci_write_config_byte(dev->pdev, 0xF4, regs->cdv.saveLBB); + pci_write_config_byte(pdev, 0xF4, regs->cdv.saveLBB); REG_WRITE(DSPCLK_GATE_D, regs->cdv.saveDSPCLK_GATE_D); REG_WRITE(RAMCLK_GATE_D, regs->cdv.saveRAMCLK_GATE_D); @@ -421,16 +428,16 @@ static int cdv_power_up(struct drm_device *dev) static void cdv_hotplug_work_func(struct work_struct *work) { struct drm_psb_private *dev_priv = container_of(work, struct drm_psb_private, - hotplug_work); + hotplug_work); struct drm_device *dev = dev_priv->dev; /* Just fire off a uevent and let userspace tell us what to do */ drm_helper_hpd_irq_event(dev); -} +} /* The core driver has received a hotplug IRQ. We are in IRQ context so extract the needed information and kick off queued processing */ - + static int cdv_hotplug_event(struct drm_device *dev) { struct drm_psb_private *dev_priv = dev->dev_private; @@ -449,7 +456,7 @@ static void cdv_hotplug_enable(struct drm_device *dev, bool on) } else { REG_WRITE(PORT_HOTPLUG_EN, 0); REG_WRITE(PORT_HOTPLUG_STAT, REG_READ(PORT_HOTPLUG_STAT)); - } + } } static const char *force_audio_names[] = { @@ -568,9 +575,10 @@ static const struct psb_offset cdv_regmap[2] = { static int cdv_chip_setup(struct drm_device *dev) { struct drm_psb_private *dev_priv = dev->dev_private; + struct pci_dev *pdev = to_pci_dev(dev->dev); INIT_WORK(&dev_priv->hotplug_work, cdv_hotplug_work_func); - if (pci_enable_msi(dev->pdev)) + if (pci_enable_msi(pdev)) dev_warn(dev->dev, "Enabling MSI failed!\n"); dev_priv->regmap = cdv_regmap; gma_get_core_freq(dev); diff --git a/drivers/gpu/drm/gma500/cdv_intel_crt.c b/drivers/gpu/drm/gma500/cdv_intel_crt.c index 88535f5aacc5d39aa0bc484a5cad41c5ae325701..c48c9d322dfbddb86b89ea1f870420a15b9ebd5b 100644 --- a/drivers/gpu/drm/gma500/cdv_intel_crt.c +++ b/drivers/gpu/drm/gma500/cdv_intel_crt.c @@ -127,7 +127,7 @@ static void cdv_intel_crt_mode_set(struct drm_encoder *encoder, } -/** +/* * Uses CRT_HOTPLUG_EN and CRT_HOTPLUG_STAT to detect CRT presence. * * \return true if CRT is connected. @@ -278,8 +278,7 @@ void cdv_intel_crt_init(struct drm_device *dev, gma_encoder->ddc_bus = psb_intel_i2c_create(dev, i2c_reg, "CRTDDC_A"); if (!gma_encoder->ddc_bus) { - dev_printk(KERN_ERR, &dev->pdev->dev, "DDC bus registration " - "failed.\n"); + dev_printk(KERN_ERR, dev->dev, "DDC bus registration failed.\n"); goto failed_ddc; } diff --git a/drivers/gpu/drm/gma500/cdv_intel_display.c b/drivers/gpu/drm/gma500/cdv_intel_display.c index 686385a66167ba49ff55028c97883dcb72faee61..5d33022497793f50a4500e1f84d6e00cab6a2d9f 100644 --- a/drivers/gpu/drm/gma500/cdv_intel_display.c +++ b/drivers/gpu/drm/gma500/cdv_intel_display.c @@ -551,7 +551,7 @@ void cdv_update_wm(struct drm_device *dev, struct drm_crtc *crtc) } } -/** +/* * Return the pipe currently connected to the panel fitter, * or -1 if the panel fitter is not present or not in use */ diff --git a/drivers/gpu/drm/gma500/cdv_intel_dp.c b/drivers/gpu/drm/gma500/cdv_intel_dp.c index bfd9a15d63b1a25a33abe256fe976478d9e4a294..6d3ada39ff8675c523f07ae4f897294c9969aaa3 100644 --- a/drivers/gpu/drm/gma500/cdv_intel_dp.c +++ b/drivers/gpu/drm/gma500/cdv_intel_dp.c @@ -306,7 +306,7 @@ static uint32_t dp_vswing_premph_table[] = { }; /** * is_edp - is the given port attached to an eDP panel (either CPU or PCH) - * @intel_dp: DP struct + * @encoder: GMA encoder struct * * If a CPU or PCH DP output is attached to an eDP panel, this function * will return true, and false otherwise. @@ -1687,7 +1687,7 @@ static enum drm_connector_status cdv_dp_detect(struct gma_encoder *encoder) return status; } -/** +/* * Uses CRT_HOTPLUG_EN and CRT_HOTPLUG_STAT to detect DP connection. * * \return true if DP port is connected. diff --git a/drivers/gpu/drm/gma500/cdv_intel_hdmi.c b/drivers/gpu/drm/gma500/cdv_intel_hdmi.c index 0d12c6ffbc4078796a9e1fcc56e3a583dba54786..e525689f84f0cf3d684d380d680e33741172a07d 100644 --- a/drivers/gpu/drm/gma500/cdv_intel_hdmi.c +++ b/drivers/gpu/drm/gma500/cdv_intel_hdmi.c @@ -22,9 +22,6 @@ * * Authors: * jim liu - * - * FIXME: - * We should probably make this generic and share it with Medfield */ #include @@ -56,7 +53,6 @@ struct mid_intel_hdmi_priv { bool has_hdmi_audio; /* Should set this when detect hotplug */ bool hdmi_device_connected; - struct mdfld_hdmi_i2c *i2c_bus; struct i2c_adapter *hdmi_i2c_adapter; /* for control functions */ struct drm_device *dev; }; diff --git a/drivers/gpu/drm/gma500/cdv_intel_lvds.c b/drivers/gpu/drm/gma500/cdv_intel_lvds.c index eaaf4efec21765a494c3bc2ef6750bc4cd080b74..5bff7d9e3aa6efab866e7e98d5b0de98f51b220b 100644 --- a/drivers/gpu/drm/gma500/cdv_intel_lvds.c +++ b/drivers/gpu/drm/gma500/cdv_intel_lvds.c @@ -74,7 +74,7 @@ static u32 cdv_intel_lvds_get_max_backlight(struct drm_device *dev) return retval; } -/** +/* * Sets the backlight level. * * level backlight level, from 0 to cdv_intel_lvds_get_max_backlight(). @@ -99,7 +99,7 @@ static void cdv_intel_lvds_set_backlight(struct drm_device *dev, int level) } } -/** +/* * Sets the power state for the panel. */ static void cdv_intel_lvds_set_power(struct drm_device *dev, @@ -291,7 +291,7 @@ static void cdv_intel_lvds_mode_set(struct drm_encoder *encoder, REG_WRITE(PFIT_CONTROL, pfit_control); } -/** +/* * Return the list of DDC modes if available, or the BIOS fixed mode otherwise. */ static int cdv_intel_lvds_get_modes(struct drm_connector *connector) @@ -471,6 +471,7 @@ static bool lvds_is_present_in_vbt(struct drm_device *dev, /** * cdv_intel_lvds_init - setup LVDS connectors on this device * @dev: drm device + * @mode_dev: PSB mode device * * Create the connector, register the LVDS DDC bus, and try to figure out what * modes we can display on the LVDS panel (if present). @@ -554,7 +555,7 @@ void cdv_intel_lvds_init(struct drm_device *dev, "LVDSBLC_B"); if (!gma_encoder->i2c_bus) { dev_printk(KERN_ERR, - &dev->pdev->dev, "I2C bus registration failed.\n"); + dev->dev, "I2C bus registration failed.\n"); goto failed_blc_i2c; } gma_encoder->i2c_bus->slave_addr = 0x2C; @@ -575,7 +576,7 @@ void cdv_intel_lvds_init(struct drm_device *dev, GPIOC, "LVDSDDC_C"); if (!gma_encoder->ddc_bus) { - dev_printk(KERN_ERR, &dev->pdev->dev, + dev_printk(KERN_ERR, dev->dev, "DDC bus registration " "failed.\n"); goto failed_ddc; } diff --git a/drivers/gpu/drm/gma500/framebuffer.c b/drivers/gpu/drm/gma500/framebuffer.c index fc4fda1d258b1daf7c6c7a5603e0d0d603463658..ebe9dccf2d830817ab43be042cda266d48d00a8f 100644 --- a/drivers/gpu/drm/gma500/framebuffer.c +++ b/drivers/gpu/drm/gma500/framebuffer.c @@ -159,7 +159,7 @@ static const struct fb_ops psbfb_unaccel_ops = { * @dev: our DRM device * @fb: framebuffer to set up * @mode_cmd: mode description - * @gt: backing object + * @obj: backing object * * Configure and fill in the boilerplate for our frame buffer. Return * 0 on success or an error code if we fail. @@ -197,7 +197,7 @@ static int psb_framebuffer_init(struct drm_device *dev, * psb_framebuffer_create - create a framebuffer backed by gt * @dev: our DRM device * @mode_cmd: the description of the requested mode - * @gt: the backing object + * @obj: the backing object * * Create a framebuffer object backed by the gt, and fill in the * boilerplate required @@ -252,7 +252,7 @@ static struct gtt_range *psbfb_alloc(struct drm_device *dev, int aligned_size) /** * psbfb_create - create a framebuffer - * @fbdev: the framebuffer device + * @fb_helper: the framebuffer helper * @sizes: specification of the layout * * Create a framebuffer to the specifications provided @@ -262,6 +262,7 @@ static int psbfb_create(struct drm_fb_helper *fb_helper, { struct drm_device *dev = fb_helper->dev; struct drm_psb_private *dev_priv = dev->dev_private; + struct pci_dev *pdev = to_pci_dev(dev->dev); struct fb_info *info; struct drm_framebuffer *fb; struct drm_mode_fb_cmd2 mode_cmd; @@ -325,8 +326,8 @@ static int psbfb_create(struct drm_fb_helper *fb_helper, drm_fb_helper_fill_info(info, fb_helper, sizes); - info->fix.mmio_start = pci_resource_start(dev->pdev, 0); - info->fix.mmio_len = pci_resource_len(dev->pdev, 0); + info->fix.mmio_start = pci_resource_start(pdev, 0); + info->fix.mmio_len = pci_resource_len(pdev, 0); /* Use default scratch pixmap (info->pixmap.flags = FB_PIXMAP_SYSTEM) */ @@ -529,6 +530,7 @@ void psb_modeset_init(struct drm_device *dev) { struct drm_psb_private *dev_priv = dev->dev_private; struct psb_intel_mode_device *mode_dev = &dev_priv->mode_dev; + struct pci_dev *pdev = to_pci_dev(dev->dev); int i; drm_mode_config_init(dev); @@ -540,8 +542,7 @@ void psb_modeset_init(struct drm_device *dev) /* set memory base */ /* Oaktrail and Poulsbo should use BAR 2*/ - pci_read_config_dword(dev->pdev, PSB_BSM, (u32 *) - &(dev->mode_config.fb_base)); + pci_read_config_dword(pdev, PSB_BSM, (u32 *)&(dev->mode_config.fb_base)); /* num pipes is 2 for PSB but 1 for Mrst */ for (i = 0; i < dev_priv->num_pipe; i++) diff --git a/drivers/gpu/drm/gma500/gem.c b/drivers/gpu/drm/gma500/gem.c index db827e591403c7db640b6ce3b6851ccb8b345155..fbf420051ef5f79986168806b6adf14d3733b8ea 100644 --- a/drivers/gpu/drm/gma500/gem.c +++ b/drivers/gpu/drm/gma500/gem.c @@ -16,6 +16,7 @@ #include #include +#include "gem.h" #include "psb_drv.h" static vm_fault_t psb_gem_fault(struct vm_fault *vmf); @@ -49,6 +50,8 @@ const struct drm_gem_object_funcs psb_gem_object_funcs = { * @dev: our device * @size: the size requested * @handlep: returned handle (opaque number) + * @stolen: unused + * @align: unused * * Create a GEM object, fill in the boilerplate and attach a handle to * it so that userspace can speak about it. This does the core work @@ -97,7 +100,7 @@ int psb_gem_create(struct drm_file *file, struct drm_device *dev, u64 size, /** * psb_gem_dumb_create - create a dumb buffer - * @drm_file: our client file + * @file: our client file * @dev: our device * @args: the requested arguments copied from userspace * @@ -116,7 +119,6 @@ int psb_gem_dumb_create(struct drm_file *file, struct drm_device *dev, /** * psb_gem_fault - pagefault handler for GEM objects - * @vma: the VMA of the GEM object * @vmf: fault detail * * Invoked when a fault occurs on an mmap of a GEM managed area. GEM diff --git a/drivers/gpu/drm/gma500/gem.h b/drivers/gpu/drm/gma500/gem.h index 3741a711b9fdecc57e8d2df255e63072ef03187d..bae6454ead292327e89c77b93154fb5242cf3ea7 100644 --- a/drivers/gpu/drm/gma500/gem.h +++ b/drivers/gpu/drm/gma500/gem.h @@ -8,6 +8,8 @@ #ifndef _GEM_H #define _GEM_H +struct drm_device; + extern const struct drm_gem_object_funcs psb_gem_object_funcs; extern int psb_gem_create(struct drm_file *file, struct drm_device *dev, diff --git a/drivers/gpu/drm/gma500/gma_device.c b/drivers/gpu/drm/gma500/gma_device.c index 869f303925667e51cecc8302af6626976982bbd7..4c91e86f4b1421b99f728650a7f4762fe381acc3 100644 --- a/drivers/gpu/drm/gma500/gma_device.c +++ b/drivers/gpu/drm/gma500/gma_device.c @@ -6,12 +6,14 @@ **************************************************************************/ #include "psb_drv.h" +#include "gma_device.h" void gma_get_core_freq(struct drm_device *dev) { uint32_t clock; + struct pci_dev *pdev = to_pci_dev(dev->dev); struct pci_dev *pci_root = - pci_get_domain_bus_and_slot(pci_domain_nr(dev->pdev->bus), + pci_get_domain_bus_and_slot(pci_domain_nr(pdev->bus), 0, 0); struct drm_psb_private *dev_priv = dev->dev_private; diff --git a/drivers/gpu/drm/gma500/gma_display.c b/drivers/gpu/drm/gma500/gma_display.c index 3df6d6e850f5200ed189fc735a4b122db2e5ecf8..b03f7b8241f2bdda2d51fb29853ecd941b1ad1ac 100644 --- a/drivers/gpu/drm/gma500/gma_display.c +++ b/drivers/gpu/drm/gma500/gma_display.c @@ -20,7 +20,7 @@ #include "psb_intel_drv.h" #include "psb_intel_reg.h" -/** +/* * Returns whether any output on the specified pipe is of the specified type */ bool gma_pipe_has_type(struct drm_crtc *crtc, int type) @@ -180,7 +180,7 @@ int gma_crtc_gamma_set(struct drm_crtc *crtc, u16 *red, u16 *green, u16 *blue, return 0; } -/** +/* * Sets the power management mode of the pipe and plane. * * This code should probably grow support for turning the cursor off and back @@ -559,14 +559,14 @@ int gma_crtc_set_config(struct drm_mode_set *set, if (!dev_priv->rpm_enabled) return drm_crtc_helper_set_config(set, ctx); - pm_runtime_forbid(&dev->pdev->dev); + pm_runtime_forbid(dev->dev); ret = drm_crtc_helper_set_config(set, ctx); - pm_runtime_allow(&dev->pdev->dev); + pm_runtime_allow(dev->dev); return ret; } -/** +/* * Save HW states of given crtc */ void gma_crtc_save(struct drm_crtc *crtc) @@ -609,7 +609,7 @@ void gma_crtc_save(struct drm_crtc *crtc) crtc_state->savePalette[i] = REG_READ(palette_reg + (i << 2)); } -/** +/* * Restore HW states of given crtc */ void gma_crtc_restore(struct drm_crtc *crtc) diff --git a/drivers/gpu/drm/gma500/gtt.c b/drivers/gpu/drm/gma500/gtt.c index d246b1f703668286f089cdade355be4cc3c75bfb..e884750bc123827f84f468deff8c1dcfb86f72ce 100644 --- a/drivers/gpu/drm/gma500/gtt.c +++ b/drivers/gpu/drm/gma500/gtt.c @@ -340,13 +340,14 @@ static void psb_gtt_alloc(struct drm_device *dev) void psb_gtt_takedown(struct drm_device *dev) { struct drm_psb_private *dev_priv = dev->dev_private; + struct pci_dev *pdev = to_pci_dev(dev->dev); if (dev_priv->gtt_map) { iounmap(dev_priv->gtt_map); dev_priv->gtt_map = NULL; } if (dev_priv->gtt_initialized) { - pci_write_config_word(dev->pdev, PSB_GMCH_CTRL, + pci_write_config_word(pdev, PSB_GMCH_CTRL, dev_priv->gmch_ctrl); PSB_WVDC32(dev_priv->pge_ctl, PSB_PGETBL_CTL); (void) PSB_RVDC32(PSB_PGETBL_CTL); @@ -358,6 +359,7 @@ void psb_gtt_takedown(struct drm_device *dev) int psb_gtt_init(struct drm_device *dev, int resume) { struct drm_psb_private *dev_priv = dev->dev_private; + struct pci_dev *pdev = to_pci_dev(dev->dev); unsigned gtt_pages; unsigned long stolen_size, vram_stolen_size; unsigned i, num_pages; @@ -376,8 +378,8 @@ int psb_gtt_init(struct drm_device *dev, int resume) pg = &dev_priv->gtt; /* Enable the GTT */ - pci_read_config_word(dev->pdev, PSB_GMCH_CTRL, &dev_priv->gmch_ctrl); - pci_write_config_word(dev->pdev, PSB_GMCH_CTRL, + pci_read_config_word(pdev, PSB_GMCH_CTRL, &dev_priv->gmch_ctrl); + pci_write_config_word(pdev, PSB_GMCH_CTRL, dev_priv->gmch_ctrl | _PSB_GMCH_ENABLED); dev_priv->pge_ctl = PSB_RVDC32(PSB_PGETBL_CTL); @@ -397,8 +399,8 @@ int psb_gtt_init(struct drm_device *dev, int resume) */ pg->mmu_gatt_start = 0xE0000000; - pg->gtt_start = pci_resource_start(dev->pdev, PSB_GTT_RESOURCE); - gtt_pages = pci_resource_len(dev->pdev, PSB_GTT_RESOURCE) + pg->gtt_start = pci_resource_start(pdev, PSB_GTT_RESOURCE); + gtt_pages = pci_resource_len(pdev, PSB_GTT_RESOURCE) >> PAGE_SHIFT; /* CDV doesn't report this. In which case the system has 64 gtt pages */ if (pg->gtt_start == 0 || gtt_pages == 0) { @@ -407,10 +409,10 @@ int psb_gtt_init(struct drm_device *dev, int resume) pg->gtt_start = dev_priv->pge_ctl; } - pg->gatt_start = pci_resource_start(dev->pdev, PSB_GATT_RESOURCE); - pg->gatt_pages = pci_resource_len(dev->pdev, PSB_GATT_RESOURCE) + pg->gatt_start = pci_resource_start(pdev, PSB_GATT_RESOURCE); + pg->gatt_pages = pci_resource_len(pdev, PSB_GATT_RESOURCE) >> PAGE_SHIFT; - dev_priv->gtt_mem = &dev->pdev->resource[PSB_GATT_RESOURCE]; + dev_priv->gtt_mem = &pdev->resource[PSB_GATT_RESOURCE]; if (pg->gatt_pages == 0 || pg->gatt_start == 0) { static struct resource fudge; /* Preferably peppermint */ @@ -431,7 +433,7 @@ int psb_gtt_init(struct drm_device *dev, int resume) dev_priv->gtt_mem = &fudge; } - pci_read_config_dword(dev->pdev, PSB_BSM, &dev_priv->stolen_base); + pci_read_config_dword(pdev, PSB_BSM, &dev_priv->stolen_base); vram_stolen_size = pg->gtt_phys_start - dev_priv->stolen_base - PAGE_SIZE; diff --git a/drivers/gpu/drm/gma500/intel_bios.c b/drivers/gpu/drm/gma500/intel_bios.c index 8ad6337eeba3a400f2c72428eb9b8e7cf78b3d54..d838369f0119471655fb99d228dfa010a116b72e 100644 --- a/drivers/gpu/drm/gma500/intel_bios.c +++ b/drivers/gpu/drm/gma500/intel_bios.c @@ -50,7 +50,7 @@ parse_edp(struct drm_psb_private *dev_priv, struct bdb_header *bdb) uint8_t panel_type; edp = find_section(bdb, BDB_EDP); - + dev_priv->edp.bpp = 18; if (!edp) { if (dev_priv->edp.support) { @@ -80,7 +80,7 @@ parse_edp(struct drm_psb_private *dev_priv, struct bdb_header *bdb) dev_priv->edp.pps = *edp_pps; DRM_DEBUG_KMS("EDP timing in vbt t1_t3 %d t8 %d t9 %d t10 %d t11_t12 %d\n", - dev_priv->edp.pps.t1_t3, dev_priv->edp.pps.t8, + dev_priv->edp.pps.t1_t3, dev_priv->edp.pps.t8, dev_priv->edp.pps.t9, dev_priv->edp.pps.t10, dev_priv->edp.pps.t11_t12); @@ -516,7 +516,7 @@ parse_device_mapping(struct drm_psb_private *dev_priv, int psb_intel_init_bios(struct drm_device *dev) { struct drm_psb_private *dev_priv = dev->dev_private; - struct pci_dev *pdev = dev->pdev; + struct pci_dev *pdev = to_pci_dev(dev->dev); struct vbt_header *vbt = NULL; struct bdb_header *bdb = NULL; u8 __iomem *bios = NULL; @@ -574,7 +574,7 @@ int psb_intel_init_bios(struct drm_device *dev) return 0; } -/** +/* * Destroy and free VBT data */ void psb_intel_destroy_bios(struct drm_device *dev) diff --git a/drivers/gpu/drm/gma500/intel_gmbus.c b/drivers/gpu/drm/gma500/intel_gmbus.c index a083fbfe35b8ad373bfa688bc72b2953b1293624..370bd6451bd9b66a655b17d7dd9844bd5aea5704 100644 --- a/drivers/gpu/drm/gma500/intel_gmbus.c +++ b/drivers/gpu/drm/gma500/intel_gmbus.c @@ -196,7 +196,7 @@ intel_gpio_create(struct drm_psb_private *dev_priv, u32 pin) "gma500 GPIO%c", "?BACDE?F"[pin]); gpio->adapter.owner = THIS_MODULE; gpio->adapter.algo_data = &gpio->algo; - gpio->adapter.dev.parent = &dev_priv->dev->pdev->dev; + gpio->adapter.dev.parent = dev_priv->dev->dev; gpio->algo.setsda = set_data; gpio->algo.setscl = set_clock; gpio->algo.getsda = get_data; @@ -417,7 +417,7 @@ int gma_intel_setup_gmbus(struct drm_device *dev) "gma500 gmbus %s", names[i]); - bus->adapter.dev.parent = &dev->pdev->dev; + bus->adapter.dev.parent = dev->dev; bus->adapter.algo_data = dev_priv; bus->adapter.algo = &gmbus_algorithm; diff --git a/drivers/gpu/drm/gma500/intel_i2c.c b/drivers/gpu/drm/gma500/intel_i2c.c index de8810188190ba03210a18156bed8e9f48ef1d8c..5e1b4d70c3171dbb2aa185fe28f4641594c50e65 100644 --- a/drivers/gpu/drm/gma500/intel_i2c.c +++ b/drivers/gpu/drm/gma500/intel_i2c.c @@ -85,7 +85,6 @@ static void set_data(void *data, int state_high) /** * psb_intel_i2c_create - instantiate an Intel i2c bus using the specified GPIO reg * @dev: DRM device - * @output: driver specific output device * @reg: GPIO reg to use * @name: name for this bus * @@ -117,7 +116,7 @@ struct psb_intel_i2c_chan *psb_intel_i2c_create(struct drm_device *dev, snprintf(chan->adapter.name, I2C_NAME_SIZE, "intel drm %s", name); chan->adapter.owner = THIS_MODULE; chan->adapter.algo_data = &chan->algo; - chan->adapter.dev.parent = &dev->pdev->dev; + chan->adapter.dev.parent = dev->dev; chan->algo.setsda = set_data; chan->algo.setscl = set_clock; chan->algo.getsda = get_data; @@ -145,7 +144,7 @@ struct psb_intel_i2c_chan *psb_intel_i2c_create(struct drm_device *dev, /** * psb_intel_i2c_destroy - unregister and free i2c bus resources - * @output: channel to free + * @chan: channel to free * * Unregister the adapter from the i2c layer, then free the structure. */ diff --git a/drivers/gpu/drm/gma500/mdfld_device.c b/drivers/gpu/drm/gma500/mdfld_device.c deleted file mode 100644 index b83d59b21de5d8c0418a4465352847073836f25b..0000000000000000000000000000000000000000 --- a/drivers/gpu/drm/gma500/mdfld_device.c +++ /dev/null @@ -1,562 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/************************************************************************** - * Copyright (c) 2011, Intel Corporation. - * All Rights Reserved. - * - **************************************************************************/ - -#include -#include - -#include - -#include "mdfld_dsi_output.h" -#include "mdfld_output.h" -#include "mid_bios.h" -#include "psb_drv.h" -#include "tc35876x-dsi-lvds.h" - -#ifdef CONFIG_BACKLIGHT_CLASS_DEVICE - -#define MRST_BLC_MAX_PWM_REG_FREQ 0xFFFF -#define BLC_PWM_PRECISION_FACTOR 100 /* 10000000 */ -#define BLC_PWM_FREQ_CALC_CONSTANT 32 -#define MHz 1000000 -#define BRIGHTNESS_MIN_LEVEL 1 -#define BRIGHTNESS_MAX_LEVEL 100 -#define BRIGHTNESS_MASK 0xFF -#define BLC_POLARITY_NORMAL 0 -#define BLC_POLARITY_INVERSE 1 -#define BLC_ADJUSTMENT_MAX 100 - -#define MDFLD_BLC_PWM_PRECISION_FACTOR 10 -#define MDFLD_BLC_MAX_PWM_REG_FREQ 0xFFFE -#define MDFLD_BLC_MIN_PWM_REG_FREQ 0x2 - -#define MDFLD_BACKLIGHT_PWM_POLARITY_BIT_CLEAR (0xFFFE) -#define MDFLD_BACKLIGHT_PWM_CTL_SHIFT (16) - -static struct backlight_device *mdfld_backlight_device; - -int mdfld_set_brightness(struct backlight_device *bd) -{ - struct drm_device *dev = - (struct drm_device *)bl_get_data(mdfld_backlight_device); - struct drm_psb_private *dev_priv = dev->dev_private; - int level = bd->props.brightness; - - DRM_DEBUG_DRIVER("backlight level set to %d\n", level); - - /* Perform value bounds checking */ - if (level < BRIGHTNESS_MIN_LEVEL) - level = BRIGHTNESS_MIN_LEVEL; - - if (gma_power_begin(dev, false)) { - u32 adjusted_level = 0; - - /* - * Adjust the backlight level with the percent in - * dev_priv->blc_adj2 - */ - adjusted_level = level * dev_priv->blc_adj2; - adjusted_level = adjusted_level / BLC_ADJUSTMENT_MAX; - dev_priv->brightness_adjusted = adjusted_level; - - if (mdfld_get_panel_type(dev, 0) == TC35876X) { - if (dev_priv->dpi_panel_on[0] || - dev_priv->dpi_panel_on[2]) - tc35876x_brightness_control(dev, - dev_priv->brightness_adjusted); - } else { - if (dev_priv->dpi_panel_on[0]) - mdfld_dsi_brightness_control(dev, 0, - dev_priv->brightness_adjusted); - } - - if (dev_priv->dpi_panel_on[2]) - mdfld_dsi_brightness_control(dev, 2, - dev_priv->brightness_adjusted); - gma_power_end(dev); - } - - /* cache the brightness for later use */ - dev_priv->brightness = level; - return 0; -} - -static int mdfld_get_brightness(struct backlight_device *bd) -{ - struct drm_device *dev = - (struct drm_device *)bl_get_data(mdfld_backlight_device); - struct drm_psb_private *dev_priv = dev->dev_private; - - DRM_DEBUG_DRIVER("brightness = 0x%x \n", dev_priv->brightness); - - /* return locally cached var instead of HW read (due to DPST etc.) */ - return dev_priv->brightness; -} - -static const struct backlight_ops mdfld_ops = { - .get_brightness = mdfld_get_brightness, - .update_status = mdfld_set_brightness, -}; - -static int device_backlight_init(struct drm_device *dev) -{ - struct drm_psb_private *dev_priv = (struct drm_psb_private *) - dev->dev_private; - - dev_priv->blc_adj1 = BLC_ADJUSTMENT_MAX; - dev_priv->blc_adj2 = BLC_ADJUSTMENT_MAX; - - return 0; -} - -static int mdfld_backlight_init(struct drm_device *dev) -{ - struct backlight_properties props; - int ret = 0; - - memset(&props, 0, sizeof(struct backlight_properties)); - props.max_brightness = BRIGHTNESS_MAX_LEVEL; - props.type = BACKLIGHT_PLATFORM; - mdfld_backlight_device = backlight_device_register("mdfld-bl", - NULL, (void *)dev, &mdfld_ops, &props); - - if (IS_ERR(mdfld_backlight_device)) - return PTR_ERR(mdfld_backlight_device); - - ret = device_backlight_init(dev); - if (ret) - return ret; - - mdfld_backlight_device->props.brightness = BRIGHTNESS_MAX_LEVEL; - mdfld_backlight_device->props.max_brightness = BRIGHTNESS_MAX_LEVEL; - backlight_update_status(mdfld_backlight_device); - return 0; -} -#endif - -struct backlight_device *mdfld_get_backlight_device(void) -{ -#ifdef CONFIG_BACKLIGHT_CLASS_DEVICE - return mdfld_backlight_device; -#else - return NULL; -#endif -} - -/* - * mdfld_save_display_registers - * - * Description: We are going to suspend so save current display - * register state. - * - * Notes: FIXME_JLIU7 need to add the support for DPI MIPI & HDMI audio - */ -static int mdfld_save_display_registers(struct drm_device *dev, int pipenum) -{ - struct drm_psb_private *dev_priv = dev->dev_private; - struct medfield_state *regs = &dev_priv->regs.mdfld; - struct psb_pipe *pipe = &dev_priv->regs.pipe[pipenum]; - const struct psb_offset *map = &dev_priv->regmap[pipenum]; - int i; - u32 *mipi_val; - - /* register */ - u32 mipi_reg = MIPI; - - switch (pipenum) { - case 0: - mipi_val = ®s->saveMIPI; - break; - case 1: - mipi_val = ®s->saveMIPI; - break; - case 2: - /* register */ - mipi_reg = MIPI_C; - /* pointer to values */ - mipi_val = ®s->saveMIPI_C; - break; - default: - DRM_ERROR("%s, invalid pipe number.\n", __func__); - return -EINVAL; - } - - /* Pipe & plane A info */ - pipe->dpll = PSB_RVDC32(map->dpll); - pipe->fp0 = PSB_RVDC32(map->fp0); - pipe->conf = PSB_RVDC32(map->conf); - pipe->htotal = PSB_RVDC32(map->htotal); - pipe->hblank = PSB_RVDC32(map->hblank); - pipe->hsync = PSB_RVDC32(map->hsync); - pipe->vtotal = PSB_RVDC32(map->vtotal); - pipe->vblank = PSB_RVDC32(map->vblank); - pipe->vsync = PSB_RVDC32(map->vsync); - pipe->src = PSB_RVDC32(map->src); - pipe->stride = PSB_RVDC32(map->stride); - pipe->linoff = PSB_RVDC32(map->linoff); - pipe->tileoff = PSB_RVDC32(map->tileoff); - pipe->size = PSB_RVDC32(map->size); - pipe->pos = PSB_RVDC32(map->pos); - pipe->surf = PSB_RVDC32(map->surf); - pipe->cntr = PSB_RVDC32(map->cntr); - pipe->status = PSB_RVDC32(map->status); - - /*save palette (gamma) */ - for (i = 0; i < 256; i++) - pipe->palette[i] = PSB_RVDC32(map->palette + (i << 2)); - - if (pipenum == 1) { - regs->savePFIT_CONTROL = PSB_RVDC32(PFIT_CONTROL); - regs->savePFIT_PGM_RATIOS = PSB_RVDC32(PFIT_PGM_RATIOS); - - regs->saveHDMIPHYMISCCTL = PSB_RVDC32(HDMIPHYMISCCTL); - regs->saveHDMIB_CONTROL = PSB_RVDC32(HDMIB_CONTROL); - return 0; - } - - *mipi_val = PSB_RVDC32(mipi_reg); - return 0; -} - -/* - * mdfld_restore_display_registers - * - * Description: We are going to resume so restore display register state. - * - * Notes: FIXME_JLIU7 need to add the support for DPI MIPI & HDMI audio - */ -static int mdfld_restore_display_registers(struct drm_device *dev, int pipenum) -{ - /* To get panel out of ULPS mode. */ - u32 temp = 0; - u32 device_ready_reg = DEVICE_READY_REG; - struct drm_psb_private *dev_priv = dev->dev_private; - struct mdfld_dsi_config *dsi_config = NULL; - struct medfield_state *regs = &dev_priv->regs.mdfld; - struct psb_pipe *pipe = &dev_priv->regs.pipe[pipenum]; - const struct psb_offset *map = &dev_priv->regmap[pipenum]; - u32 i; - u32 dpll; - u32 timeout = 0; - - /* register */ - u32 mipi_reg = MIPI; - - /* values */ - u32 dpll_val = pipe->dpll; - u32 mipi_val = regs->saveMIPI; - - switch (pipenum) { - case 0: - dpll_val &= ~DPLL_VCO_ENABLE; - dsi_config = dev_priv->dsi_configs[0]; - break; - case 1: - dpll_val &= ~DPLL_VCO_ENABLE; - break; - case 2: - mipi_reg = MIPI_C; - mipi_val = regs->saveMIPI_C; - dsi_config = dev_priv->dsi_configs[1]; - break; - default: - DRM_ERROR("%s, invalid pipe number.\n", __func__); - return -EINVAL; - } - - /*make sure VGA plane is off. it initializes to on after reset!*/ - PSB_WVDC32(0x80000000, VGACNTRL); - - if (pipenum == 1) { - PSB_WVDC32(dpll_val & ~DPLL_VCO_ENABLE, map->dpll); - PSB_RVDC32(map->dpll); - - PSB_WVDC32(pipe->fp0, map->fp0); - } else { - - dpll = PSB_RVDC32(map->dpll); - - if (!(dpll & DPLL_VCO_ENABLE)) { - - /* When ungating power of DPLL, needs to wait 0.5us - before enable the VCO */ - if (dpll & MDFLD_PWR_GATE_EN) { - dpll &= ~MDFLD_PWR_GATE_EN; - PSB_WVDC32(dpll, map->dpll); - /* FIXME_MDFLD PO - change 500 to 1 after PO */ - udelay(500); - } - - PSB_WVDC32(pipe->fp0, map->fp0); - PSB_WVDC32(dpll_val, map->dpll); - /* FIXME_MDFLD PO - change 500 to 1 after PO */ - udelay(500); - - dpll_val |= DPLL_VCO_ENABLE; - PSB_WVDC32(dpll_val, map->dpll); - PSB_RVDC32(map->dpll); - - /* wait for DSI PLL to lock */ - while (timeout < 20000 && - !(PSB_RVDC32(map->conf) & PIPECONF_DSIPLL_LOCK)) { - udelay(150); - timeout++; - } - - if (timeout == 20000) { - DRM_ERROR("%s, can't lock DSIPLL.\n", - __func__); - return -EINVAL; - } - } - } - /* Restore mode */ - PSB_WVDC32(pipe->htotal, map->htotal); - PSB_WVDC32(pipe->hblank, map->hblank); - PSB_WVDC32(pipe->hsync, map->hsync); - PSB_WVDC32(pipe->vtotal, map->vtotal); - PSB_WVDC32(pipe->vblank, map->vblank); - PSB_WVDC32(pipe->vsync, map->vsync); - PSB_WVDC32(pipe->src, map->src); - PSB_WVDC32(pipe->status, map->status); - - /*set up the plane*/ - PSB_WVDC32(pipe->stride, map->stride); - PSB_WVDC32(pipe->linoff, map->linoff); - PSB_WVDC32(pipe->tileoff, map->tileoff); - PSB_WVDC32(pipe->size, map->size); - PSB_WVDC32(pipe->pos, map->pos); - PSB_WVDC32(pipe->surf, map->surf); - - if (pipenum == 1) { - /* restore palette (gamma) */ - /* udelay(50000); */ - for (i = 0; i < 256; i++) - PSB_WVDC32(pipe->palette[i], map->palette + (i << 2)); - - PSB_WVDC32(regs->savePFIT_CONTROL, PFIT_CONTROL); - PSB_WVDC32(regs->savePFIT_PGM_RATIOS, PFIT_PGM_RATIOS); - - /*TODO: resume HDMI port */ - - /*TODO: resume pipe*/ - - /*enable the plane*/ - PSB_WVDC32(pipe->cntr & ~DISPLAY_PLANE_ENABLE, map->cntr); - - return 0; - } - - /*set up pipe related registers*/ - PSB_WVDC32(mipi_val, mipi_reg); - - /*setup MIPI adapter + MIPI IP registers*/ - if (dsi_config) - mdfld_dsi_controller_init(dsi_config, pipenum); - - if (in_atomic() || in_interrupt()) - mdelay(20); - else - msleep(20); - - /*enable the plane*/ - PSB_WVDC32(pipe->cntr, map->cntr); - - if (in_atomic() || in_interrupt()) - mdelay(20); - else - msleep(20); - - /* LP Hold Release */ - temp = REG_READ(mipi_reg); - temp |= LP_OUTPUT_HOLD_RELEASE; - REG_WRITE(mipi_reg, temp); - mdelay(1); - - - /* Set DSI host to exit from Utra Low Power State */ - temp = REG_READ(device_ready_reg); - temp &= ~ULPS_MASK; - temp |= 0x3; - temp |= EXIT_ULPS_DEV_READY; - REG_WRITE(device_ready_reg, temp); - mdelay(1); - - temp = REG_READ(device_ready_reg); - temp &= ~ULPS_MASK; - temp |= EXITING_ULPS; - REG_WRITE(device_ready_reg, temp); - mdelay(1); - - /*enable the pipe*/ - PSB_WVDC32(pipe->conf, map->conf); - - /* restore palette (gamma) */ - /* udelay(50000); */ - for (i = 0; i < 256; i++) - PSB_WVDC32(pipe->palette[i], map->palette + (i << 2)); - - return 0; -} - -static int mdfld_save_registers(struct drm_device *dev) -{ - /* mdfld_save_cursor_overlay_registers(dev); */ - mdfld_save_display_registers(dev, 0); - mdfld_save_display_registers(dev, 2); - mdfld_disable_crtc(dev, 0); - mdfld_disable_crtc(dev, 2); - - return 0; -} - -static int mdfld_restore_registers(struct drm_device *dev) -{ - mdfld_restore_display_registers(dev, 2); - mdfld_restore_display_registers(dev, 0); - /* mdfld_restore_cursor_overlay_registers(dev); */ - - return 0; -} - -static int mdfld_power_down(struct drm_device *dev) -{ - /* FIXME */ - return 0; -} - -static int mdfld_power_up(struct drm_device *dev) -{ - /* FIXME */ - return 0; -} - -/* Medfield */ -static const struct psb_offset mdfld_regmap[3] = { - { - .fp0 = MRST_FPA0, - .fp1 = MRST_FPA1, - .cntr = DSPACNTR, - .conf = PIPEACONF, - .src = PIPEASRC, - .dpll = MRST_DPLL_A, - .htotal = HTOTAL_A, - .hblank = HBLANK_A, - .hsync = HSYNC_A, - .vtotal = VTOTAL_A, - .vblank = VBLANK_A, - .vsync = VSYNC_A, - .stride = DSPASTRIDE, - .size = DSPASIZE, - .pos = DSPAPOS, - .surf = DSPASURF, - .addr = MRST_DSPABASE, - .status = PIPEASTAT, - .linoff = DSPALINOFF, - .tileoff = DSPATILEOFF, - .palette = PALETTE_A, - }, - { - .fp0 = MDFLD_DPLL_DIV0, - .cntr = DSPBCNTR, - .conf = PIPEBCONF, - .src = PIPEBSRC, - .dpll = MDFLD_DPLL_B, - .htotal = HTOTAL_B, - .hblank = HBLANK_B, - .hsync = HSYNC_B, - .vtotal = VTOTAL_B, - .vblank = VBLANK_B, - .vsync = VSYNC_B, - .stride = DSPBSTRIDE, - .size = DSPBSIZE, - .pos = DSPBPOS, - .surf = DSPBSURF, - .addr = MRST_DSPBBASE, - .status = PIPEBSTAT, - .linoff = DSPBLINOFF, - .tileoff = DSPBTILEOFF, - .palette = PALETTE_B, - }, - { - .fp0 = MRST_FPA0, /* This is what the old code did ?? */ - .cntr = DSPCCNTR, - .conf = PIPECCONF, - .src = PIPECSRC, - /* No DPLL_C */ - .dpll = MRST_DPLL_A, - .htotal = HTOTAL_C, - .hblank = HBLANK_C, - .hsync = HSYNC_C, - .vtotal = VTOTAL_C, - .vblank = VBLANK_C, - .vsync = VSYNC_C, - .stride = DSPCSTRIDE, - .size = DSPBSIZE, - .pos = DSPCPOS, - .surf = DSPCSURF, - .addr = MDFLD_DSPCBASE, - .status = PIPECSTAT, - .linoff = DSPCLINOFF, - .tileoff = DSPCTILEOFF, - .palette = PALETTE_C, - }, -}; - -/* - * The GPIO lines for resetting DSI pipe 0 and 2 are available in the - * PCI device 0000:00:0c.0 on the Medfield. - */ -static struct gpiod_lookup_table mdfld_dsi_pipe_gpio_table = { - .table = { - GPIO_LOOKUP("0000:00:0c.0", 128, "dsi-pipe0-reset", - GPIO_ACTIVE_HIGH), - GPIO_LOOKUP("0000:00:0c.0", 34, "dsi-pipe2-reset", - GPIO_ACTIVE_HIGH), - { }, - }, -}; - -static int mdfld_chip_setup(struct drm_device *dev) -{ - struct drm_psb_private *dev_priv = dev->dev_private; - if (pci_enable_msi(dev->pdev)) - dev_warn(dev->dev, "Enabling MSI failed!\n"); - dev_priv->regmap = mdfld_regmap; - - /* Associate the GPIO lines with the DRM device */ - mdfld_dsi_pipe_gpio_table.dev_id = dev_name(dev->dev); - gpiod_add_lookup_table(&mdfld_dsi_pipe_gpio_table); - - return mid_chip_setup(dev); -} - -const struct psb_ops mdfld_chip_ops = { - .name = "mdfld", - .pipes = 3, - .crtcs = 3, - .lvds_mask = (1 << 1), - .hdmi_mask = (1 << 1), - .cursor_needs_phys = 0, - .sgx_offset = MRST_SGX_OFFSET, - - .chip_setup = mdfld_chip_setup, - .crtc_helper = &mdfld_helper_funcs, - .crtc_funcs = &psb_intel_crtc_funcs, - - .output_init = mdfld_output_init, - -#ifdef CONFIG_BACKLIGHT_CLASS_DEVICE - .backlight_init = mdfld_backlight_init, -#endif - - .save_regs = mdfld_save_registers, - .restore_regs = mdfld_restore_registers, - .save_crtc = gma_crtc_save, - .restore_crtc = gma_crtc_restore, - .power_down = mdfld_power_down, - .power_up = mdfld_power_up, -}; diff --git a/drivers/gpu/drm/gma500/mdfld_dsi_dpi.c b/drivers/gpu/drm/gma500/mdfld_dsi_dpi.c deleted file mode 100644 index ae1223f631a7f8af9e4957430917d8cd076cd811..0000000000000000000000000000000000000000 --- a/drivers/gpu/drm/gma500/mdfld_dsi_dpi.c +++ /dev/null @@ -1,1017 +0,0 @@ -/* - * Copyright © 2010 Intel Corporation - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - * DEALINGS IN THE SOFTWARE. - * - * Authors: - * jim liu - * Jackie Li - */ - -#include - -#include - -#include "mdfld_dsi_dpi.h" -#include "mdfld_dsi_pkg_sender.h" -#include "mdfld_output.h" -#include "psb_drv.h" -#include "tc35876x-dsi-lvds.h" - -static void mdfld_dsi_dpi_shut_down(struct mdfld_dsi_dpi_output *output, - int pipe); - -static void mdfld_wait_for_HS_DATA_FIFO(struct drm_device *dev, u32 pipe) -{ - u32 gen_fifo_stat_reg = MIPI_GEN_FIFO_STAT_REG(pipe); - int timeout = 0; - - udelay(500); - - /* This will time out after approximately 2+ seconds */ - while ((timeout < 20000) && - (REG_READ(gen_fifo_stat_reg) & DSI_FIFO_GEN_HS_DATA_FULL)) { - udelay(100); - timeout++; - } - - if (timeout == 20000) - DRM_INFO("MIPI: HS Data FIFO was never cleared!\n"); -} - -static void mdfld_wait_for_HS_CTRL_FIFO(struct drm_device *dev, u32 pipe) -{ - u32 gen_fifo_stat_reg = MIPI_GEN_FIFO_STAT_REG(pipe); - int timeout = 0; - - udelay(500); - - /* This will time out after approximately 2+ seconds */ - while ((timeout < 20000) && (REG_READ(gen_fifo_stat_reg) - & DSI_FIFO_GEN_HS_CTRL_FULL)) { - udelay(100); - timeout++; - } - if (timeout == 20000) - DRM_INFO("MIPI: HS CMD FIFO was never cleared!\n"); -} - -static void mdfld_wait_for_DPI_CTRL_FIFO(struct drm_device *dev, u32 pipe) -{ - u32 gen_fifo_stat_reg = MIPI_GEN_FIFO_STAT_REG(pipe); - int timeout = 0; - - udelay(500); - - /* This will time out after approximately 2+ seconds */ - while ((timeout < 20000) && ((REG_READ(gen_fifo_stat_reg) & - DPI_FIFO_EMPTY) != DPI_FIFO_EMPTY)) { - udelay(100); - timeout++; - } - - if (timeout == 20000) - DRM_ERROR("MIPI: DPI FIFO was never cleared\n"); -} - -static void mdfld_wait_for_SPL_PKG_SENT(struct drm_device *dev, u32 pipe) -{ - u32 intr_stat_reg = MIPI_INTR_STAT_REG(pipe); - int timeout = 0; - - udelay(500); - - /* This will time out after approximately 2+ seconds */ - while ((timeout < 20000) && (!(REG_READ(intr_stat_reg) - & DSI_INTR_STATE_SPL_PKG_SENT))) { - udelay(100); - timeout++; - } - - if (timeout == 20000) - DRM_ERROR("MIPI: SPL_PKT_SENT_INTERRUPT was not sent successfully!\n"); -} - -/* For TC35876X */ - -static void dsi_set_device_ready_state(struct drm_device *dev, int state, - int pipe) -{ - REG_FLD_MOD(MIPI_DEVICE_READY_REG(pipe), !!state, 0, 0); -} - -static void dsi_set_pipe_plane_enable_state(struct drm_device *dev, - int state, int pipe) -{ - struct drm_psb_private *dev_priv = dev->dev_private; - u32 pipeconf_reg = PIPEACONF; - u32 dspcntr_reg = DSPACNTR; - - u32 dspcntr = dev_priv->dspcntr[pipe]; - u32 mipi = MIPI_PORT_EN | PASS_FROM_SPHY_TO_AFE | SEL_FLOPPED_HSTX; - - if (pipe) { - pipeconf_reg = PIPECCONF; - dspcntr_reg = DSPCCNTR; - } else - mipi &= (~0x03); - - if (state) { - /*Set up pipe */ - REG_WRITE(pipeconf_reg, BIT(31)); - - if (REG_BIT_WAIT(pipeconf_reg, 1, 30)) - dev_err(&dev->pdev->dev, "%s: Pipe enable timeout\n", - __func__); - - /*Set up display plane */ - REG_WRITE(dspcntr_reg, dspcntr); - } else { - u32 dspbase_reg = pipe ? MDFLD_DSPCBASE : MRST_DSPABASE; - - /* Put DSI lanes to ULPS to disable pipe */ - REG_FLD_MOD(MIPI_DEVICE_READY_REG(pipe), 2, 2, 1); - REG_READ(MIPI_DEVICE_READY_REG(pipe)); /* posted write? */ - - /* LP Hold */ - REG_FLD_MOD(MIPI_PORT_CONTROL(pipe), 0, 16, 16); - REG_READ(MIPI_PORT_CONTROL(pipe)); /* posted write? */ - - /* Disable display plane */ - REG_FLD_MOD(dspcntr_reg, 0, 31, 31); - - /* Flush the plane changes ??? posted write? */ - REG_WRITE(dspbase_reg, REG_READ(dspbase_reg)); - REG_READ(dspbase_reg); - - /* Disable PIPE */ - REG_FLD_MOD(pipeconf_reg, 0, 31, 31); - - if (REG_BIT_WAIT(pipeconf_reg, 0, 30)) - dev_err(&dev->pdev->dev, "%s: Pipe disable timeout\n", - __func__); - - if (REG_BIT_WAIT(MIPI_GEN_FIFO_STAT_REG(pipe), 1, 28)) - dev_err(&dev->pdev->dev, "%s: FIFO not empty\n", - __func__); - } -} - -static void mdfld_dsi_configure_down(struct mdfld_dsi_encoder *dsi_encoder, - int pipe) -{ - struct mdfld_dsi_dpi_output *dpi_output = - MDFLD_DSI_DPI_OUTPUT(dsi_encoder); - struct mdfld_dsi_config *dsi_config = - mdfld_dsi_encoder_get_config(dsi_encoder); - struct drm_device *dev = dsi_config->dev; - struct drm_psb_private *dev_priv = dev->dev_private; - - if (!dev_priv->dpi_panel_on[pipe]) { - dev_err(dev->dev, "DPI panel is already off\n"); - return; - } - tc35876x_toshiba_bridge_panel_off(dev); - tc35876x_set_bridge_reset_state(dev, 1); - dsi_set_pipe_plane_enable_state(dev, 0, pipe); - mdfld_dsi_dpi_shut_down(dpi_output, pipe); - dsi_set_device_ready_state(dev, 0, pipe); -} - -static void mdfld_dsi_configure_up(struct mdfld_dsi_encoder *dsi_encoder, - int pipe) -{ - struct mdfld_dsi_dpi_output *dpi_output = - MDFLD_DSI_DPI_OUTPUT(dsi_encoder); - struct mdfld_dsi_config *dsi_config = - mdfld_dsi_encoder_get_config(dsi_encoder); - struct drm_device *dev = dsi_config->dev; - struct drm_psb_private *dev_priv = dev->dev_private; - - if (dev_priv->dpi_panel_on[pipe]) { - dev_err(dev->dev, "DPI panel is already on\n"); - return; - } - - /* For resume path sequence */ - mdfld_dsi_dpi_shut_down(dpi_output, pipe); - dsi_set_device_ready_state(dev, 0, pipe); - - dsi_set_device_ready_state(dev, 1, pipe); - tc35876x_set_bridge_reset_state(dev, 0); - tc35876x_configure_lvds_bridge(dev); - mdfld_dsi_dpi_turn_on(dpi_output, pipe); /* Send turn on command */ - dsi_set_pipe_plane_enable_state(dev, 1, pipe); -} -/* End for TC35876X */ - -/* ************************************************************************* *\ - * FUNCTION: mdfld_dsi_tpo_ic_init - * - * DESCRIPTION: This function is called only by mrst_dsi_mode_set and - * restore_display_registers. since this function does not - * acquire the mutex, it is important that the calling function - * does! -\* ************************************************************************* */ -static void mdfld_dsi_tpo_ic_init(struct mdfld_dsi_config *dsi_config, u32 pipe) -{ - struct drm_device *dev = dsi_config->dev; - u32 dcsChannelNumber = dsi_config->channel_num; - u32 gen_data_reg = MIPI_HS_GEN_DATA_REG(pipe); - u32 gen_ctrl_reg = MIPI_HS_GEN_CTRL_REG(pipe); - u32 gen_ctrl_val = GEN_LONG_WRITE; - - DRM_INFO("Enter mrst init TPO MIPI display.\n"); - - gen_ctrl_val |= dcsChannelNumber << DCS_CHANNEL_NUMBER_POS; - - /* Flip page order */ - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x00008036); - mdfld_wait_for_HS_CTRL_FIFO(dev, pipe); - REG_WRITE(gen_ctrl_reg, gen_ctrl_val | (0x02 << WORD_COUNTS_POS)); - - /* 0xF0 */ - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x005a5af0); - mdfld_wait_for_HS_CTRL_FIFO(dev, pipe); - REG_WRITE(gen_ctrl_reg, gen_ctrl_val | (0x03 << WORD_COUNTS_POS)); - - /* Write protection key */ - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x005a5af1); - mdfld_wait_for_HS_CTRL_FIFO(dev, pipe); - REG_WRITE(gen_ctrl_reg, gen_ctrl_val | (0x03 << WORD_COUNTS_POS)); - - /* 0xFC */ - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x005a5afc); - mdfld_wait_for_HS_CTRL_FIFO(dev, pipe); - REG_WRITE(gen_ctrl_reg, gen_ctrl_val | (0x03 << WORD_COUNTS_POS)); - - /* 0xB7 */ - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x770000b7); - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x00000044); - mdfld_wait_for_HS_CTRL_FIFO(dev, pipe); - REG_WRITE(gen_ctrl_reg, gen_ctrl_val | (0x05 << WORD_COUNTS_POS)); - - /* 0xB6 */ - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x000a0ab6); - mdfld_wait_for_HS_CTRL_FIFO(dev, pipe); - REG_WRITE(gen_ctrl_reg, gen_ctrl_val | (0x03 << WORD_COUNTS_POS)); - - /* 0xF2 */ - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x081010f2); - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x4a070708); - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x000000c5); - mdfld_wait_for_HS_CTRL_FIFO(dev, pipe); - REG_WRITE(gen_ctrl_reg, gen_ctrl_val | (0x09 << WORD_COUNTS_POS)); - - /* 0xF8 */ - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x024003f8); - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x01030a04); - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x0e020220); - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x00000004); - mdfld_wait_for_HS_CTRL_FIFO(dev, pipe); - REG_WRITE(gen_ctrl_reg, gen_ctrl_val | (0x0d << WORD_COUNTS_POS)); - - /* 0xE2 */ - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x398fc3e2); - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x0000916f); - mdfld_wait_for_HS_CTRL_FIFO(dev, pipe); - REG_WRITE(gen_ctrl_reg, gen_ctrl_val | (0x06 << WORD_COUNTS_POS)); - - /* 0xB0 */ - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x000000b0); - mdfld_wait_for_HS_CTRL_FIFO(dev, pipe); - REG_WRITE(gen_ctrl_reg, gen_ctrl_val | (0x02 << WORD_COUNTS_POS)); - - /* 0xF4 */ - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x240242f4); - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x78ee2002); - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x2a071050); - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x507fee10); - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x10300710); - mdfld_wait_for_HS_CTRL_FIFO(dev, pipe); - REG_WRITE(gen_ctrl_reg, gen_ctrl_val | (0x14 << WORD_COUNTS_POS)); - - /* 0xBA */ - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x19fe07ba); - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x101c0a31); - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x00000010); - mdfld_wait_for_HS_CTRL_FIFO(dev, pipe); - REG_WRITE(gen_ctrl_reg, gen_ctrl_val | (0x09 << WORD_COUNTS_POS)); - - /* 0xBB */ - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x28ff07bb); - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x24280a31); - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x00000034); - mdfld_wait_for_HS_CTRL_FIFO(dev, pipe); - REG_WRITE(gen_ctrl_reg, gen_ctrl_val | (0x09 << WORD_COUNTS_POS)); - - /* 0xFB */ - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x535d05fb); - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x1b1a2130); - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x221e180e); - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x131d2120); - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x535d0508); - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x1c1a2131); - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x231f160d); - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x111b2220); - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x535c2008); - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x1f1d2433); - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x2c251a10); - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x2c34372d); - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x00000023); - mdfld_wait_for_HS_CTRL_FIFO(dev, pipe); - REG_WRITE(gen_ctrl_reg, gen_ctrl_val | (0x31 << WORD_COUNTS_POS)); - - /* 0xFA */ - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x525c0bfa); - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x1c1c232f); - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x2623190e); - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x18212625); - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x545d0d0e); - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x1e1d2333); - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x26231a10); - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x1a222725); - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x545d280f); - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x21202635); - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x31292013); - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x31393d33); - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x00000029); - mdfld_wait_for_HS_CTRL_FIFO(dev, pipe); - REG_WRITE(gen_ctrl_reg, gen_ctrl_val | (0x31 << WORD_COUNTS_POS)); - - /* Set DM */ - mdfld_wait_for_HS_DATA_FIFO(dev, pipe); - REG_WRITE(gen_data_reg, 0x000100f7); - mdfld_wait_for_HS_CTRL_FIFO(dev, pipe); - REG_WRITE(gen_ctrl_reg, gen_ctrl_val | (0x03 << WORD_COUNTS_POS)); -} - -static u16 mdfld_dsi_dpi_to_byte_clock_count(int pixel_clock_count, - int num_lane, int bpp) -{ - return (u16)((pixel_clock_count * bpp) / (num_lane * 8)); -} - -/* - * Calculate the dpi time basing on a given drm mode @mode - * return 0 on success. - * FIXME: I was using proposed mode value for calculation, may need to - * use crtc mode values later - */ -int mdfld_dsi_dpi_timing_calculation(struct drm_display_mode *mode, - struct mdfld_dsi_dpi_timing *dpi_timing, - int num_lane, int bpp) -{ - int pclk_hsync, pclk_hfp, pclk_hbp, pclk_hactive; - int pclk_vsync, pclk_vfp, pclk_vbp; - - pclk_hactive = mode->hdisplay; - pclk_hfp = mode->hsync_start - mode->hdisplay; - pclk_hsync = mode->hsync_end - mode->hsync_start; - pclk_hbp = mode->htotal - mode->hsync_end; - - pclk_vfp = mode->vsync_start - mode->vdisplay; - pclk_vsync = mode->vsync_end - mode->vsync_start; - pclk_vbp = mode->vtotal - mode->vsync_end; - - /* - * byte clock counts were calculated by following formula - * bclock_count = pclk_count * bpp / num_lane / 8 - */ - dpi_timing->hsync_count = mdfld_dsi_dpi_to_byte_clock_count( - pclk_hsync, num_lane, bpp); - dpi_timing->hbp_count = mdfld_dsi_dpi_to_byte_clock_count( - pclk_hbp, num_lane, bpp); - dpi_timing->hfp_count = mdfld_dsi_dpi_to_byte_clock_count( - pclk_hfp, num_lane, bpp); - dpi_timing->hactive_count = mdfld_dsi_dpi_to_byte_clock_count( - pclk_hactive, num_lane, bpp); - dpi_timing->vsync_count = mdfld_dsi_dpi_to_byte_clock_count( - pclk_vsync, num_lane, bpp); - dpi_timing->vbp_count = mdfld_dsi_dpi_to_byte_clock_count( - pclk_vbp, num_lane, bpp); - dpi_timing->vfp_count = mdfld_dsi_dpi_to_byte_clock_count( - pclk_vfp, num_lane, bpp); - - return 0; -} - -void mdfld_dsi_dpi_controller_init(struct mdfld_dsi_config *dsi_config, - int pipe) -{ - struct drm_device *dev = dsi_config->dev; - int lane_count = dsi_config->lane_count; - struct mdfld_dsi_dpi_timing dpi_timing; - struct drm_display_mode *mode = dsi_config->mode; - u32 val; - - /*un-ready device*/ - REG_FLD_MOD(MIPI_DEVICE_READY_REG(pipe), 0, 0, 0); - - /*init dsi adapter before kicking off*/ - REG_WRITE(MIPI_CTRL_REG(pipe), 0x00000018); - - /*enable all interrupts*/ - REG_WRITE(MIPI_INTR_EN_REG(pipe), 0xffffffff); - - /*set up func_prg*/ - val = lane_count; - val |= dsi_config->channel_num << DSI_DPI_VIRT_CHANNEL_OFFSET; - - switch (dsi_config->bpp) { - case 16: - val |= DSI_DPI_COLOR_FORMAT_RGB565; - break; - case 18: - val |= DSI_DPI_COLOR_FORMAT_RGB666; - break; - case 24: - val |= DSI_DPI_COLOR_FORMAT_RGB888; - break; - default: - DRM_ERROR("unsupported color format, bpp = %d\n", - dsi_config->bpp); - } - REG_WRITE(MIPI_DSI_FUNC_PRG_REG(pipe), val); - - REG_WRITE(MIPI_HS_TX_TIMEOUT_REG(pipe), - (mode->vtotal * mode->htotal * dsi_config->bpp / - (8 * lane_count)) & DSI_HS_TX_TIMEOUT_MASK); - REG_WRITE(MIPI_LP_RX_TIMEOUT_REG(pipe), - 0xffff & DSI_LP_RX_TIMEOUT_MASK); - - /*max value: 20 clock cycles of txclkesc*/ - REG_WRITE(MIPI_TURN_AROUND_TIMEOUT_REG(pipe), - 0x14 & DSI_TURN_AROUND_TIMEOUT_MASK); - - /*min 21 txclkesc, max: ffffh*/ - REG_WRITE(MIPI_DEVICE_RESET_TIMER_REG(pipe), - 0xffff & DSI_RESET_TIMER_MASK); - - REG_WRITE(MIPI_DPI_RESOLUTION_REG(pipe), - mode->vdisplay << 16 | mode->hdisplay); - - /*set DPI timing registers*/ - mdfld_dsi_dpi_timing_calculation(mode, &dpi_timing, - dsi_config->lane_count, dsi_config->bpp); - - REG_WRITE(MIPI_HSYNC_COUNT_REG(pipe), - dpi_timing.hsync_count & DSI_DPI_TIMING_MASK); - REG_WRITE(MIPI_HBP_COUNT_REG(pipe), - dpi_timing.hbp_count & DSI_DPI_TIMING_MASK); - REG_WRITE(MIPI_HFP_COUNT_REG(pipe), - dpi_timing.hfp_count & DSI_DPI_TIMING_MASK); - REG_WRITE(MIPI_HACTIVE_COUNT_REG(pipe), - dpi_timing.hactive_count & DSI_DPI_TIMING_MASK); - REG_WRITE(MIPI_VSYNC_COUNT_REG(pipe), - dpi_timing.vsync_count & DSI_DPI_TIMING_MASK); - REG_WRITE(MIPI_VBP_COUNT_REG(pipe), - dpi_timing.vbp_count & DSI_DPI_TIMING_MASK); - REG_WRITE(MIPI_VFP_COUNT_REG(pipe), - dpi_timing.vfp_count & DSI_DPI_TIMING_MASK); - - REG_WRITE(MIPI_HIGH_LOW_SWITCH_COUNT_REG(pipe), 0x46); - - /*min: 7d0 max: 4e20*/ - REG_WRITE(MIPI_INIT_COUNT_REG(pipe), 0x000007d0); - - /*set up video mode*/ - val = dsi_config->video_mode | DSI_DPI_COMPLETE_LAST_LINE; - REG_WRITE(MIPI_VIDEO_MODE_FORMAT_REG(pipe), val); - - REG_WRITE(MIPI_EOT_DISABLE_REG(pipe), 0x00000000); - - REG_WRITE(MIPI_LP_BYTECLK_REG(pipe), 0x00000004); - - /*TODO: figure out how to setup these registers*/ - if (mdfld_get_panel_type(dev, pipe) == TC35876X) - REG_WRITE(MIPI_DPHY_PARAM_REG(pipe), 0x2A0c6008); - else - REG_WRITE(MIPI_DPHY_PARAM_REG(pipe), 0x150c3408); - - REG_WRITE(MIPI_CLK_LANE_SWITCH_TIME_CNT_REG(pipe), (0xa << 16) | 0x14); - - if (mdfld_get_panel_type(dev, pipe) == TC35876X) - tc35876x_set_bridge_reset_state(dev, 0); /*Pull High Reset */ - - /*set device ready*/ - REG_FLD_MOD(MIPI_DEVICE_READY_REG(pipe), 1, 0, 0); -} - -void mdfld_dsi_dpi_turn_on(struct mdfld_dsi_dpi_output *output, int pipe) -{ - struct drm_device *dev = output->dev; - - /* clear special packet sent bit */ - if (REG_READ(MIPI_INTR_STAT_REG(pipe)) & DSI_INTR_STATE_SPL_PKG_SENT) - REG_WRITE(MIPI_INTR_STAT_REG(pipe), - DSI_INTR_STATE_SPL_PKG_SENT); - - /*send turn on package*/ - REG_WRITE(MIPI_DPI_CONTROL_REG(pipe), DSI_DPI_CTRL_HS_TURN_ON); - - /*wait for SPL_PKG_SENT interrupt*/ - mdfld_wait_for_SPL_PKG_SENT(dev, pipe); - - if (REG_READ(MIPI_INTR_STAT_REG(pipe)) & DSI_INTR_STATE_SPL_PKG_SENT) - REG_WRITE(MIPI_INTR_STAT_REG(pipe), - DSI_INTR_STATE_SPL_PKG_SENT); - - output->panel_on = 1; - - /* FIXME the following is disabled to WA the X slow start issue - for TMD panel - if (pipe == 2) - dev_priv->dpi_panel_on2 = true; - else if (pipe == 0) - dev_priv->dpi_panel_on = true; */ -} - -static void mdfld_dsi_dpi_shut_down(struct mdfld_dsi_dpi_output *output, - int pipe) -{ - struct drm_device *dev = output->dev; - - /*if output is on, or mode setting didn't happen, ignore this*/ - if ((!output->panel_on) || output->first_boot) { - output->first_boot = 0; - return; - } - - /* Wait for dpi fifo to empty */ - mdfld_wait_for_DPI_CTRL_FIFO(dev, pipe); - - /* Clear the special packet interrupt bit if set */ - if (REG_READ(MIPI_INTR_STAT_REG(pipe)) & DSI_INTR_STATE_SPL_PKG_SENT) - REG_WRITE(MIPI_INTR_STAT_REG(pipe), - DSI_INTR_STATE_SPL_PKG_SENT); - - if (REG_READ(MIPI_DPI_CONTROL_REG(pipe)) == DSI_DPI_CTRL_HS_SHUTDOWN) - goto shutdown_out; - - REG_WRITE(MIPI_DPI_CONTROL_REG(pipe), DSI_DPI_CTRL_HS_SHUTDOWN); - -shutdown_out: - output->panel_on = 0; - output->first_boot = 0; - - /* FIXME the following is disabled to WA the X slow start issue - for TMD panel - if (pipe == 2) - dev_priv->dpi_panel_on2 = false; - else if (pipe == 0) - dev_priv->dpi_panel_on = false; */ -} - -static void mdfld_dsi_dpi_set_power(struct drm_encoder *encoder, bool on) -{ - struct mdfld_dsi_encoder *dsi_encoder = mdfld_dsi_encoder(encoder); - struct mdfld_dsi_dpi_output *dpi_output = - MDFLD_DSI_DPI_OUTPUT(dsi_encoder); - struct mdfld_dsi_config *dsi_config = - mdfld_dsi_encoder_get_config(dsi_encoder); - int pipe = mdfld_dsi_encoder_get_pipe(dsi_encoder); - struct drm_device *dev = dsi_config->dev; - struct drm_psb_private *dev_priv = dev->dev_private; - - /*start up display island if it was shutdown*/ - if (!gma_power_begin(dev, true)) - return; - - if (on) { - if (mdfld_get_panel_type(dev, pipe) == TMD_VID) - mdfld_dsi_dpi_turn_on(dpi_output, pipe); - else if (mdfld_get_panel_type(dev, pipe) == TC35876X) - mdfld_dsi_configure_up(dsi_encoder, pipe); - else { - /*enable mipi port*/ - REG_WRITE(MIPI_PORT_CONTROL(pipe), - REG_READ(MIPI_PORT_CONTROL(pipe)) | BIT(31)); - REG_READ(MIPI_PORT_CONTROL(pipe)); - - mdfld_dsi_dpi_turn_on(dpi_output, pipe); - mdfld_dsi_tpo_ic_init(dsi_config, pipe); - } - dev_priv->dpi_panel_on[pipe] = true; - } else { - if (mdfld_get_panel_type(dev, pipe) == TMD_VID) - mdfld_dsi_dpi_shut_down(dpi_output, pipe); - else if (mdfld_get_panel_type(dev, pipe) == TC35876X) - mdfld_dsi_configure_down(dsi_encoder, pipe); - else { - mdfld_dsi_dpi_shut_down(dpi_output, pipe); - - /*disable mipi port*/ - REG_WRITE(MIPI_PORT_CONTROL(pipe), - REG_READ(MIPI_PORT_CONTROL(pipe)) & ~BIT(31)); - REG_READ(MIPI_PORT_CONTROL(pipe)); - } - dev_priv->dpi_panel_on[pipe] = false; - } - gma_power_end(dev); -} - -void mdfld_dsi_dpi_dpms(struct drm_encoder *encoder, int mode) -{ - mdfld_dsi_dpi_set_power(encoder, mode == DRM_MODE_DPMS_ON); -} - -bool mdfld_dsi_dpi_mode_fixup(struct drm_encoder *encoder, - const struct drm_display_mode *mode, - struct drm_display_mode *adjusted_mode) -{ - struct mdfld_dsi_encoder *dsi_encoder = mdfld_dsi_encoder(encoder); - struct mdfld_dsi_config *dsi_config = - mdfld_dsi_encoder_get_config(dsi_encoder); - struct drm_display_mode *fixed_mode = dsi_config->fixed_mode; - - if (fixed_mode) { - adjusted_mode->hdisplay = fixed_mode->hdisplay; - adjusted_mode->hsync_start = fixed_mode->hsync_start; - adjusted_mode->hsync_end = fixed_mode->hsync_end; - adjusted_mode->htotal = fixed_mode->htotal; - adjusted_mode->vdisplay = fixed_mode->vdisplay; - adjusted_mode->vsync_start = fixed_mode->vsync_start; - adjusted_mode->vsync_end = fixed_mode->vsync_end; - adjusted_mode->vtotal = fixed_mode->vtotal; - adjusted_mode->clock = fixed_mode->clock; - drm_mode_set_crtcinfo(adjusted_mode, CRTC_INTERLACE_HALVE_V); - } - return true; -} - -void mdfld_dsi_dpi_prepare(struct drm_encoder *encoder) -{ - mdfld_dsi_dpi_set_power(encoder, false); -} - -void mdfld_dsi_dpi_commit(struct drm_encoder *encoder) -{ - mdfld_dsi_dpi_set_power(encoder, true); -} - -/* For TC35876X */ -/* This functionality was implemented in FW in iCDK */ -/* But removed in DV0 and later. So need to add here. */ -static void mipi_set_properties(struct mdfld_dsi_config *dsi_config, int pipe) -{ - struct drm_device *dev = dsi_config->dev; - - REG_WRITE(MIPI_CTRL_REG(pipe), 0x00000018); - REG_WRITE(MIPI_INTR_EN_REG(pipe), 0xffffffff); - REG_WRITE(MIPI_HS_TX_TIMEOUT_REG(pipe), 0xffffff); - REG_WRITE(MIPI_LP_RX_TIMEOUT_REG(pipe), 0xffffff); - REG_WRITE(MIPI_TURN_AROUND_TIMEOUT_REG(pipe), 0x14); - REG_WRITE(MIPI_DEVICE_RESET_TIMER_REG(pipe), 0xff); - REG_WRITE(MIPI_HIGH_LOW_SWITCH_COUNT_REG(pipe), 0x25); - REG_WRITE(MIPI_INIT_COUNT_REG(pipe), 0xf0); - REG_WRITE(MIPI_EOT_DISABLE_REG(pipe), 0x00000000); - REG_WRITE(MIPI_LP_BYTECLK_REG(pipe), 0x00000004); - REG_WRITE(MIPI_DBI_BW_CTRL_REG(pipe), 0x00000820); - REG_WRITE(MIPI_CLK_LANE_SWITCH_TIME_CNT_REG(pipe), (0xa << 16) | 0x14); -} - -static void mdfld_mipi_set_video_timing(struct mdfld_dsi_config *dsi_config, - int pipe) -{ - struct drm_device *dev = dsi_config->dev; - struct mdfld_dsi_dpi_timing dpi_timing; - struct drm_display_mode *mode = dsi_config->mode; - - mdfld_dsi_dpi_timing_calculation(mode, &dpi_timing, - dsi_config->lane_count, - dsi_config->bpp); - - REG_WRITE(MIPI_DPI_RESOLUTION_REG(pipe), - mode->vdisplay << 16 | mode->hdisplay); - REG_WRITE(MIPI_HSYNC_COUNT_REG(pipe), - dpi_timing.hsync_count & DSI_DPI_TIMING_MASK); - REG_WRITE(MIPI_HBP_COUNT_REG(pipe), - dpi_timing.hbp_count & DSI_DPI_TIMING_MASK); - REG_WRITE(MIPI_HFP_COUNT_REG(pipe), - dpi_timing.hfp_count & DSI_DPI_TIMING_MASK); - REG_WRITE(MIPI_HACTIVE_COUNT_REG(pipe), - dpi_timing.hactive_count & DSI_DPI_TIMING_MASK); - REG_WRITE(MIPI_VSYNC_COUNT_REG(pipe), - dpi_timing.vsync_count & DSI_DPI_TIMING_MASK); - REG_WRITE(MIPI_VBP_COUNT_REG(pipe), - dpi_timing.vbp_count & DSI_DPI_TIMING_MASK); - REG_WRITE(MIPI_VFP_COUNT_REG(pipe), - dpi_timing.vfp_count & DSI_DPI_TIMING_MASK); -} - -static void mdfld_mipi_config(struct mdfld_dsi_config *dsi_config, int pipe) -{ - struct drm_device *dev = dsi_config->dev; - int lane_count = dsi_config->lane_count; - - if (pipe) { - REG_WRITE(MIPI_PORT_CONTROL(0), 0x00000002); - REG_WRITE(MIPI_PORT_CONTROL(2), 0x80000000); - } else { - REG_WRITE(MIPI_PORT_CONTROL(0), 0x80010000); - REG_WRITE(MIPI_PORT_CONTROL(2), 0x00); - } - - REG_WRITE(MIPI_DPHY_PARAM_REG(pipe), 0x150A600F); - REG_WRITE(MIPI_VIDEO_MODE_FORMAT_REG(pipe), 0x0000000F); - - /* lane_count = 3 */ - REG_WRITE(MIPI_DSI_FUNC_PRG_REG(pipe), 0x00000200 | lane_count); - - mdfld_mipi_set_video_timing(dsi_config, pipe); -} - -static void mdfld_set_pipe_timing(struct mdfld_dsi_config *dsi_config, int pipe) -{ - struct drm_device *dev = dsi_config->dev; - struct drm_display_mode *mode = dsi_config->mode; - - REG_WRITE(HTOTAL_A, ((mode->htotal - 1) << 16) | (mode->hdisplay - 1)); - REG_WRITE(HBLANK_A, ((mode->htotal - 1) << 16) | (mode->hdisplay - 1)); - REG_WRITE(HSYNC_A, - ((mode->hsync_end - 1) << 16) | (mode->hsync_start - 1)); - - REG_WRITE(VTOTAL_A, ((mode->vtotal - 1) << 16) | (mode->vdisplay - 1)); - REG_WRITE(VBLANK_A, ((mode->vtotal - 1) << 16) | (mode->vdisplay - 1)); - REG_WRITE(VSYNC_A, - ((mode->vsync_end - 1) << 16) | (mode->vsync_start - 1)); - - REG_WRITE(PIPEASRC, - ((mode->hdisplay - 1) << 16) | (mode->vdisplay - 1)); -} -/* End for TC35876X */ - -void mdfld_dsi_dpi_mode_set(struct drm_encoder *encoder, - struct drm_display_mode *mode, - struct drm_display_mode *adjusted_mode) -{ - struct mdfld_dsi_encoder *dsi_encoder = mdfld_dsi_encoder(encoder); - struct mdfld_dsi_dpi_output *dpi_output = - MDFLD_DSI_DPI_OUTPUT(dsi_encoder); - struct mdfld_dsi_config *dsi_config = - mdfld_dsi_encoder_get_config(dsi_encoder); - struct drm_device *dev = dsi_config->dev; - struct drm_psb_private *dev_priv = dev->dev_private; - int pipe = mdfld_dsi_encoder_get_pipe(dsi_encoder); - u32 pipeconf_reg = PIPEACONF; - u32 dspcntr_reg = DSPACNTR; - u32 pipeconf, dspcntr; - - u32 mipi = MIPI_PORT_EN | PASS_FROM_SPHY_TO_AFE | SEL_FLOPPED_HSTX; - - if (WARN_ON(pipe < 0)) - return; - - pipeconf = dev_priv->pipeconf[pipe]; - dspcntr = dev_priv->dspcntr[pipe]; - - if (pipe) { - pipeconf_reg = PIPECCONF; - dspcntr_reg = DSPCCNTR; - } else { - if (mdfld_get_panel_type(dev, pipe) == TC35876X) - mipi &= (~0x03); /* Use all four lanes */ - else - mipi |= 2; - } - - /*start up display island if it was shutdown*/ - if (!gma_power_begin(dev, true)) - return; - - if (mdfld_get_panel_type(dev, pipe) == TC35876X) { - /* - * The following logic is required to reset the bridge and - * configure. This also starts the DSI clock at 200MHz. - */ - tc35876x_set_bridge_reset_state(dev, 0); /*Pull High Reset */ - tc35876x_toshiba_bridge_panel_on(dev); - udelay(100); - /* Now start the DSI clock */ - REG_WRITE(MRST_DPLL_A, 0x00); - REG_WRITE(MRST_FPA0, 0xC1); - REG_WRITE(MRST_DPLL_A, 0x00800000); - udelay(500); - REG_WRITE(MRST_DPLL_A, 0x80800000); - - if (REG_BIT_WAIT(pipeconf_reg, 1, 29)) - dev_err(&dev->pdev->dev, "%s: DSI PLL lock timeout\n", - __func__); - - REG_WRITE(MIPI_DPHY_PARAM_REG(pipe), 0x2A0c6008); - - mipi_set_properties(dsi_config, pipe); - mdfld_mipi_config(dsi_config, pipe); - mdfld_set_pipe_timing(dsi_config, pipe); - - REG_WRITE(DSPABASE, 0x00); - REG_WRITE(DSPASIZE, - ((mode->vdisplay - 1) << 16) | (mode->hdisplay - 1)); - - REG_WRITE(DSPACNTR, 0x98000000); - REG_WRITE(DSPASURF, 0x00); - - REG_WRITE(VGACNTRL, 0x80000000); - REG_WRITE(DEVICE_READY_REG, 0x00000001); - - REG_WRITE(MIPI_PORT_CONTROL(pipe), 0x80810000); - } else { - /*set up mipi port FIXME: do at init time */ - REG_WRITE(MIPI_PORT_CONTROL(pipe), mipi); - } - REG_READ(MIPI_PORT_CONTROL(pipe)); - - if (mdfld_get_panel_type(dev, pipe) == TMD_VID) { - /* NOP */ - } else if (mdfld_get_panel_type(dev, pipe) == TC35876X) { - /* set up DSI controller DPI interface */ - mdfld_dsi_dpi_controller_init(dsi_config, pipe); - - /* Configure MIPI Bridge and Panel */ - tc35876x_configure_lvds_bridge(dev); - dev_priv->dpi_panel_on[pipe] = true; - } else { - /*turn on DPI interface*/ - mdfld_dsi_dpi_turn_on(dpi_output, pipe); - } - - /*set up pipe*/ - REG_WRITE(pipeconf_reg, pipeconf); - REG_READ(pipeconf_reg); - - /*set up display plane*/ - REG_WRITE(dspcntr_reg, dspcntr); - REG_READ(dspcntr_reg); - - msleep(20); /* FIXME: this should wait for vblank */ - - if (mdfld_get_panel_type(dev, pipe) == TMD_VID) { - /* NOP */ - } else if (mdfld_get_panel_type(dev, pipe) == TC35876X) { - mdfld_dsi_dpi_turn_on(dpi_output, pipe); - } else { - /* init driver ic */ - mdfld_dsi_tpo_ic_init(dsi_config, pipe); - /*init backlight*/ - mdfld_dsi_brightness_init(dsi_config, pipe); - } - - gma_power_end(dev); -} - -/* - * Init DSI DPI encoder. - * Allocate an mdfld_dsi_encoder and attach it to given @dsi_connector - * return pointer of newly allocated DPI encoder, NULL on error - */ -struct mdfld_dsi_encoder *mdfld_dsi_dpi_init(struct drm_device *dev, - struct mdfld_dsi_connector *dsi_connector, - const struct panel_funcs *p_funcs) -{ - struct mdfld_dsi_dpi_output *dpi_output = NULL; - struct mdfld_dsi_config *dsi_config; - struct drm_connector *connector = NULL; - struct drm_encoder *encoder = NULL; - int pipe; - u32 data; - int ret; - - pipe = dsi_connector->pipe; - - if (mdfld_get_panel_type(dev, pipe) != TC35876X) { - dsi_config = mdfld_dsi_get_config(dsi_connector); - - /* panel hard-reset */ - if (p_funcs->reset) { - ret = p_funcs->reset(dev, pipe); - if (ret) { - DRM_ERROR("Panel %d hard-reset failed\n", pipe); - return NULL; - } - } - - /* panel drvIC init */ - if (p_funcs->drv_ic_init) - p_funcs->drv_ic_init(dsi_config, pipe); - - /* panel power mode detect */ - ret = mdfld_dsi_get_power_mode(dsi_config, &data, false); - if (ret) { - DRM_ERROR("Panel %d get power mode failed\n", pipe); - dsi_connector->status = connector_status_disconnected; - } else { - DRM_INFO("pipe %d power mode 0x%x\n", pipe, data); - dsi_connector->status = connector_status_connected; - } - } - - dpi_output = kzalloc(sizeof(struct mdfld_dsi_dpi_output), GFP_KERNEL); - if (!dpi_output) { - DRM_ERROR("No memory\n"); - return NULL; - } - - dpi_output->panel_on = 0; - dpi_output->dev = dev; - if (mdfld_get_panel_type(dev, pipe) != TC35876X) - dpi_output->p_funcs = p_funcs; - dpi_output->first_boot = 1; - - /*get fixed mode*/ - dsi_config = mdfld_dsi_get_config(dsi_connector); - - /*create drm encoder object*/ - connector = &dsi_connector->base.base; - encoder = &dpi_output->base.base.base; - drm_simple_encoder_init(dev, encoder, DRM_MODE_ENCODER_LVDS); - drm_encoder_helper_add(encoder, - p_funcs->encoder_helper_funcs); - - /*attach to given connector*/ - drm_connector_attach_encoder(connector, encoder); - - /*set possible crtcs and clones*/ - if (dsi_connector->pipe) { - encoder->possible_crtcs = (1 << 2); - encoder->possible_clones = 0; - } else { - encoder->possible_crtcs = (1 << 0); - encoder->possible_clones = 0; - } - - dsi_connector->base.encoder = &dpi_output->base.base; - - return &dpi_output->base; -} diff --git a/drivers/gpu/drm/gma500/mdfld_dsi_dpi.h b/drivers/gpu/drm/gma500/mdfld_dsi_dpi.h deleted file mode 100644 index 2b40663e16964dba0a038e3a55493bc137034616..0000000000000000000000000000000000000000 --- a/drivers/gpu/drm/gma500/mdfld_dsi_dpi.h +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright © 2010 Intel Corporation - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - * DEALINGS IN THE SOFTWARE. - * - * Authors: - * jim liu - * Jackie Li - */ - -#ifndef __MDFLD_DSI_DPI_H__ -#define __MDFLD_DSI_DPI_H__ - -#include "mdfld_dsi_output.h" -#include "mdfld_output.h" - -struct mdfld_dsi_dpi_timing { - u16 hsync_count; - u16 hbp_count; - u16 hfp_count; - u16 hactive_count; - u16 vsync_count; - u16 vbp_count; - u16 vfp_count; -}; - -struct mdfld_dsi_dpi_output { - struct mdfld_dsi_encoder base; - struct drm_device *dev; - - int panel_on; - int first_boot; - - const struct panel_funcs *p_funcs; -}; - -#define MDFLD_DSI_DPI_OUTPUT(dsi_encoder)\ - container_of(dsi_encoder, struct mdfld_dsi_dpi_output, base) - -/* Export functions */ -extern int mdfld_dsi_dpi_timing_calculation(struct drm_display_mode *mode, - struct mdfld_dsi_dpi_timing *dpi_timing, - int num_lane, int bpp); -extern struct mdfld_dsi_encoder *mdfld_dsi_dpi_init(struct drm_device *dev, - struct mdfld_dsi_connector *dsi_connector, - const struct panel_funcs *p_funcs); - -/* MDFLD DPI helper functions */ -extern void mdfld_dsi_dpi_dpms(struct drm_encoder *encoder, int mode); -extern bool mdfld_dsi_dpi_mode_fixup(struct drm_encoder *encoder, - const struct drm_display_mode *mode, - struct drm_display_mode *adjusted_mode); -extern void mdfld_dsi_dpi_prepare(struct drm_encoder *encoder); -extern void mdfld_dsi_dpi_commit(struct drm_encoder *encoder); -extern void mdfld_dsi_dpi_mode_set(struct drm_encoder *encoder, - struct drm_display_mode *mode, - struct drm_display_mode *adjusted_mode); -extern void mdfld_dsi_dpi_turn_on(struct mdfld_dsi_dpi_output *output, - int pipe); -extern void mdfld_dsi_dpi_controller_init(struct mdfld_dsi_config *dsi_config, - int pipe); -#endif /*__MDFLD_DSI_DPI_H__*/ diff --git a/drivers/gpu/drm/gma500/mdfld_dsi_output.c b/drivers/gpu/drm/gma500/mdfld_dsi_output.c deleted file mode 100644 index 4aab76613bd9e7977a52eb2c38afbb7fd8f002bd..0000000000000000000000000000000000000000 --- a/drivers/gpu/drm/gma500/mdfld_dsi_output.c +++ /dev/null @@ -1,603 +0,0 @@ -/* - * Copyright © 2010 Intel Corporation - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - * DEALINGS IN THE SOFTWARE. - * - * Authors: - * jim liu - * Jackie Li - */ - -#include -#include -#include -#include - -#include - -#include "mdfld_dsi_dpi.h" -#include "mdfld_dsi_output.h" -#include "mdfld_dsi_pkg_sender.h" -#include "mdfld_output.h" -#include "tc35876x-dsi-lvds.h" - -/* get the LABC from command line. */ -static int LABC_control = 1; - -#ifdef MODULE -module_param(LABC_control, int, 0644); -#else - -static int __init parse_LABC_control(char *arg) -{ - /* LABC control can be passed in as a cmdline parameter */ - /* to enable this feature add LABC=1 to cmdline */ - /* to disable this feature add LABC=0 to cmdline */ - if (!arg) - return -EINVAL; - - if (!strcasecmp(arg, "0")) - LABC_control = 0; - else if (!strcasecmp(arg, "1")) - LABC_control = 1; - - return 0; -} -early_param("LABC", parse_LABC_control); -#endif - -/** - * Check and see if the generic control or data buffer is empty and ready. - */ -void mdfld_dsi_gen_fifo_ready(struct drm_device *dev, u32 gen_fifo_stat_reg, - u32 fifo_stat) -{ - u32 GEN_BF_time_out_count; - - /* Check MIPI Adatper command registers */ - for (GEN_BF_time_out_count = 0; - GEN_BF_time_out_count < GEN_FB_TIME_OUT; - GEN_BF_time_out_count++) { - if ((REG_READ(gen_fifo_stat_reg) & fifo_stat) == fifo_stat) - break; - udelay(100); - } - - if (GEN_BF_time_out_count == GEN_FB_TIME_OUT) - DRM_ERROR("mdfld_dsi_gen_fifo_ready, Timeout. gen_fifo_stat_reg = 0x%x.\n", - gen_fifo_stat_reg); -} - -/** - * Manage the DSI MIPI keyboard and display brightness. - * FIXME: this is exported to OSPM code. should work out an specific - * display interface to OSPM. - */ - -void mdfld_dsi_brightness_init(struct mdfld_dsi_config *dsi_config, int pipe) -{ - struct mdfld_dsi_pkg_sender *sender = - mdfld_dsi_get_pkg_sender(dsi_config); - struct drm_device *dev; - struct drm_psb_private *dev_priv; - u32 gen_ctrl_val; - - if (!sender) { - DRM_ERROR("No sender found\n"); - return; - } - - dev = sender->dev; - dev_priv = dev->dev_private; - - /* Set default display backlight value to 85% (0xd8)*/ - mdfld_dsi_send_mcs_short(sender, write_display_brightness, 0xd8, 1, - true); - - /* Set minimum brightness setting of CABC function to 20% (0x33)*/ - mdfld_dsi_send_mcs_short(sender, write_cabc_min_bright, 0x33, 1, true); - - /* Enable backlight or/and LABC */ - gen_ctrl_val = BRIGHT_CNTL_BLOCK_ON | DISPLAY_DIMMING_ON | - BACKLIGHT_ON; - if (LABC_control == 1) - gen_ctrl_val |= DISPLAY_DIMMING_ON | DISPLAY_BRIGHTNESS_AUTO - | GAMMA_AUTO; - - if (LABC_control == 1) - gen_ctrl_val |= AMBIENT_LIGHT_SENSE_ON; - - dev_priv->mipi_ctrl_display = gen_ctrl_val; - - mdfld_dsi_send_mcs_short(sender, write_ctrl_display, (u8)gen_ctrl_val, - 1, true); - - mdfld_dsi_send_mcs_short(sender, write_ctrl_cabc, UI_IMAGE, 1, true); -} - -void mdfld_dsi_brightness_control(struct drm_device *dev, int pipe, int level) -{ - struct mdfld_dsi_pkg_sender *sender; - struct drm_psb_private *dev_priv; - struct mdfld_dsi_config *dsi_config; - u32 gen_ctrl_val = 0; - int p_type = TMD_VID; - - if (!dev || (pipe != 0 && pipe != 2)) { - DRM_ERROR("Invalid parameter\n"); - return; - } - - p_type = mdfld_get_panel_type(dev, 0); - - dev_priv = dev->dev_private; - - if (pipe) - dsi_config = dev_priv->dsi_configs[1]; - else - dsi_config = dev_priv->dsi_configs[0]; - - sender = mdfld_dsi_get_pkg_sender(dsi_config); - - if (!sender) { - DRM_ERROR("No sender found\n"); - return; - } - - gen_ctrl_val = (level * 0xff / MDFLD_DSI_BRIGHTNESS_MAX_LEVEL) & 0xff; - - dev_dbg(sender->dev->dev, "pipe = %d, gen_ctrl_val = %d.\n", - pipe, gen_ctrl_val); - - if (p_type == TMD_VID) { - /* Set display backlight value */ - mdfld_dsi_send_mcs_short(sender, tmd_write_display_brightness, - (u8)gen_ctrl_val, 1, true); - } else { - /* Set display backlight value */ - mdfld_dsi_send_mcs_short(sender, write_display_brightness, - (u8)gen_ctrl_val, 1, true); - - /* Enable backlight control */ - if (level == 0) - gen_ctrl_val = 0; - else - gen_ctrl_val = dev_priv->mipi_ctrl_display; - - mdfld_dsi_send_mcs_short(sender, write_ctrl_display, - (u8)gen_ctrl_val, 1, true); - } -} - -static int mdfld_dsi_get_panel_status(struct mdfld_dsi_config *dsi_config, - u8 dcs, u32 *data, bool hs) -{ - struct mdfld_dsi_pkg_sender *sender - = mdfld_dsi_get_pkg_sender(dsi_config); - - if (!sender || !data) { - DRM_ERROR("Invalid parameter\n"); - return -EINVAL; - } - - return mdfld_dsi_read_mcs(sender, dcs, data, 1, hs); -} - -int mdfld_dsi_get_power_mode(struct mdfld_dsi_config *dsi_config, u32 *mode, - bool hs) -{ - if (!dsi_config || !mode) { - DRM_ERROR("Invalid parameter\n"); - return -EINVAL; - } - - return mdfld_dsi_get_panel_status(dsi_config, 0x0a, mode, hs); -} - -/* - * NOTE: this function was used by OSPM. - * TODO: will be removed later, should work out display interfaces for OSPM - */ -void mdfld_dsi_controller_init(struct mdfld_dsi_config *dsi_config, int pipe) -{ - if (!dsi_config || ((pipe != 0) && (pipe != 2))) { - DRM_ERROR("Invalid parameters\n"); - return; - } - - mdfld_dsi_dpi_controller_init(dsi_config, pipe); -} - -static void mdfld_dsi_connector_save(struct drm_connector *connector) -{ -} - -static void mdfld_dsi_connector_restore(struct drm_connector *connector) -{ -} - -/* FIXME: start using the force parameter */ -static enum drm_connector_status -mdfld_dsi_connector_detect(struct drm_connector *connector, bool force) -{ - struct mdfld_dsi_connector *dsi_connector - = mdfld_dsi_connector(connector); - - dsi_connector->status = connector_status_connected; - - return dsi_connector->status; -} - -static int mdfld_dsi_connector_set_property(struct drm_connector *connector, - struct drm_property *property, - uint64_t value) -{ - struct drm_encoder *encoder = connector->encoder; - - if (!strcmp(property->name, "scaling mode") && encoder) { - struct gma_crtc *gma_crtc = to_gma_crtc(encoder->crtc); - bool centerechange; - uint64_t val; - - if (!gma_crtc) - goto set_prop_error; - - switch (value) { - case DRM_MODE_SCALE_FULLSCREEN: - break; - case DRM_MODE_SCALE_NO_SCALE: - break; - case DRM_MODE_SCALE_ASPECT: - break; - default: - goto set_prop_error; - } - - if (drm_object_property_get_value(&connector->base, property, &val)) - goto set_prop_error; - - if (val == value) - goto set_prop_done; - - if (drm_object_property_set_value(&connector->base, - property, value)) - goto set_prop_error; - - centerechange = (val == DRM_MODE_SCALE_NO_SCALE) || - (value == DRM_MODE_SCALE_NO_SCALE); - - if (gma_crtc->saved_mode.hdisplay != 0 && - gma_crtc->saved_mode.vdisplay != 0) { - if (centerechange) { - if (!drm_crtc_helper_set_mode(encoder->crtc, - &gma_crtc->saved_mode, - encoder->crtc->x, - encoder->crtc->y, - encoder->crtc->primary->fb)) - goto set_prop_error; - } else { - const struct drm_encoder_helper_funcs *funcs = - encoder->helper_private; - funcs->mode_set(encoder, - &gma_crtc->saved_mode, - &gma_crtc->saved_adjusted_mode); - } - } - } else if (!strcmp(property->name, "backlight") && encoder) { - if (drm_object_property_set_value(&connector->base, property, - value)) - goto set_prop_error; - else - gma_backlight_set(encoder->dev, value); - } -set_prop_done: - return 0; -set_prop_error: - return -1; -} - -static void mdfld_dsi_connector_destroy(struct drm_connector *connector) -{ - struct mdfld_dsi_connector *dsi_connector = - mdfld_dsi_connector(connector); - struct mdfld_dsi_pkg_sender *sender; - - if (!dsi_connector) - return; - drm_connector_unregister(connector); - drm_connector_cleanup(connector); - sender = dsi_connector->pkg_sender; - mdfld_dsi_pkg_sender_destroy(sender); - kfree(dsi_connector); -} - -static int mdfld_dsi_connector_get_modes(struct drm_connector *connector) -{ - struct mdfld_dsi_connector *dsi_connector = - mdfld_dsi_connector(connector); - struct mdfld_dsi_config *dsi_config = - mdfld_dsi_get_config(dsi_connector); - struct drm_display_mode *fixed_mode = dsi_config->fixed_mode; - struct drm_display_mode *dup_mode = NULL; - struct drm_device *dev = connector->dev; - - if (fixed_mode) { - dev_dbg(dev->dev, "fixed_mode %dx%d\n", - fixed_mode->hdisplay, fixed_mode->vdisplay); - dup_mode = drm_mode_duplicate(dev, fixed_mode); - drm_mode_probed_add(connector, dup_mode); - return 1; - } - DRM_ERROR("Didn't get any modes!\n"); - return 0; -} - -static enum drm_mode_status mdfld_dsi_connector_mode_valid(struct drm_connector *connector, - struct drm_display_mode *mode) -{ - struct mdfld_dsi_connector *dsi_connector = - mdfld_dsi_connector(connector); - struct mdfld_dsi_config *dsi_config = - mdfld_dsi_get_config(dsi_connector); - struct drm_display_mode *fixed_mode = dsi_config->fixed_mode; - - if (mode->flags & DRM_MODE_FLAG_DBLSCAN) - return MODE_NO_DBLESCAN; - - if (mode->flags & DRM_MODE_FLAG_INTERLACE) - return MODE_NO_INTERLACE; - - /** - * FIXME: current DC has no fitting unit, reject any mode setting - * request - * Will figure out a way to do up-scaling(panel fitting) later. - **/ - if (fixed_mode) { - if (mode->hdisplay != fixed_mode->hdisplay) - return MODE_PANEL; - - if (mode->vdisplay != fixed_mode->vdisplay) - return MODE_PANEL; - } - - return MODE_OK; -} - -static struct drm_encoder *mdfld_dsi_connector_best_encoder( - struct drm_connector *connector) -{ - struct mdfld_dsi_connector *dsi_connector = - mdfld_dsi_connector(connector); - struct mdfld_dsi_config *dsi_config = - mdfld_dsi_get_config(dsi_connector); - return &dsi_config->encoder->base.base; -} - -/*DSI connector funcs*/ -static const struct drm_connector_funcs mdfld_dsi_connector_funcs = { - .dpms = drm_helper_connector_dpms, - .detect = mdfld_dsi_connector_detect, - .fill_modes = drm_helper_probe_single_connector_modes, - .set_property = mdfld_dsi_connector_set_property, - .destroy = mdfld_dsi_connector_destroy, -}; - -/*DSI connector helper funcs*/ -static const struct drm_connector_helper_funcs - mdfld_dsi_connector_helper_funcs = { - .get_modes = mdfld_dsi_connector_get_modes, - .mode_valid = mdfld_dsi_connector_mode_valid, - .best_encoder = mdfld_dsi_connector_best_encoder, -}; - -static int mdfld_dsi_get_default_config(struct drm_device *dev, - struct mdfld_dsi_config *config, int pipe) -{ - if (!dev || !config) { - DRM_ERROR("Invalid parameters"); - return -EINVAL; - } - - config->bpp = 24; - if (mdfld_get_panel_type(dev, pipe) == TC35876X) - config->lane_count = 4; - else - config->lane_count = 2; - config->channel_num = 0; - - if (mdfld_get_panel_type(dev, pipe) == TMD_VID) - config->video_mode = MDFLD_DSI_VIDEO_NON_BURST_MODE_SYNC_PULSE; - else if (mdfld_get_panel_type(dev, pipe) == TC35876X) - config->video_mode = - MDFLD_DSI_VIDEO_NON_BURST_MODE_SYNC_EVENTS; - else - config->video_mode = MDFLD_DSI_VIDEO_BURST_MODE; - - return 0; -} - -int mdfld_dsi_panel_reset(struct drm_device *ddev, int pipe) -{ - struct device *dev = ddev->dev; - struct gpio_desc *gpiod; - - /* - * Raise the GPIO reset line for the corresponding pipe to HIGH, - * this is probably because it is active low so this takes the - * respective pipe out of reset. (We have no code to put it back - * into reset in this driver.) - */ - switch (pipe) { - case 0: - gpiod = gpiod_get(dev, "dsi-pipe0-reset", GPIOD_OUT_HIGH); - if (IS_ERR(gpiod)) - return PTR_ERR(gpiod); - break; - case 2: - gpiod = gpiod_get(dev, "dsi-pipe2-reset", GPIOD_OUT_HIGH); - if (IS_ERR(gpiod)) - return PTR_ERR(gpiod); - break; - default: - DRM_DEV_ERROR(dev, "Invalid output pipe\n"); - return -EINVAL; - } - gpiod_put(gpiod); - - /* Flush posted writes on the device */ - gpiod = gpiod_get(dev, "dsi-pipe0-reset", GPIOD_ASIS); - if (IS_ERR(gpiod)) - return PTR_ERR(gpiod); - gpiod_get_value(gpiod); - gpiod_put(gpiod); - - return 0; -} - -/* - * MIPI output init - * @dev drm device - * @pipe pipe number. 0 or 2 - * @config - * - * Do the initialization of a MIPI output, including create DRM mode objects - * initialization of DSI output on @pipe - */ -void mdfld_dsi_output_init(struct drm_device *dev, - int pipe, - const struct panel_funcs *p_vid_funcs) -{ - struct mdfld_dsi_config *dsi_config; - struct mdfld_dsi_connector *dsi_connector; - struct drm_connector *connector; - struct mdfld_dsi_encoder *encoder; - struct drm_psb_private *dev_priv = dev->dev_private; - struct panel_info dsi_panel_info; - u32 width_mm, height_mm; - - dev_dbg(dev->dev, "init DSI output on pipe %d\n", pipe); - - if (pipe != 0 && pipe != 2) { - DRM_ERROR("Invalid parameter\n"); - return; - } - - /*create a new connector*/ - dsi_connector = kzalloc(sizeof(struct mdfld_dsi_connector), GFP_KERNEL); - if (!dsi_connector) { - DRM_ERROR("No memory"); - return; - } - - dsi_connector->pipe = pipe; - - dsi_config = kzalloc(sizeof(struct mdfld_dsi_config), - GFP_KERNEL); - if (!dsi_config) { - DRM_ERROR("cannot allocate memory for DSI config\n"); - goto dsi_init_err0; - } - mdfld_dsi_get_default_config(dev, dsi_config, pipe); - - dsi_connector->private = dsi_config; - - dsi_config->changed = 1; - dsi_config->dev = dev; - - dsi_config->fixed_mode = p_vid_funcs->get_config_mode(dev); - if (p_vid_funcs->get_panel_info(dev, pipe, &dsi_panel_info)) - goto dsi_init_err0; - - width_mm = dsi_panel_info.width_mm; - height_mm = dsi_panel_info.height_mm; - - dsi_config->mode = dsi_config->fixed_mode; - dsi_config->connector = dsi_connector; - - if (!dsi_config->fixed_mode) { - DRM_ERROR("No panel fixed mode was found\n"); - goto dsi_init_err0; - } - - if (pipe && dev_priv->dsi_configs[0]) { - dsi_config->dvr_ic_inited = 0; - dev_priv->dsi_configs[1] = dsi_config; - } else if (pipe == 0) { - dsi_config->dvr_ic_inited = 1; - dev_priv->dsi_configs[0] = dsi_config; - } else { - DRM_ERROR("Trying to init MIPI1 before MIPI0\n"); - goto dsi_init_err0; - } - - - connector = &dsi_connector->base.base; - dsi_connector->base.save = mdfld_dsi_connector_save; - dsi_connector->base.restore = mdfld_dsi_connector_restore; - - drm_connector_init(dev, connector, &mdfld_dsi_connector_funcs, - DRM_MODE_CONNECTOR_LVDS); - drm_connector_helper_add(connector, &mdfld_dsi_connector_helper_funcs); - - connector->display_info.subpixel_order = SubPixelHorizontalRGB; - connector->display_info.width_mm = width_mm; - connector->display_info.height_mm = height_mm; - connector->interlace_allowed = false; - connector->doublescan_allowed = false; - - /*attach properties*/ - drm_object_attach_property(&connector->base, - dev->mode_config.scaling_mode_property, - DRM_MODE_SCALE_FULLSCREEN); - drm_object_attach_property(&connector->base, - dev_priv->backlight_property, - MDFLD_DSI_BRIGHTNESS_MAX_LEVEL); - - /*init DSI package sender on this output*/ - if (mdfld_dsi_pkg_sender_init(dsi_connector, pipe)) { - DRM_ERROR("Package Sender initialization failed on pipe %d\n", - pipe); - goto dsi_init_err0; - } - - encoder = mdfld_dsi_dpi_init(dev, dsi_connector, p_vid_funcs); - if (!encoder) { - DRM_ERROR("Create DPI encoder failed\n"); - goto dsi_init_err1; - } - encoder->private = dsi_config; - dsi_config->encoder = encoder; - encoder->base.type = (pipe == 0) ? INTEL_OUTPUT_MIPI : - INTEL_OUTPUT_MIPI2; - drm_connector_register(connector); - return; - - /*TODO: add code to destroy outputs on error*/ -dsi_init_err1: - /*destroy sender*/ - mdfld_dsi_pkg_sender_destroy(dsi_connector->pkg_sender); - - drm_connector_cleanup(connector); - - kfree(dsi_config->fixed_mode); - kfree(dsi_config); -dsi_init_err0: - kfree(dsi_connector); -} diff --git a/drivers/gpu/drm/gma500/mdfld_dsi_output.h b/drivers/gpu/drm/gma500/mdfld_dsi_output.h deleted file mode 100644 index 5c0db3c2903ff8762cff3b35c8607b05597aa7d8..0000000000000000000000000000000000000000 --- a/drivers/gpu/drm/gma500/mdfld_dsi_output.h +++ /dev/null @@ -1,377 +0,0 @@ -/* - * Copyright © 2010 Intel Corporation - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - * DEALINGS IN THE SOFTWARE. - * - * Authors: - * jim liu - * Jackie Li - */ - -#ifndef __MDFLD_DSI_OUTPUT_H__ -#define __MDFLD_DSI_OUTPUT_H__ - -#include - -#include - -#include -#include -#include - -#include "mdfld_output.h" -#include "psb_drv.h" -#include "psb_intel_drv.h" -#include "psb_intel_reg.h" - -#define FLD_MASK(start, end) (((1 << ((start) - (end) + 1)) - 1) << (end)) -#define FLD_VAL(val, start, end) (((val) << (end)) & FLD_MASK(start, end)) -#define FLD_GET(val, start, end) (((val) & FLD_MASK(start, end)) >> (end)) -#define FLD_MOD(orig, val, start, end) \ - (((orig) & ~FLD_MASK(start, end)) | FLD_VAL(val, start, end)) - -#define REG_FLD_MOD(reg, val, start, end) \ - REG_WRITE(reg, FLD_MOD(REG_READ(reg), val, start, end)) - -static inline int REGISTER_FLD_WAIT(struct drm_device *dev, u32 reg, - u32 val, int start, int end) -{ - int t = 100000; - - while (FLD_GET(REG_READ(reg), start, end) != val) { - if (--t == 0) - return 1; - } - - return 0; -} - -#define REG_FLD_WAIT(reg, val, start, end) \ - REGISTER_FLD_WAIT(dev, reg, val, start, end) - -#define REG_BIT_WAIT(reg, val, bitnum) \ - REGISTER_FLD_WAIT(dev, reg, val, bitnum, bitnum) - -#define MDFLD_DSI_BRIGHTNESS_MAX_LEVEL 100 - -#ifdef DEBUG -#define CHECK_PIPE(pipe) ({ \ - const typeof(pipe) __pipe = (pipe); \ - BUG_ON(__pipe != 0 && __pipe != 2); \ - __pipe; }) -#else -#define CHECK_PIPE(pipe) (pipe) -#endif - -/* - * Actual MIPIA->MIPIC reg offset is 0x800, value 0x400 is valid for 0 and 2 - */ -#define REG_OFFSET(pipe) (CHECK_PIPE(pipe) * 0x400) - -/* mdfld DSI controller registers */ -#define MIPI_DEVICE_READY_REG(pipe) (0xb000 + REG_OFFSET(pipe)) -#define MIPI_INTR_STAT_REG(pipe) (0xb004 + REG_OFFSET(pipe)) -#define MIPI_INTR_EN_REG(pipe) (0xb008 + REG_OFFSET(pipe)) -#define MIPI_DSI_FUNC_PRG_REG(pipe) (0xb00c + REG_OFFSET(pipe)) -#define MIPI_HS_TX_TIMEOUT_REG(pipe) (0xb010 + REG_OFFSET(pipe)) -#define MIPI_LP_RX_TIMEOUT_REG(pipe) (0xb014 + REG_OFFSET(pipe)) -#define MIPI_TURN_AROUND_TIMEOUT_REG(pipe) (0xb018 + REG_OFFSET(pipe)) -#define MIPI_DEVICE_RESET_TIMER_REG(pipe) (0xb01c + REG_OFFSET(pipe)) -#define MIPI_DPI_RESOLUTION_REG(pipe) (0xb020 + REG_OFFSET(pipe)) -#define MIPI_DBI_FIFO_THROTTLE_REG(pipe) (0xb024 + REG_OFFSET(pipe)) -#define MIPI_HSYNC_COUNT_REG(pipe) (0xb028 + REG_OFFSET(pipe)) -#define MIPI_HBP_COUNT_REG(pipe) (0xb02c + REG_OFFSET(pipe)) -#define MIPI_HFP_COUNT_REG(pipe) (0xb030 + REG_OFFSET(pipe)) -#define MIPI_HACTIVE_COUNT_REG(pipe) (0xb034 + REG_OFFSET(pipe)) -#define MIPI_VSYNC_COUNT_REG(pipe) (0xb038 + REG_OFFSET(pipe)) -#define MIPI_VBP_COUNT_REG(pipe) (0xb03c + REG_OFFSET(pipe)) -#define MIPI_VFP_COUNT_REG(pipe) (0xb040 + REG_OFFSET(pipe)) -#define MIPI_HIGH_LOW_SWITCH_COUNT_REG(pipe) (0xb044 + REG_OFFSET(pipe)) -#define MIPI_DPI_CONTROL_REG(pipe) (0xb048 + REG_OFFSET(pipe)) -#define MIPI_DPI_DATA_REG(pipe) (0xb04c + REG_OFFSET(pipe)) -#define MIPI_INIT_COUNT_REG(pipe) (0xb050 + REG_OFFSET(pipe)) -#define MIPI_MAX_RETURN_PACK_SIZE_REG(pipe) (0xb054 + REG_OFFSET(pipe)) -#define MIPI_VIDEO_MODE_FORMAT_REG(pipe) (0xb058 + REG_OFFSET(pipe)) -#define MIPI_EOT_DISABLE_REG(pipe) (0xb05c + REG_OFFSET(pipe)) -#define MIPI_LP_BYTECLK_REG(pipe) (0xb060 + REG_OFFSET(pipe)) -#define MIPI_LP_GEN_DATA_REG(pipe) (0xb064 + REG_OFFSET(pipe)) -#define MIPI_HS_GEN_DATA_REG(pipe) (0xb068 + REG_OFFSET(pipe)) -#define MIPI_LP_GEN_CTRL_REG(pipe) (0xb06c + REG_OFFSET(pipe)) -#define MIPI_HS_GEN_CTRL_REG(pipe) (0xb070 + REG_OFFSET(pipe)) -#define MIPI_GEN_FIFO_STAT_REG(pipe) (0xb074 + REG_OFFSET(pipe)) -#define MIPI_HS_LS_DBI_ENABLE_REG(pipe) (0xb078 + REG_OFFSET(pipe)) -#define MIPI_DPHY_PARAM_REG(pipe) (0xb080 + REG_OFFSET(pipe)) -#define MIPI_DBI_BW_CTRL_REG(pipe) (0xb084 + REG_OFFSET(pipe)) -#define MIPI_CLK_LANE_SWITCH_TIME_CNT_REG(pipe) (0xb088 + REG_OFFSET(pipe)) - -#define MIPI_CTRL_REG(pipe) (0xb104 + REG_OFFSET(pipe)) -#define MIPI_DATA_ADD_REG(pipe) (0xb108 + REG_OFFSET(pipe)) -#define MIPI_DATA_LEN_REG(pipe) (0xb10c + REG_OFFSET(pipe)) -#define MIPI_CMD_ADD_REG(pipe) (0xb110 + REG_OFFSET(pipe)) -#define MIPI_CMD_LEN_REG(pipe) (0xb114 + REG_OFFSET(pipe)) - -/* non-uniform reg offset */ -#define MIPI_PORT_CONTROL(pipe) (CHECK_PIPE(pipe) ? MIPI_C : MIPI) - -#define DSI_DEVICE_READY (0x1) -#define DSI_POWER_STATE_ULPS_ENTER (0x2 << 1) -#define DSI_POWER_STATE_ULPS_EXIT (0x1 << 1) -#define DSI_POWER_STATE_ULPS_OFFSET (0x1) - - -#define DSI_ONE_DATA_LANE (0x1) -#define DSI_TWO_DATA_LANE (0x2) -#define DSI_THREE_DATA_LANE (0X3) -#define DSI_FOUR_DATA_LANE (0x4) -#define DSI_DPI_VIRT_CHANNEL_OFFSET (0x3) -#define DSI_DBI_VIRT_CHANNEL_OFFSET (0x5) -#define DSI_DPI_COLOR_FORMAT_RGB565 (0x01 << 7) -#define DSI_DPI_COLOR_FORMAT_RGB666 (0x02 << 7) -#define DSI_DPI_COLOR_FORMAT_RGB666_UNPACK (0x03 << 7) -#define DSI_DPI_COLOR_FORMAT_RGB888 (0x04 << 7) -#define DSI_DBI_COLOR_FORMAT_OPTION2 (0x05 << 13) - -#define DSI_INTR_STATE_RXSOTERROR BIT(0) - -#define DSI_INTR_STATE_SPL_PKG_SENT BIT(30) -#define DSI_INTR_STATE_TE BIT(31) - -#define DSI_HS_TX_TIMEOUT_MASK (0xffffff) - -#define DSI_LP_RX_TIMEOUT_MASK (0xffffff) - -#define DSI_TURN_AROUND_TIMEOUT_MASK (0x3f) - -#define DSI_RESET_TIMER_MASK (0xffff) - -#define DSI_DBI_FIFO_WM_HALF (0x0) -#define DSI_DBI_FIFO_WM_QUARTER (0x1) -#define DSI_DBI_FIFO_WM_LOW (0x2) - -#define DSI_DPI_TIMING_MASK (0xffff) - -#define DSI_INIT_TIMER_MASK (0xffff) - -#define DSI_DBI_RETURN_PACK_SIZE_MASK (0x3ff) - -#define DSI_LP_BYTECLK_MASK (0x0ffff) - -#define DSI_HS_CTRL_GEN_SHORT_W0 (0x03) -#define DSI_HS_CTRL_GEN_SHORT_W1 (0x13) -#define DSI_HS_CTRL_GEN_SHORT_W2 (0x23) -#define DSI_HS_CTRL_GEN_R0 (0x04) -#define DSI_HS_CTRL_GEN_R1 (0x14) -#define DSI_HS_CTRL_GEN_R2 (0x24) -#define DSI_HS_CTRL_GEN_LONG_W (0x29) -#define DSI_HS_CTRL_MCS_SHORT_W0 (0x05) -#define DSI_HS_CTRL_MCS_SHORT_W1 (0x15) -#define DSI_HS_CTRL_MCS_R0 (0x06) -#define DSI_HS_CTRL_MCS_LONG_W (0x39) -#define DSI_HS_CTRL_VC_OFFSET (0x06) -#define DSI_HS_CTRL_WC_OFFSET (0x08) - -#define DSI_FIFO_GEN_HS_DATA_FULL BIT(0) -#define DSI_FIFO_GEN_HS_DATA_HALF_EMPTY BIT(1) -#define DSI_FIFO_GEN_HS_DATA_EMPTY BIT(2) -#define DSI_FIFO_GEN_LP_DATA_FULL BIT(8) -#define DSI_FIFO_GEN_LP_DATA_HALF_EMPTY BIT(9) -#define DSI_FIFO_GEN_LP_DATA_EMPTY BIT(10) -#define DSI_FIFO_GEN_HS_CTRL_FULL BIT(16) -#define DSI_FIFO_GEN_HS_CTRL_HALF_EMPTY BIT(17) -#define DSI_FIFO_GEN_HS_CTRL_EMPTY BIT(18) -#define DSI_FIFO_GEN_LP_CTRL_FULL BIT(24) -#define DSI_FIFO_GEN_LP_CTRL_HALF_EMPTY BIT(25) -#define DSI_FIFO_GEN_LP_CTRL_EMPTY BIT(26) -#define DSI_FIFO_DBI_EMPTY BIT(27) -#define DSI_FIFO_DPI_EMPTY BIT(28) - -#define DSI_DBI_HS_LP_SWITCH_MASK (0x1) - -#define DSI_HS_LP_SWITCH_COUNTER_OFFSET (0x0) -#define DSI_LP_HS_SWITCH_COUNTER_OFFSET (0x16) - -#define DSI_DPI_CTRL_HS_SHUTDOWN (0x00000001) -#define DSI_DPI_CTRL_HS_TURN_ON (0x00000002) - -/*dsi power modes*/ -#define DSI_POWER_MODE_DISPLAY_ON BIT(2) -#define DSI_POWER_MODE_NORMAL_ON BIT(3) -#define DSI_POWER_MODE_SLEEP_OUT BIT(4) -#define DSI_POWER_MODE_PARTIAL_ON BIT(5) -#define DSI_POWER_MODE_IDLE_ON BIT(6) - -enum { - MDFLD_DSI_VIDEO_NON_BURST_MODE_SYNC_PULSE = 1, - MDFLD_DSI_VIDEO_NON_BURST_MODE_SYNC_EVENTS = 2, - MDFLD_DSI_VIDEO_BURST_MODE = 3, -}; - -#define DSI_DPI_COMPLETE_LAST_LINE BIT(2) -#define DSI_DPI_DISABLE_BTA BIT(3) - -struct mdfld_dsi_connector { - struct gma_connector base; - - int pipe; - void *private; - void *pkg_sender; - - /* Connection status */ - enum drm_connector_status status; -}; - -struct mdfld_dsi_encoder { - struct gma_encoder base; - void *private; -}; - -/* - * DSI config, consists of one DSI connector, two DSI encoders. - * DRM will pick up on DSI encoder basing on differents configs. - */ -struct mdfld_dsi_config { - struct drm_device *dev; - struct drm_display_mode *fixed_mode; - struct drm_display_mode *mode; - - struct mdfld_dsi_connector *connector; - struct mdfld_dsi_encoder *encoder; - - int changed; - - int bpp; - int lane_count; - /*Virtual channel number for this encoder*/ - int channel_num; - /*video mode configure*/ - int video_mode; - - int dvr_ic_inited; -}; - -static inline struct mdfld_dsi_connector *mdfld_dsi_connector( - struct drm_connector *connector) -{ - struct gma_connector *gma_connector; - - gma_connector = to_gma_connector(connector); - - return container_of(gma_connector, struct mdfld_dsi_connector, base); -} - -static inline struct mdfld_dsi_encoder *mdfld_dsi_encoder( - struct drm_encoder *encoder) -{ - struct gma_encoder *gma_encoder; - - gma_encoder = to_gma_encoder(encoder); - - return container_of(gma_encoder, struct mdfld_dsi_encoder, base); -} - -static inline struct mdfld_dsi_config * - mdfld_dsi_get_config(struct mdfld_dsi_connector *connector) -{ - if (!connector) - return NULL; - return (struct mdfld_dsi_config *)connector->private; -} - -static inline void *mdfld_dsi_get_pkg_sender(struct mdfld_dsi_config *config) -{ - struct mdfld_dsi_connector *dsi_connector; - - if (!config) - return NULL; - - dsi_connector = config->connector; - - if (!dsi_connector) - return NULL; - - return dsi_connector->pkg_sender; -} - -static inline struct mdfld_dsi_config * - mdfld_dsi_encoder_get_config(struct mdfld_dsi_encoder *encoder) -{ - if (!encoder) - return NULL; - return (struct mdfld_dsi_config *)encoder->private; -} - -static inline struct mdfld_dsi_connector * - mdfld_dsi_encoder_get_connector(struct mdfld_dsi_encoder *encoder) -{ - struct mdfld_dsi_config *config; - - if (!encoder) - return NULL; - - config = mdfld_dsi_encoder_get_config(encoder); - if (!config) - return NULL; - - return config->connector; -} - -static inline void *mdfld_dsi_encoder_get_pkg_sender( - struct mdfld_dsi_encoder *encoder) -{ - struct mdfld_dsi_config *dsi_config; - - dsi_config = mdfld_dsi_encoder_get_config(encoder); - if (!dsi_config) - return NULL; - - return mdfld_dsi_get_pkg_sender(dsi_config); -} - -static inline int mdfld_dsi_encoder_get_pipe(struct mdfld_dsi_encoder *encoder) -{ - struct mdfld_dsi_connector *connector; - - if (!encoder) - return -1; - - connector = mdfld_dsi_encoder_get_connector(encoder); - if (!connector) - return -1; - return connector->pipe; -} - -/* Export functions */ -extern void mdfld_dsi_gen_fifo_ready(struct drm_device *dev, - u32 gen_fifo_stat_reg, u32 fifo_stat); -extern void mdfld_dsi_brightness_init(struct mdfld_dsi_config *dsi_config, - int pipe); -extern void mdfld_dsi_brightness_control(struct drm_device *dev, int pipe, - int level); -extern void mdfld_dsi_output_init(struct drm_device *dev, - int pipe, - const struct panel_funcs *p_vid_funcs); -extern void mdfld_dsi_controller_init(struct mdfld_dsi_config *dsi_config, - int pipe); - -extern int mdfld_dsi_get_power_mode(struct mdfld_dsi_config *dsi_config, - u32 *mode, bool hs); -extern int mdfld_dsi_panel_reset(struct drm_device *dev, int pipe); - -#endif /*__MDFLD_DSI_OUTPUT_H__*/ diff --git a/drivers/gpu/drm/gma500/mdfld_dsi_pkg_sender.c b/drivers/gpu/drm/gma500/mdfld_dsi_pkg_sender.c deleted file mode 100644 index 6e0de83e9f7d247162d1e7e9b0e11c2abc4d7ff8..0000000000000000000000000000000000000000 --- a/drivers/gpu/drm/gma500/mdfld_dsi_pkg_sender.c +++ /dev/null @@ -1,679 +0,0 @@ -/* - * Copyright © 2010 Intel Corporation - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - * DEALINGS IN THE SOFTWARE. - * - * Authors: - * Jackie Li - */ - -#include -#include - -#include