diff options
205 files changed, 10084 insertions, 5776 deletions
diff --git a/Documentation/ABI/testing/debugfs-driver-qat_telemetry b/Documentation/ABI/testing/debugfs-driver-qat_telemetry new file mode 100644 index 000000000000..eacee2072088 --- /dev/null +++ b/Documentation/ABI/testing/debugfs-driver-qat_telemetry @@ -0,0 +1,228 @@ +What: /sys/kernel/debug/qat_<device>_<BDF>/telemetry/control +Date: March 2024 +KernelVersion: 6.8 +Contact: qat-linux@intel.com +Description: (RW) Enables/disables the reporting of telemetry metrics. + + Allowed values to write: + ======================== + * 0: disable telemetry + * 1: enable telemetry + * 2, 3, 4: enable telemetry and calculate minimum, maximum + and average for each counter over 2, 3 or 4 samples + + Returned values: + ================ + * 1-4: telemetry is enabled and running + * 0: telemetry is disabled + + Example. + + Writing '3' to this file starts the collection of + telemetry metrics. Samples are collected every second and + stored in a circular buffer of size 3. These values are then + used to calculate the minimum, maximum and average for each + counter. After enabling, counters can be retrieved through + the ``device_data`` file:: + + echo 3 > /sys/kernel/debug/qat_4xxx_0000:6b:00.0/telemetry/control + + Writing '0' to this file stops the collection of telemetry + metrics:: + + echo 0 > /sys/kernel/debug/qat_4xxx_0000:6b:00.0/telemetry/control + + This attribute is only available for qat_4xxx devices. + +What: /sys/kernel/debug/qat_<device>_<BDF>/telemetry/device_data +Date: March 2024 +KernelVersion: 6.8 +Contact: qat-linux@intel.com +Description: (RO) Reports device telemetry counters. + Reads report metrics about performance and utilization of + a QAT device: + + ======================= ======================================== + Field Description + ======================= ======================================== + sample_cnt number of acquisitions of telemetry data + from the device. Reads are performed + every 1000 ms. + pci_trans_cnt number of PCIe partial transactions + max_rd_lat maximum logged read latency [ns] (could + be any read operation) + rd_lat_acc_avg average read latency [ns] + max_gp_lat max get to put latency [ns] (only takes + samples for AE0) + gp_lat_acc_avg average get to put latency [ns] + bw_in PCIe, write bandwidth [Mbps] + bw_out PCIe, read bandwidth [Mbps] + at_page_req_lat_avg Address Translator(AT), average page + request latency [ns] + at_trans_lat_avg AT, average page translation latency [ns] + at_max_tlb_used AT, maximum uTLB used + util_cpr<N> utilization of Compression slice N [%] + exec_cpr<N> execution count of Compression slice N + util_xlt<N> utilization of Translator slice N [%] + exec_xlt<N> execution count of Translator slice N + util_dcpr<N> utilization of Decompression slice N [%] + exec_dcpr<N> execution count of Decompression slice N + util_pke<N> utilization of PKE N [%] + exec_pke<N> execution count of PKE N + util_ucs<N> utilization of UCS slice N [%] + exec_ucs<N> execution count of UCS slice N + util_wat<N> utilization of Wireless Authentication + slice N [%] + exec_wat<N> execution count of Wireless Authentication + slice N + util_wcp<N> utilization of Wireless Cipher slice N [%] + exec_wcp<N> execution count of Wireless Cipher slice N + util_cph<N> utilization of Cipher slice N [%] + exec_cph<N> execution count of Cipher slice N + util_ath<N> utilization of Authentication slice N [%] + exec_ath<N> execution count of Authentication slice N + ======================= ======================================== + + The telemetry report file can be read with the following command:: + + cat /sys/kernel/debug/qat_4xxx_0000:6b:00.0/telemetry/device_data + + If ``control`` is set to 1, only the current values of the + counters are displayed:: + + <counter_name> <current> + + If ``control`` is 2, 3 or 4, counters are displayed in the + following format:: + + <counter_name> <current> <min> <max> <avg> + + If a device lacks of a specific accelerator, the corresponding + attribute is not reported. + + This attribute is only available for qat_4xxx devices. + +What: /sys/kernel/debug/qat_<device>_<BDF>/telemetry/rp_<A/B/C/D>_data +Date: March 2024 +KernelVersion: 6.8 +Contact: qat-linux@intel.com +Description: (RW) Selects up to 4 Ring Pairs (RP) to monitor, one per file, + and report telemetry counters related to each. + + Allowed values to write: + ======================== + * 0 to ``<num_rps - 1>``: + Ring pair to be monitored. The value of ``num_rps`` can be + retrieved through ``/sys/bus/pci/devices/<BDF>/qat/num_rps``. + See Documentation/ABI/testing/sysfs-driver-qat. + + Reads report metrics about performance and utilization of + the selected RP: + + ======================= ======================================== + Field Description + ======================= ======================================== + sample_cnt number of acquisitions of telemetry data + from the device. Reads are performed + every 1000 ms + rp_num RP number associated with slot <A/B/C/D> + service_type service associated to the RP + pci_trans_cnt number of PCIe partial transactions + gp_lat_acc_avg average get to put latency [ns] + bw_in PCIe, write bandwidth [Mbps] + bw_out PCIe, read bandwidth [Mbps] + at_glob_devtlb_hit Message descriptor DevTLB hit rate + at_glob_devtlb_miss Message descriptor DevTLB miss rate + tl_at_payld_devtlb_hit Payload DevTLB hit rate + tl_at_payld_devtlb_miss Payload DevTLB miss rate + ======================= ======================================== + + Example. + + Writing the value '32' to the file ``rp_C_data`` starts the + collection of telemetry metrics for ring pair 32:: + + echo 32 > /sys/kernel/debug/qat_4xxx_0000:6b:00.0/telemetry/rp_C_data + + Once a ring pair is selected, statistics can be read accessing + the file:: + + cat /sys/kernel/debug/qat_4xxx_0000:6b:00.0/telemetry/rp_C_data + + If ``control`` is set to 1, only the current values of the + counters are displayed:: + + <counter_name> <current> + + If ``control`` is 2, 3 or 4, counters are displayed in the + following format:: + + <counter_name> <current> <min> <max> <avg> + + + On QAT GEN4 devices there are 64 RPs on a PF, so the allowed + values are 0..63. This number is absolute to the device. + If Virtual Functions (VF) are used, the ring pair number can + be derived from the Bus, Device, Function of the VF: + + ============ ====== ====== ====== ====== + PCI BDF/VF RP0 RP1 RP2 RP3 + ============ ====== ====== ====== ====== + 0000:6b:0.1 RP 0 RP 1 RP 2 RP 3 + 0000:6b:0.2 RP 4 RP 5 RP 6 RP 7 + 0000:6b:0.3 RP 8 RP 9 RP 10 RP 11 + 0000:6b:0.4 RP 12 RP 13 RP 14 RP 15 + 0000:6b:0.5 RP 16 RP 17 RP 18 RP 19 + 0000:6b:0.6 RP 20 RP 21 RP 22 RP 23 + 0000:6b:0.7 RP 24 RP 25 RP 26 RP 27 + 0000:6b:1.0 RP 28 RP 29 RP 30 RP 31 + 0000:6b:1.1 RP 32 RP 33 RP 34 RP 35 + 0000:6b:1.2 RP 36 RP 37 RP 38 RP 39 + 0000:6b:1.3 RP 40 RP 41 RP 42 RP 43 + 0000:6b:1.4 RP 44 RP 45 RP 46 RP 47 + 0000:6b:1.5 RP 48 RP 49 RP 50 RP 51 + 0000:6b:1.6 RP 52 RP 53 RP 54 RP 55 + 0000:6b:1.7 RP 56 RP 57 RP 58 RP 59 + 0000:6b:2.0 RP 60 RP 61 RP 62 RP 63 + ============ ====== ====== ====== ====== + + The mapping is only valid for the BDFs of VFs on the host. + + + The service provided on a ring-pair varies depending on the + configuration. The configuration for a given device can be + queried and set using ``cfg_services``. + See Documentation/ABI/testing/sysfs-driver-qat for details. + + The following table reports how ring pairs are mapped to VFs + on the PF 0000:6b:0.0 configured for `sym;asym` or `asym;sym`: + + =========== ============ =========== ============ =========== + PCI BDF/VF RP0/service RP1/service RP2/service RP3/service + =========== ============ =========== ============ =========== + 0000:6b:0.1 RP 0 asym RP 1 sym RP 2 asym RP 3 sym + 0000:6b:0.2 RP 4 asym RP 5 sym RP 6 asym RP 7 sym + 0000:6b:0.3 RP 8 asym RP 9 sym RP10 asym RP11 sym + ... ... ... ... ... + =========== ============ =========== ============ =========== + + All VFs follow the same pattern. + + + The following table reports how ring pairs are mapped to VFs on + the PF 0000:6b:0.0 configured for `dc`: + + =========== ============ =========== ============ =========== + PCI BDF/VF RP0/service RP1/service RP2/service RP3/service + =========== ============ =========== ============ =========== + 0000:6b:0.1 RP 0 dc RP 1 dc RP 2 dc RP 3 dc + 0000:6b:0.2 RP 4 dc RP 5 dc RP 6 dc RP 7 dc + 0000:6b:0.3 RP 8 dc RP 9 dc RP10 dc RP11 dc + ... ... ... ... ... + =========== ============ =========== ============ =========== + + The mapping of a RP to a service can be retrieved using + ``rp2srv`` from sysfs. + See Documentation/ABI/testing/sysfs-driver-qat for details. + + This attribute is only available for qat_4xxx devices. diff --git a/Documentation/ABI/testing/debugfs-hisi-hpre b/Documentation/ABI/testing/debugfs-hisi-hpre index 82abf92df429..8e8de49c5cc6 100644 --- a/Documentation/ABI/testing/debugfs-hisi-hpre +++ b/Documentation/ABI/testing/debugfs-hisi-hpre @@ -101,7 +101,7 @@ What: /sys/kernel/debug/hisi_hpre/<bdf>/qm/status Date: Apr 2020 Contact: linux-crypto@vger.kernel.org Description: Dump the status of the QM. - Four states: initiated, started, stopped and closed. + Two states: work, stop. Available for both PF and VF, and take no other effect on HPRE. What: /sys/kernel/debug/hisi_hpre/<bdf>/qm/diff_regs diff --git a/Documentation/ABI/testing/debugfs-hisi-sec b/Documentation/ABI/testing/debugfs-hisi-sec index 93c530d1bf0f..deeefe2c735e 100644 --- a/Documentation/ABI/testing/debugfs-hisi-sec +++ b/Documentation/ABI/testing/debugfs-hisi-sec @@ -81,7 +81,7 @@ What: /sys/kernel/debug/hisi_sec2/<bdf>/qm/status Date: Apr 2020 Contact: linux-crypto@vger.kernel.org Description: Dump the status of the QM. - Four states: initiated, started, stopped and closed. + Two states: work, stop. Available for both PF and VF, and take no other effect on SEC. What: /sys/kernel/debug/hisi_sec2/<bdf>/qm/diff_regs diff --git a/Documentation/ABI/testing/debugfs-hisi-zip b/Documentation/ABI/testing/debugfs-hisi-zip index fd3f314cf8d1..593714afaed2 100644 --- a/Documentation/ABI/testing/debugfs-hisi-zip +++ b/Documentation/ABI/testing/debugfs-hisi-zip @@ -94,7 +94,7 @@ What: /sys/kernel/debug/hisi_zip/<bdf>/qm/status Date: Apr 2020 Contact: linux-crypto@vger.kernel.org Description: Dump the status of the QM. - Four states: initiated, started, stopped and closed. + Two states: work, stop. Available for both PF and VF, and take no other effect on ZIP. What: /sys/kernel/debug/hisi_zip/<bdf>/qm/diff_regs diff --git a/Documentation/crypto/device_drivers/index.rst b/Documentation/crypto/device_drivers/index.rst new file mode 100644 index 000000000000..c81d311ac61b --- /dev/null +++ b/Documentation/crypto/device_drivers/index.rst @@ -0,0 +1,9 @@ +.. SPDX-License-Identifier: GPL-2.0 + +Hardware Device Driver Specific Documentation +--------------------------------------------- + +.. toctree:: + :maxdepth: 1 + + octeontx2 diff --git a/Documentation/crypto/device_drivers/octeontx2.rst b/Documentation/crypto/device_drivers/octeontx2.rst new file mode 100644 index 000000000000..7e469b173ac8 --- /dev/null +++ b/Documentation/crypto/device_drivers/octeontx2.rst @@ -0,0 +1,25 @@ +.. SPDX-License-Identifier: GPL-2.0 + +========================= +octeontx2 devlink support +========================= + +This document describes the devlink features implemented by the ``octeontx2 CPT`` +device drivers. + +Parameters +========== + +The ``octeontx2`` driver implements the following driver-specific parameters. + +.. list-table:: Driver-specific parameters implemented + :widths: 5 5 5 85 + + * - Name + - Type + - Mode + - Description + * - ``t106_mode`` + - u8 + - runtime + - Used to configure CN10KA B0/CN10KB CPT to work as CN10KA A0/A1. diff --git a/Documentation/crypto/index.rst b/Documentation/crypto/index.rst index da5d5ad2bdf3..945ca1505ad9 100644 --- a/Documentation/crypto/index.rst +++ b/Documentation/crypto/index.rst @@ -28,3 +28,4 @@ for cryptographic use cases, as well as programming examples. api api-samples descore-readme + device_drivers/index diff --git a/Documentation/devicetree/bindings/crypto/inside-secure,safexcel.yaml b/Documentation/devicetree/bindings/crypto/inside-secure,safexcel.yaml new file mode 100644 index 000000000000..ef07258d16c1 --- /dev/null +++ b/Documentation/devicetree/bindings/crypto/inside-secure,safexcel.yaml @@ -0,0 +1,86 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/crypto/inside-secure,safexcel.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Inside Secure SafeXcel cryptographic engine + +maintainers: + - Antoine Tenart <atenart@kernel.org> + +properties: + compatible: + oneOf: + - const: inside-secure,safexcel-eip197b + - const: inside-secure,safexcel-eip197d + - const: inside-secure,safexcel-eip97ies + - const: inside-secure,safexcel-eip197 + description: Equivalent of inside-secure,safexcel-eip197b + deprecated: true + - const: inside-secure,safexcel-eip97 + description: Equivalent of inside-secure,safexcel-eip97ies + deprecated: true + + reg: + maxItems: 1 + + interrupts: + maxItems: 6 + + interrupt-names: + items: + - const: ring0 + - const: ring1 + - const: ring2 + - const: ring3 + - const: eip + - const: mem + + clocks: + minItems: 1 + maxItems: 2 + + clock-names: + minItems: 1 + items: + - const: core + - const: reg + +required: + - reg + - interrupts + - interrupt-names + +allOf: + - if: + properties: + clocks: + minItems: 2 + then: + properties: + clock-names: + minItems: 2 + required: + - clock-names + +additionalProperties: false + +examples: + - | + #include <dt-bindings/interrupt-controller/arm-gic.h> + #include <dt-bindings/interrupt-controller/irq.h> + + crypto@800000 { + compatible = "inside-secure,safexcel-eip197b"; + reg = <0x800000 0x200000>; + interrupts = <GIC_SPI 54 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 55 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 56 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 57 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 58 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 34 IRQ_TYPE_LEVEL_HIGH>; + interrupt-names = "ring0", "ring1", "ring2", "ring3", "eip", "mem"; + clocks = <&cpm_syscon0 1 26>; + clock-names = "core"; + }; diff --git a/Documentation/devicetree/bindings/crypto/inside-secure-safexcel.txt b/Documentation/devicetree/bindings/crypto/inside-secure-safexcel.txt deleted file mode 100644 index 3bbf144c9988..000000000000 --- a/Documentation/devicetree/bindings/crypto/inside-secure-safexcel.txt +++ /dev/null @@ -1,40 +0,0 @@ -Inside Secure SafeXcel cryptographic engine - -Required properties: -- compatible: Should be "inside-secure,safexcel-eip197b", - "inside-secure,safexcel-eip197d" or - "inside-secure,safexcel-eip97ies". -- reg: Base physical address of the engine and length of memory mapped region. -- interrupts: Interrupt numbers for the rings and engine. -- interrupt-names: Should be "ring0", "ring1", "ring2", "ring3", "eip", "mem". - -Optional properties: -- clocks: Reference to the crypto engine clocks, the second clock is - needed for the Armada 7K/8K SoCs. -- clock-names: mandatory if there is a second clock, in this case the - name must be "core" for the first clock and "reg" for - the second one. - -Backward compatibility: -Two compatibles are kept for backward compatibility, but shouldn't be used for -new submissions: -- "inside-secure,safexcel-eip197" is equivalent to - "inside-secure,safexcel-eip197b". -- "inside-secure,safexcel-eip97" is equivalent to - "inside-secure,safexcel-eip97ies". - -Example: - - crypto: crypto@800000 { - compatible = "inside-secure,safexcel-eip197b"; - reg = <0x800000 0x200000>; - interrupts = <GIC_SPI 34 IRQ_TYPE_LEVEL_HIGH>, - <GIC_SPI 54 IRQ_TYPE_LEVEL_HIGH>, - <GIC_SPI 55 IRQ_TYPE_LEVEL_HIGH>, - <GIC_SPI 56 IRQ_TYPE_LEVEL_HIGH>, - <GIC_SPI 57 IRQ_TYPE_LEVEL_HIGH>, - <GIC_SPI 58 IRQ_TYPE_LEVEL_HIGH>; - interrupt-names = "mem", "ring0", "ring1", "ring2", "ring3", - "eip"; - clocks = <&cpm_syscon0 1 26>; - }; diff --git a/Documentation/devicetree/bindings/crypto/qcom,inline-crypto-engine.yaml b/Documentation/devicetree/bindings/crypto/qcom,inline-crypto-engine.yaml index ca4f7d1cefaa..09e43157cc71 100644 --- a/Documentation/devicetree/bindings/crypto/qcom,inline-crypto-engine.yaml +++ b/Documentation/devicetree/bindings/crypto/qcom,inline-crypto-engine.yaml @@ -16,6 +16,7 @@ properties: - qcom,sa8775p-inline-crypto-engine - qcom,sm8450-inline-crypto-engine - qcom,sm8550-inline-crypto-engine + - qcom,sm8650-inline-crypto-engine - const: qcom,inline-crypto-engine reg: diff --git a/Documentation/devicetree/bindings/crypto/qcom,prng.yaml b/Documentation/devicetree/bindings/crypto/qcom,prng.yaml index 13070db0f70c..89c88004b41b 100644 --- a/Documentation/devicetree/bindings/crypto/qcom,prng.yaml +++ b/Documentation/devicetree/bindings/crypto/qcom,prng.yaml @@ -21,6 +21,7 @@ properties: - qcom,sc7280-trng - qcom,sm8450-trng - qcom,sm8550-trng + - qcom,sm8650-trng - const: qcom,trng reg: diff --git a/Documentation/devicetree/bindings/crypto/qcom-qce.yaml b/Documentation/devicetree/bindings/crypto/qcom-qce.yaml index 8e665d910e6e..a48bd381063a 100644 --- a/Documentation/devicetree/bindings/crypto/qcom-qce.yaml +++ b/Documentation/devicetree/bindings/crypto/qcom-qce.yaml @@ -44,10 +44,12 @@ properties: - items: - enum: + - qcom,sc7280-qce - qcom,sm8250-qce - qcom,sm8350-qce - qcom,sm8450-qce - qcom,sm8550-qce + - qcom,sm8650-qce - const: qcom,sm8150-qce - const: qcom,qce @@ -96,6 +98,7 @@ allOf: - qcom,crypto-v5.4 - qcom,ipq6018-qce - qcom,ipq8074-qce + - qcom,ipq9574-qce - qcom,msm8996-qce - qcom,sdm845-qce then: @@ -129,6 +132,17 @@ allOf: - clocks - clock-names + - if: + properties: + compatible: + contains: + enum: + - qcom,sm8150-qce + then: + properties: + clocks: false + clock-names: false + required: - compatible - reg diff --git a/Documentation/devicetree/bindings/rng/starfive,jh7110-trng.yaml b/Documentation/devicetree/bindings/rng/starfive,jh7110-trng.yaml index 2b76ce25acc4..4639247e9e51 100644 --- a/Documentation/devicetree/bindings/rng/starfive,jh7110-trng.yaml +++ b/Documentation/devicetree/bindings/rng/starfive,jh7110-trng.yaml @@ -11,7 +11,11 @@ maintainers: properties: compatible: - const: starfive,jh7110-trng + oneOf: + - items: + - const: starfive,jh8100-trng + - const: starfive,jh7110-trng + - const: starfive,jh7110-trng reg: maxItems: 1 diff --git a/Documentation/driver-api/crypto/iaa/iaa-crypto.rst b/Documentation/driver-api/crypto/iaa/iaa-crypto.rst new file mode 100644 index 000000000000..de587cf9cbed --- /dev/null +++ b/Documentation/driver-api/crypto/iaa/iaa-crypto.rst @@ -0,0 +1,824 @@ +.. SPDX-License-Identifier: GPL-2.0 + +========================================= +IAA Compression Accelerator Crypto Driver +========================================= + +Tom Zanussi <tom.zanussi@linux.intel.com> + +The IAA crypto driver supports compression/decompression compatible +with the DEFLATE compression standard described in RFC 1951, which is +the compression/decompression algorithm exported by this module. + +The IAA hardware spec can be found here: + + https://cdrdv2.intel.com/v1/dl/getContent/721858 + +The iaa_crypto driver is designed to work as a layer underneath +higher-level compression devices such as zswap. + +Users can select IAA compress/decompress acceleration by specifying +one of the supported IAA compression algorithms in whatever facility +allows compression algorithms to be selected. + +For example, a zswap device can select the IAA 'fixed' mode +represented by selecting the 'deflate-iaa' crypto compression +algorithm:: + + # echo deflate-iaa > /sys/module/zswap/parameters/compressor + +This will tell zswap to use the IAA 'fixed' compression mode for all +compresses and decompresses. + +Currently, there is only one compression modes available, 'fixed' +mode. + +The 'fixed' compression mode implements the compression scheme +specified by RFC 1951 and is given the crypto algorithm name +'deflate-iaa'. (Because the IAA hardware has a 4k history-window +limitation, only buffers <= 4k, or that have been compressed using a +<= 4k history window, are technically compliant with the deflate spec, +which allows for a window of up to 32k. Because of this limitation, +the IAA fixed mode deflate algorithm is given its own algorithm name +rather than simply 'deflate'). + + +Config options and other setup +============================== + +The IAA crypto driver is available via menuconfig using the following +path:: + + Cryptographic API -> Hardware crypto devices -> Support for Intel(R) IAA Compression Accelerator + +In the configuration file the option called CONFIG_CRYPTO_DEV_IAA_CRYPTO. + +The IAA crypto driver also supports statistics, which are available +via menuconfig using the following path:: + + Cryptographic API -> Hardware crypto devices -> Support for Intel(R) IAA Compression -> Enable Intel(R) IAA Compression Accelerator Statistics + +In the configuration file the option called CONFIG_CRYPTO_DEV_IAA_CRYPTO_STATS. + +The following config options should also be enabled:: + + CONFIG_IRQ_REMAP=y + CONFIG_INTEL_IOMMU=y + CONFIG_INTEL_IOMMU_SVM=y + CONFIG_PCI_ATS=y + CONFIG_PCI_PRI=y + CONFIG_PCI_PASID=y + CONFIG_INTEL_IDXD=m + CONFIG_INTEL_IDXD_SVM=y + +IAA is one of the first Intel accelerator IPs that can work in +conjunction with the Intel IOMMU. There are multiple modes that exist +for testing. Based on IOMMU configuration, there are 3 modes:: + + - Scalable + - Legacy + - No IOMMU + + +Scalable mode +------------- + +Scalable mode supports Shared Virtual Memory (SVM or SVA). It is +entered when using the kernel boot commandline:: + + intel_iommu=on,sm_on + +with VT-d turned on in BIOS. + +With scalable mode, both shared and dedicated workqueues are available +for use. + +For scalable mode, the following BIOS settings should be enabled:: + + Socket Configuration > IIO Configuration > Intel VT for Directed I/O (VT-d) > Intel VT for Directed I/O + + Socket Configuration > IIO Configuration > PCIe ENQCMD > ENQCMDS + + +Legacy mode +----------- + +Legacy mode is entered when using the kernel boot commandline:: + + intel_iommu=off + +or VT-d is not turned on in BIOS. + +If you have booted into Linux and not sure if VT-d is on, do a "dmesg +| grep -i dmar". If you don't see a number of DMAR devices enumerated, +most likely VT-d is not on. + +With legacy mode, only dedicated workqueues are available for use. + + +No IOMMU mode +------------- + +No IOMMU mode is entered when using the kernel boot commandline:: + + iommu=off. + +With no IOMMU mode, only dedicated workqueues are available for use. + + +Usage +===== + +accel-config +------------ + +When loaded, the iaa_crypto driver automatically creates a default +configuration and enables it, and assigns default driver attributes. +If a different configuration or set of driver attributes is required, +the user must first disable the IAA devices and workqueues, reset the +configuration, and then re-register the deflate-iaa algorithm with the +crypto subsystem by removing and reinserting the iaa_crypto module. + +The :ref:`iaa_disable_script` in the 'Use Cases' +section below can be used to disable the default configuration. + +See :ref:`iaa_default_config` below for details of the default +configuration. + +More likely than not, however, and because of the complexity and +configurability of the accelerator devices, the user will want to +configure the device and manually enable the desired devices and +workqueues. + +The userspace tool to help doing that is called accel-config. Using +accel-config to configure device or loading a previously saved config +is highly recommended. The device can be controlled via sysfs +directly but comes with the warning that you should do this ONLY if +you know exactly what you are doing. The following sections will not +cover the sysfs interface but assumes you will be using accel-config. + +The :ref:`iaa_sysfs_config` section in the appendix below can be +consulted for the sysfs interface details if interested. + +The accel-config tool along with instructions for building it can be +found here: + + https://github.com/intel/idxd-config/#readme + +Typical usage +------------- + +In order for the iaa_crypto module to actually do any +compression/decompression work on behalf of a facility, one or more +IAA workqueues need to be bound to the iaa_crypto driver. + +For instance, here's an example of configuring an IAA workqueue and +binding it to the iaa_crypto driver (note that device names are +specified as 'iax' rather than 'iaa' - this is because upstream still +has the old 'iax' device naming in place) :: + + # configure wq1.0 + + accel-config config-wq --group-id=0 --mode=dedicated --type=kernel --name="iaa_crypto" --device_name="crypto" iax1/wq1.0 + + # enable IAA device iax1 + + accel-config enable-device iax1 + + # enable wq1.0 on IAX device iax1 + + accel-config enable-wq iax1/wq1.0 + +Whenever a new workqueue is bound to or unbound from the iaa_crypto +driver, the available workqueues are 'rebalanced' such that work +submitted from a particular CPU is given to the most appropriate +workqueue available. Current best practice is to configure and bind +at least one workqueue for each IAA device, but as long as there is at +least one workqueue configured and bound to any IAA device in the +system, the iaa_crypto driver will work, albeit most likely not as +efficiently. + +The IAA crypto algorigthms is operational and compression and +decompression operations are fully enabled following the successful +binding of the first IAA workqueue to the iaa_crypto driver. + +Similarly, the IAA crypto algorithm is not operational and compression +and decompression operations are disabled following the unbinding of +the last IAA worqueue to the iaa_crypto driver. + +As a result, the IAA crypto algorithms and thus the IAA hardware are +only available when one or more workques are bound to the iaa_crypto +driver. + +When there are no IAA workqueues bound to the driver, the IAA crypto +algorithms can be unregistered by removing the module. + + +Driver attributes +----------------- + +There are a couple user-configurable driver attributes that can be +used to configure various modes of operation. They're listed below, +along with their default values. To set any of these attributes, echo +the appropriate values to the attribute file located under +/sys/bus/dsa/drivers/crypto/ + +The attribute settings at the time the IAA algorithms are registered +are captured in each algorithm's crypto_ctx and used for all compresses +and decompresses when using that algorithm. + +The available attributes are: + + - verify_compress + + Toggle compression verification. If set, each compress will be + internally decompressed and the contents verified, returning error + codes if unsuccessful. This can be toggled with 0/1:: + + echo 0 > /sys/bus/dsa/drivers/crypto/verify_compress + + The default setting is '1' - verify all compresses. + + - sync_mode + + Select mode to be used to wait for completion of each compresses + and decompress operation. + + The crypto async interface support implemented by iaa_crypto + provides an implementation that satisfies the interface but does + so in a synchronous manner - it fills and submits the IDXD + descriptor and then loops around waiting for it to complete before + returning. This isn't a problem at the moment, since all existing + callers (e.g. zswap) wrap any asynchronous callees in a + synchronous wrapper anyway. + + The iaa_crypto driver does however provide true asynchronous + support for callers that can make use of it. In this mode, it + fills and submits the IDXD descriptor, then returns immediately + with -EINPROGRESS. The caller can then either poll for completion + itself, which requires specific code in the caller which currently + nothing in the upstream kernel implements, or go to sleep and wait + for an interrupt signaling completion. This latter mode is + supported by current users in the kernel such as zswap via + synchronous wrappers. Although it is supported this mode is + significantly slower than the synchronous mode that does the + polling in the iaa_crypto driver previously mentioned. + + This mode can be enabled by writing 'async_irq' to the sync_mode + iaa_crypto driver attribute:: + + echo async_irq > /sys/bus/dsa/drivers/crypto/sync_mode + + Async mode without interrupts (caller must poll) can be enabled by + writing 'async' to it:: + + echo async > /sys/bus/dsa/drivers/crypto/sync_mode + + The mode that does the polling in the iaa_crypto driver can be + enabled by writing 'sync' to it:: + + echo sync > /sys/bus/dsa/drivers/crypto/sync_mode + + The default mode is 'sync'. + +.. _iaa_default_config: + +IAA Default Configuration +------------------------- + +When the iaa_crypto driver is loaded, each IAA device has a single +work queue configured for it, with the following attributes:: + + mode "dedicated" + threshold 0 + size Total WQ Size from WQCAP + priority 10 + type IDXD_WQT_KERNEL + group 0 + name "iaa_crypto" + driver_name "crypto" + +The devices and workqueues are also enabled and therefore the driver +is ready to be used without any additional configuration. + +The default driver attributes in effect when the driver is loaded are:: + + sync_mode "sync" + verify_compress 1 + +In order to change either the device/work queue or driver attributes, +the enabled devices and workqueues must first be disabled. In order +to have the new configuration applied to the deflate-iaa crypto +algorithm, it needs to be re-registered by removing and reinserting +the iaa_crypto module. The :ref:`iaa_disable_script` in the 'Use +Cases' section below can be used to disable the default configuration. + +Statistics +========== + +If the optional debugfs statistics support is enabled, the IAA crypto +driver will generate statistics which can be accessed in debugfs at:: + + # ls -al /sys/kernel/debug/iaa-crypto/ + total 0 + drwxr-xr-x 2 root root 0 Mar 3 09:35 . + drwx------ 47 root root 0 Mar 3 09:35 .. + -rw-r--r-- 1 root root 0 Mar 3 09:35 max_acomp_delay_ns + -rw-r--r-- 1 root root 0 Mar 3 09:35 max_adecomp_delay_ns + -rw-r--r-- 1 root root 0 Mar 3 09:35 max_comp_delay_ns + -rw-r--r-- 1 root root 0 Mar 3 09:35 max_decomp_delay_ns + -rw-r--r-- 1 root root 0 Mar 3 09:35 stats_reset + -rw-r--r-- 1 root root 0 Mar 3 09:35 total_comp_bytes_out + -rw-r--r-- 1 root root 0 Mar 3 09:35 total_comp_calls + -rw-r--r-- 1 root root 0 Mar 3 09:35 total_decomp_bytes_in + -rw-r--r-- 1 root root 0 Mar 3 09:35 total_decomp_calls + -rw-r--r-- 1 root root 0 Mar 3 09:35 wq_stats + +Most of the above statisticss are self-explanatory. The wq_stats file +shows per-wq stats, a set for each iaa device and wq in addition to +some global stats:: + + # cat wq_stats + global stats: + total_comp_calls: 100 + total_decomp_calls: 100 + total_comp_bytes_out: 22800 + total_decomp_bytes_in: 22800 + total_completion_einval_errors: 0 + total_completion_timeout_errors: 0 + total_completion_comp_buf_overflow_errors: 0 + + iaa device: + id: 1 + n_wqs: 1 + comp_calls: 0 + comp_bytes: 0 + decomp_calls: 0 + decomp_bytes: 0 + wqs: + name: iaa_crypto + comp_calls: 0 + comp_bytes: 0 + decomp_calls: 0 + decomp_bytes: 0 + + iaa device: + id: 3 + n_wqs: 1 + comp_calls: 0 + comp_bytes: 0 + decomp_calls: 0 + decomp_bytes: 0 + wqs: + name: iaa_crypto + comp_calls: 0 + comp_bytes: 0 + decomp_calls: 0 + decomp_bytes: 0 + + iaa device: + id: 5 + n_wqs: 1 + comp_calls: 100 + comp_bytes: 22800 + decomp_calls: 100 + decomp_bytes: 22800 + wqs: + name: iaa_crypto + comp_calls: 100 + comp_bytes: 22800 + decomp_calls: 100 + decomp_bytes: 22800 + +Writing 0 to 'stats_reset' resets all the stats, including the +per-device and per-wq stats:: + + # echo 0 > stats_reset + # cat wq_stats + global stats: + total_comp_calls: 0 + total_decomp_calls: 0 + total_comp_bytes_out: 0 + total_decomp_bytes_in: 0 + total_completion_einval_errors: 0 + total_completion_timeout_errors: 0 + total_completion_comp_buf_overflow_errors: 0 + ... + + +Use cases +========= + +Simple zswap test +----------------- + +For this example, the kernel should be configured according to the +dedicated mode options described above, and zswap should be enabled as +well:: + + CONFIG_ZSWAP=y + +This is a simple test that uses iaa_compress as the compressor for a +swap (zswap) device. It sets up the zswap device and then uses the +memory_memadvise program listed below to forcibly swap out and in a +specified number of pages, demonstrating both compress and decompress. + +The zswap test expects the work queues for each IAA device on the +system to be configured properly as a kernel workqueue with a +workqueue driver_name of "crypto". + +The first step is to make sure the iaa_crypto module is loaded:: + + modprobe iaa_crypto + +If the IAA devices and workqueues haven't previously been disabled and +reconfigured, then the default configuration should be in place and no +further IAA configuration is necessary. See :ref:`iaa_default_config` +below for details of the default configuration. + +If the default configuration is in place, you should see the iaa +devices and wq0s enabled:: + + # cat /sys/bus/dsa/devices/iax1/state + enabled + # cat /sys/bus/dsa/devices/iax1/wq1.0/state + enabled + +To demonstrate that the following steps work as expected, these +commands can be used to enable debug output:: + + # echo -n 'module iaa_crypto +p' > /sys/kernel/debug/dynamic_debug/control + # echo -n 'module idxd +p' > /sys/kernel/debug/dynamic_debug/control + +Use the following commands to enable zswap:: + + # echo 0 > /sys/module/zswap/parameters/enabled + # echo 50 > /sys/module/zswap/parameters/max_pool_percent + # echo deflate-iaa > /sys/module/zswap/parameters/compressor + # echo zsmalloc > /sys/module/zswap/parameters/zpool + # echo 1 > /sys/module/zswap/parameters/enabled + # echo 0 > /sys/module/zswap/parameters/same_filled_pages_enabled + # echo 100 > /proc/sys/vm/swappiness + # echo never > /sys/kernel/mm/transparent_hugepage/enabled + # echo 1 > /proc/sys/vm/overcommit_memory + +Now you can now run the zswap workload you want to measure. For +example, using the memory_memadvise code below, the following command +will swap in and out 100 pages:: + + ./memory_madvise 100 + + Allocating 100 pages to swap in/out + Swapping out 100 pages + Swapping in 100 pages + Swapped out and in 100 pages + +You should see something like the following in the dmesg output:: + + [ 404.202972] idxd 0000:e7:02.0: iaa_comp_acompress: dma_map_sg, src_addr 223925c000, nr_sgs 1, req->src 00000000ee7cb5e6, req->slen 4096, sg_dma_len(sg) 4096 + [ 404.202973] idxd 0000:e7:02.0: iaa_comp_acompress: dma_map_sg, dst_addr 21dadf8000, nr_sgs 1, req->dst 000000008d6acea8, req->dlen 4096, sg_dma_len(sg) 8192 + [ 404.202975] idxd 0000:e7:02.0: iaa_compress: desc->src1_addr 223925c000, desc->src1_size 4096, desc->dst_addr 21dadf8000, desc->max_dst_size 4096, desc->src2_addr 2203543000, desc->src2_size 1568 + [ 404.202981] idxd 0000:e7:02.0: iaa_compress_verify: (verify) desc->src1_addr 21dadf8000, desc->src1_size 228, desc->dst_addr 223925c000, desc->max_dst_size 4096, desc->src2_addr 0, desc->src2_size 0 + ... + +Now that basic functionality has been demonstrated, the defaults can +be erased and replaced with a different configuration. To do that, +first disable zswap:: + + # echo lzo > /sys/module/zswap/parameters/compressor + # swapoff -a + # echo 0 > /sys/module/zswap/parameters/accept_threshold_percent + # echo 0 > /sys/module/zswap/parameters/max_pool_percent + # echo 0 > /sys/module/zswap/parameters/enabled + # echo 0 > /sys/module/zswap/parameters/enabled + +Then run the :ref:`iaa_disable_script` in the 'Use Cases' section +below to disable the default configuration. + +Finally turn swap back on:: + + # swapon -a + +Following all that the IAA device(s) can now be re-configured and +enabled as desired for further testing. Below is one example. + +The zswap test expects the work queues for each IAA device on the +system to be configured properly as a kernel workqueue with a +workqueue driver_name of "crypto". + +The below script automatically does that:: + + #!/bin/bash + + echo "IAA devices:" + lspci -d:0cfe + echo "# IAA devices:" + lspci -d:0cfe | wc -l + + # + # count iaa instances + # + iaa_dev_id="0cfe" + num_iaa=$(lspci -d:${iaa_dev_id} | wc -l) + echo "Found ${num_iaa} IAA instances" + + # + # disable iaa wqs and devices + # + echo "Disable IAA" + + for ((i = 1; i < ${num_iaa} * 2; i += 2)); do + echo disable wq iax${i}/wq${i}.0 + accel-config disable-wq iax${i}/wq${i}.0 + echo disable iaa iax${i} + accel-config disable-device iax${i} + done + + echo "End Disable IAA" + + # + # configure iaa wqs and devices + # + echo "Configure IAA" + for ((i = 1; i < ${num_iaa} * 2; i += 2)); do + accel-config config-wq --group-id=0 --mode=dedicated --size=128 --priority=10 --type=kernel --name="iaa_crypto" --driver_name="crypto" iax${i}/wq${i} + done + + echo "End Configure IAA" + + # + # enable iaa wqs and devices + # + echo "Enable IAA" + + for ((i = 1; i < ${num_iaa} * 2; i += 2)); do + echo enable iaa iaa${i} + accel-config enable-device iaa${i} + echo enable wq iaa${i}/wq${i}.0 + accel-config enable-wq iaa${i}/wq${i}.0 + done + + echo "End Enable IAA" + +When the workqueues are bound to the iaa_crypto driver, you should +see something similar to the following in dmesg output if you've +enabled debug output (echo -n 'module iaa_crypto +p' > +/sys/kernel/debug/dynamic_debug/control):: + + [ 60.752344] idxd 0000:f6:02.0: add_iaa_wq: added wq 000000004068d14d to iaa 00000000c9585ba2, n_wq 1 + [ 60.752346] iaa_crypto: rebalance_wq_table: nr_nodes=2, nr_cpus 160, nr_iaa 8, cpus_per_iaa 20 + [ 60.752347] iaa_crypto: rebalance_wq_table: iaa=0 + [ 60.752349] idxd 0000:6a:02.0: request_iaa_wq: getting wq from iaa_device 0000000042d7bc52 (0) + [ 60.752350] idxd 0000:6a:02.0: request_iaa_wq: returning unused wq 00000000c8bb4452 (0) from iaa device 0000000042d7bc52 (0) + [ 60.752352] iaa_crypto: rebalance_wq_table: assigned wq for cpu=0, node=0 = wq 00000000c8bb4452 + [ 60.752354] iaa_crypto: rebalance_wq_table: iaa=0 + [ 60.752355] idxd 0000:6a:02.0: request_iaa_wq: getting wq from iaa_device 0000000042d7bc52 (0) + [ 60.752356] idxd 0000:6a:02.0: request_iaa_wq: returning unused wq 00000000c8bb4452 (0) from iaa device 0000000042d7bc52 (0) + [ 60.752358] iaa_crypto: rebalance_wq_table: assigned wq for cpu=1, node=0 = wq 00000000c8bb4452 + [ 60.752359] iaa_crypto: rebalance_wq_table: iaa=0 + [ 60.752360] idxd 0000:6a:02.0: request_iaa_wq: getting wq from iaa_device 0000000042d7bc52 (0) + [ 60.752361] idxd 0000:6a:02.0: request_iaa_wq: returning unused wq 00000000c8bb4452 (0) from iaa device 0000000042d7bc52 (0) + [ 60.752362] iaa_crypto: rebalance_wq_table: assigned wq for cpu=2, node=0 = wq 00000000c8bb4452 + [ 60.752364] iaa_crypto: rebalance_wq_table: iaa=0 + . + . + . + +Once the workqueues and devices have been enabled, the IAA crypto +algorithms are enabled and available. When the IAA crypto algorithms +have been successfully enabled, you should see the following dmesg +output:: + + [ 64.893759] iaa_crypto: iaa_crypto_enable: iaa_crypto now ENABLED + +Now run the following zswap-specific setup commands to have zswap use +the 'fixed' compression mode:: + + echo 0 > /sys/module/zswap/parameters/enabled + echo 50 > /sys/module/zswap/parameters/max_pool_percent + echo deflate-iaa > /sys/module/zswap/parameters/compressor + echo zsmalloc > /sys/module/zswap/parameters/zpool + echo 1 > /sys/module/zswap/parameters/enabled + echo 0 > /sys/module/zswap/parameters/same_filled_pages_enabled + + echo 100 > /proc/sys/vm/swappiness + echo never > /sys/kernel/mm/transparent_hugepage/enabled + echo 1 > /proc/sys/vm/overcommit_memory + +Finally, you can now run the zswap workload you want to measure. For +example, using the code below, the following command will swap in and +out 100 pages:: + + ./memory_madvise 100 + + Allocating 100 pages to swap in/out + Swapping out 100 pages + Swapping in 100 pages + Swapped out and in 100 pages + +You should see something like the following in the dmesg output if +you've enabled debug output (echo -n 'module iaa_crypto +p' > +/sys/kernel/debug/dynamic_debug/control):: + + [ 404.202972] idxd 0000:e7:02.0: iaa_comp_acompress: dma_map_sg, src_addr 223925c000, nr_sgs 1, req->src 00000000ee7cb5e6, req->slen 4096, sg_dma_len(sg) 4096 + [ 404.202973] idxd 0000:e7:02.0: iaa_comp_acompress: dma_map_sg, dst_addr 21dadf8000, nr_sgs 1, req->dst 000000008d6acea8, req->dlen 4096, sg_dma_len(sg) 8192 + [ 404.202975] idxd 0000:e7:02.0: iaa_compress: desc->src1_addr 223925c000, desc->src1_size 4096, desc->dst_addr 21dadf8000, desc->max_dst_size 4096, desc->src2_addr 2203543000, desc->src2_size 1568 + [ 404.202981] idxd 0000:e7:02.0: iaa_compress_verify: (verify) desc->src1_addr 21dadf8000, desc->src1_size 228, desc->dst_addr 223925c000, desc->max_dst_size 4096, desc->src2_addr 0, desc->src2_size 0 + [ 409.203227] idxd 0000:e7:02.0: iaa_comp_adecompress: dma_map_sg, src_addr 21ddd8b100, nr_sgs 1, req->src 0000000084adab64, req->slen 228, sg_dma_len(sg) 228 + [ 409.203235] idxd 0000:e7:02.0: iaa_comp_adecompress: dma_map_sg, dst_addr 21ee3dc000, nr_sgs 1, req->dst 000000004e2990d0, req->dlen 4096, sg_dma_len(sg) 4096 + [ 409.203239] idxd 0000:e7:02.0: iaa_decompress: desc->src1_addr 21ddd8b100, desc->src1_size 228, desc->dst_addr 21ee3dc000, desc->max_dst_size 4096, desc->src2_addr 0, desc->src2_size 0 + [ 409.203254] idxd 0000:e7:02.0: iaa_comp_adecompress: dma_map_sg, src_addr 21ddd8b100, nr_sgs 1, req->src 0000000084adab64, req->slen 228, sg_dma_len(sg) 228 + [ 409.203256] idxd 0000:e7:02.0: iaa_comp_adecompress: dma_map_sg, dst_addr 21f1551000, nr_sgs 1, req->dst 000000004e2990d0, req->dlen 4096, sg_dma_len(sg) 4096 + [ 409.203257] idxd 0000:e7:02.0: iaa_decompress: desc->src1_addr 21ddd8b100, desc->src1_size 228, desc->dst_addr 21f1551000, desc->max_dst_size 4096, desc->src2_addr 0, desc->src2_size 0 + +In order to unregister the IAA crypto algorithms, and register new +ones using different parameters, any users of the current algorithm +should be stopped and the IAA workqueues and devices disabled. + +In the case of zswap, remove the IAA crypto algorithm as the +compressor and turn off swap (to remove all references to +iaa_crypto):: + + echo lzo > /sys/module/zswap/parameters/compressor + swapoff -a + + echo 0 > /sys/module/zswap/parameters/accept_threshold_percent + echo 0 > /sys/module/zswap/parameters/max_pool_percent + echo 0 > /sys/module/zswap/parameters/enabled + +Once zswap is disabled and no longer using iaa_crypto, the IAA wqs and +devices can be disabled. + +.. _iaa_disable_script: + +IAA disable script +------------------ + +The below script automatically does that:: + + #!/bin/bash + + echo "IAA devices:" + lspci -d:0cfe + echo "# IAA devices:" + lspci -d:0cfe | wc -l + + # + # count iaa instances + # + iaa_dev_id="0cfe" + num_iaa=$(lspci -d:${iaa_dev_id} | wc -l) + echo "Found ${num_iaa} IAA instances" + + # + # disable iaa wqs and devices + # + echo "Disable IAA" + + for ((i = 1; i < ${num_iaa} * 2; i += 2)); do + echo disable wq iax${i}/wq${i}.0 + accel-config disable-wq iax${i}/wq${i}.0 + echo disable iaa iax${i} + accel-config disable-device iax${i} + done + + echo "End Disable IAA" + +Finally, at this point the iaa_crypto module can be removed, which +will unregister the current IAA crypto algorithms:: + + rmmod iaa_crypto + + +memory_madvise.c (gcc -o memory_memadvise memory_madvise.c):: + + #include <stdio.h> + #include <stdlib.h> + #include <string.h> + #include <unistd.h> + #include <sys/mman.h> + #include <linux/mman.h> + + #ifndef MADV_PAGEOUT + #define MADV_PAGEOUT 21 /* force pages out immediately */ + #endif + + #define PG_SZ 4096 + + int main(int argc, char **argv) + { + int i, nr_pages = 1; + int64_t *dump_ptr; + char *addr, *a; + int loop = 1; + + if (argc > 1) + nr_pages = atoi(argv[1]); + + printf("Allocating %d pages to swap in/out\n", nr_pages); + + /* allocate pages */ + addr = mmap(NULL, nr_pages * PG_SZ, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0); + *addr = 1; + + /* initialize data in page to all '*' chars */ + memset(addr, '*', nr_pages * PG_SZ); + + printf("Swapping out %d pages\n", nr_pages); + + /* Tell kernel to swap it out */ + madvise(addr, nr_pages * PG_SZ, MADV_PAGEOUT); + + while (loop > 0) { + /* Wait for swap out to finish */ + sleep(5); + + a = addr; + + printf("Swapping in %d pages\n", nr_pages); + + /* Access the page ... this will swap it back in again */ + for (i = 0; i < nr_pages; i++) { + if (a[0] != '*') { + printf("Bad data from decompress!!!!!\n"); + + dump_ptr = (int64_t *)a; + for (int j = 0; j < 100; j++) { + printf(" page %d data: %#llx\n", i, *dump_ptr); + dump_ptr++; + } + } + + a += PG_SZ; + } + + loop --; + } + + printf("Swapped out and in %d pages\n", nr_pages); + +Appendix +======== + +.. _iaa_sysfs_config: + +IAA sysfs config interface +-------------------------- + +Below is a description of the IAA sysfs interface, which as mentioned +in the main document, should only be used if you know exactly what you +are doing. Even then, there's no compelling reason to use it directly +since accel-config can do everything the sysfs interface can and in +fact accel-config is based on it under the covers. + +The 'IAA config path' is /sys/bus/dsa/devices and contains +subdirectories representing each IAA device, workqueue, engine, and +group. Note that in the sysfs interface, the IAA devices are actually +named using iax e.g. iax1, iax3, etc. (Note that IAA devices are the +odd-numbered devices; the even-numbered devices are DSA devices and +can be ignored for IAA). + +The 'IAA device bind path' is /sys/bus/dsa/drivers/idxd/bind and is +the file that is written to enable an IAA device. + +The 'IAA workqueue bind path' is /sys/bus/dsa/drivers/crypto/bind and +is the file that is written to enable an IAA workqueue. + +Similarly /sys/bus/dsa/drivers/idxd/unbind and +/sys/bus/dsa/drivers/crypto/unbind are used to disable IAA devices and +workqueues. + +The basic sequence of commands needed to set up the IAA devices and +workqueues is: + +For each device:: + 1) Disable any workqueues enabled on the device. For example to + disable workques 0 and 1 on IAA device 3:: + + # echo wq3.0 > /sys/bus/dsa/drivers/crypto/unbind + # echo wq3.1 > /sys/bus/dsa/drivers/crypto/unbind + + 2) Disable the device. For example to disable IAA device 3:: + + # echo iax3 > /sys/bus/dsa/drivers/idxd/unbind + + 3) configure the desired workqueues. For example, to configure + workqueue 3 on IAA device 3:: + + # echo dedicated > /sys/bus/dsa/devices/iax3/wq3.3/mode + # echo 128 > /sys/bus/dsa/devices/iax3/wq3.3/size + # echo 0 > /sys/bus/dsa/devices/iax3/wq3.3/group_id + # echo 10 > /sys/bus/dsa/devices/iax3/wq3.3/priority + # echo "kernel" > /sys/bus/dsa/devices/iax3/wq3.3/type + # echo "iaa_crypto" > /sys/bus/dsa/devices/iax3/wq3.3/name + # echo "crypto" > /sys/bus/dsa/devices/iax3/wq3.3/driver_name + + 4) Enable the device. For example to enable IAA device 3:: + + # echo iax3 > /sys/bus/dsa/drivers/idxd/bind + + 5) Enable the desired workqueues on the device. For example to + enable workques 0 and 1 on IAA device 3:: + + # echo wq3.0 > /sys/bus/dsa/drivers/crypto/bind + # echo wq3.1 > /sys/bus/dsa/drivers/crypto/bind diff --git a/Documentation/driver-api/crypto/iaa/index.rst b/Documentation/driver-api/crypto/iaa/index.rst new file mode 100644 index 000000000000..aa6837e27264 --- /dev/null +++ b/Documentation/driver-api/crypto/iaa/index.rst @@ -0,0 +1,20 @@ +.. SPDX-License-Identifier: GPL-2.0 + +================================= +IAA (Intel Analytics Accelerator) +================================= + +IAA provides hardware compression and decompression via the crypto +API. + +.. toctree:: + :maxdepth: 1 + + iaa-crypto + +.. only:: subproject and html + + Indices + ======= + + * :ref:`genindex` diff --git a/Documentation/driver-api/crypto/index.rst b/Documentation/driver-api/crypto/index.rst new file mode 100644 index 000000000000..fb9709b98bea --- /dev/null +++ b/Documentation/driver-api/crypto/index.rst @@ -0,0 +1,20 @@ +.. SPDX-License-Identifier: GPL-2.0 + +============== +Crypto Drivers +============== + +Documentation for crypto drivers that may need more involved setup and +configuration. + +.. toctree:: + :maxdepth: 1 + + iaa/index + +.. only:: subproject and html + + Indices + ======= + + * :ref:`genindex` diff --git a/Documentation/driver-api/index.rst b/Documentation/driver-api/index.rst index 8bc4ebe7a36f..3cc0e1a517c0 100644 --- a/Documentation/driver-api/index.rst +++ b/Documentation/driver-api/index.rst @@ -116,6 +116,7 @@ available subsections can be seen below. wmi dpll wbrf + crypto/index .. only:: subproject and html diff --git a/MAINTAINERS b/MAINTAINERS index 83240dcd4065..c4bdd50e670b 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -5538,6 +5538,12 @@ F: include/crypto/ F: include/linux/crypto* F: lib/crypto/ +CRYPTO SPEED TEST COMPARE +M: Wang Jinchao <wangjinchao@xfusion.com> +L: linux-crypto@vger.kernel.org +S: Maintained +F: tools/crypto/tcrypt/tcrypt_speed_compare.py + CRYPTOGRAPHIC RANDOM NUMBER GENERATOR M: Neil Horman <nhorman@tuxdriver.com> L: linux-crypto@vger.kernel.org @@ -9526,6 +9532,7 @@ F: Documentation/devicetree/bindings/gpio/hisilicon,ascend910-gpio.yaml F: drivers/gpio/gpio-hisi.c HISILICON HIGH PERFORMANCE RSA ENGINE DRIVER (HPRE) +M: Zhiqi Song <songzhiqi1@huawei.com> M: Longfang Liu <liulongfang@huawei.com> L: linux-crypto@vger.kernel.org S: Maintained @@ -9628,7 +9635,6 @@ F: Documentation/devicetree/bindings/scsi/hisilicon-sas.txt F: drivers/scsi/hisi_sas/ HISILICON SECURITY ENGINE V2 DRIVER (SEC2) -M: Kai Ye <yekai13@huawei.com> M: Longfang Liu <liulongfang@huawei.com> L: linux-crypto@vger.kernel.org S: Maintained @@ -10697,6 +10703,13 @@ S: Supported Q: https://patchwork.kernel.org/project/linux-dmaengine/list/ F: drivers/dma/ioat* +INTEL IAA CRYPTO DRIVER +M: Tom Zanussi <tom.zanussi@linux.intel.com> +L: linux-crypto@vger.kernel.org +S: Supported +F: Documentation/driver-api/crypto/iaa/iaa-crypto.rst +F: drivers/crypto/intel/iaa/* + INTEL IDLE DRIVER M: Jacob Pan <jacob.jun.pan@linux.intel.com> M: Len Brown <lenb@kernel.org> diff --git a/arch/arm64/crypto/Kconfig b/arch/arm64/crypto/Kconfig index 6d06b448a66e..eb7b423ba463 100644 --- a/arch/arm64/crypto/Kconfig +++ b/arch/arm64/crypto/Kconfig @@ -231,7 +231,7 @@ config CRYPTO_SM4_ARM64_CE - NEON (Advanced SIMD) extensions config CRYPTO_SM4_ARM64_CE_BLK - tristate "Ciphers: SM4, modes: ECB/CBC/CFB/CTR/XTS (ARMv8 Crypto Extensions)" + tristate "Ciphers: SM4, modes: ECB/CBC/CTR/XTS (ARMv8 Crypto Extensions)" depends on KERNEL_MODE_NEON select CRYPTO_SKCIPHER select CRYPTO_SM4 @@ -240,7 +240,6 @@ config CRYPTO_SM4_ARM64_CE_BLK with block cipher modes: - ECB (Electronic Codebook) mode (NIST SP800-38A) - CBC (Cipher Block Chaining) mode (NIST SP800-38A) - - CFB (Cipher Feedback) mode (NIST SP800-38A) - CTR (Counter) mode (NIST SP800-38A) - XTS (XOR Encrypt XOR with ciphertext stealing) mode (NIST SP800-38E and IEEE 1619) @@ -250,7 +249,7 @@ config CRYPTO_SM4_ARM64_CE_BLK - NEON (Advanced SIMD) extensions config CRYPTO_SM4_ARM64_NEON_BLK - tristate "Ciphers: SM4, modes: ECB/CBC/CFB/CTR (NEON)" + tristate "Ciphers: SM4, modes: ECB/CBC/CTR (NEON)" depends on KERNEL_MODE_NEON select CRYPTO_SKCIPHER select CRYPTO_SM4 @@ -259,7 +258,6 @@ config CRYPTO_SM4_ARM64_NEON_BLK with block cipher modes: - ECB (Electronic Codebook) mode (NIST SP800-38A) - CBC (Cipher Block Chaining) mode (NIST SP800-38A) - - CFB (Cipher Feedback) mode (NIST SP800-38A) - CTR (Counter) mode (NIST SP800-38A) Architecture: arm64 using: diff --git a/arch/arm64/crypto/sm4-ce-core.S b/arch/arm64/crypto/sm4-ce-core.S index 877b80c54a0d..1f3625c2c67e 100644 --- a/arch/arm64/crypto/sm4-ce-core.S +++ b/arch/arm64/crypto/sm4-ce-core.S @@ -403,164 +403,6 @@ SYM_FUNC_START(sm4_ce_cbc_cts_dec) SYM_FUNC_END(sm4_ce_cbc_cts_dec) .align 3 -SYM_FUNC_START(sm4_ce_cfb_enc) - /* input: - * x0: round key array, CTX - * x1: dst - * x2: src - * x3: iv (big endian, 128 bit) - * w4: nblocks - */ - SM4_PREPARE(x0) - - ld1 {RIV.16b}, [x3] - -.Lcfb_enc_loop_4x: - cmp w4, #4 - blt .Lcfb_enc_loop_1x - - sub w4, w4, #4 - - ld1 {v0.16b-v3.16b}, [x2], #64 - - rev32 v8.16b, RIV.16b - SM4_CRYPT_BLK_BE(v8) - eor v0.16b, v0.16b, v8.16b - - rev32 v8.16b, v0.16b - SM4_CRYPT_BLK_BE(v8) - eor v1.16b, v1.16b, v8.16b - - rev32 v8.16b, v1.16b - SM4_CRYPT_BLK_BE(v8) - eor v2.16b, v2.16b, v8.16b - - rev32 v8.16b, v2.16b - SM4_CRYPT_BLK_BE(v8) - eor v3.16b, v3.16b, v8.16b - - st1 {v0.16b-v3.16b}, [x1], #64 - mov RIV.16b, v3.16b - - cbz w4, .Lcfb_enc_end - b .Lcfb_enc_loop_4x - -.Lcfb_enc_loop_1x: - sub w4, w4, #1 - - ld1 {v0.16b}, [x2], #16 - - SM4_CRYPT_BLK(RIV) - eor RIV.16b, RIV.16b, v0.16b - - st1 {RIV.16b}, [x1], #16 - - cbnz w4, .Lcfb_enc_loop_1x - -.Lcfb_enc_end: - /* store new IV */ - st1 {RIV.16b}, [x3] - - ret -SYM_FUNC_END(sm4_ce_cfb_enc) - -.align 3 -SYM_FUNC_START(sm4_ce_cfb_dec) - /* input: - * x0: round key array, CTX - * x1: dst - * x2: src - * x3: iv (big endian, 128 bit) - * w4: nblocks - */ - SM4_PREPARE(x0) - - ld1 {RIV.16b}, [x3] - -.Lcfb_dec_loop_8x: - sub w4, w4, #8 - tbnz w4, #31, .Lcfb_dec_4x - - ld1 {v0.16b-v3.16b}, [x2], #64 - ld1 {v4.16b-v7.16b}, [x2], #64 - - rev32 v8.16b, RIV.16b - rev32 v9.16b, v0.16b - rev32 v10.16b, v1.16b - rev32 v11.16b, v2.16b - rev32 v12.16b, v3.16b - rev32 v13.16b, v4.16b - rev32 v14.16b, v5.16b - rev32 v15.16b, v6.16b - - SM4_CRYPT_BLK8_BE(v8, v9, v10, v11, v12, v13, v14, v15) - - mov RIV.16b, v7.16b - - eor v0.16b, v0.16b, v8.16b - eor v1.16b, v1.16b, v9.16b - eor v2.16b, v2.16b, v10.16b - eor v3.16b, v3.16b, v11.16b - eor v4.16b, v4.16b, v12.16b - eor v5.16b, v5.16b, v13.16b - eor v6.16b, v6.16b, v14.16b - eor v7.16b, v7.16b, v15.16b - - st1 {v0.16b-v3.16b}, [x1], #64 - st1 {v4.16b-v7.16b}, [x1], #64 - - cbz w4, .Lcfb_dec_end - b .Lcfb_dec_loop_8x - -.Lcfb_dec_4x: - add w4, w4, #8 - cmp w4, #4 - blt .Lcfb_dec_loop_1x - - sub w4, w4, #4 - - ld1 {v0.16b-v3.16b}, [x2], #64 - - rev32 v8.16b, RIV.16b - rev32 v9.16b, v0.16b - rev32 v10.16b, v1.16b - rev32 v11.16b, v2.16b - - SM4_CRYPT_BLK4_BE(v8, v9, v10, v11) - - mov RIV.16b, v3.16b - - eor v0.16b, v0.16b, v8.16b - eor v1.16b, v1.16b, v9.16b - eor v2.16b, v2.16b, v10.16b - eor v3.16b, v3.16b, v11.16b - - st1 {v0.16b-v3.16b}, [x1], #64 - - cbz w4, .Lcfb_dec_end - -.Lcfb_dec_loop_1x: - sub w4, w4, #1 - - ld1 {v0.16b}, [x2], #16 - - SM4_CRYPT_BLK(RIV) - - eor RIV.16b, RIV.16b, v0.16b - st1 {RIV.16b}, [x1], #16 - - mov RIV.16b, v0.16b - - cbnz w4, .Lcfb_dec_loop_1x - -.Lcfb_dec_end: - /* store new IV */ - st1 {RIV.16b}, [x3] - - ret -SYM_FUNC_END(sm4_ce_cfb_dec) - -.align 3 SYM_FUNC_START(sm4_ce_ctr_enc) /* input: * x0: round key array, CTX diff --git a/arch/arm64/crypto/sm4-ce-glue.c b/arch/arm64/crypto/sm4-ce-glue.c index 0a2d32ed3bde..43741bed874e 100644 --- a/arch/arm64/crypto/sm4-ce-glue.c +++ b/arch/arm64/crypto/sm4-ce-glue.c @@ -37,10 +37,6 @@ asmlinkage void sm4_ce_cbc_cts_enc(const u32 *rkey, u8 *dst, const u8 *src, u8 *iv, unsigned int nbytes); asmlinkage void sm4_ce_cbc_cts_dec(const u32 *rkey, u8 *dst, const u8 *src, u8 *iv, unsigned int nbytes); -asmlinkage void sm4_ce_cfb_enc(const u32 *rkey, u8 *dst, const u8 *src, - u8 *iv, unsigned int nblks); -asmlinkage void sm4_ce_cfb_dec(const u32 *rkey, u8 *dst, const u8 *src, - u8 *iv, unsigned int nblks); asmlinkage void sm4_ce_ctr_enc(const u32 *rkey, u8 *dst, const u8 *src, u8 *iv, unsigned int nblks); asmlinkage void sm4_ce_xts_enc(const u32 *rkey1, u8 *dst, const u8 *src, @@ -56,7 +52,6 @@ asmlinkage void sm4_ce_mac_update(const u32 *rkey_enc, u8 *digest, EXPORT_SYMBOL(sm4_ce_expand_key); EXPORT_SYMBOL(sm4_ce_crypt_block); EXPORT_SYMBOL(sm4_ce_cbc_enc); -EXPORT_SYMBOL(sm4_ce_cfb_enc); struct sm4_xts_ctx { struct sm4_ctx key1; @@ -280,90 +275,6 @@ static int sm4_cbc_cts_decrypt(struct skcipher_request *req) return sm4_cbc_cts_crypt(req, false); } -static int sm4_cfb_encrypt(struct skcipher_request *req) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm); - struct skcipher_walk walk; - unsigned int nbytes; - int err; - - err = skcipher_walk_virt(&walk, req, false); - - while ((nbytes = walk.nbytes) > 0) { - const u8 *src = walk.src.virt.addr; - u8 *dst = walk.dst.virt.addr; - unsigned int nblks; - - kernel_neon_begin(); - - nblks = BYTES2BLKS(nbytes); - if (nblks) { - sm4_ce_cfb_enc(ctx->rkey_enc, dst, src, walk.iv, nblks); - dst += nblks * SM4_BLOCK_SIZE; - src += nblks * SM4_BLOCK_SIZE; - nbytes -= nblks * SM4_BLOCK_SIZE; - } - - /* tail */ - if (walk.nbytes == walk.total && nbytes > 0) { - u8 keystream[SM4_BLOCK_SIZE]; - - sm4_ce_crypt_block(ctx->rkey_enc, keystream, walk.iv); - crypto_xor_cpy(dst, src, keystream, nbytes); - nbytes = 0; - } - - kernel_neon_end(); - - err = skcipher_walk_done(&walk, nbytes); - } - - return err; -} - -static int sm4_cfb_decrypt(struct skcipher_request *req) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm); - struct skcipher_walk walk; - unsigned int nbytes; - int err; - - err = skcipher_walk_virt(&walk, req, false); - - while ((nbytes = walk.nbytes) > 0) { - const u8 *src = walk.src.virt.addr; - u8 *dst = walk.dst.virt.addr; - unsigned int nblks; - - kernel_neon_begin(); - - nblks = BYTES2BLKS(nbytes); - if (nblks) { - sm4_ce_cfb_dec(ctx->rkey_enc, dst, src, walk.iv, nblks); - dst += nblks * SM4_BLOCK_SIZE; - src += nblks * SM4_BLOCK_SIZE; - nbytes -= nblks * SM4_BLOCK_SIZE; - } - - /* tail */ - if (walk.nbytes == walk.total && nbytes > 0) { - u8 keystream[SM4_BLOCK_SIZE]; - - sm4_ce_crypt_block(ctx->rkey_enc, keystream, walk.iv); - crypto_xor_cpy(dst, src, keystream, nbytes); - nbytes = 0; - } - - kernel_neon_end(); - - err = skcipher_walk_done(&walk, nbytes); - } - - return err; -} - static int sm4_ctr_crypt(struct skcipher_request *req) { struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); @@ -544,22 +455,6 @@ static struct skcipher_alg sm4_algs[] = { .decrypt = sm4_cbc_decrypt, }, { .base = { - .cra_name = "cfb(sm4)", - .cra_driver_name = "cfb-sm4-ce", - .cra_priority = 400, - .cra_blocksize = 1, - .cra_ctxsize = sizeof(struct sm4_ctx), - .cra_module = THIS_MODULE, - }, - .min_keysize = SM4_KEY_SIZE, - .max_keysize = SM4_KEY_SIZE, - .ivsize = SM4_BLOCK_SIZE, - .chunksize = SM4_BLOCK_SIZE, - .setkey = sm4_setkey, - .encrypt = sm4_cfb_encrypt, - .decrypt = sm4_cfb_decrypt, - }, { - .base = { .cra_name = "ctr(sm4)", .cra_driver_name = "ctr-sm4-ce", .cra_priority = 400, @@ -869,12 +764,11 @@ static void __exit sm4_exit(void) module_cpu_feature_match(SM4, sm4_init); module_exit(sm4_exit); -MODULE_DESCRIPTION("SM4 ECB/CBC/CFB/CTR/XTS using ARMv8 Crypto Extensions"); +MODULE_DESCRIPTION("SM4 ECB/CBC/CTR/XTS using ARMv8 Crypto Extensions"); MODULE_ALIAS_CRYPTO("sm4-ce"); MODULE_ALIAS_CRYPTO("sm4"); MODULE_ALIAS_CRYPTO("ecb(sm4)"); MODULE_ALIAS_CRYPTO("cbc(sm4)"); -MODULE_ALIAS_CRYPTO("cfb(sm4)"); MODULE_ALIAS_CRYPTO("ctr(sm4)"); MODULE_ALIAS_CRYPTO("cts(cbc(sm4))"); MODULE_ALIAS_CRYPTO("xts(sm4)"); diff --git a/arch/arm64/crypto/sm4-ce.h b/arch/arm64/crypto/sm4-ce.h index 109c21b37590..1e235c4371eb 100644 --- a/arch/arm64/crypto/sm4-ce.h +++ b/arch/arm64/crypto/sm4-ce.h @@ -11,6 +11,3 @@ void sm4_ce_crypt_block(const u32 *rkey, u8 *dst, const u8 *src); void sm4_ce_cbc_enc(const u32 *rkey_enc, u8 *dst, const u8 *src, u8 *iv, unsigned int nblocks); - -void sm4_ce_cfb_enc(const u32 *rkey_enc, u8 *dst, const u8 *src, - u8 *iv, unsigned int nblocks); diff --git a/arch/arm64/crypto/sm4-neon-core.S b/arch/arm64/crypto/sm4-neon-core.S index f295b4b7d70a..734dc7193610 100644 --- a/arch/arm64/crypto/sm4-neon-core.S +++ b/arch/arm64/crypto/sm4-neon-core.S @@ -438,119 +438,6 @@ SYM_FUNC_START(sm4_neon_cbc_dec) SYM_FUNC_END(sm4_neon_cbc_dec) .align 3 -SYM_FUNC_START(sm4_neon_cfb_dec) - /* input: - * x0: round key array, CTX - * x1: dst - * x2: src - * x3: iv (big endian, 128 bit) - * w4: nblocks - */ - SM4_PREPARE() - - ld1 {v0.16b}, [x3] - -.Lcfb_dec_loop_8x: - sub w4, w4, #8 - tbnz w4, #31, .Lcfb_dec_4x - - ld1 {v1.16b-v3.16b}, [x2], #48 - ld4 {v4.4s-v7.4s}, [x2] - - transpose_4x4(v0, v1, v2, v3) - - SM4_CRYPT_BLK8(v0, v1, v2, v3, v4, v5, v6, v7) - - sub x2, x2, #48 - ld1 {RTMP0.16b-RTMP3.16b}, [x2], #64 - ld1 {RTMP4.16b-RTMP7.16b}, [x2], #64 - - eor v0.16b, v0.16b, RTMP0.16b - eor v1.16b, v1.16b, RTMP1.16b - eor v2.16b, v2.16b, RTMP2.16b - eor v3.16b, v3.16b, RTMP3.16b - eor v4.16b, v4.16b, RTMP4.16b - eor v5.16b, v5.16b, RTMP5.16b - eor v6.16b, v6.16b, RTMP6.16b - eor v7.16b, v7.16b, RTMP7.16b - - st1 {v0.16b-v3.16b}, [x1], #64 - st1 {v4.16b-v7.16b}, [x1], #64 - - mov v0.16b, RTMP7.16b - - cbz w4, .Lcfb_dec_end - b .Lcfb_dec_loop_8x - -.Lcfb_dec_4x: - add w4, w4, #8 - cmp w4, #4 - blt .Lcfb_dec_tail - - sub w4, w4, #4 - - ld1 {v4.16b-v7.16b}, [x2], #64 - - rev32 v0.16b, v0.16b /* v0 is IV register */ - rev32 v1.16b, v4.16b - rev32 v2.16b, v5.16b - rev32 v3.16b, v6.16b - - transpose_4x4(v0, v1, v2, v3) - - SM4_CRYPT_BLK4_BE(v0, v1, v2, v3) - - eor v0.16b, v0.16b, v4.16b - eor v1.16b, v1.16b, v5.16b - eor v2.16b, v2.16b, v6.16b - eor v3.16b, v3.16b, v7.16b - - st1 {v0.16b-v3.16b}, [x1], #64 - - mov v0.16b, v7.16b - - cbz w4, .Lcfb_dec_end - -.Lcfb_dec_tail: - cmp w4, #2 - ld1 {v4.16b}, [x2], #16 - blt .Lcfb_dec_tail_load_done - ld1 {v5.16b}, [x2], #16 - beq .Lcfb_dec_tail_load_done - ld1 {v6.16b}, [x2], #16 - -.Lcfb_dec_tail_load_done: - rev32 v0.16b, v0.16b /* v0 is IV register */ - rev32 v1.16b, v4.16b - rev32 v2.16b, v5.16b - - transpose_4x4(v0, v1, v2, v3) - - SM4_CRYPT_BLK4_BE(v0, v1, v2, v3) - - cmp w4, #2 - eor v0.16b, v0.16b, v4.16b - st1 {v0.16b}, [x1], #16 - mov v0.16b, v4.16b - blt .Lcfb_dec_end - - eor v1.16b, v1.16b, v5.16b - st1 {v1.16b}, [x1], #16 - mov v0.16b, v5.16b - beq .Lcfb_dec_end - - eor v2.16b, v2.16b, v6.16b - st1 {v2.16b}, [x1], #16 - mov v0.16b, v6.16b - -.Lcfb_dec_end: - /* store new IV */ - st1 {v0.16b}, [x3] - - ret -SYM_FUNC_END(sm4_neon_cfb_dec) - -.align 3 SYM_FUNC_START(sm4_neon_ctr_crypt) /* input: * x0: round key array, CTX diff --git a/arch/arm64/crypto/sm4-neon-glue.c b/arch/arm64/crypto/sm4-neon-glue.c index 7b19accf5c03..e3500aca2d18 100644 --- a/arch/arm64/crypto/sm4-neon-glue.c +++ b/arch/arm64/crypto/sm4-neon-glue.c @@ -22,8 +22,6 @@ asmlinkage void sm4_neon_crypt(const u32 *rkey, u8 *dst, const u8 *src, unsigned int nblocks); asmlinkage void sm4_neon_cbc_dec(const u32 *rkey_dec, u8 *dst, const u8 *src, u8 *iv, unsigned int nblocks); -asmlinkage void sm4_neon_cfb_dec(const u32 *rkey_enc, u8 *dst, const u8 *src, - u8 *iv, unsigned int nblocks); asmlinkage void sm4_neon_ctr_crypt(const u32 *rkey_enc, u8 *dst, const u8 *src, u8 *iv, unsigned int nblocks); @@ -142,90 +140,6 @@ static int sm4_cbc_decrypt(struct skcipher_request *req) return err; } -static int sm4_cfb_encrypt(struct skcipher_request *req) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm); - struct skcipher_walk walk; - unsigned int nbytes; - int err; - - err = skcipher_walk_virt(&walk, req, false); - - while ((nbytes = walk.nbytes) > 0) { - u8 keystream[SM4_BLOCK_SIZE]; - const u8 *iv = walk.iv; - const u8 *src = walk.src.virt.addr; - u8 *dst = walk.dst.virt.addr; - - while (nbytes >= SM4_BLOCK_SIZE) { - sm4_crypt_block(ctx->rkey_enc, keystream, iv); - crypto_xor_cpy(dst, src, keystream, SM4_BLOCK_SIZE); - iv = dst; - src += SM4_BLOCK_SIZE; - dst += SM4_BLOCK_SIZE; - nbytes -= SM4_BLOCK_SIZE; - } - if (iv != walk.iv) - memcpy(walk.iv, iv, SM4_BLOCK_SIZE); - - /* tail */ - if (walk.nbytes == walk.total && nbytes > 0) { - sm4_crypt_block(ctx->rkey_enc, keystream, walk.iv); - crypto_xor_cpy(dst, src, keystream, nbytes); - nbytes = 0; - } - - err = skcipher_walk_done(&walk, nbytes); - } - - return err; -} - -static int sm4_cfb_decrypt(struct skcipher_request *req) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm); - struct skcipher_walk walk; - unsigned int nbytes; - int err; - - err = skcipher_walk_virt(&walk, req, false); - - while ((nbytes = walk.nbytes) > 0) { - const u8 *src = walk.src.virt.addr; - u8 *dst = walk.dst.virt.addr; - unsigned int nblocks; - - nblocks = nbytes / SM4_BLOCK_SIZE; - if (nblocks) { - kernel_neon_begin(); - - sm4_neon_cfb_dec(ctx->rkey_enc, dst, src, - walk.iv, nblocks); - - kernel_neon_end(); - - dst += nblocks * SM4_BLOCK_SIZE; - src += nblocks * SM4_BLOCK_SIZE; - nbytes -= nblocks * SM4_BLOCK_SIZE; - } - - /* tail */ - if (walk.nbytes == walk.total && nbytes > 0) { - u8 keystream[SM4_BLOCK_SIZE]; - - sm4_crypt_block(ctx->rkey_enc, keystream, walk.iv); - crypto_xor_cpy(dst, src, keystream, nbytes); - nbytes = 0; - } - - err = skcipher_walk_done(&walk, nbytes); - } - - return err; -} - static int sm4_ctr_crypt(struct skcipher_request *req) { struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); @@ -303,22 +217,6 @@ static struct skcipher_alg sm4_algs[] = { .decrypt = sm4_cbc_decrypt, }, { .base = { - .cra_name = "cfb(sm4)", - .cra_driver_name = "cfb-sm4-neon", - .cra_priority = 200, - .cra_blocksize = 1, - .cra_ctxsize = sizeof(struct sm4_ctx), - .cra_module = THIS_MODULE, - }, - .min_keysize = SM4_KEY_SIZE, - .max_keysize = SM4_KEY_SIZE, - .ivsize = SM4_BLOCK_SIZE, - .chunksize = SM4_BLOCK_SIZE, - .setkey = sm4_setkey, - .encrypt = sm4_cfb_encrypt, - .decrypt = sm4_cfb_decrypt, - }, { - .base = { .cra_name = "ctr(sm4)", .cra_driver_name = "ctr-sm4-neon", .cra_priority = 200, @@ -349,12 +247,11 @@ static void __exit sm4_exit(void) module_init(sm4_init); module_exit(sm4_exit); -MODULE_DESCRIPTION("SM4 ECB/CBC/CFB/CTR using ARMv8 NEON"); +MODULE_DESCRIPTION("SM4 ECB/CBC/CTR using ARMv8 NEON"); MODULE_ALIAS_CRYPTO("sm4-neon"); MODULE_ALIAS_CRYPTO("sm4"); MODULE_ALIAS_CRYPTO("ecb(sm4)"); MODULE_ALIAS_CRYPTO("cbc(sm4)"); -MODULE_ALIAS_CRYPTO("cfb(sm4)"); MODULE_ALIAS_CRYPTO("ctr(sm4)"); MODULE_AUTHOR("Tianjia Zhang <tianjia.zhang@linux.alibaba.com>"); MODULE_LICENSE("GPL v2"); diff --git a/arch/powerpc/crypto/aes-gcm-p10-glue.c b/arch/powerpc/crypto/aes-gcm-p10-glue.c index 4b6e899895e7..f62ee54076c0 100644 --- a/arch/powerpc/crypto/aes-gcm-p10-glue.c +++ b/arch/powerpc/crypto/aes-gcm-p10-glue.c @@ -37,7 +37,7 @@ asmlinkage void aes_p10_gcm_encrypt(u8 *in, u8 *out, size_t len, void *rkey, u8 *iv, void *Xi); asmlinkage void aes_p10_gcm_decrypt(u8 *in, u8 *out, size_t len, void *rkey, u8 *iv, void *Xi); -asmlinkage void gcm_init_htable(unsigned char htable[256], unsigned char Xi[16]); +asmlinkage void gcm_init_htable(unsigned char htable[], unsigned char Xi[]); asmlinkage void gcm_ghash_p10(unsigned char *Xi, unsigned char *Htable, unsigned char *aad, unsigned int alen); diff --git a/arch/s390/crypto/aes_s390.c b/arch/s390/crypto/aes_s390.c index c773820e4af9..c6fe5405de4a 100644 --- a/arch/s390/crypto/aes_s390.c +++ b/arch/s390/crypto/aes_s390.c @@ -597,7 +597,9 @@ static int ctr_aes_crypt(struct skcipher_request *req) * final block may be < AES_BLOCK_SIZE, copy only nbytes */ if (nbytes) { - cpacf_kmctr(sctx->fc, sctx->key, buf, walk.src.virt.addr, + memset(buf, 0, AES_BLOCK_SIZE); + memcpy(buf, walk.src.virt.addr, nbytes); + cpacf_kmctr(sctx->fc, sctx->key, buf, buf, AES_BLOCK_SIZE, walk.iv); memcpy(walk.dst.virt.addr, buf, nbytes); crypto_inc(walk.iv, AES_BLOCK_SIZE); diff --git a/arch/s390/crypto/paes_s390.c b/arch/s390/crypto/paes_s390.c index 8b541e44151d..55ee5567a5ea 100644 --- a/arch/s390/crypto/paes_s390.c +++ b/arch/s390/crypto/paes_s390.c @@ -693,9 +693,11 @@ static int ctr_paes_crypt(struct skcipher_request *req) * final block may be < AES_BLOCK_SIZE, copy only nbytes */ if (nbytes) { + memset(buf, 0, AES_BLOCK_SIZE); + memcpy(buf, walk.src.virt.addr, nbytes); while (1) { if (cpacf_kmctr(ctx->fc, ¶m, buf, - walk.src.virt.addr, AES_BLOCK_SIZE, + buf, AES_BLOCK_SIZE, walk.iv) == AES_BLOCK_SIZE) break; if (__paes_convert_key(ctx)) diff --git a/arch/x86/crypto/Kconfig b/arch/x86/crypto/Kconfig index 9bbfd01cfa2f..c9e59589a1ce 100644 --- a/arch/x86/crypto/Kconfig +++ b/arch/x86/crypto/Kconfig @@ -189,7 +189,7 @@ config CRYPTO_SERPENT_AVX2_X86_64 Processes 16 blocks in parallel. config CRYPTO_SM4_AESNI_AVX_X86_64 - tristate "Ciphers: SM4 with modes: ECB, CBC, CFB, CTR (AES-NI/AVX)" + tristate "Ciphers: SM4 with modes: ECB, CBC, CTR (AES-NI/AVX)" depends on X86 && 64BIT select CRYPTO_SKCIPHER select CRYPTO_SIMD @@ -197,7 +197,7 @@ config CRYPTO_SM4_AESNI_AVX_X86_64 select CRYPTO_SM4 help Length-preserving ciphers: SM4 cipher algorithms - (OSCCA GB/T 32907-2016) with ECB, CBC, CFB, and CTR modes + (OSCCA GB/T 32907-2016) with ECB, CBC, and CTR modes Architecture: x86_64 using: - AES-NI (AES New Instructions) @@ -210,7 +210,7 @@ config CRYPTO_SM4_AESNI_AVX_X86_64 If unsure, say N. config CRYPTO_SM4_AESNI_AVX2_X86_64 - tristate "Ciphers: SM4 with modes: ECB, CBC, CFB, CTR (AES-NI/AVX2)" + tristate "Ciphers: SM4 with modes: ECB, CBC, CTR (AES-NI/AVX2)" depends on X86 && 64BIT select CRYPTO_SKCIPHER select CRYPTO_SIMD @@ -219,7 +219,7 @@ config CRYPTO_SM4_AESNI_AVX2_X86_64 select CRYPTO_SM4_AESNI_AVX_X86_64 help Length-preserving ciphers: SM4 cipher algorithms - (OSCCA GB/T 32907-2016) with ECB, CBC, CFB, and CTR modes + (OSCCA GB/T 32907-2016) with ECB, CBC, and CTR modes Architecture: x86_64 using: - AES-NI (AES New Instructions) diff --git a/arch/x86/crypto/sha1_ssse3_glue.c b/arch/x86/crypto/sha1_ssse3_glue.c index 959afa705e95..ab8bc54f254d 100644 --- a/arch/x86/crypto/sha1_ssse3_glue.c +++ b/arch/x86/crypto/sha1_ssse3_glue.c @@ -2,8 +2,8 @@ /* * Cryptographic API. * - * Glue code for the SHA1 Secure Hash Algorithm assembler implementation using - * Supplemental SSE3 instructions. + * Glue code for the SHA1 Secure Hash Algorithm assembler implementations + * using SSSE3, AVX, AVX2, and SHA-NI instructions. * * This file is based on sha1_generic.c * @@ -28,6 +28,9 @@ #include <asm/simd.h> static const struct x86_cpu_id module_cpu_ids[] = { +#ifdef CONFIG_AS_SHA1_NI + X86_MATCH_FEATURE(X86_FEATURE_SHA_NI, NULL), +#endif X86_MATCH_FEATURE(X86_FEATURE_AVX2, NULL), X86_MATCH_FEATURE(X86_FEATURE_AVX, NULL), X86_MATCH_FEATURE(X86_FEATURE_SSSE3, NULL), diff --git a/arch/x86/crypto/sha256_ssse3_glue.c b/arch/x86/crypto/sha256_ssse3_glue.c index 4c0383a90e11..e04a43d9f7d5 100644 --- a/arch/x86/crypto/sha256_ssse3_glue.c +++ b/arch/x86/crypto/sha256_ssse3_glue.c @@ -1,8 +1,8 @@ /* * Cryptographic API. * - * Glue code for the SHA256 Secure Hash Algorithm assembler - * implementation using supplemental SSE3 / AVX / AVX2 instructions. + * Glue code for the SHA256 Secure Hash Algorithm assembler implementations + * using SSSE3, AVX, AVX2, and SHA-NI instructions. * * This file is based on sha256_generic.c * @@ -45,6 +45,9 @@ asmlinkage void sha256_transform_ssse3(struct sha256_state *state, const u8 *data, int blocks); static const struct x86_cpu_id module_cpu_ids[] = { +#ifdef CONFIG_AS_SHA256_NI + X86_MATCH_FEATURE(X86_FEATURE_SHA_NI, NULL), +#endif X86_MATCH_FEATURE(X86_FEATURE_AVX2, NULL), X86_MATCH_FEATURE(X86_FEATURE_AVX, NULL), X86_MATCH_FEATURE(X86_FEATURE_SSSE3, NULL), diff --git a/arch/x86/crypto/sm4-aesni-avx-asm_64.S b/arch/x86/crypto/sm4-aesni-avx-asm_64.S index e2668d2fe6ce..2bf611eaa191 100644 --- a/arch/x86/crypto/sm4-aesni-avx-asm_64.S +++ b/arch/x86/crypto/sm4-aesni-avx-asm_64.S @@ -534,55 +534,3 @@ SYM_TYPED_FUNC_START(sm4_aesni_avx_cbc_dec_blk8) FRAME_END RET; SYM_FUNC_END(sm4_aesni_avx_cbc_dec_blk8) - -/* - * void sm4_aesni_avx_cfb_dec_blk8(const u32 *rk, u8 *dst, - * const u8 *src, u8 *iv) - */ -SYM_TYPED_FUNC_START(sm4_aesni_avx_cfb_dec_blk8) - /* input: - * %rdi: round key array, CTX - * %rsi: dst (8 blocks) - * %rdx: src (8 blocks) - * %rcx: iv - */ - FRAME_BEGIN - - /* Load input */ - vmovdqu (%rcx), RA0; - vmovdqu 0 * 16(%rdx), RA1; - vmovdqu 1 * 16(%rdx), RA2; - vmovdqu 2 * 16(%rdx), RA3; - vmovdqu 3 * 16(%rdx), RB0; - vmovdqu 4 * 16(%rdx), RB1; - vmovdqu 5 * 16(%rdx), RB2; - vmovdqu 6 * 16(%rdx), RB3; - - /* Update IV */ - vmovdqu 7 * 16(%rdx), RNOT; - vmovdqu RNOT, (%rcx); - - call __sm4_crypt_blk8; - - vpxor (0 * 16)(%rdx), RA0, RA0; - vpxor (1 * 16)(%rdx), RA1, RA1; - vpxor (2 * 16)(%rdx), RA2, RA2; - vpxor (3 * 16)(%rdx), RA3, RA3; - vpxor (4 * 16)(%rdx), RB0, RB0; - vpxor (5 * 16)(%rdx), RB1, RB1; - vpxor (6 * 16)(%rdx), RB2, RB2; - vpxor (7 * 16)(%rdx), RB3, RB3; - - vmovdqu RA0, (0 * 16)(%rsi); - vmovdqu RA1, (1 * 16)(%rsi); - vmovdqu RA2, (2 * 16)(%rsi); - vmovdqu RA3, (3 * 16)(%rsi); - vmovdqu RB0, (4 * 16)(%rsi); - vmovdqu RB1, (5 * 16)(%rsi); - vmovdqu RB2, (6 * 16)(%rsi); - vmovdqu RB3, (7 * 16)(%rsi); - - vzeroall; - FRAME_END - RET; -SYM_FUNC_END(sm4_aesni_avx_cfb_dec_blk8) diff --git a/arch/x86/crypto/sm4-aesni-avx2-asm_64.S b/arch/x86/crypto/sm4-aesni-avx2-asm_64.S index 98ede9459287..9ff5ba075591 100644 --- a/arch/x86/crypto/sm4-aesni-avx2-asm_64.S +++ b/arch/x86/crypto/sm4-aesni-avx2-asm_64.S @@ -439,58 +439,3 @@ SYM_TYPED_FUNC_START(sm4_aesni_avx2_cbc_dec_blk16) FRAME_END RET; SYM_FUNC_END(sm4_aesni_avx2_cbc_dec_blk16) - -/* - * void sm4_aesni_avx2_cfb_dec_blk16(const u32 *rk, u8 *dst, - * const u8 *src, u8 *iv) - */ -SYM_TYPED_FUNC_START(sm4_aesni_avx2_cfb_dec_blk16) - /* input: - * %rdi: round key array, CTX - * %rsi: dst (16 blocks) - * %rdx: src (16 blocks) - * %rcx: iv - */ - FRAME_BEGIN - - vzeroupper; - - /* Load input */ - vmovdqu (%rcx), RNOTx; - vinserti128 $1, (%rdx), RNOT, RA0; - vmovdqu (0 * 32 + 16)(%rdx), RA1; - vmovdqu (1 * 32 + 16)(%rdx), RA2; - vmovdqu (2 * 32 + 16)(%rdx), RA3; - vmovdqu (3 * 32 + 16)(%rdx), RB0; - vmovdqu (4 * 32 + 16)(%rdx), RB1; - vmovdqu (5 * 32 + 16)(%rdx), RB2; - vmovdqu (6 * 32 + 16)(%rdx), RB3; - - /* Update IV */ - vmovdqu (7 * 32 + 16)(%rdx), RNOTx; - vmovdqu RNOTx, (%rcx); - - call __sm4_crypt_blk16; - - vpxor (0 * 32)(%rdx), RA0, RA0; - vpxor (1 * 32)(%rdx), RA1, RA1; - vpxor (2 * 32)(%rdx), RA2, RA2; - vpxor (3 * 32)(%rdx), RA3, RA3; - vpxor (4 * 32)(%rdx), RB0, RB0; - vpxor (5 * 32)(%rdx), RB1, RB1; - vpxor (6 * 32)(%rdx), RB2, RB2; - vpxor (7 * 32)(%rdx), RB3, RB3; - - vmovdqu RA0, (0 * 32)(%rsi); - vmovdqu RA1, (1 * 32)(%rsi); - vmovdqu RA2, (2 * 32)(%rsi); - vmovdqu RA3, (3 * 32)(%rsi); - vmovdqu RB0, (4 * 32)(%rsi); - vmovdqu RB1, (5 * 32)(%rsi); - vmovdqu RB2, (6 * 32)(%rsi); - vmovdqu RB3, (7 * 32)(%rsi); - - vzeroall; - FRAME_END - RET; -SYM_FUNC_END(sm4_aesni_avx2_cfb_dec_blk16) diff --git a/arch/x86/crypto/sm4-avx.h b/arch/x86/crypto/sm4-avx.h index 1bceab7516aa..b5b5e67e40ed 100644 --- a/arch/x86/crypto/sm4-avx.h +++ b/arch/x86/crypto/sm4-avx.h @@ -14,10 +14,6 @@ int sm4_cbc_encrypt(struct skcipher_request *req); int sm4_avx_cbc_decrypt(struct skcipher_request *req, unsigned int bsize, sm4_crypt_func func); -int sm4_cfb_encrypt(struct skcipher_request *req); -int sm4_avx_cfb_decrypt(struct skcipher_request *req, - unsigned int bsize, sm4_crypt_func func); - int sm4_avx_ctr_crypt(struct skcipher_request *req, unsigned int bsize, sm4_crypt_func func); diff --git a/arch/x86/crypto/sm4_aesni_avx2_glue.c b/arch/x86/crypto/sm4_aesni_avx2_glue.c index 84bc718f49a3..1148fd4cd57f 100644 --- a/arch/x86/crypto/sm4_aesni_avx2_glue.c +++ b/arch/x86/crypto/sm4_aesni_avx2_glue.c @@ -23,8 +23,6 @@ asmlinkage void sm4_aesni_avx2_ctr_enc_blk16(const u32 *rk, u8 *dst, const u8 *src, u8 *iv); asmlinkage void sm4_aesni_avx2_cbc_dec_blk16(const u32 *rk, u8 *dst, const u8 *src, u8 *iv); -asmlinkage void sm4_aesni_avx2_cfb_dec_blk16(const u32 *rk, u8 *dst, - const u8 *src, u8 *iv); static int sm4_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key, unsigned int key_len) @@ -41,12 +39,6 @@ static int cbc_decrypt(struct skcipher_request *req) } -static int cfb_decrypt(struct skcipher_request *req) -{ - return sm4_avx_cfb_decrypt(req, SM4_CRYPT16_BLOCK_SIZE, - sm4_aesni_avx2_cfb_dec_blk16); -} - static int ctr_crypt(struct skcipher_request *req) { return sm4_avx_ctr_crypt(req, SM4_CRYPT16_BLOCK_SIZE, @@ -89,24 +81,6 @@ static struct skcipher_alg sm4_aesni_avx2_skciphers[] = { .decrypt = cbc_decrypt, }, { .base = { - .cra_name = "__cfb(sm4)", - .cra_driver_name = "__cfb-sm4-aesni-avx2", - .cra_priority = 500, - .cra_flags = CRYPTO_ALG_INTERNAL, - .cra_blocksize = 1, - .cra_ctxsize = sizeof(struct sm4_ctx), - .cra_module = THIS_MODULE, - }, - .min_keysize = SM4_KEY_SIZE, - .max_keysize = SM4_KEY_SIZE, - .ivsize = SM4_BLOCK_SIZE, - .chunksize = SM4_BLOCK_SIZE, - .walksize = 16 * SM4_BLOCK_SIZE, - .setkey = sm4_skcipher_setkey, - .encrypt = sm4_cfb_encrypt, - .decrypt = cfb_decrypt, - }, { - .base = { .cra_name = "__ctr(sm4)", .cra_driver_name = "__ctr-sm4-aesni-avx2", .cra_priority = 500, diff --git a/arch/x86/crypto/sm4_aesni_avx_glue.c b/arch/x86/crypto/sm4_aesni_avx_glue.c index 7800f77d68ad..85b4ca78b47b 100644 --- a/arch/x86/crypto/sm4_aesni_avx_glue.c +++ b/arch/x86/crypto/sm4_aesni_avx_glue.c @@ -27,8 +27,6 @@ asmlinkage void sm4_aesni_avx_ctr_enc_blk8(const u32 *rk, u8 *dst, const u8 *src, u8 *iv); asmlinkage void sm4_aesni_avx_cbc_dec_blk8(const u32 *rk, u8 *dst, const u8 *src, u8 *iv); -asmlinkage void sm4_aesni_avx_cfb_dec_blk8(const u32 *rk, u8 *dst, - const u8 *src, u8 *iv); static int sm4_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key, unsigned int key_len) @@ -188,116 +186,6 @@ static int cbc_decrypt(struct skcipher_request *req) sm4_aesni_avx_cbc_dec_blk8); } -int sm4_cfb_encrypt(struct skcipher_request *req) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm); - struct skcipher_walk walk; - unsigned int nbytes; - int err; - - err = skcipher_walk_virt(&walk, req, false); - - while ((nbytes = walk.nbytes) > 0) { - u8 keystream[SM4_BLOCK_SIZE]; - const u8 *iv = walk.iv; - const u8 *src = walk.src.virt.addr; - u8 *dst = walk.dst.virt.addr; - - while (nbytes >= SM4_BLOCK_SIZE) { - sm4_crypt_block(ctx->rkey_enc, keystream, iv); - crypto_xor_cpy(dst, src, keystream, SM4_BLOCK_SIZE); - iv = dst; - src += SM4_BLOCK_SIZE; - dst += SM4_BLOCK_SIZE; - nbytes -= SM4_BLOCK_SIZE; - } - if (iv != walk.iv) - memcpy(walk.iv, iv, SM4_BLOCK_SIZE); - - /* tail */ - if (walk.nbytes == walk.total && nbytes > 0) { - sm4_crypt_block(ctx->rkey_enc, keystream, walk.iv); - crypto_xor_cpy(dst, src, keystream, nbytes); - nbytes = 0; - } - - err = skcipher_walk_done(&walk, nbytes); - } - - return err; -} -EXPORT_SYMBOL_GPL(sm4_cfb_encrypt); - -int sm4_avx_cfb_decrypt(struct skcipher_request *req, - unsigned int bsize, sm4_crypt_func func) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm); - struct skcipher_walk walk; - unsigned int nbytes; - int err; - - err = skcipher_walk_virt(&walk, req, false); - - while ((nbytes = walk.nbytes) > 0) { - const u8 *src = walk.src.virt.addr; - u8 *dst = walk.dst.virt.addr; - - kernel_fpu_begin(); - - while (nbytes >= bsize) { - func(ctx->rkey_enc, dst, src, walk.iv); - dst += bsize; - src += bsize; - nbytes -= bsize; - } - - while (nbytes >= SM4_BLOCK_SIZE) { - u8 keystream[SM4_BLOCK_SIZE * 8]; - unsigned int nblocks = min(nbytes >> 4, 8u); - - memcpy(keystream, walk.iv, SM4_BLOCK_SIZE); - if (nblocks > 1) - memcpy(&keystream[SM4_BLOCK_SIZE], src, - (nblocks - 1) * SM4_BLOCK_SIZE); - memcpy(walk.iv, src + (nblocks - 1) * SM4_BLOCK_SIZE, - SM4_BLOCK_SIZE); - - sm4_aesni_avx_crypt8(ctx->rkey_enc, keystream, - keystream, nblocks); - - crypto_xor_cpy(dst, src, keystream, - nblocks * SM4_BLOCK_SIZE); - dst += nblocks * SM4_BLOCK_SIZE; - src += nblocks * SM4_BLOCK_SIZE; - nbytes -= nblocks * SM4_BLOCK_SIZE; - } - - kernel_fpu_end(); - - /* tail */ - if (walk.nbytes == walk.total && nbytes > 0) { - u8 keystream[SM4_BLOCK_SIZE]; - - sm4_crypt_block(ctx->rkey_enc, keystream, walk.iv); - crypto_xor_cpy(dst, src, keystream, nbytes); - nbytes = 0; - } - - err = skcipher_walk_done(&walk, nbytes); - } - - return err; -} -EXPORT_SYMBOL_GPL(sm4_avx_cfb_decrypt); - -static int cfb_decrypt(struct skcipher_request *req) -{ - return sm4_avx_cfb_decrypt(req, SM4_CRYPT8_BLOCK_SIZE, - sm4_aesni_avx_cfb_dec_blk8); -} - int sm4_avx_ctr_crypt(struct skcipher_request *req, unsigned int bsize, sm4_crypt_func func) { @@ -408,24 +296,6 @@ static struct skcipher_alg sm4_aesni_avx_skciphers[] = { .decrypt = cbc_decrypt, }, { .base = { - .cra_name = "__cfb(sm4)", - .cra_driver_name = "__cfb-sm4-aesni-avx", - .cra_priority = 400, - .cra_flags = CRYPTO_ALG_INTERNAL, - .cra_blocksize = 1, - .cra_ctxsize = sizeof(struct sm4_ctx), - .cra_module = THIS_MODULE, - }, - .min_keysize = SM4_KEY_SIZE, - .max_keysize = SM4_KEY_SIZE, - .ivsize = SM4_BLOCK_SIZE, - .chunksize = SM4_BLOCK_SIZE, - .walksize = 8 * SM4_BLOCK_SIZE, - .setkey = sm4_skcipher_setkey, - .encrypt = sm4_cfb_encrypt, - .decrypt = cfb_decrypt, - }, { - .base = { .cra_name = "__ctr(sm4)", .cra_driver_name = "__ctr-sm4-aesni-avx", .cra_priority = 400, diff --git a/crypto/Kconfig b/crypto/Kconfig index 70661f58ee41..7d156c75f15f 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -661,15 +661,6 @@ config CRYPTO_CBC This block cipher mode is required for IPSec ESP (XFRM_ESP). -config CRYPTO_CFB - tristate "CFB (Cipher Feedback)" - select CRYPTO_SKCIPHER - select CRYPTO_MANAGER - help - CFB (Cipher Feedback) mode (NIST SP800-38A) - - This block cipher mode is required for TPM2 Cryptography. - config CRYPTO_CTR tristate "CTR (Counter)" select CRYPTO_SKCIPHER @@ -735,20 +726,6 @@ config CRYPTO_LRW See https://people.csail.mit.edu/rivest/pubs/LRW02.pdf -config CRYPTO_OFB - tristate "OFB (Output Feedback)" - select CRYPTO_SKCIPHER - select CRYPTO_MANAGER - help - OFB (Output Feedback) mode (NIST SP800-38A) - - This mode makes a block cipher into a synchronous - stream cipher. It generates keystream blocks, which are then XORed - with the plaintext blocks to get the ciphertext. Flipping a bit in the - ciphertext produces a flipped bit in the plaintext at the same - location. This property allows many error correcting codes to function - normally even when applied before encryption. - config CRYPTO_PCBC tristate "PCBC (Propagating Cipher Block Chaining)" select CRYPTO_SKCIPHER diff --git a/crypto/Makefile b/crypto/Makefile index 5ac6876f935a..408f0a1f9ab9 100644 --- a/crypto/Makefile +++ b/crypto/Makefile @@ -92,7 +92,6 @@ obj-$(CONFIG_CRYPTO_BLAKE2B) += blake2b_generic.o CFLAGS_blake2b_generic.o := -Wframe-larger-than=4096 # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=105930 obj-$(CONFIG_CRYPTO_ECB) += ecb.o obj-$(CONFIG_CRYPTO_CBC) += cbc.o -obj-$(CONFIG_CRYPTO_CFB) += cfb.o obj-$(CONFIG_CRYPTO_PCBC) += pcbc.o obj-$(CONFIG_CRYPTO_CTS) += cts.o obj-$(CONFIG_CRYPTO_LRW) += lrw.o @@ -186,7 +185,6 @@ obj-$(CONFIG_CRYPTO_USER_API_SKCIPHER) += algif_skcipher.o obj-$(CONFIG_CRYPTO_USER_API_RNG) += algif_rng.o obj-$(CONFIG_CRYPTO_USER_API_AEAD) += algif_aead.o obj-$(CONFIG_CRYPTO_ZSTD) += zstd.o -obj-$(CONFIG_CRYPTO_OFB) += ofb.o obj-$(CONFIG_CRYPTO_ECC) += ecc.o obj-$(CONFIG_CRYPTO_ESSIV) += essiv.o obj-$(CONFIG_CRYPTO_CURVE25519) += curve25519-generic.o diff --git a/crypto/af_alg.c b/crypto/af_alg.c index ea6fb8e89d06..68cc9290cabe 100644 --- a/crypto/af_alg.c +++ b/crypto/af_alg.c @@ -1116,9 +1116,13 @@ EXPORT_SYMBOL_GPL(af_alg_sendmsg); void af_alg_free_resources(struct af_alg_async_req *areq) { struct sock *sk = areq->sk; + struct af_alg_ctx *ctx; af_alg_free_areq_sgls(areq); sock_kfree_s(sk, areq, areq->areqlen); + + ctx = alg_sk(sk)->private; + ctx->inflight = false; } EXPORT_SYMBOL_GPL(af_alg_free_resources); @@ -1188,11 +1192,19 @@ EXPORT_SYMBOL_GPL(af_alg_poll); struct af_alg_async_req *af_alg_alloc_areq(struct sock *sk, unsigned int areqlen) { - struct af_alg_async_req *areq = sock_kmalloc(sk, areqlen, GFP_KERNEL); + struct af_alg_ctx *ctx = alg_sk(sk)->private; + struct af_alg_async_req *areq; + + /* Only one AIO request can be in flight. */ + if (ctx->inflight) + return ERR_PTR(-EBUSY); + areq = sock_kmalloc(sk, areqlen, GFP_KERNEL); if (unlikely(!areq)) return ERR_PTR(-ENOMEM); + ctx->inflight = true; + areq->areqlen = areqlen; areq->sk = sk; areq->first_rsgl.sgl.sgt.sgl = areq->first_rsgl.sgl.sgl; diff --git a/crypto/algapi.c b/crypto/algapi.c index 4fe95c448047..85bc279b4233 100644 --- a/crypto/algapi.c +++ b/crypto/algapi.c @@ -341,6 +341,7 @@ __crypto_register_alg(struct crypto_alg *alg, struct list_head *algs_to_put) } if (!strcmp(q->cra_driver_name, alg->cra_name) || + !strcmp(q->cra_driver_name, alg->cra_driver_name) || !strcmp(q->cra_name, alg->cra_driver_name)) goto err; } diff --git a/crypto/algif_skcipher.c b/crypto/algif_skcipher.c index 9ada9b741af8..02cea2149504 100644 --- a/crypto/algif_skcipher.c +++ b/crypto/algif_skcipher.c @@ -47,6 +47,52 @@ static int skcipher_sendmsg(struct socket *sock, struct msghdr *msg, return af_alg_sendmsg(sock, msg, size, ivsize); } +static int algif_skcipher_export(struct sock *sk, struct skcipher_request *req) +{ + struct alg_sock *ask = alg_sk(sk); + struct crypto_skcipher *tfm; + struct af_alg_ctx *ctx; + struct alg_sock *pask; + unsigned statesize; + struct sock *psk; + int err; + + if (!(req->base.flags & CRYPTO_SKCIPHER_REQ_NOTFINAL)) + return 0; + + ctx = ask->private; + psk = ask->parent; + pask = alg_sk(psk); + tfm = pask->private; + + statesize = crypto_skcipher_statesize(tfm); + ctx->state = sock_kmalloc(sk, statesize, GFP_ATOMIC); + if (!ctx->state) + return -ENOMEM; + + err = crypto_skcipher_export(req, ctx->state); + if (err) { + sock_kzfree_s(sk, ctx->state, statesize); + ctx->state = NULL; + } + + return err; +} + +static void algif_skcipher_done(void *data, int err) +{ + struct af_alg_async_req *areq = data; + struct sock *sk = areq->sk; + + if (err) + goto out; + + err = algif_skcipher_export(sk, &areq->cra_u.skcipher_req); + +out: + af_alg_async_cb(data, err); +} + static int _skcipher_recvmsg(struct socket *sock, struct msghdr *msg, size_t ignored, int flags) { @@ -58,6 +104,7 @@ static int _skcipher_recvmsg(struct socket *sock, struct msghdr *msg, struct crypto_skcipher *tfm = pask->private; unsigned int bs = crypto_skcipher_chunksize(tfm); struct af_alg_async_req *areq; + unsigned cflags = 0; int err = 0; size_t len = 0; @@ -82,8 +129,10 @@ static int _skcipher_recvmsg(struct socket *sock, struct msghdr *msg, * If more buffers are to be expected to be processed, process only * full block size buffers. */ - if (ctx->more || len < ctx->used) + if (ctx->more || len < ctx->used) { len -= len % bs; + cflags |= CRYPTO_SKCIPHER_REQ_NOTFINAL; + } /* * Create a per request TX SGL for this request which tracks the @@ -107,6 +156,16 @@ static int _skcipher_recvmsg(struct socket *sock, struct msghdr *msg, skcipher_request_set_crypt(&areq->cra_u.skcipher_req, areq->tsgl, areq->first_rsgl.sgl.sgt.sgl, len, ctx->iv); + if (ctx->state) { + err = crypto_skcipher_import(&areq->cra_u.skcipher_req, + ctx->state); + sock_kzfree_s(sk, ctx->state, crypto_skcipher_statesize(tfm)); + ctx->state = NULL; + if (err) + goto free; + cflags |= CRYPTO_SKCIPHER_REQ_CONT; + } + if (msg->msg_iocb && !is_sync_kiocb(msg->msg_iocb)) { /* AIO operation */ sock_hold(sk); @@ -116,8 +175,9 @@ static int _skcipher_recvmsg(struct socket *sock, struct msghdr *msg, areq->outlen = len; skcipher_request_set_callback(&areq->cra_u.skcipher_req, + cflags | CRYPTO_TFM_REQ_MAY_SLEEP, - af_alg_async_cb, areq); + algif_skcipher_done, areq); err = ctx->enc ? crypto_skcipher_encrypt(&areq->cra_u.skcipher_req) : crypto_skcipher_decrypt(&areq->cra_u.skcipher_req); @@ -130,6 +190,7 @@ static int _skcipher_recvmsg(struct socket *sock, struct msghdr *msg, } else { /* Synchronous operation */ skcipher_request_set_callback(&areq->cra_u.skcipher_req, + cflags | CRYPTO_TFM_REQ_MAY_SLEEP | CRYPTO_TFM_REQ_MAY_BACKLOG, crypto_req_done, &ctx->wait); @@ -137,8 +198,11 @@ static int _skcipher_recvmsg(struct socket *sock, struct msghdr *msg, crypto_skcipher_encrypt(&areq->cra_u.skcipher_req) : crypto_skcipher_decrypt(&areq->cra_u.skcipher_req), &ctx->wait); - } + if (!err) + err = algif_skcipher_export( + sk, &areq->cra_u.skcipher_req); + } free: af_alg_free_resources(areq); @@ -301,6 +365,8 @@ static void skcipher_sock_destruct(struct sock *sk) af_alg_pull_tsgl(sk, ctx->used, NULL, 0); sock_kzfree_s(sk, ctx->iv, crypto_skcipher_ivsize(tfm)); + if (ctx->state) + sock_kzfree_s(sk, ctx->state, crypto_skcipher_statesize(tfm)); sock_kfree_s(sk, ctx, ctx->len); af_alg_release_parent(sk); } diff --git a/crypto/arc4.c b/crypto/arc4.c index eb3590dc9282..1a4825c97c5a 100644 --- a/crypto/arc4.c +++ b/crypto/arc4.c @@ -14,6 +14,8 @@ #include <linux/module.h> #include <linux/sched.h> +#define ARC4_ALIGN __alignof__(struct arc4_ctx) + static int crypto_arc4_setkey(struct crypto_lskcipher *tfm, const u8 *in_key, unsigned int key_len) { @@ -23,10 +25,15 @@ static int crypto_arc4_setkey(struct crypto_lskcipher *tfm, const u8 *in_key, } static int crypto_arc4_crypt(struct crypto_lskcipher *tfm, const u8 *src, - u8 *dst, unsigned nbytes, u8 *iv, bool final) + u8 *dst, unsigned nbytes, u8 *siv, u32 flags) { struct arc4_ctx *ctx = crypto_lskcipher_ctx(tfm); + if (!(flags & CRYPTO_LSKCIPHER_FLAG_CONT)) + memcpy(siv, ctx, sizeof(*ctx)); + + ctx = (struct arc4_ctx *)siv; + arc4_crypt(ctx, dst, src, nbytes); return 0; } @@ -45,9 +52,11 @@ static struct lskcipher_alg arc4_alg = { .co.base.cra_priority = 100, .co.base.cra_blocksize = ARC4_BLOCK_SIZE, .co.base.cra_ctxsize = sizeof(struct arc4_ctx), + .co.base.cra_alignmask = ARC4_ALIGN - 1, .co.base.cra_module = THIS_MODULE, .co.min_keysize = ARC4_MIN_KEY_SIZE, .co.max_keysize = ARC4_MAX_KEY_SIZE, + .co.statesize = sizeof(struct arc4_ctx), .setkey = crypto_arc4_setkey, .encrypt = crypto_arc4_crypt, .decrypt = crypto_arc4_crypt, diff --git a/crypto/cbc.c b/crypto/cbc.c index 28345b8d921c..eedddef9ce40 100644 --- a/crypto/cbc.c +++ b/crypto/cbc.c @@ -51,9 +51,10 @@ out: } static int crypto_cbc_encrypt(struct crypto_lskcipher *tfm, const u8 *src, - u8 *dst, unsigned len, u8 *iv, bool final) + u8 *dst, unsigned len, u8 *iv, u32 flags) { struct crypto_lskcipher **ctx = crypto_lskcipher_ctx(tfm); + bool final = flags & CRYPTO_LSKCIPHER_FLAG_FINAL; struct crypto_lskcipher *cipher = *ctx; int rem; @@ -119,9 +120,10 @@ out: } static int crypto_cbc_decrypt(struct crypto_lskcipher *tfm, const u8 *src, - u8 *dst, unsigned len, u8 *iv, bool final) + u8 *dst, unsigned len, u8 *iv, u32 flags) { struct crypto_lskcipher **ctx = crypto_lskcipher_ctx(tfm); + bool final = flags & CRYPTO_LSKCIPHER_FLAG_FINAL; struct crypto_lskcipher *cipher = *ctx; int rem; diff --git a/crypto/cfb.c b/crypto/cfb.c deleted file mode 100644 index 5c36b7b65e2a..000000000000 --- a/crypto/cfb.c +++ /dev/null @@ -1,254 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * CFB: Cipher FeedBack mode - * - * Copyright (c) 2018 James.Bottomley@HansenPartnership.com - * - * CFB is a stream cipher mode which is layered on to a block - * encryption scheme. It works very much like a one time pad where - * the pad is generated initially from the encrypted IV and then - * subsequently from the encrypted previous block of ciphertext. The - * pad is XOR'd into the plain text to get the final ciphertext. - * - * The scheme of CFB is best described by wikipedia: - * - * https://en.wikipedia.org/wiki/Block_cipher_mode_of_operation#CFB - * - * Note that since the pad for both encryption and decryption is - * generated by an encryption operation, CFB never uses the block - * decryption function. - */ - -#include <crypto/algapi.h> -#include <crypto/internal/cipher.h> -#include <crypto/internal/skcipher.h> -#include <linux/err.h> -#include <linux/init.h> -#include <linux/kernel.h> -#include <linux/module.h> -#include <linux/string.h> - -static unsigned int crypto_cfb_bsize(struct crypto_skcipher *tfm) -{ - return crypto_cipher_blocksize(skcipher_cipher_simple(tfm)); -} - -static void crypto_cfb_encrypt_one(struct crypto_skcipher *tfm, - const u8 *src, u8 *dst) -{ - crypto_cipher_encrypt_one(skcipher_cipher_simple(tfm), dst, src); -} - -/* final encrypt and decrypt is the same */ -static void crypto_cfb_final(struct skcipher_walk *walk, - struct crypto_skcipher *tfm) -{ - const unsigned long alignmask = crypto_skcipher_alignmask(tfm); - u8 tmp[MAX_CIPHER_BLOCKSIZE + MAX_CIPHER_ALIGNMASK]; - u8 *stream = PTR_ALIGN(tmp + 0, alignmask + 1); - u8 *src = walk->src.virt.addr; - u8 *dst = walk->dst.virt.addr; - u8 *iv = walk->iv; - unsigned int nbytes = walk->nbytes; - - crypto_cfb_encrypt_one(tfm, iv, stream); - crypto_xor_cpy(dst, stream, src, nbytes); -} - -static int crypto_cfb_encrypt_segment(struct skcipher_walk *walk, - struct crypto_skcipher *tfm) -{ - const unsigned int bsize = crypto_cfb_bsize(tfm); - unsigned int nbytes = walk->nbytes; - u8 *src = walk->src.virt.addr; - u8 *dst = walk->dst.virt.addr; - u8 *iv = walk->iv; - - do { - crypto_cfb_encrypt_one(tfm, iv, dst); - crypto_xor(dst, src, bsize); - iv = dst; - - src += bsize; - dst += bsize; - } while ((nbytes -= bsize) >= bsize); - - memcpy(walk->iv, iv, bsize); - - return nbytes; -} - -static int crypto_cfb_encrypt_inplace(struct skcipher_walk *walk, - struct crypto_skcipher *tfm) -{ - const unsigned int bsize = crypto_cfb_bsize(tfm); - unsigned int nbytes = walk->nbytes; - u8 *src = walk->src.virt.addr; - u8 *iv = walk->iv; - u8 tmp[MAX_CIPHER_BLOCKSIZE]; - - do { - crypto_cfb_encrypt_one(tfm, iv, tmp); - crypto_xor(src, tmp, bsize); - iv = src; - - src += bsize; - } while ((nbytes -= bsize) >= bsize); - - memcpy(walk->iv, iv, bsize); - - return nbytes; -} - -static int crypto_cfb_encrypt(struct skcipher_request *req) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct skcipher_walk walk; - unsigned int bsize = crypto_cfb_bsize(tfm); - int err; - - err = skcipher_walk_virt(&walk, req, false); - - while (walk.nbytes >= bsize) { - if (walk.src.virt.addr == walk.dst.virt.addr) - err = crypto_cfb_encrypt_inplace(&walk, tfm); - else - err = crypto_cfb_encrypt_segment(&walk, tfm); - err = skcipher_walk_done(&walk, err); - } - - if (walk.nbytes) { - crypto_cfb_final(&walk, tfm); - err = skcipher_walk_done(&walk, 0); - } - - return err; -} - -static int crypto_cfb_decrypt_segment(struct skcipher_walk *walk, - struct crypto_skcipher *tfm) -{ - const unsigned int bsize = crypto_cfb_bsize(tfm); - unsigned int nbytes = walk->nbytes; - u8 *src = walk->src.virt.addr; - u8 *dst = walk->dst.virt.addr; - u8 *iv = walk->iv; - - do { - crypto_cfb_encrypt_one(tfm, iv, dst); - crypto_xor(dst, src, bsize); - iv = src; - - src += bsize; - dst += bsize; - } while ((nbytes -= bsize) >= bsize); - - memcpy(walk->iv, iv, bsize); - - return nbytes; -} - -static int crypto_cfb_decrypt_inplace(struct skcipher_walk *walk, - struct crypto_skcipher *tfm) -{ - const unsigned int bsize = crypto_cfb_bsize(tfm); - unsigned int nbytes = walk->nbytes; - u8 *src = walk->src.virt.addr; - u8 * const iv = walk->iv; - u8 tmp[MAX_CIPHER_BLOCKSIZE]; - - do { - crypto_cfb_encrypt_one(tfm, iv, tmp); - memcpy(iv, src, bsize); - crypto_xor(src, tmp, bsize); - src += bsize; - } while ((nbytes -= bsize) >= bsize); - - return nbytes; -} - -static int crypto_cfb_decrypt_blocks(struct skcipher_walk *walk, - struct crypto_skcipher *tfm) -{ - if (walk->src.virt.addr == walk->dst.virt.addr) - return crypto_cfb_decrypt_inplace(walk, tfm); - else - return crypto_cfb_decrypt_segment(walk, tfm); -} - -static int crypto_cfb_decrypt(struct skcipher_request *req) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct skcipher_walk walk; - const unsigned int bsize = crypto_cfb_bsize(tfm); - int err; - - err = skcipher_walk_virt(&walk, req, false); - - while (walk.nbytes >= bsize) { - err = crypto_cfb_decrypt_blocks(&walk, tfm); - err = skcipher_walk_done(&walk, err); - } - - if (walk.nbytes) { - crypto_cfb_final(&walk, tfm); - err = skcipher_walk_done(&walk, 0); - } - - return err; -} - -static int crypto_cfb_create(struct crypto_template *tmpl, struct rtattr **tb) -{ - struct skcipher_instance *inst; - struct crypto_alg *alg; - int err; - - inst = skcipher_alloc_instance_simple(tmpl, tb); - if (IS_ERR(inst)) - return PTR_ERR(inst); - - alg = skcipher_ialg_simple(inst); - - /* CFB mode is a stream cipher. */ - inst->alg.base.cra_blocksize = 1; - - /* - * To simplify the implementation, configure the skcipher walk to only - * give a partial block at the very end, never earlier. - */ - inst->alg.chunksize = alg->cra_blocksize; - - inst->alg.encrypt = crypto_cfb_encrypt; - inst->alg.decrypt = crypto_cfb_decrypt; - - err = skcipher_register_instance(tmpl, inst); - if (err) - inst->free(inst); - - return err; -} - -static struct crypto_template crypto_cfb_tmpl = { - .name = "cfb", - .create = crypto_cfb_create, - .module = THIS_MODULE, -}; - -static int __init crypto_cfb_module_init(void) -{ - return crypto_register_template(&crypto_cfb_tmpl); -} - -static void __exit crypto_cfb_module_exit(void) -{ - crypto_unregister_template(&crypto_cfb_tmpl); -} - -subsys_initcall(crypto_cfb_module_init); -module_exit(crypto_cfb_module_exit); - -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("CFB block cipher mode of operation"); -MODULE_ALIAS_CRYPTO("cfb"); -MODULE_IMPORT_NS(CRYPTO_INTERNAL); diff --git a/crypto/drbg.c b/crypto/drbg.c index e01f8c7769d0..3addce90930c 100644 --- a/crypto/drbg.c +++ b/crypto/drbg.c @@ -111,9 +111,9 @@ * as stdrng. Each DRBG receives an increasing cra_priority values the later * they are defined in this array (see drbg_fill_array). * - * HMAC DRBGs are favored over Hash DRBGs over CTR DRBGs, and - * the SHA256 / AES 256 over other ciphers. Thus, the favored - * DRBGs are the latest entries in this array. + * HMAC DRBGs are favored over Hash DRBGs over CTR DRBGs, and the + * HMAC-SHA512 / SHA256 / AES 256 over other ciphers. Thus, the + * favored DRBGs are the latest entries in this array. */ static const struct drbg_core drbg_cores[] = { #ifdef CONFIG_CRYPTO_DRBG_CTR @@ -139,12 +139,6 @@ static const struct drbg_core drbg_cores[] = { #endif /* CONFIG_CRYPTO_DRBG_CTR */ #ifdef CONFIG_CRYPTO_DRBG_HASH { - .flags = DRBG_HASH | DRBG_STRENGTH128, - .statelen = 55, /* 440 bits */ - .blocklen_bytes = 20, - .cra_name = "sha1", - .backend_cra_name = "sha1", - }, { .flags = DRBG_HASH | DRBG_STRENGTH256, .statelen = 111, /* 888 bits */ .blocklen_bytes = 48, @@ -166,12 +160,6 @@ static const struct drbg_core drbg_cores[] = { #endif /* CONFIG_CRYPTO_DRBG_HASH */ #ifdef CONFIG_CRYPTO_DRBG_HMAC { - .flags = DRBG_HMAC | DRBG_STRENGTH128, - .statelen = 20, /* block length of cipher */ - .blocklen_bytes = 20, - .cra_name = "hmac_sha1", - .backend_cra_name = "hmac(sha1)", - }, { .flags = DRBG_HMAC | DRBG_STRENGTH256, .statelen = 48, /* block length of cipher */ .blocklen_bytes = 48, @@ -648,8 +636,6 @@ MODULE_ALIAS_CRYPTO("drbg_pr_hmac_sha384"); MODULE_ALIAS_CRYPTO("drbg_nopr_hmac_sha384"); MODULE_ALIAS_CRYPTO("drbg_pr_hmac_sha256"); MODULE_ALIAS_CRYPTO("drbg_nopr_hmac_sha256"); -MODULE_ALIAS_CRYPTO("drbg_pr_hmac_sha1"); -MODULE_ALIAS_CRYPTO("drbg_nopr_hmac_sha1"); /* update function of HMAC DRBG as defined in 10.1.2.2 */ static int drbg_hmac_update(struct drbg_state *drbg, struct list_head *seed, @@ -768,8 +754,6 @@ MODULE_ALIAS_CRYPTO("drbg_pr_sha384"); MODULE_ALIAS_CRYPTO("drbg_nopr_sha384"); MODULE_ALIAS_CRYPTO("drbg_pr_sha256"); MODULE_ALIAS_CRYPTO("drbg_nopr_sha256"); -MODULE_ALIAS_CRYPTO("drbg_pr_sha1"); -MODULE_ALIAS_CRYPTO("drbg_nopr_sha1"); /* * Increment buffer @@ -1475,11 +1459,11 @@ static int drbg_generate(struct drbg_state *drbg, int err = 0; pr_devel("DRBG: start to perform self test\n"); if (drbg->core->flags & DRBG_HMAC) - err = alg_test("drbg_pr_hmac_sha256", - "drbg_pr_hmac_sha256", 0, 0); + err = alg_test("drbg_pr_hmac_sha512", + "drbg_pr_hmac_sha512", 0, 0); else if (drbg->core->flags & DRBG_CTR) - err = alg_test("drbg_pr_ctr_aes128", - "drbg_pr_ctr_aes128", 0, 0); + err = alg_test("drbg_pr_ctr_aes256", + "drbg_pr_ctr_aes256", 0, 0); else err = alg_test("drbg_pr_sha256", "drbg_pr_sha256", 0, 0); @@ -2017,11 +2001,13 @@ static inline int __init drbg_healthcheck_sanity(void) return 0; #ifdef CONFIG_CRYPTO_DRBG_CTR - drbg_convert_tfm_core("drbg_nopr_ctr_aes128", &coreref, &pr); -#elif defined CONFIG_CRYPTO_DRBG_HASH + drbg_convert_tfm_core("drbg_nopr_ctr_aes256", &coreref, &pr); +#endif +#ifdef CONFIG_CRYPTO_DRBG_HASH drbg_convert_tfm_core("drbg_nopr_sha256", &coreref, &pr); -#else - drbg_convert_tfm_core("drbg_nopr_hmac_sha256", &coreref, &pr); +#endif +#ifdef CONFIG_CRYPTO_DRBG_HMAC + drbg_convert_tfm_core("drbg_nopr_hmac_sha512", &coreref, &pr); #endif drbg = kzalloc(sizeof(struct drbg_state), GFP_KERNEL); diff --git a/crypto/ecb.c b/crypto/ecb.c index cc7625d1a475..e3a67789050e 100644 --- a/crypto/ecb.c +++ b/crypto/ecb.c @@ -32,22 +32,24 @@ static int crypto_ecb_crypt(struct crypto_cipher *cipher, const u8 *src, } static int crypto_ecb_encrypt2(struct crypto_lskcipher *tfm, const u8 *src, - u8 *dst, unsigned len, u8 *iv, bool final) + u8 *dst, unsigned len, u8 *iv, u32 flags) { struct crypto_cipher **ctx = crypto_lskcipher_ctx(tfm); struct crypto_cipher *cipher = *ctx; - return crypto_ecb_crypt(cipher, src, dst, len, final, + return crypto_ecb_crypt(cipher, src, dst, len, + flags & CRYPTO_LSKCIPHER_FLAG_FINAL, crypto_cipher_alg(cipher)->cia_encrypt); } static int crypto_ecb_decrypt2(struct crypto_lskcipher *tfm, const u8 *src, - u8 *dst, unsigned len, u8 *iv, bool final) + u8 *dst, unsigned len, u8 *iv, u32 flags) { struct crypto_cipher **ctx = crypto_lskcipher_ctx(tfm); struct crypto_cipher *cipher = *ctx; - return crypto_ecb_crypt(cipher, src, dst, len, final, + return crypto_ecb_crypt(cipher, src, dst, len, + flags & CRYPTO_LSKCIPHER_FLAG_FINAL, crypto_cipher_alg(cipher)->cia_decrypt); } diff --git a/crypto/lskcipher.c b/crypto/lskcipher.c index 9edc89730951..0b6dd8aa21f2 100644 --- a/crypto/lskcipher.c +++ b/crypto/lskcipher.c @@ -88,8 +88,9 @@ EXPORT_SYMBOL_GPL(crypto_lskcipher_setkey); static int crypto_lskcipher_crypt_unaligned( struct crypto_lskcipher *tfm, const u8 *src, u8 *dst, unsigned len, u8 *iv, int (*crypt)(struct crypto_lskcipher *tfm, const u8 *src, - u8 *dst, unsigned len, u8 *iv, bool final)) + u8 *dst, unsigned len, u8 *iv, u32 flags)) { + unsigned statesize = crypto_lskcipher_statesize(tfm); unsigned ivsize = crypto_lskcipher_ivsize(tfm); unsigned bs = crypto_lskcipher_blocksize(tfm); unsigned cs = crypto_lskcipher_chunksize(tfm); @@ -104,7 +105,7 @@ static int crypto_lskcipher_crypt_unaligned( if (!tiv) return -ENOMEM; - memcpy(tiv, iv, ivsize); + memcpy(tiv, iv, ivsize + statesize); p = kmalloc(PAGE_SIZE, GFP_ATOMIC); err = -ENOMEM; @@ -119,7 +120,7 @@ static int crypto_lskcipher_crypt_unaligned( chunk &= ~(cs - 1); memcpy(p, src, chunk); - err = crypt(tfm, p, p, chunk, tiv, true); + err = crypt(tfm, p, p, chunk, tiv, CRYPTO_LSKCIPHER_FLAG_FINAL); if (err) goto out; @@ -132,7 +133,7 @@ static int crypto_lskcipher_crypt_unaligned( err = len ? -EINVAL : 0; out: - memcpy(iv, tiv, ivsize); + memcpy(iv, tiv, ivsize + statesize); kfree_sensitive(p); kfree_sensitive(tiv); return err; @@ -143,7 +144,7 @@ static int crypto_lskcipher_crypt(struct crypto_lskcipher *tfm, const u8 *src, int (*crypt)(struct crypto_lskcipher *tfm, const u8 *src, u8 *dst, unsigned len, u8 *iv, - bool final)) + u32 flags)) { unsigned long alignmask = crypto_lskcipher_alignmask(tfm); struct lskcipher_alg *alg = crypto_lskcipher_alg(tfm); @@ -156,7 +157,7 @@ static int crypto_lskcipher_crypt(struct crypto_lskcipher *tfm, const u8 *src, goto out; } - ret = crypt(tfm, src, dst, len, iv, true); + ret = crypt(tfm, src, dst, len, iv, CRYPTO_LSKCIPHER_FLAG_FINAL); out: return crypto_lskcipher_errstat(alg, ret); @@ -197,23 +198,45 @@ EXPORT_SYMBOL_GPL(crypto_lskcipher_decrypt); static int crypto_lskcipher_crypt_sg(struct skcipher_request *req, int (*crypt)(struct crypto_lskcipher *tfm, const u8 *src, u8 *dst, - unsigned len, u8 *iv, - bool final)) + unsigned len, u8 *ivs, + u32 flags)) { struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); struct crypto_lskcipher **ctx = crypto_skcipher_ctx(skcipher); + u8 *ivs = skcipher_request_ctx(req); struct crypto_lskcipher *tfm = *ctx; struct skcipher_walk walk; + unsigned ivsize; + u32 flags; int err; + ivsize = crypto_lskcipher_ivsize(tfm); + ivs = PTR_ALIGN(ivs, crypto_skcipher_alignmask(skcipher) + 1); + + flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; + + if (req->base.flags & CRYPTO_SKCIPHER_REQ_CONT) + flags |= CRYPTO_LSKCIPHER_FLAG_CONT; + else + memcpy(ivs, req->iv, ivsize); + + if (!(req->base.flags & CRYPTO_SKCIPHER_REQ_NOTFINAL)) + flags |= CRYPTO_LSKCIPHER_FLAG_FINAL; + err = skcipher_walk_virt(&walk, req, false); while (walk.nbytes) { err = crypt(tfm, walk.src.virt.addr, walk.dst.virt.addr, - walk.nbytes, walk.iv, walk.nbytes == walk.total); + walk.nbytes, ivs, + flags & ~(walk.nbytes == walk.total ? + 0 : CRYPTO_LSKCIPHER_FLAG_FINAL)); err = skcipher_walk_done(&walk, err); + flags |= CRYPTO_LSKCIPHER_FLAG_CONT; } + if (flags & CRYPTO_LSKCIPHER_FLAG_FINAL) + memcpy(req->iv, ivs, ivsize); + return err; } @@ -276,6 +299,7 @@ static void __maybe_unused crypto_lskcipher_show( seq_printf(m, "max keysize : %u\n", skcipher->co.max_keysize); seq_printf(m, "ivsize : %u\n", skcipher->co.ivsize); seq_printf(m, "chunksize : %u\n", skcipher->co.chunksize); + seq_printf(m, "statesize : %u\n", skcipher->co.statesize); } static int __maybe_unused crypto_lskcipher_report( @@ -618,6 +642,7 @@ struct lskcipher_instance *lskcipher_alloc_instance_simple( inst->alg.co.min_keysize = cipher_alg->co.min_keysize; inst->alg.co.max_keysize = cipher_alg->co.max_keysize; inst->alg.co.ivsize = cipher_alg->co.base.cra_blocksize; + inst->alg.co.statesize = cipher_alg->co.statesize; /* Use struct crypto_lskcipher * by default, can be overridden */ inst->alg.co.base.cra_ctxsize = sizeof(struct crypto_lskcipher *); diff --git a/crypto/ofb.c b/crypto/ofb.c deleted file mode 100644 index b630fdecceee..000000000000 --- a/crypto/ofb.c +++ /dev/null @@ -1,106 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 - -/* - * OFB: Output FeedBack mode - * - * Copyright (C) 2018 ARM Limited or its affiliates. - * All rights reserved. - */ - -#include <crypto/algapi.h> -#include <crypto/internal/cipher.h> -#include <crypto/internal/skcipher.h> -#include <linux/err.h> -#include <linux/init.h> -#include <linux/kernel.h> -#include <linux/module.h> - -static int crypto_ofb_crypt(struct skcipher_request *req) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct crypto_cipher *cipher = skcipher_cipher_simple(tfm); - const unsigned int bsize = crypto_cipher_blocksize(cipher); - struct skcipher_walk walk; - int err; - - err = skcipher_walk_virt(&walk, req, false); - - while (walk.nbytes >= bsize) { - const u8 *src = walk.src.virt.addr; - u8 *dst = walk.dst.virt.addr; - u8 * const iv = walk.iv; - unsigned int nbytes = walk.nbytes; - - do { - crypto_cipher_encrypt_one(cipher, iv, iv); - crypto_xor_cpy(dst, src, iv, bsize); - dst += bsize; - src += bsize; - } while ((nbytes -= bsize) >= bsize); - - err = skcipher_walk_done(&walk, nbytes); - } - - if (walk.nbytes) { - crypto_cipher_encrypt_one(cipher, walk.iv, walk.iv); - crypto_xor_cpy(walk.dst.virt.addr, walk.src.virt.addr, walk.iv, - walk.nbytes); - err = skcipher_walk_done(&walk, 0); - } - return err; -} - -static int crypto_ofb_create(struct crypto_template *tmpl, struct rtattr **tb) -{ - struct skcipher_instance *inst; - struct crypto_alg *alg; - int err; - - inst = skcipher_alloc_instance_simple(tmpl, tb); - if (IS_ERR(inst)) - return PTR_ERR(inst); - - alg = skcipher_ialg_simple(inst); - - /* OFB mode is a stream cipher. */ - inst->alg.base.cra_blocksize = 1; - - /* - * To simplify the implementation, configure the skcipher walk to only - * give a partial block at the very end, never earlier. - */ - inst->alg.chunksize = alg->cra_blocksize; - - inst->alg.encrypt = crypto_ofb_crypt; - inst->alg.decrypt = crypto_ofb_crypt; - - err = skcipher_register_instance(tmpl, inst); - if (err) - inst->free(inst); - - return err; -} - -static struct crypto_template crypto_ofb_tmpl = { - .name = "ofb", - .create = crypto_ofb_create, - .module = THIS_MODULE, -}; - -static int __init crypto_ofb_module_init(void) -{ - return crypto_register_template(&crypto_ofb_tmpl); -} - -static void __exit crypto_ofb_module_exit(void) -{ - crypto_unregister_template(&crypto_ofb_tmpl); -} - -subsys_initcall(crypto_ofb_module_init); -module_exit(crypto_ofb_module_exit); - -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("OFB block cipher mode of operation"); -MODULE_ALIAS_CRYPTO("ofb"); -MODULE_IMPORT_NS(CRYPTO_INTERNAL); diff --git a/crypto/rsa.c b/crypto/rsa.c index c79613cdce6e..b9cd11fb7d36 100644 --- a/crypto/rsa.c +++ b/crypto/rsa.c @@ -220,6 +220,8 @@ static int rsa_check_exponent_fips(MPI e) } e_max = mpi_alloc(0); + if (!e_max) + return -ENOMEM; mpi_set_bit(e_max, 256); if (mpi_cmp(e, e_max) >= 0) { diff --git a/crypto/scompress.c b/crypto/scompress.c index 442a82c9de7d..b108a30a7600 100644 --- a/crypto/scompress.c +++ b/crypto/scompress.c @@ -117,6 +117,7 @@ static int scomp_acomp_comp_decomp(struct acomp_req *req, int dir) struct crypto_scomp *scomp = *tfm_ctx; void **ctx = acomp_request_ctx(req); struct scomp_scratch *scratch; + unsigned int dlen; int ret; if (!req->src || !req->slen || req->slen > SCOMP_SCRATCH_SIZE) @@ -128,6 +129,8 @@ static int scomp_acomp_comp_decomp(struct acomp_req *req, int dir) if (!req->dlen || req->dlen > SCOMP_SCRATCH_SIZE) req->dlen = SCOMP_SCRATCH_SIZE; + dlen = req->dlen; + scratch = raw_cpu_ptr(&scomp_scratch); spin_lock(&scratch->lock); @@ -145,6 +148,9 @@ static int scomp_acomp_comp_decomp(struct acomp_req *req, int dir) ret = -ENOMEM; goto out; } + } else if (req->dlen > dlen) { + ret = -ENOSPC; + goto out; } scatterwalk_map_and_copy(scratch->dst, req->dst, 0, req->dlen, 1); diff --git a/crypto/shash.c b/crypto/shash.c index d5194221c88c..c3f7f6a25280 100644 --- a/crypto/shash.c +++ b/crypto/shash.c @@ -23,12 +23,8 @@ static inline struct crypto_istat_hash *shash_get_stat(struct shash_alg *alg) static inline int crypto_shash_errstat(struct shash_alg *alg, int err) { - if (!IS_ENABLED(CONFIG_CRYPTO_STATS)) - return err; - - if (err && err != -EINPROGRESS && err != -EBUSY) + if (IS_ENABLED(CONFIG_CRYPTO_STATS) && err) atomic64_inc(&shash_get_stat(alg)->err_cnt); - return err; } diff --git a/crypto/skcipher.c b/crypto/skcipher.c index ac8b8c042654..bc70e159d27d 100644 --- a/crypto/skcipher.c +++ b/crypto/skcipher.c @@ -698,6 +698,64 @@ int crypto_skcipher_decrypt(struct skcipher_request *req) } EXPORT_SYMBOL_GPL(crypto_skcipher_decrypt); +static int crypto_lskcipher_export(struct skcipher_request *req, void *out) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + u8 *ivs = skcipher_request_ctx(req); + + ivs = PTR_ALIGN(ivs, crypto_skcipher_alignmask(tfm) + 1); + + memcpy(out, ivs + crypto_skcipher_ivsize(tfm), + crypto_skcipher_statesize(tfm)); + + return 0; +} + +static int crypto_lskcipher_import(struct skcipher_request *req, const void *in) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + u8 *ivs = skcipher_request_ctx(req); + + ivs = PTR_ALIGN(ivs, crypto_skcipher_alignmask(tfm) + 1); + + memcpy(ivs + crypto_skcipher_ivsize(tfm), in, + crypto_skcipher_statesize(tfm)); + + return 0; +} + +static int skcipher_noexport(struct skcipher_request *req, void *out) +{ + return 0; +} + +static int skcipher_noimport(struct skcipher_request *req, const void *in) +{ + return 0; +} + +int crypto_skcipher_export(struct skcipher_request *req, void *out) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct skcipher_alg *alg = crypto_skcipher_alg(tfm); + + if (alg->co.base.cra_type != &crypto_skcipher_type) + return crypto_lskcipher_export(req, out); + return alg->export(req, out); +} +EXPORT_SYMBOL_GPL(crypto_skcipher_export); + +int crypto_skcipher_import(struct skcipher_request *req, const void *in) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct skcipher_alg *alg = crypto_skcipher_alg(tfm); + + if (alg->co.base.cra_type != &crypto_skcipher_type) + return crypto_lskcipher_import(req, in); + return alg->import(req, in); +} +EXPORT_SYMBOL_GPL(crypto_skcipher_import); + static void crypto_skcipher_exit_tfm(struct crypto_tfm *tfm) { struct crypto_skcipher *skcipher = __crypto_skcipher_cast(tfm); @@ -713,8 +771,17 @@ static int crypto_skcipher_init_tfm(struct crypto_tfm *tfm) skcipher_set_needkey(skcipher); - if (tfm->__crt_alg->cra_type != &crypto_skcipher_type) + if (tfm->__crt_alg->cra_type != &crypto_skcipher_type) { + unsigned am = crypto_skcipher_alignmask(skcipher); + unsigned reqsize; + + reqsize = am & ~(crypto_tfm_ctx_alignment() - 1); + reqsize += crypto_skcipher_ivsize(skcipher); + reqsize += crypto_skcipher_statesize(skcipher); + crypto_skcipher_set_reqsize(skcipher, reqsize); + return crypto_init_lskcipher_ops_sg(tfm); + } if (alg->exit) skcipher->base.exit = crypto_skcipher_exit_tfm; @@ -756,6 +823,7 @@ static void crypto_skcipher_show(struct seq_file *m, struct crypto_alg *alg) seq_printf(m, "ivsize : %u\n", skcipher->ivsize); seq_printf(m, "chunksize : %u\n", skcipher->chunksize); seq_printf(m, "walksize : %u\n", skcipher->walksize); + seq_printf(m, "statesize : %u\n", skcipher->statesize); } static int __maybe_unused crypto_skcipher_report( @@ -870,7 +938,9 @@ int skcipher_prepare_alg_common(struct skcipher_alg_common *alg) struct crypto_istat_cipher *istat = skcipher_get_stat_common(alg); struct crypto_alg *base = &alg->base; - if (alg->ivsize > PAGE_SIZE / 8 || alg->chunksize > PAGE_SIZE / 8) + if (alg->ivsize > PAGE_SIZE / 8 || alg->chunksize > PAGE_SIZE / 8 || + alg->statesize > PAGE_SIZE / 2 || + (alg->ivsize + alg->statesize) > PAGE_SIZE / 2) return -EINVAL; if (!alg->chunksize) @@ -899,6 +969,12 @@ static int skcipher_prepare_alg(struct skcipher_alg *alg) if (!alg->walksize) alg->walksize = alg->chunksize; + if (!alg->statesize) { + alg->import = skcipher_noimport; + alg->export = skcipher_noexport; + } else if (!(alg->import && alg->export)) + return -EINVAL; + base->cra_type = &crypto_skcipher_type; base->cra_flags |= CRYPTO_ALG_TYPE_SKCIPHER; diff --git a/crypto/tcrypt.c b/crypto/tcrypt.c index 202ca1a3105d..ea4d1cea9c06 100644 --- a/crypto/tcrypt.c +++ b/crypto/tcrypt.c @@ -1524,8 +1524,6 @@ static int do_test(const char *alg, u32 type, u32 mask, int m, u32 num_mb) ret = min(ret, tcrypt_test("xts(aes)")); ret = min(ret, tcrypt_test("ctr(aes)")); ret = min(ret, tcrypt_test("rfc3686(ctr(aes))")); - ret = min(ret, tcrypt_test("ofb(aes)")); - ret = min(ret, tcrypt_test("cfb(aes)")); ret = min(ret, tcrypt_test("xctr(aes)")); break; @@ -1845,14 +1843,12 @@ static int do_test(const char *alg, u32 type, u32 mask, int m, u32 num_mb) case 191: ret = min(ret, tcrypt_test("ecb(sm4)")); ret = min(ret, tcrypt_test("cbc(sm4)")); - ret = min(ret, tcrypt_test("cfb(sm4)")); ret = min(ret, tcrypt_test("ctr(sm4)")); ret = min(ret, tcrypt_test("xts(sm4)")); break; case 192: ret = min(ret, tcrypt_test("ecb(aria)")); ret = min(ret, tcrypt_test("cbc(aria)")); - ret = min(ret, tcrypt_test("cfb(aria)")); ret = min(ret, tcrypt_test("ctr(aria)")); break; case 200: @@ -1880,10 +1876,6 @@ static int do_test(const char *alg, u32 type, u32 mask, int m, u32 num_mb) speed_template_16_24_32); test_cipher_speed("ctr(aes)", DECRYPT, sec, NULL, 0, speed_template_16_24_32); - test_cipher_speed("cfb(aes)", ENCRYPT, sec, NULL, 0, - speed_template_16_24_32); - test_cipher_speed("cfb(aes)", DECRYPT, sec, NULL, 0, - speed_template_16_24_32); break; case 201: @@ -2115,10 +2107,6 @@ static int do_test(const char *alg, u32 type, u32 mask, int m, u32 num_mb) speed_template_16); test_cipher_speed("cts(cbc(sm4))", DECRYPT, sec, NULL, 0, speed_template_16); - test_cipher_speed("cfb(sm4)", ENCRYPT, sec, NULL, 0, - speed_template_16); - test_cipher_speed("cfb(sm4)", DECRYPT, sec, NULL, 0, - speed_template_16); test_cipher_speed("ctr(sm4)", ENCRYPT, sec, NULL, 0, speed_template_16); test_cipher_speed("ctr(sm4)", DECRYPT, sec, NULL, 0, @@ -2198,10 +2186,6 @@ static int do_test(const char *alg, u32 type, u32 mask, int m, u32 num_mb) speed_template_16_24_32); test_cipher_speed("cbc(aria)", DECRYPT, sec, NULL, 0, speed_template_16_24_32); - test_cipher_speed("cfb(aria)", ENCRYPT, sec, NULL, 0, - speed_template_16_24_32); - test_cipher_speed("cfb(aria)", DECRYPT, sec, NULL, 0, - speed_template_16_24_32); test_cipher_speed("ctr(aria)", ENCRYPT, sec, NULL, 0, speed_template_16_24_32); test_cipher_speed("ctr(aria)", DECRYPT, sec, NULL, 0, @@ -2436,14 +2420,6 @@ static int do_test(const char *alg, u32 type, u32 mask, int m, u32 num_mb) speed_template_16_24_32); test_acipher_speed("ctr(aes)", DECRYPT, sec, NULL, 0, speed_template_16_24_32); - test_acipher_speed("cfb(aes)", ENCRYPT, sec, NULL, 0, - speed_template_16_24_32); - test_acipher_speed("cfb(aes)", DECRYPT, sec, NULL, 0, - speed_template_16_24_32); - test_acipher_speed("ofb(aes)", ENCRYPT, sec, NULL, 0, - speed_template_16_24_32); - test_acipher_speed("ofb(aes)", DECRYPT, sec, NULL, 0, - speed_template_16_24_32); test_acipher_speed("rfc3686(ctr(aes))", ENCRYPT, sec, NULL, 0, speed_template_20_28_36); test_acipher_speed("rfc3686(ctr(aes))", DECRYPT, sec, NULL, 0, @@ -2463,18 +2439,6 @@ static int do_test(const char *alg, u32 type, u32 mask, int m, u32 num_mb) test_acipher_speed("cbc(des3_ede)", DECRYPT, sec, des3_speed_template, DES3_SPEED_VECTORS, speed_template_24); - test_acipher_speed("cfb(des3_ede)", ENCRYPT, sec, - des3_speed_template, DES3_SPEED_VECTORS, - speed_template_24); - test_acipher_speed("cfb(des3_ede)", DECRYPT, sec, - des3_speed_template, DES3_SPEED_VECTORS, - speed_template_24); - test_acipher_speed("ofb(des3_ede)", ENCRYPT, sec, - des3_speed_template, DES3_SPEED_VECTORS, - speed_template_24); - test_acipher_speed("ofb(des3_ede)", DECRYPT, sec, - des3_speed_template, DES3_SPEED_VECTORS, - speed_template_24); break; case 502: @@ -2486,14 +2450,6 @@ static int do_test(const char *alg, u32 type, u32 mask, int m, u32 num_mb) speed_template_8); test_acipher_speed("cbc(des)", DECRYPT, sec, NULL, 0, speed_template_8); - test_acipher_speed("cfb(des)", ENCRYPT, sec, NULL, 0, - speed_template_8); - test_acipher_speed("cfb(des)", DECRYPT, sec, NULL, 0, - speed_template_8); - test_acipher_speed("ofb(des)", ENCRYPT, sec, NULL, 0, - speed_template_8); - test_acipher_speed("ofb(des)", DECRYPT, sec, NULL, 0, - speed_template_8); break; case 503: @@ -2632,10 +2588,6 @@ static int do_test(const char *alg, u32 type, u32 mask, int m, u32 num_mb) speed_template_16); test_acipher_speed("cbc(sm4)", DECRYPT, sec, NULL, 0, speed_template_16); - test_acipher_speed("cfb(sm4)", ENCRYPT, sec, NULL, 0, - speed_template_16); - test_acipher_speed("cfb(sm4)", DECRYPT, sec, NULL, 0, - speed_template_16); test_acipher_speed("ctr(sm4)", ENCRYPT, sec, NULL, 0, speed_template_16); test_acipher_speed("ctr(sm4)", DECRYPT, sec, NULL, 0, @@ -2682,14 +2634,6 @@ static int do_test(const char *alg, u32 type, u32 mask, int m, u32 num_mb) speed_template_16_24_32, num_mb); test_mb_skcipher_speed("ctr(aes)", DECRYPT, sec, NULL, 0, speed_template_16_24_32, num_mb); - test_mb_skcipher_speed("cfb(aes)", ENCRYPT, sec, NULL, 0, - speed_template_16_24_32, num_mb); - test_mb_skcipher_speed("cfb(aes)", DECRYPT, sec, NULL, 0, - speed_template_16_24_32, num_mb); - test_mb_skcipher_speed("ofb(aes)", ENCRYPT, sec, NULL, 0, - speed_template_16_24_32, num_mb); - test_mb_skcipher_speed("ofb(aes)", DECRYPT, sec, NULL, 0, - speed_template_16_24_32, num_mb); test_mb_skcipher_speed("rfc3686(ctr(aes))", ENCRYPT, sec, NULL, 0, speed_template_20_28_36, num_mb); test_mb_skcipher_speed("rfc3686(ctr(aes))", DECRYPT, sec, NULL, @@ -2709,18 +2653,6 @@ static int do_test(const char *alg, u32 type, u32 mask, int m, u32 num_mb) test_mb_skcipher_speed("cbc(des3_ede)", DECRYPT, sec, des3_speed_template, DES3_SPEED_VECTORS, speed_template_24, num_mb); - test_mb_skcipher_speed("cfb(des3_ede)", ENCRYPT, sec, - des3_speed_template, DES3_SPEED_VECTORS, - speed_template_24, num_mb); - test_mb_skcipher_speed("cfb(des3_ede)", DECRYPT, sec, - des3_speed_template, DES3_SPEED_VECTORS, - speed_template_24, num_mb); - test_mb_skcipher_speed("ofb(des3_ede)", ENCRYPT, sec, - des3_speed_template, DES3_SPEED_VECTORS, - speed_template_24, num_mb); - test_mb_skcipher_speed("ofb(des3_ede)", DECRYPT, sec, - des3_speed_template, DES3_SPEED_VECTORS, - speed_template_24, num_mb); break; case 602: @@ -2732,14 +2664,6 @@ static int do_test(const char *alg, u32 type, u32 mask, int m, u32 num_mb) speed_template_8, num_mb); test_mb_skcipher_speed("cbc(des)", DECRYPT, sec, NULL, 0, speed_template_8, num_mb); - test_mb_skcipher_speed("cfb(des)", ENCRYPT, sec, NULL, 0, - speed_template_8, num_mb); - test_mb_skcipher_speed("cfb(des)", DECRYPT, sec, NULL, 0, - speed_template_8, num_mb); - test_mb_skcipher_speed("ofb(des)", ENCRYPT, sec, NULL, 0, - speed_template_8, num_mb); - test_mb_skcipher_speed("ofb(des)", DECRYPT, sec, NULL, 0, - speed_template_8, num_mb); break; case 603: diff --git a/crypto/testmgr.c b/crypto/testmgr.c index 15c7a3011269..c26aeda85787 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -4609,25 +4609,6 @@ static const struct alg_test_desc alg_test_descs[] = { } } }, { - .alg = "cfb(aes)", - .test = alg_test_skcipher, - .fips_allowed = 1, - .suite = { - .cipher = __VECS(aes_cfb_tv_template) - }, - }, { - .alg = "cfb(aria)", - .test = alg_test_skcipher, - .suite = { - .cipher = __VECS(aria_cfb_tv_template) - }, - }, { - .alg = "cfb(sm4)", - .test = alg_test_skcipher, - .suite = { - .cipher = __VECS(sm4_cfb_tv_template) - } - }, { .alg = "chacha20", .test = alg_test_skcipher, .suite = { @@ -4816,6 +4797,16 @@ static const struct alg_test_desc alg_test_descs[] = { } } }, { + .alg = "deflate-iaa", + .test = alg_test_comp, + .fips_allowed = 1, + .suite = { + .comp = { + .comp = __VECS(deflate_comp_tv_template), + .decomp = __VECS(deflate_decomp_tv_template) + } + } + }, { .alg = "dh", .test = alg_test_kpp, .suite = { @@ -4846,14 +4837,6 @@ static const struct alg_test_desc alg_test_descs[] = { .drbg = __VECS(drbg_nopr_ctr_aes256_tv_template) } }, { - /* - * There is no need to specifically test the DRBG with every - * backend cipher -- covered by drbg_nopr_hmac_sha256 test - */ - .alg = "drbg_nopr_hmac_sha1", - .fips_allowed = 1, - .test = alg_test_null, - }, { .alg = "drbg_nopr_hmac_sha256", .test = alg_test_drbg, .fips_allowed = 1, @@ -4861,7 +4844,10 @@ static const struct alg_test_desc alg_test_descs[] = { .drbg = __VECS(drbg_nopr_hmac_sha256_tv_template) } }, { - /* covered by drbg_nopr_hmac_sha256 test */ + /* + * There is no need to specifically test the DRBG with every + * backend cipher -- covered by drbg_nopr_hmac_sha512 test + */ .alg = "drbg_nopr_hmac_sha384", .test = alg_test_null, }, { @@ -4872,10 +4858,6 @@ static const struct alg_test_desc alg_test_descs[] = { .drbg = __VECS(drbg_nopr_hmac_sha512_tv_template) } }, { - .alg = "drbg_nopr_sha1", - .fips_allowed = 1, - .test = alg_test_null, - }, { .alg = "drbg_nopr_sha256", .test = alg_test_drbg, .fips_allowed = 1, @@ -4907,10 +4889,6 @@ static const struct alg_test_desc alg_test_descs[] = { .fips_allowed = 1, .test = alg_test_null, }, { - .alg = "drbg_pr_hmac_sha1", - .fips_allowed = 1, - .test = alg_test_null, - }, { .alg = "drbg_pr_hmac_sha256", .test = alg_test_drbg, .fips_allowed = 1, @@ -4926,10 +4904,6 @@ static const struct alg_test_desc alg_test_descs[] = { .test = alg_test_null, .fips_allowed = 1, }, { - .alg = "drbg_pr_sha1", - .fips_allowed = 1, - .test = alg_test_null, - }, { .alg = "drbg_pr_sha256", .test = alg_test_drbg, .fips_allowed = 1, @@ -5420,26 +5394,6 @@ static const struct alg_test_desc alg_test_descs[] = { .hash = __VECS(nhpoly1305_tv_template) } }, { - .alg = "ofb(aes)", - .test = alg_test_skcipher, - .fips_allowed = 1, - .suite = { - .cipher = __VECS(aes_ofb_tv_template) - } - }, { - /* Same as ofb(aes) except the key is stored in - * hardware secure memory which we reference by index - */ - .alg = "ofb(paes)", - .test = alg_test_null, - .fips_allowed = 1, - }, { - .alg = "ofb(sm4)", - .test = alg_test_skcipher, - .suite = { - .cipher = __VECS(sm4_ofb_tv_template) - } - }, { .alg = "pcbc(fcrypt)", .test = alg_test_skcipher, .suite = { diff --git a/crypto/testmgr.h b/crypto/testmgr.h index d7e98397549b..986f331a5fc2 100644 --- a/crypto/testmgr.h +++ b/crypto/testmgr.h @@ -14704,104 +14704,6 @@ static const struct cipher_testvec sm4_ctr_rfc3686_tv_template[] = { } }; -static const struct cipher_testvec sm4_ofb_tv_template[] = { - { /* From: draft-ribose-cfrg-sm4-02, paragraph 12.2.3 */ - .key = "\x01\x23\x45\x67\x89\xab\xcd\xef" - "\xfe\xdc\xba\x98\x76\x54\x32\x10", - .klen = 16, - .iv = "\x01\x23\x45\x67\x89\xab\xcd\xef" - "\xfe\xdc\xba\x98\x76\x54\x32\x10", - .ptext = "\x01\x23\x45\x67\x89\xab\xcd\xef" - "\xfe\xdc\xba\x98\x76\x54\x32\x10" - "\x01\x23\x45\x67\x89\xab\xcd\xef" - "\xfe\xdc\xba\x98\x76\x54\x32\x10", - .ctext = "\x69\x3d\x9a\x53\x5b\xad\x5b\xb1" - "\x78\x6f\x53\xd7\x25\x3a\x70\x56" - "\xf2\x07\x5d\x28\xb5\x23\x5f\x58" - "\xd5\x00\x27\xe4\x17\x7d\x2b\xce", - .len = 32, - }, { /* From: draft-ribose-cfrg-sm4-09, appendix A.2.3, Example 1 */ - .key = "\x01\x23\x45\x67\x89\xab\xcd\xef" - "\xfe\xdc\xba\x98\x76\x54\x32\x10", - .klen = 16, - .iv = "\x00\x01\x02\x03\x04\x05\x06\x07" - "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", - .ptext = "\xaa\xaa\xaa\xaa\xbb\xbb\xbb\xbb" - "\xcc\xcc\xcc\xcc\xdd\xdd\xdd\xdd" - "\xee\xee\xee\xee\xff\xff\xff\xff" - "\xaa\xaa\xaa\xaa\xbb\xbb\xbb\xbb", - .ctext = "\xac\x32\x36\xcb\x86\x1d\xd3\x16" - "\xe6\x41\x3b\x4e\x3c\x75\x24\xb7" - "\x1d\x01\xac\xa2\x48\x7c\xa5\x82" - "\xcb\xf5\x46\x3e\x66\x98\x53\x9b", - .len = 32, - }, { /* From: draft-ribose-cfrg-sm4-09, appendix A.2.3, Example 2 */ - .key = "\xfe\xdc\xba\x98\x76\x54\x32\x10" - "\x01\x23\x45\x67\x89\xab\xcd\xef", - .klen = 16, - .iv = "\x00\x01\x02\x03\x04\x05\x06\x07" - "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", - .ptext = "\xaa\xaa\xaa\xaa\xbb\xbb\xbb\xbb" - "\xcc\xcc\xcc\xcc\xdd\xdd\xdd\xdd" - "\xee\xee\xee\xee\xff\xff\xff\xff" - "\xaa\xaa\xaa\xaa\xbb\xbb\xbb\xbb", - .ctext = "\x5d\xcc\xcd\x25\xa8\x4b\xa1\x65" - "\x60\xd7\xf2\x65\x88\x70\x68\x49" - "\x33\xfa\x16\xbd\x5c\xd9\xc8\x56" - "\xca\xca\xa1\xe1\x01\x89\x7a\x97", - .len = 32, - } -}; - -static const struct cipher_testvec sm4_cfb_tv_template[] = { - { /* From: draft-ribose-cfrg-sm4-02, paragraph 12.2.4 */ - .key = "\x01\x23\x45\x67\x89\xab\xcd\xef" - "\xfe\xdc\xba\x98\x76\x54\x32\x10", - .klen = 16, - .iv = "\x01\x23\x45\x67\x89\xab\xcd\xef" - "\xfe\xdc\xba\x98\x76\x54\x32\x10", - .ptext = "\x01\x23\x45\x67\x89\xab\xcd\xef" - "\xfe\xdc\xba\x98\x76\x54\x32\x10" - "\x01\x23\x45\x67\x89\xab\xcd\xef" - "\xfe\xdc\xba\x98\x76\x54\x32\x10", - .ctext = "\x69\x3d\x9a\x53\x5b\xad\x5b\xb1" - "\x78\x6f\x53\xd7\x25\x3a\x70\x56" - "\x9e\xd2\x58\xa8\x5a\x04\x67\xcc" - "\x92\xaa\xb3\x93\xdd\x97\x89\x95", - .len = 32, - }, { /* From: draft-ribose-cfrg-sm4-09, appendix A.2.4, Example 1 */ - .key = "\x01\x23\x45\x67\x89\xab\xcd\xef" - "\xfe\xdc\xba\x98\x76\x54\x32\x10", - .klen = 16, - .iv = "\x00\x01\x02\x03\x04\x05\x06\x07" - "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", - .ptext = "\xaa\xaa\xaa\xaa\xbb\xbb\xbb\xbb" - "\xcc\xcc\xcc\xcc\xdd\xdd\xdd\xdd" - "\xee\xee\xee\xee\xff\xff\xff\xff" - "\xaa\xaa\xaa\xaa\xbb\xbb\xbb\xbb", - .ctext = "\xac\x32\x36\xcb\x86\x1d\xd3\x16" - "\xe6\x41\x3b\x4e\x3c\x75\x24\xb7" - "\x69\xd4\xc5\x4e\xd4\x33\xb9\xa0" - "\x34\x60\x09\xbe\xb3\x7b\x2b\x3f", - .len = 32, - }, { /* From: draft-ribose-cfrg-sm4-09, appendix A.2.4, Example 2 */ - .key = "\xfe\xdc\xba\x98\x76\x54\x32\x10" - "\x01\x23\x45\x67\x89\xab\xcd\xef", - .klen = 16, - .iv = "\x00\x01\x02\x03\x04\x05\x06\x07" - "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", - .ptext = "\xaa\xaa\xaa\xaa\xbb\xbb\xbb\xbb" - "\xcc\xcc\xcc\xcc\xdd\xdd\xdd\xdd" - "\xee\xee\xee\xee\xff\xff\xff\xff" - "\xaa\xaa\xaa\xaa\xbb\xbb\xbb\xbb", - .ctext = "\x5d\xcc\xcd\x25\xa8\x4b\xa1\x65" - "\x60\xd7\xf2\x65\x88\x70\x68\x49" - "\x0d\x9b\x86\xff\x20\xc3\xbf\xe1" - "\x15\xff\xa0\x2c\xa6\x19\x2c\xc5", - .len = 32, - } -}; - static const struct cipher_testvec sm4_cts_tv_template[] = { /* Generated from AES-CTS test vectors */ { @@ -17064,104 +16966,6 @@ static const struct cipher_testvec aes_cbc_tv_template[] = { }, }; -static const struct cipher_testvec aes_cfb_tv_template[] = { - { /* From NIST SP800-38A */ - .key = "\x2b\x7e\x15\x16\x28\xae\xd2\xa6" - "\xab\xf7\x15\x88\x09\xcf\x4f\x3c", - .klen = 16, - .iv = "\x00\x01\x02\x03\x04\x05\x06\x07" - "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", - .ptext = "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96" - "\xe9\x3d\x7e\x11\x73\x93\x17\x2a" - "\xae\x2d\x8a\x57\x1e\x03\xac\x9c" - "\x9e\xb7\x6f\xac\x45\xaf\x8e\x51" - "\x30\xc8\x1c\x46\xa3\x5c\xe4\x11" - "\xe5\xfb\xc1\x19\x1a\x0a\x52\xef" - "\xf6\x9f\x24\x45\xdf\x4f\x9b\x17" - "\xad\x2b\x41\x7b\xe6\x6c\x37\x10", - .ctext = "\x3b\x3f\xd9\x2e\xb7\x2d\xad\x20" - "\x33\x34\x49\xf8\xe8\x3c\xfb\x4a" - "\xc8\xa6\x45\x37\xa0\xb3\xa9\x3f" - "\xcd\xe3\xcd\xad\x9f\x1c\xe5\x8b" - "\x26\x75\x1f\x67\xa3\xcb\xb1\x40" - "\xb1\x80\x8c\xf1\x87\xa4\xf4\xdf" - "\xc0\x4b\x05\x35\x7c\x5d\x1c\x0e" - "\xea\xc4\xc6\x6f\x9f\xf7\xf2\xe6", - .len = 64, - }, { - .key = "\x8e\x73\xb0\xf7\xda\x0e\x64\x52" - "\xc8\x10\xf3\x2b\x80\x90\x79\xe5" - "\x62\xf8\xea\xd2\x52\x2c\x6b\x7b", - .klen = 24, - .iv = "\x00\x01\x02\x03\x04\x05\x06\x07" - "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", - .ptext = "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96" - "\xe9\x3d\x7e\x11\x73\x93\x17\x2a" - "\xae\x2d\x8a\x57\x1e\x03\xac\x9c" - "\x9e\xb7\x6f\xac\x45\xaf\x8e\x51" - "\x30\xc8\x1c\x46\xa3\x5c\xe4\x11" - "\xe5\xfb\xc1\x19\x1a\x0a\x52\xef" - "\xf6\x9f\x24\x45\xdf\x4f\x9b\x17" - "\xad\x2b\x41\x7b\xe6\x6c\x37\x10", - .ctext = "\xcd\xc8\x0d\x6f\xdd\xf1\x8c\xab" - "\x34\xc2\x59\x09\xc9\x9a\x41\x74" - "\x67\xce\x7f\x7f\x81\x17\x36\x21" - "\x96\x1a\x2b\x70\x17\x1d\x3d\x7a" - "\x2e\x1e\x8a\x1d\xd5\x9b\x88\xb1" - "\xc8\xe6\x0f\xed\x1e\xfa\xc4\xc9" - "\xc0\x5f\x9f\x9c\xa9\x83\x4f\xa0" - "\x42\xae\x8f\xba\x58\x4b\x09\xff", - .len = 64, - }, { - .key = "\x60\x3d\xeb\x10\x15\xca\x71\xbe" - "\x2b\x73\xae\xf0\x85\x7d\x77\x81" - "\x1f\x35\x2c\x07\x3b\x61\x08\xd7" - "\x2d\x98\x10\xa3\x09\x14\xdf\xf4", - .klen = 32, - .iv = "\x00\x01\x02\x03\x04\x05\x06\x07" - "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", - .ptext = "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96" - "\xe9\x3d\x7e\x11\x73\x93\x17\x2a" - "\xae\x2d\x8a\x57\x1e\x03\xac\x9c" - "\x9e\xb7\x6f\xac\x45\xaf\x8e\x51" - "\x30\xc8\x1c\x46\xa3\x5c\xe4\x11" - "\xe5\xfb\xc1\x19\x1a\x0a\x52\xef" - "\xf6\x9f\x24\x45\xdf\x4f\x9b\x17" - "\xad\x2b\x41\x7b\xe6\x6c\x37\x10", - .ctext = "\xdc\x7e\x84\xbf\xda\x79\x16\x4b" - "\x7e\xcd\x84\x86\x98\x5d\x38\x60" - "\x39\xff\xed\x14\x3b\x28\xb1\xc8" - "\x32\x11\x3c\x63\x31\xe5\x40\x7b" - "\xdf\x10\x13\x24\x15\xe5\x4b\x92" - "\xa1\x3e\xd0\xa8\x26\x7a\xe2\xf9" - "\x75\xa3\x85\x74\x1a\xb9\xce\xf8" - "\x20\x31\x62\x3d\x55\xb1\xe4\x71", - .len = 64, - }, { /* > 16 bytes, not a multiple of 16 bytes */ - .key = "\x2b\x7e\x15\x16\x28\xae\xd2\xa6" - "\xab\xf7\x15\x88\x09\xcf\x4f\x3c", - .klen = 16, - .iv = "\x00\x01\x02\x03\x04\x05\x06\x07" - "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", - .ptext = "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96" - "\xe9\x3d\x7e\x11\x73\x93\x17\x2a" - "\xae", - .ctext = "\x3b\x3f\xd9\x2e\xb7\x2d\xad\x20" - "\x33\x34\x49\xf8\xe8\x3c\xfb\x4a" - "\xc8", - .len = 17, - }, { /* < 16 bytes */ - .key = "\x2b\x7e\x15\x16\x28\xae\xd2\xa6" - "\xab\xf7\x15\x88\x09\xcf\x4f\x3c", - .klen = 16, - .iv = "\x00\x01\x02\x03\x04\x05\x06\x07" - "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", - .ptext = "\x6b\xc1\xbe\xe2\x2e\x40\x9f", - .ctext = "\x3b\x3f\xd9\x2e\xb7\x2d\xad", - .len = 7, - }, -}; - static const struct aead_testvec hmac_md5_ecb_cipher_null_tv_template[] = { { /* Input data from RFC 2410 Case 1 */ #ifdef __LITTLE_ENDIAN @@ -20852,55 +20656,6 @@ static const struct cipher_testvec aes_ctr_rfc3686_tv_template[] = { }, }; -static const struct cipher_testvec aes_ofb_tv_template[] = { - { /* From NIST Special Publication 800-38A, Appendix F.5 */ - .key = "\x2b\x7e\x15\x16\x28\xae\xd2\xa6" - "\xab\xf7\x15\x88\x09\xcf\x4f\x3c", - .klen = 16, - .iv = "\x00\x01\x02\x03\x04\x05\x06\x07\x08" - "\x09\x0a\x0b\x0c\x0d\x0e\x0f", - .ptext = "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96" - "\xe9\x3d\x7e\x11\x73\x93\x17\x2a" - "\xae\x2d\x8a\x57\x1e\x03\xac\x9c" - "\x9e\xb7\x6f\xac\x45\xaf\x8e\x51" - "\x30\xc8\x1c\x46\xa3\x5c\xe4\x11" - "\xe5\xfb\xc1\x19\x1a\x0a\x52\xef" - "\xf6\x9f\x24\x45\xdf\x4f\x9b\x17" - "\xad\x2b\x41\x7b\xe6\x6c\x37\x10", - .ctext = "\x3b\x3f\xd9\x2e\xb7\x2d\xad\x20" - "\x33\x34\x49\xf8\xe8\x3c\xfb\x4a" - "\x77\x89\x50\x8d\x16\x91\x8f\x03\xf5" - "\x3c\x52\xda\xc5\x4e\xd8\x25" - "\x97\x40\x05\x1e\x9c\x5f\xec\xf6\x43" - "\x44\xf7\xa8\x22\x60\xed\xcc" - "\x30\x4c\x65\x28\xf6\x59\xc7\x78" - "\x66\xa5\x10\xd9\xc1\xd6\xae\x5e", - .len = 64, - }, { /* > 16 bytes, not a multiple of 16 bytes */ - .key = "\x2b\x7e\x15\x16\x28\xae\xd2\xa6" - "\xab\xf7\x15\x88\x09\xcf\x4f\x3c", - .klen = 16, - .iv = "\x00\x01\x02\x03\x04\x05\x06\x07" - "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", - .ptext = "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96" - "\xe9\x3d\x7e\x11\x73\x93\x17\x2a" - "\xae", - .ctext = "\x3b\x3f\xd9\x2e\xb7\x2d\xad\x20" - "\x33\x34\x49\xf8\xe8\x3c\xfb\x4a" - "\x77", - .len = 17, - }, { /* < 16 bytes */ - .key = "\x2b\x7e\x15\x16\x28\xae\xd2\xa6" - "\xab\xf7\x15\x88\x09\xcf\x4f\x3c", - .klen = 16, - .iv = "\x00\x01\x02\x03\x04\x05\x06\x07" - "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", - .ptext = "\x6b\xc1\xbe\xe2\x2e\x40\x9f", - .ctext = "\x3b\x3f\xd9\x2e\xb7\x2d\xad", - .len = 7, - } -}; - static const struct aead_testvec aes_gcm_tv_template[] = { { /* From McGrew & Viega - http://citeseer.ist.psu.edu/656989.html */ .key = zeroed_string, @@ -29349,909 +29104,6 @@ static const struct cipher_testvec aria_ctr_tv_template[] = { } }; -static const struct cipher_testvec aria_cfb_tv_template[] = { - { - .key = "\x7f\x92\xd5\x06\x30\x6b\xc0\x23" - "\x87\xa8\x8e\x6d\xc7\xc5\xd7\xf1", - .klen = 16, - .iv = "\x5f\xce\x89\xb3\xd5\x7f\x7f\xf0" - "\xfd\xab\x56\xa6\x6e\xda\x7c\x57", - .ptext = "\x36\x36\x89\x09\xcd\xa8\xd3\x91" - "\x48\x3e\x3c\x11\xcf\xd0\x4f\xc0", - .ctext = "\x19\x28\xb5\xf2\x1c\xbc\xf8\xaf" - "\xb9\xae\x1b\x23\x4f\xe1\x6e\x40", - }, { - .key = "\x51\xe3\x8c\xe9\x76\xcd\xff\x37" - "\xd6\x1a\x18\x2f\x68\x2f\xb6\xfe", - .klen = 16, - .iv = "\x3d\x2d\x85\x75\x6e\x18\x8a\x52" - "\x53\x39\xfc\xc1\xf5\xc0\x56\x22", - .ptext = "\xc6\xae\xaa\x0d\x90\xf2\x38\x93" - "\xac\xd2\x3f\xc7\x74\x8d\x13\x7e" - "\xfa\x3f\x70\x52\xfb\x04\x0e\xed" - "\x0e\x60\x75\x84\x21\xdf\x13\xa1", - .ctext = "\x3f\x8c\xa9\x19\xd6\xb4\xfb\xed" - "\x9c\x6d\xaa\x1b\xe1\xc1\xe6\xa8" - "\x47\x35\x7d\xa3\x96\x7d\x53\x60" - "\xa9\x33\x9c\x34\xae\x7d\x7c\x74", - .len = 32, - }, { - .key = "\x26\xf8\x8c\x26\x0a\x37\x51\x8f" - "\xe7\x9c\x74\x77\x7a\x3e\xbb\x5d", - .klen = 16, - .iv = "\xd7\x33\xf3\xa9\x5b\xb4\x86\xea" - "\xe3\x7d\x50\x62\x3b\x73\xaf\xc4", - .ptext = "\xda\x89\xd9\x3c\xcc\xe4\x73\xb0" - "\xef\x3e\x5f\x46\x62\x88\xd5\x26" - "\x3b\xd3\xb5\x81\x78\x70\x1b\xd2" - "\x39\x56\x34\x63\x2c\xc5\x51\x13" - "\x48\x29\x3a\x58\xbe\x41\xc5\x80" - "\x2c\x80\xa7\x3c\x14\xb4\x89\x5e", - .ctext = "\x28\xd8\xa7\xf8\x74\x98\x00\xfc" - "\xd6\x48\xad\xbd\xbe\x3f\x0e\x7b" - "\xa3\xec\x03\x6a\xfb\xc9\x01\x83" - "\xb3\x2f\xda\x5e\x66\xa0\xc3\xec" - "\xe9\xd4\x72\x2a\xa2\x90\x41\xcf" - "\xde\x30\x79\xc3\x82\x10\x51\xe1", - .len = 48, - }, { - .key = "\x8e\xe5\x5f\xe2\x39\x80\xf5\x2b" - "\x77\xb5\xca\x90\xda\x1d\x22\x17", - .klen = 16, - .iv = "\xd9\xa0\x57\x80\xc8\x96\x70\x86" - "\x07\x2c\xf4\x61\x79\x09\x01\x8f", - .ptext = "\x37\x32\x98\xd4\x86\x2b\x3b\x80" - "\x07\x60\xba\xf0\x2e\xc3\x4a\x57" - "\xf5\xb5\xd7\xbf\xd2\x2a\x9b\x4a" - "\xe6\x08\xf0\xbe\x77\xd1\x62\x40" - "\xa0\x82\x09\x60\x47\xbb\x16\x56" - "\x50\x1f\xab\x8b\x10\xfe\xf0\x5c" - "\x05\x32\x63\x1a\xc4\x46\x6f\x55" - "\x32\xde\x41\x5a\xf7\x52\xd7\xfa", - .ctext = "\x29\x31\x55\xd2\xe5\x0b\x81\x39" - "\xf9\xbc\x63\xe2\xfa\x26\x99\xde" - "\x5c\xd3\x0a\x56\xe5\xfc\x83\xdd" - "\xab\x26\x90\x7d\xa8\x0f\x01\xa6" - "\x0e\x01\xdc\x1f\xfa\xa7\xdd\x09" - "\xf9\xbf\x12\xf4\xc6\x9f\xbd\x57" - "\x23\x68\x54\x0f\xe0\xcf\x1c\x6d" - "\xe1\x5e\x0b\x4a\x1e\x71\x1d\xaa", - .len = 64, - }, { - .key = "\x30\x9d\x59\x8d\x64\x76\xad\x37" - "\xba\xbc\x46\x6a\x69\x17\x3c\xac", - .klen = 16, - .iv = "\x6f\xdd\xa2\x9b\x86\x32\x14\x2e" - "\x54\x74\x8f\x3d\xe2\xd6\x85\x44", - .ptext = "\x4f\x4a\x31\x64\xc6\xa5\x29\xaa" - "\xad\xfd\x32\x94\x1f\x56\x57\xd1" - "\x9d\x7e\x3d\x49\x00\x36\xb1\x5d" - "\xb2\x92\x83\x70\x1e\xa3\x97\xa6" - "\x65\x53\x39\xeb\x53\x8f\xb1\x38" - "\x91\xac\x17\x11\x1c\x03\x69\x53" - "\xf5\xdf\xdb\x2c\x1b\x9a\x6e\x6b" - "\xb6\x02\xc4\xfa\x95\x01\x33\xa8" - "\xda\x7e\x18\x2c\xf4\x7e\x6e\x67" - "\xce\x8f\x9f\xea\x46\x66\x99\xb8", - .ctext = "\x38\xbc\xf5\x9d\x0e\x26\xa6\x18" - "\x95\x0b\x23\x54\x09\xa1\xf9\x46" - "\x7a\x31\xa0\xd7\x4a\xec\xb3\x10" - "\x8a\x8e\x99\x78\x6c\x6e\x76\xf2" - "\x63\x8a\x3b\x90\xaa\xd5\x64\x65" - "\x5a\x52\xb0\x36\x4c\xce\xed\xc7" - "\x51\x3c\x06\xb0\xee\x54\xec\x10" - "\xc0\x5f\xfd\xa9\x44\x9a\x29\x32" - "\x19\x79\x7d\x2b\x14\x26\x96\x13" - "\x9d\xa5\x61\xbd\xb6\x72\x37\x26", - .len = 80, - }, { - .key = "\xe1\xc7\x25\x4d\xbd\xa5\x74\xdf" - "\xc7\x8b\xfb\xe3\x2d\x3a\x82\xd3", - .klen = 16, - .iv = "\x17\x94\x77\x2f\x92\xb8\x87\xc2" - "\xcc\x6f\x70\x26\x87\xc7\x10\x8a", - .ptext = "\xc8\xfd\xc2\xb3\xcf\xa0\xeb\x41" - "\x4c\xf4\xd0\x34\xd0\x95\xab\xae" - "\x82\x5c\xfd\xfa\x13\x86\x25\xce" - "\xf4\x13\x32\xcd\xc6\x6d\xf6\x50" - "\x12\x4a\x5b\x66\x3a\xd3\xfb\x1a" - "\xaf\x06\xea\xf4\x65\x59\xd6\xc2" - "\x84\xa0\x53\x97\x61\x30\x70\x15" - "\xac\x45\x8e\xe8\xeb\xa1\x72\x93" - "\x26\x76\x98\x6f\xe4\x86\xca\xf0" - "\x57\x89\xf2\x2b\xd4\xcf\x2d\x95" - "\x86\x26\x20\x0e\x62\xfe\x8f\x1e" - "\x5d\xcb\x2b\x7e\xdd\xab\xac\xda", - .ctext = "\xdf\x79\x58\x30\x6f\x47\x12\x78" - "\x04\xb2\x0b\x1a\x62\x22\xe2\x9f" - "\xfe\x90\x50\x41\x1b\x6a\x6a\x9c" - "\x4e\x77\x8f\xca\xd1\x68\x31\xcd" - "\x41\x82\xa5\x5b\xc0\x08\x2b\x37" - "\x62\xec\x95\xf1\x56\x12\x38\x66" - "\x84\x82\x72\xda\x00\x21\x96\x82" - "\x33\xd4\x99\xaa\xb9\xeb\xd5\xc3" - "\x2b\xa8\xf7\xdc\x13\x0e\x21\x9f" - "\x4b\xf9\x42\x58\xa8\x39\x10\xd5" - "\x86\xa5\xc6\x78\x3b\x34\x05\x03" - "\x54\x43\x2b\x80\xa9\x53\x4d\x0e", - .len = 96, - }, { - .key = "\x6e\x49\x20\xd5\xb7\x01\x83\x4e" - "\xac\x45\x8f\xe1\x05\x3f\xd5\xb1", - .klen = 16, - .iv = "\xee\xb7\x0d\x65\x00\x38\xab\x71" - "\x70\x6e\xb3\x97\x86\xd3\xcd\xad", - .ptext = "\x51\x8b\x9c\xa0\x9a\x8b\x4c\xb9" - "\x16\x01\x6a\x1f\xdf\xf0\xf9\x9e" - "\x25\x1b\xc2\xa6\x21\x25\xeb\x97" - "\x4b\xf6\xcb\x3b\xcd\x61\xfd\x94" - "\x37\x03\xb3\xd9\x74\x6e\x4d\xbb" - "\xfd\x87\x2b\xec\x4c\x2c\xbf\xe2" - "\x94\x1a\xe6\xd9\xaf\x0e\x78\x17" - "\x58\x2b\x1d\x73\x9a\x9c\x63\x18" - "\x88\x7a\x0e\x87\x2f\xf0\xb0\xdb" - "\xc9\x9d\x79\x51\x34\x39\x4f\x07" - "\xa2\x7c\x21\x04\x91\x3b\x79\x79" - "\xfe\xd5\x51\x46\xd5\xcd\x28\xc0" - "\xad\xb8\x55\xb2\xb2\x5a\x9a\xa2" - "\xe2\x0c\xfc\x55\x7d\x60\xd2\x95", - .ctext = "\xe4\x25\x0d\x22\xeb\xbe\x5e\x90" - "\x01\xe5\xae\xc9\x94\xbd\x93\x89" - "\x5e\x5a\x5a\x2f\xf6\xdf\xf8\x16" - "\xd3\xb2\xed\x29\x51\xe2\x75\xb0" - "\x1a\x48\xb5\xe6\xd3\x58\x40\xc7" - "\x6f\x6f\xcf\x57\x82\x43\x5a\x36" - "\xef\x27\xe1\x34\x85\x01\xec\x98" - "\x00\xbd\x94\x6f\x12\x39\xa8\x13" - "\xfe\x3c\x39\xc0\xc6\xe1\xcc\x05" - "\x0e\xd5\xc9\xda\xbd\xdd\xdb\xaa" - "\x5a\xaa\x8e\xe8\xa8\x0a\xc5\x18" - "\xb4\x1d\x13\x81\xc9\xc4\xaa\x61" - "\xa9\xbd\xaa\x03\x12\x93\xbb\xed" - "\x0c\x6e\xbd\x1c\x05\x16\x8a\x59", - .len = 112, - }, { - .key = "\xb6\x08\x1d\x31\xaf\xf4\x17\x46" - "\xa4\xbb\x0f\xbd\x67\x3c\x73\x15", - .klen = 16, - .iv = "\x0c\x85\x2f\x62\xe5\xf4\x35\x96" - "\xb1\x9b\x5d\x00\x10\xe9\x70\x12", - .ptext = "\x3a\x87\x7f\x67\xf1\x81\x7a\x05" - "\xb4\xa6\xfe\xdf\x36\x31\x6d\x9e" - "\x0e\xa9\x44\xa0\xb0\x05\xa9\x41" - "\x9c\x14\x44\x5a\xd5\x1c\x50\x08" - "\x95\xc2\xf2\xaf\x3f\x29\xc9\x3e" - "\x95\x5e\xc6\xb4\x2b\xf4\x3e\xe3" - "\x1b\xeb\x3d\x73\xfb\xd7\x1e\x2b" - "\x0c\x3d\x58\x6c\xb4\x41\x9b\xfe" - "\x2f\x7e\x1c\x10\x81\x36\x2d\x79" - "\xaf\xab\x10\x44\x2e\xcc\x0d\x6c" - "\x9c\x14\xc2\xe4\xae\xb0\xbb\xda" - "\x6a\xe0\x42\x3d\x96\x9f\x78\x7d" - "\x70\x86\xa5\x92\x9f\xee\xcd\x3f" - "\x6a\x55\x84\x98\x28\x03\x02\xc2" - "\xf7\xec\x7a\xfa\xb1\xd9\xa8\xd8" - "\x1c\xc3\xaa\xd5\x61\x7f\x10\x0c", - .ctext = "\xa7\x4c\x96\x55\x7c\x07\xce\xb2" - "\x6f\x63\x9f\xc6\x8b\x6f\xc6\x4a" - "\x85\xf2\x4b\xdf\x62\x0c\x6c\x8d" - "\x13\x5d\xd3\x40\x58\xa6\xf9\x03" - "\xd9\xf2\x48\x4e\x12\x64\x9a\x55" - "\xa2\xa3\xd0\x19\xe5\x5b\xaa\x62" - "\x7b\xe9\x2a\x23\xab\xb5\xa6\xcf" - "\x53\x59\x70\xc6\xb8\x92\x12\x3b" - "\x93\x68\x24\xba\x7d\xd6\xc0\x5b" - "\x06\x2e\x7f\x2e\x32\x5d\x42\x9c" - "\x13\x8e\x92\x3c\x99\x20\x32\x2b" - "\x4a\x41\xb2\x4a\x81\xe8\x6e\x7f" - "\x5b\x8e\xca\x4d\xd7\x29\x96\xde" - "\x30\x9c\xa6\x84\x90\xe7\xc2\xae" - "\xf4\x7e\x73\x32\x4c\x25\xec\xef" - "\x58\x69\x63\x3f\x4e\x71\x4b\x1c", - .len = 128, - }, { - .key = "\xc0\xa1\x36\x3d\x81\x9a\xd2\x17" - "\x2e\x23\xc9\xb7\xff\xdf\x47\x6c", - .klen = 16, - .iv = "\x96\x3b\x0e\xbd\xec\x9a\x0e\xad" - "\x8c\xaf\x36\x3d\xff\x29\x8b\x33", - .ptext = "\x87\x96\x77\x1a\x10\x81\x63\x8a" - "\x63\xde\x88\xa9\x9d\xa9\x01\xf2" - "\xdf\xc9\x25\x35\x48\x3a\x15\xdf" - "\x20\x6b\x91\x7c\x56\xe5\x10\x7a" - "\x2d\x2e\x0f\x30\x32\xed\xa9\x1f" - "\x71\x4e\x68\x77\xe8\xa8\x5b\xdd" - "\x3c\x5e\x68\x6b\xab\x03\xe4\xf8" - "\x42\xc1\x61\x9a\x50\xfb\xc7\x6a" - "\x1a\x31\xa7\x87\xd0\x24\xcb\x5e" - "\xc0\x3b\x12\x28\xca\x26\x7b\xb3" - "\x14\xc1\x7f\x66\xff\x3b\xa4\x80" - "\x59\x77\x4f\xa0\xd4\xb2\xd9\x8a" - "\xb6\x67\xe6\x28\xd3\x6f\xf2\xcf" - "\xb8\x6d\x2d\xc4\x2a\x69\x89\xff" - "\xcf\xbb\x11\x2e\x2a\x2b\x7c\xfd" - "\xcd\x56\x02\x95\xc9\x54\x6e\x62" - "\x6a\x97\x75\x1a\x21\x16\x46\xfb" - "\xc2\xab\x62\x54\xef\xba\xae\x46", - .ctext = "\x11\x7f\xea\x49\xaf\x24\x52\xa2" - "\xde\x60\x99\x58\x23\xf9\x9e\x91" - "\x94\x52\x31\xa3\x28\x07\x14\xad" - "\x00\x24\x4a\x4a\xe7\x18\xd7\x24" - "\xcc\x8b\x66\x53\x82\x65\x31\xa5" - "\x54\x76\x59\x0b\x69\x6f\x90\x2c" - "\x8d\xa5\x2b\x61\x05\x80\xfb\xe0" - "\xf9\x6e\xaf\xb9\xc4\x15\x67\xcc" - "\x15\xce\xa0\xc0\xf2\xae\xa6\x15" - "\x24\x9a\xe5\xcb\x09\x42\xcf\x41" - "\x95\xa4\x8d\xbf\xe8\xb8\x40\xcd" - "\xb0\x33\x2c\xb3\xc4\xdd\xf9\x45" - "\xda\xb2\xeb\xb3\xf8\xfa\x7f\xe3" - "\xc0\x3a\x98\xe7\x17\x4a\x0c\x60" - "\xb2\x22\xba\x3b\x21\x85\x27\x56" - "\xe0\xb2\xf7\x2a\x59\xb1\x56\x20" - "\x0b\xa9\x13\x73\xe0\x6f\x61\x32" - "\xa5\x38\x14\xb3\xe3\xaa\x70\x44", - .len = 144, - }, { - .key = "\xd4\x14\xc6\xcc\x16\x1b\x95\xf9" - "\x05\x26\x23\x81\x19\x27\xad\x7b", - .klen = 16, - .iv = "\x9c\x8b\xfb\x65\xa4\x61\xee\x69" - "\x44\xbf\x59\xde\x03\x61\x11\x12", - .ptext = "\x8d\x94\x48\x47\xa9\x52\x16\xfb" - "\x6b\xaf\x59\x6d\xab\x74\xbf\x5c" - "\xb6\x09\x21\x12\x42\x98\x13\xa1" - "\xa8\x6f\xb9\x6d\x4d\xa6\xdc\xea" - "\x61\x02\x3c\xa7\xcd\x1a\x28\x8c" - "\x66\xb8\x4d\x60\x67\x82\xcc\x8d" - "\x1e\xda\x8f\x28\xe5\x02\xdc\x2c" - "\x54\x84\x2a\x06\xb5\xd1\x34\x57" - "\xb8\x28\x4d\xf5\x69\xb9\xf3\x33" - "\x5e\x0b\xa6\x62\x35\x9b\xfb\x97" - "\x3e\xc6\xec\xaf\x74\xe8\x72\x91" - "\xb2\xc6\x56\xb3\x23\x29\x43\xe0" - "\xfb\xcc\x21\x38\x64\x78\x9e\x78" - "\xbb\x6e\x0d\x7b\xfd\x05\x74\x01" - "\x7c\x94\xe0\xb0\xd7\x92\xfc\x58" - "\x28\xfc\xe2\x7b\x7f\xf7\x31\x0d" - "\x90\xb7\x60\x78\xa8\x9f\x52\xe3" - "\xe6\xaa\x2a\xb4\xa7\x09\x60\x53" - "\x42\x0e\x15\x31\xf6\x48\xa3\x0a" - "\x20\xf0\x79\x67\xb1\x83\x26\x66", - .ctext = "\x5b\xc0\xe8\x17\xa4\xf9\xea\xce" - "\x9e\xf9\xe0\xb1\xac\x37\xe9\x41" - "\xc8\x06\xf9\x1c\x1a\xfc\xe8\x7a" - "\x38\xf2\x80\x66\xc2\x70\x59\x4e" - "\xe0\x32\x5b\x27\x39\xf5\xfb\x03" - "\xc8\xaf\xd6\x7e\x57\xc7\xc6\x71" - "\xd9\xd0\x48\x39\xb1\x0d\xa8\x1a" - "\x23\x8a\x3d\x05\xe2\x90\x7e\x18" - "\xd7\x20\x04\x3b\x82\x76\x3f\xaa" - "\xc2\x89\xb6\x9e\x14\x2f\x46\xcd" - "\x51\x9b\xa8\x7b\x62\x7b\x9c\x17" - "\xc4\xe1\x8b\x3f\xb5\x4d\xac\x66" - "\x49\xf6\xb6\x4c\x3e\x16\x46\xb0" - "\xca\x04\xef\x72\x5c\x03\x0a\xe5" - "\x2f\x4e\x36\x38\x36\x9f\xf4\xe2" - "\x81\x7a\x4c\xdf\x36\x27\xd5\x9d" - "\x03\xad\x1d\x3a\xe9\x2a\x99\xb0" - "\x2c\xba\x13\x75\xc8\x37\x97\x11" - "\xf4\x15\x0f\xb7\x75\x26\xa1\x14" - "\x79\xec\x1f\xab\xd2\x10\x8c\x5f", - .len = 160, - }, { - .key = "\x7f\x92\xd5\x06\x30\x6b\xc0\x23" - "\x87\xa8\x8e\x6d\xc7\xc5\xd7\xf1" - "\x5f\xce\x89\xb3\xd5\x7f\x7f\xf0", - .klen = 24, - .iv = "\xfd\xab\x56\xa6\x6e\xda\x7c\x57" - "\x36\x36\x89\x09\xcd\xa8\xd3\x91", - .ptext = "\x48\x3e\x3c\x11\xcf\xd0\x4f\xc0" - "\x51\xe3\x8c\xe9\x76\xcd\xff\x37", - .ctext = "\xa4\x12\x2f\xc4\xf0\x6d\xd9\x46" - "\xe4\xe6\xd1\x0b\x6d\x14\xf0\x8f", - .len = 16, - }, { - .key = "\xd6\x1a\x18\x2f\x68\x2f\xb6\xfe" - "\x3d\x2d\x85\x75\x6e\x18\x8a\x52" - "\x53\x39\xfc\xc1\xf5\xc0\x56\x22", - .klen = 24, - .iv = "\xc6\xae\xaa\x0d\x90\xf2\x38\x93" - "\xac\xd2\x3f\xc7\x74\x8d\x13\x7e", - .ptext = "\xfa\x3f\x70\x52\xfb\x04\x0e\xed" - "\x0e\x60\x75\x84\x21\xdf\x13\xa1" - "\x26\xf8\x8c\x26\x0a\x37\x51\x8f" - "\xe7\x9c\x74\x77\x7a\x3e\xbb\x5d", - .ctext = "\x80\x2b\xf0\x88\xb9\x4b\x8d\xf5" - "\xc3\x0e\x15\x5b\xea\x5d\x5b\xa8" - "\x52\xe7\x83\x3c\xa1\x51\x1c\x1f" - "\x38\xd9\x7c\x88\x3c\x3a\xcd\x3e", - .len = 32, - }, { - .key = "\xd7\x33\xf3\xa9\x5b\xb4\x86\xea" - "\xe3\x7d\x50\x62\x3b\x73\xaf\xc4" - "\xda\x89\xd9\x3c\xcc\xe4\x73\xb0", - .klen = 24, - .iv = "\xef\x3e\x5f\x46\x62\x88\xd5\x26" - "\x3b\xd3\xb5\x81\x78\x70\x1b\xd2", - .ptext = "\x39\x56\x34\x63\x2c\xc5\x51\x13" - "\x48\x29\x3a\x58\xbe\x41\xc5\x80" - "\x2c\x80\xa7\x3c\x14\xb4\x89\x5e" - "\x8e\xe5\x5f\xe2\x39\x80\xf5\x2b" - "\x77\xb5\xca\x90\xda\x1d\x22\x17" - "\xd9\xa0\x57\x80\xc8\x96\x70\x86", - .ctext = "\x65\x01\x3c\xb0\xac\x4c\x63\xb6" - "\xe7\xf1\xf4\x61\x35\xf4\x36\xde" - "\xeb\x0f\x8c\x34\xd1\x78\xb4\x00" - "\xb2\xc1\x7c\x28\xb2\xb7\xbb\xa3" - "\xc6\xb7\x27\xf7\x6d\x56\x79\xfa" - "\x61\x57\xba\x30\x6f\x56\xe9\x8c", - .len = 48, - }, { - .key = "\x07\x2c\xf4\x61\x79\x09\x01\x8f" - "\x37\x32\x98\xd4\x86\x2b\x3b\x80" - "\x07\x60\xba\xf0\x2e\xc3\x4a\x57", - .klen = 24, - .iv = "\xf5\xb5\xd7\xbf\xd2\x2a\x9b\x4a" - "\xe6\x08\xf0\xbe\x77\xd1\x62\x40", - .ptext = "\xa0\x82\x09\x60\x47\xbb\x16\x56" - "\x50\x1f\xab\x8b\x10\xfe\xf0\x5c" - "\x05\x32\x63\x1a\xc4\x46\x6f\x55" - "\x32\xde\x41\x5a\xf7\x52\xd7\xfa" - "\x30\x9d\x59\x8d\x64\x76\xad\x37" - "\xba\xbc\x46\x6a\x69\x17\x3c\xac" - "\x6f\xdd\xa2\x9b\x86\x32\x14\x2e" - "\x54\x74\x8f\x3d\xe2\xd6\x85\x44", - .ctext = "\x5a\xfb\xb1\x2c\x6e\xe5\xb8\xe0" - "\x80\xb6\x77\xa8\xfe\x10\x3a\x99" - "\xbf\xc0\x2a\xfe\x6f\x38\xf2\x1d" - "\x53\x6c\x05\x83\xb1\x13\x00\x87" - "\x92\x92\x42\x70\xcf\x9f\xf7\x8f" - "\x53\x55\x18\x6f\x35\x68\x35\x50" - "\x3a\xc8\x45\x3e\xa3\xf1\x33\x2e" - "\xa1\x65\x42\xe2\x6d\x31\x8c\x4b", - .len = 64, - }, { - .key = "\x4f\x4a\x31\x64\xc6\xa5\x29\xaa" - "\xad\xfd\x32\x94\x1f\x56\x57\xd1" - "\x9d\x7e\x3d\x49\x00\x36\xb1\x5d", - .klen = 24, - .iv = "\xb2\x92\x83\x70\x1e\xa3\x97\xa6" - "\x65\x53\x39\xeb\x53\x8f\xb1\x38", - .ptext = "\x91\xac\x17\x11\x1c\x03\x69\x53" - "\xf5\xdf\xdb\x2c\x1b\x9a\x6e\x6b" - "\xb6\x02\xc4\xfa\x95\x01\x33\xa8" - "\xda\x7e\x18\x2c\xf4\x7e\x6e\x67" - "\xce\x8f\x9f\xea\x46\x66\x99\xb8" - "\xe1\xc7\x25\x4d\xbd\xa5\x74\xdf" - "\xc7\x8b\xfb\xe3\x2d\x3a\x82\xd3" - "\x17\x94\x77\x2f\x92\xb8\x87\xc2" - "\xcc\x6f\x70\x26\x87\xc7\x10\x8a" - "\xc8\xfd\xc2\xb3\xcf\xa0\xeb\x41", - .ctext = "\xc9\x5f\xe0\x60\x61\x38\x7e\x79" - "\x52\x68\x64\x8f\x55\x9b\x6b\x72" - "\xa5\x17\x61\xb7\xce\x02\xa9\xa4" - "\x5c\x73\x45\x33\xd1\x07\x5e\xdc" - "\xe5\xbe\xa7\xde\x69\xa0\x97\x98" - "\x02\xef\xa4\x67\x51\x60\x69\x4f" - "\x03\xf5\xa8\x5f\x03\x69\xbc\xc2" - "\x34\x59\x7e\xd4\xd2\xb3\x32\x2f" - "\x0c\xb4\x37\xca\xc4\xc7\x93\xf4" - "\xa4\xab\x01\x3f\x91\x29\x55\x98", - .len = 80, - }, { - .key = "\x4c\xf4\xd0\x34\xd0\x95\xab\xae" - "\x82\x5c\xfd\xfa\x13\x86\x25\xce" - "\xf4\x13\x32\xcd\xc6\x6d\xf6\x50", - .klen = 24, - .iv = "\x12\x4a\x5b\x66\x3a\xd3\xfb\x1a" - "\xaf\x06\xea\xf4\x65\x59\xd6\xc2", - .ptext = "\x84\xa0\x53\x97\x61\x30\x70\x15" - "\xac\x45\x8e\xe8\xeb\xa1\x72\x93" - "\x26\x76\x98\x6f\xe4\x86\xca\xf0" - "\x57\x89\xf2\x2b\xd4\xcf\x2d\x95" - "\x86\x26\x20\x0e\x62\xfe\x8f\x1e" - "\x5d\xcb\x2b\x7e\xdd\xab\xac\xda" - "\x6e\x49\x20\xd5\xb7\x01\x83\x4e" - "\xac\x45\x8f\xe1\x05\x3f\xd5\xb1" - "\xee\xb7\x0d\x65\x00\x38\xab\x71" - "\x70\x6e\xb3\x97\x86\xd3\xcd\xad" - "\x51\x8b\x9c\xa0\x9a\x8b\x4c\xb9" - "\x16\x01\x6a\x1f\xdf\xf0\xf9\x9e", - .ctext = "\x03\x2c\x39\x24\x99\xb5\xf6\x79" - "\x91\x89\xb7\xf8\x89\x68\x37\x9d" - "\xa2\x80\x95\x74\x87\x64\xb9\xeb" - "\x85\x28\x92\x9a\x6e\xd3\x3b\x50" - "\x4c\x80\x5b\xe4\xf2\x7e\xda\x2a" - "\xd4\xf8\xcb\xe3\x6f\xdf\xae\x0e" - "\xc5\x6c\x0b\x49\x2e\x29\x1c\xf2" - "\x3f\x44\x44\x12\x67\xa6\xff\x44" - "\xe0\xec\xd8\xf7\x32\xde\x21\x15" - "\xab\x8f\x98\x4d\xed\xb0\x42\xfd" - "\x83\x94\xe2\xcc\x69\x6d\xe8\xdb" - "\x62\x93\x1f\xd0\xf4\x8c\x62\xc0", - .len = 96, - }, { - .key = "\x25\x1b\xc2\xa6\x21\x25\xeb\x97" - "\x4b\xf6\xcb\x3b\xcd\x61\xfd\x94" - "\x37\x03\xb3\xd9\x74\x6e\x4d\xbb", - .klen = 24, - .iv = "\xfd\x87\x2b\xec\x4c\x2c\xbf\xe2" - "\x94\x1a\xe6\xd9\xaf\x0e\x78\x17", - .ptext = "\x58\x2b\x1d\x73\x9a\x9c\x63\x18" - "\x88\x7a\x0e\x87\x2f\xf0\xb0\xdb" - "\xc9\x9d\x79\x51\x34\x39\x4f\x07" - "\xa2\x7c\x21\x04\x91\x3b\x79\x79" - "\xfe\xd5\x51\x46\xd5\xcd\x28\xc0" - "\xad\xb8\x55\xb2\xb2\x5a\x9a\xa2" - "\xe2\x0c\xfc\x55\x7d\x60\xd2\x95" - "\xb6\x08\x1d\x31\xaf\xf4\x17\x46" - "\xa4\xbb\x0f\xbd\x67\x3c\x73\x15" - "\x0c\x85\x2f\x62\xe5\xf4\x35\x96" - "\xb1\x9b\x5d\x00\x10\xe9\x70\x12" - "\x3a\x87\x7f\x67\xf1\x81\x7a\x05" - "\xb4\xa6\xfe\xdf\x36\x31\x6d\x9e" - "\x0e\xa9\x44\xa0\xb0\x05\xa9\x41", - .ctext = "\xd4\x9a\x04\x54\x05\xd2\xe6\x3f" - "\xb0\xa4\x36\x5e\x1e\x9c\x35\xb0" - "\xc0\x89\xbd\x1c\xaa\x45\xa6\xc8" - "\x16\x68\x4a\x06\x93\x67\x88\xd7" - "\x72\x6e\x48\x0a\x17\xa3\x52\x8b" - "\x96\x5f\x41\xf6\x17\x64\x55\x8b" - "\xac\xce\xf6\x8c\xce\xd2\xd4\xd4" - "\x8d\x92\x32\xe0\x0d\xb4\xf7\x4a" - "\x90\xaf\x7b\x85\x21\x46\x2e\xa6" - "\x9e\xac\x0d\x22\xf2\x26\xf6\xd3" - "\x27\xcd\x59\xa0\xe2\xbb\x22\xcd" - "\x35\xb6\x28\x45\x0a\x46\xb0\x3a" - "\xac\x3e\xd3\x5b\xc6\x54\xa2\xa3" - "\x6d\xbb\xb3\xcd\xc5\x64\x62\x92", - .len = 112, - }, { - .key = "\x9c\x14\x44\x5a\xd5\x1c\x50\x08" - "\x95\xc2\xf2\xaf\x3f\x29\xc9\x3e" - "\x95\x5e\xc6\xb4\x2b\xf4\x3e\xe3", - .klen = 24, - .iv = "\x1b\xeb\x3d\x73\xfb\xd7\x1e\x2b" - "\x0c\x3d\x58\x6c\xb4\x41\x9b\xfe", - .ptext = "\x2f\x7e\x1c\x10\x81\x36\x2d\x79" - "\xaf\xab\x10\x44\x2e\xcc\x0d\x6c" - "\x9c\x14\xc2\xe4\xae\xb0\xbb\xda" - "\x6a\xe0\x42\x3d\x96\x9f\x78\x7d" - "\x70\x86\xa5\x92\x9f\xee\xcd\x3f" - "\x6a\x55\x84\x98\x28\x03\x02\xc2" - "\xf7\xec\x7a\xfa\xb1\xd9\xa8\xd8" - "\x1c\xc3\xaa\xd5\x61\x7f\x10\x0c" - "\xc0\xa1\x36\x3d\x81\x9a\xd2\x17" - "\x2e\x23\xc9\xb7\xff\xdf\x47\x6c" - "\x96\x3b\x0e\xbd\xec\x9a\x0e\xad" - "\x8c\xaf\x36\x3d\xff\x29\x8b\x33" - "\x87\x96\x77\x1a\x10\x81\x63\x8a" - "\x63\xde\x88\xa9\x9d\xa9\x01\xf2" - "\xdf\xc9\x25\x35\x48\x3a\x15\xdf" - "\x20\x6b\x91\x7c\x56\xe5\x10\x7a", - .ctext = "\xbc\x57\x2a\x88\x0a\xd0\x06\x4f" - "\xdb\x7b\x03\x9f\x97\x1a\x20\xfe" - "\x15\x91\xb4\xed\x5d\x78\x89\x2a" - "\x67\x6b\x9c\x47\x36\xc2\x80\x0e" - "\x03\x8d\x6f\xfc\x94\xc7\xc5\xc2" - "\xeb\x43\x74\x5d\xfe\xc4\x5a\xa1" - "\x80\x51\x8a\x63\xd1\x27\x1b\x0a" - "\x88\x2c\xc4\x7f\x1a\xa3\x28\xe5" - "\xfd\xd0\x8a\xd4\x36\xa6\x19\xd5" - "\xff\x41\x7a\x8b\x6e\x9a\x97\x14" - "\x2a\xc8\xd0\xb8\xa3\x8e\x64\x32" - "\xb7\x2d\x76\x9b\x3b\xe2\x3f\x91" - "\xb4\x64\xbf\x59\x67\x14\xc3\xf5" - "\xa8\x92\x4b\x85\xdf\x80\xcb\xb5" - "\xc7\x80\xf9\x4a\xbc\xed\x67\x5a" - "\x0b\x58\x65\x1f\xc9\x6e\x9b\x0a", - .len = 128, - }, { - .key = "\x2d\x2e\x0f\x30\x32\xed\xa9\x1f" - "\x71\x4e\x68\x77\xe8\xa8\x5b\xdd" - "\x3c\x5e\x68\x6b\xab\x03\xe4\xf8", - .klen = 24, - .iv = "\x42\xc1\x61\x9a\x50\xfb\xc7\x6a" - "\x1a\x31\xa7\x87\xd0\x24\xcb\x5e", - .ptext = "\xc0\x3b\x12\x28\xca\x26\x7b\xb3" - "\x14\xc1\x7f\x66\xff\x3b\xa4\x80" - "\x59\x77\x4f\xa0\xd4\xb2\xd9\x8a" - "\xb6\x67\xe6\x28\xd3\x6f\xf2\xcf" - "\xb8\x6d\x2d\xc4\x2a\x69\x89\xff" - "\xcf\xbb\x11\x2e\x2a\x2b\x7c\xfd" - "\xcd\x56\x02\x95\xc9\x54\x6e\x62" - "\x6a\x97\x75\x1a\x21\x16\x46\xfb" - "\xc2\xab\x62\x54\xef\xba\xae\x46" - "\xd4\x14\xc6\xcc\x16\x1b\x95\xf9" - "\x05\x26\x23\x81\x19\x27\xad\x7b" - "\x9c\x8b\xfb\x65\xa4\x61\xee\x69" - "\x44\xbf\x59\xde\x03\x61\x11\x12" - "\x8d\x94\x48\x47\xa9\x52\x16\xfb" - "\x6b\xaf\x59\x6d\xab\x74\xbf\x5c" - "\xb6\x09\x21\x12\x42\x98\x13\xa1" - "\xa8\x6f\xb9\x6d\x4d\xa6\xdc\xea" - "\x61\x02\x3c\xa7\xcd\x1a\x28\x8c", - .ctext = "\xd7\xb4\xfc\xcc\x1f\xf7\xfc\x7d" - "\x69\xfa\xcb\x01\x60\xf3\x5a\x14" - "\xfe\x8c\x4e\xfa\x09\xb5\x0d\xda" - "\xff\xdd\xba\xdf\xa3\x6b\x3a\x87" - "\x21\xbb\xf8\x62\x14\x22\xdd\x9b" - "\x92\x23\xaa\xd7\xcc\xb2\x15\xd0" - "\xbd\x81\x95\x24\xc2\xc6\x53\x5b" - "\xf7\x3c\xa0\xf7\x36\xbc\xbf\xf3" - "\xfc\x1c\x6e\xe0\x71\x8d\xa1\x3d" - "\x8e\x1a\xc5\xba\xd5\x68\xd4\x7a" - "\xe0\x4f\x0a\x14\x89\x0b\xa6\x2f" - "\x18\xc5\x38\x76\xf1\xe7\x5c\xae" - "\x7a\xbb\x27\x1c\xf0\x7c\x6c\x14" - "\x07\xb7\x49\x6e\x29\x04\x38\x31" - "\x91\xe8\x1d\x0f\xfc\x3b\xb8\x20" - "\x58\x64\x11\xa1\xf5\xba\xa3\x62" - "\x92\xcf\x44\x63\x2c\xe8\x10\xb5" - "\xf0\x97\x86\xcb\x5f\xc1\x80\x7a", - .len = 144, - }, { - .key = "\x66\xb8\x4d\x60\x67\x82\xcc\x8d" - "\x1e\xda\x8f\x28\xe5\x02\xdc\x2c" - "\x54\x84\x2a\x06\xb5\xd1\x34\x57", - .klen = 24, - .iv = "\xb8\x28\x4d\xf5\x69\xb9\xf3\x33" - "\x5e\x0b\xa6\x62\x35\x9b\xfb\x97", - .ptext = "\x3e\xc6\xec\xaf\x74\xe8\x72\x91" - "\xb2\xc6\x56\xb3\x23\x29\x43\xe0" - "\xfb\xcc\x21\x38\x64\x78\x9e\x78" - "\xbb\x6e\x0d\x7b\xfd\x05\x74\x01" - "\x7c\x94\xe0\xb0\xd7\x92\xfc\x58" - "\x28\xfc\xe2\x7b\x7f\xf7\x31\x0d" - "\x90\xb7\x60\x78\xa8\x9f\x52\xe3" - "\xe6\xaa\x2a\xb4\xa7\x09\x60\x53" - "\x42\x0e\x15\x31\xf6\x48\xa3\x0a" - "\x20\xf0\x79\x67\xb1\x83\x26\x66" - "\xe0\xb1\xb3\xbd\x1c\x76\x36\xfd" - "\x45\x87\xa4\x14\x1b\xef\xe7\x16" - "\xf7\xfa\x30\x3d\xb9\x52\x8f\x2e" - "\x01\x68\xc1\x7d\xa2\x15\x49\x74" - "\x53\x82\xc2\x10\xa8\x45\x73\x4d" - "\x41\xcc\x24\xa3\x42\xff\x30\xd1" - "\x02\x21\xdc\xd9\x08\xf7\xe7\x4c" - "\x33\x2d\x62\xc7\x38\xf5\xc2\xbe" - "\x52\xf1\x34\x78\x34\x53\x30\x5b" - "\x43\x43\x51\x6a\x02\x81\x64\x0c", - .ctext = "\x71\xf6\x96\x02\x07\x71\x1a\x08" - "\x7c\xfe\x33\xc4\xc9\xbe\xe2\xed" - "\xd0\xcc\x5d\x27\x75\xb4\x5d\x8d" - "\x24\x03\xe4\x96\x31\x94\x0e\x38" - "\x14\x4f\xad\x16\x58\x0d\x73\xdc" - "\xbe\x5b\xcb\x38\xeb\x4d\xbc\x9a" - "\x44\x69\x7a\x12\x91\x14\x52\xfa" - "\xd2\xa2\xc5\x66\xd7\xaf\x4d\xb9" - "\xb1\x58\x24\x10\xde\x6a\xee\x7e" - "\x45\xf3\x76\xea\x47\x8a\xe6\x96" - "\x41\xf2\x96\x2d\x3c\xec\xcf\xc6" - "\x1d\xf4\x26\xc0\xea\x90\x27\x6e" - "\x87\xef\xb5\x39\x38\xdb\xad\xbf" - "\x57\x9a\x1d\xbc\x1d\xe5\x16\x91" - "\x41\x45\xbe\x67\x6c\x42\x0f\xad" - "\xcf\xfb\xcd\xf1\x4c\xd8\x73\xe7" - "\x24\x3b\xd7\x03\xeb\xd1\xb1\x1b" - "\x7d\xc9\x3d\x34\xd7\xb8\x69\x03" - "\x76\x95\x32\x26\xed\x88\x76\x89" - "\x13\xc6\xc8\xa6\x60\xf9\x73\x4d", - .len = 160, - }, { - .key = "\x82\x8e\x9e\x06\x7b\xc2\xe9\xb3" - "\x06\xa3\xfa\x99\x42\x67\x87\xac" - "\x21\xc7\xb0\x98\x6c\xf8\x26\x57" - "\x08\xdd\x92\x02\x77\x7b\x35\xe7", - .klen = 32, - .iv = "\xa1\xad\xcb\xdd\xd5\x19\xb6\xd4" - "\x0b\x62\x58\xb0\x6c\xa0\xc1\x58", - .ptext = "\x14\x0d\x8a\x09\x16\x00\x00\xf1" - "\xc0\x20\x86\xf9\x21\xd1\x34\xe2", - .ctext = "\x05\xe3\x34\xaf\x6c\x83\x14\x8b" - "\x9d\x1c\xd6\x87\x74\x91\xdf\x17", - .len = 16, - }, { - .key = "\xc9\xf3\xc4\x93\xd0\xcc\xaf\xb1" - "\x1a\x42\x93\x71\xd8\x4e\xd8\xaa" - "\x52\xad\x93\x2f\xe5\xd9\xaa\x5b" - "\x47\x37\x3a\xed\x13\x92\x35\x16", - .klen = 32, - .iv = "\x81\xc8\x50\xd1\x74\xc3\x1c\x73" - "\xbb\xab\x72\x83\x90\x5a\x15\xcb", - .ptext = "\x65\x11\x93\xaf\xe1\x69\x6c\xbe" - "\x25\x8c\x76\x87\x53\xa4\x80\xae" - "\x51\x94\x36\x3f\xca\xe7\x45\x41" - "\x76\x05\xbf\x8f\x9c\xad\xc0\xe3", - .ctext = "\x6B\x00\x6E\x49\x7A\x6D\xE3\x04" - "\x4E\xF7\x9F\x8A\x1F\x14\xBD\xB1" - "\xD3\x5D\xA4\x30\x26\x85\x85\xEF" - "\x12\xBC\xC7\xA1\x65\x82\xA7\x74", - .len = 32, - }, { - .key = "\xd5\x9f\x52\x34\x12\x99\x8e\x42" - "\xe0\x85\x04\x6f\xeb\xf1\x5d\xd0" - "\xc1\xbf\x3f\x84\xd9\x1e\x71\x44" - "\xd4\xb9\x40\x3c\x02\x2e\x21\x19", - .klen = 32, - .iv = "\x28\xc1\x97\x64\x81\x52\x57\x0e" - "\x02\x8c\xab\x4c\xe2\x60\x14\xa5", - .ptext = "\x5a\xb1\x33\x48\xaa\x51\xe9\xa4" - "\x5c\x2d\xbe\x33\xcc\xc4\x7f\x96" - "\xe8\xde\x2b\xe7\x35\x7a\x11\x4b" - "\x13\x08\x32\xc6\x41\xd8\xec\x54" - "\xa3\xd3\xda\x35\x43\x69\xf6\x88" - "\x97\xca\x00\x1b\x02\x59\x24\x82", - .ctext = "\x03\xaf\x76\xbd\x5e\x5b\xca\xc0" - "\xae\x44\xa2\x2f\xc2\x76\x2f\x50" - "\x6a\x73\x28\xf2\xba\xe8\xb2\xb8" - "\x43\x61\x41\x92\xff\xac\xcb\xa6" - "\x84\x31\xe3\x34\xd0\x37\x81\xab" - "\x2b\x0e\x97\x3c\x4a\x2d\xa4\x83", - .len = 48, - }, { - .key = "\x9c\x5d\xd7\x66\x36\xfa\x02\x20" - "\x99\x61\x62\x86\x0f\x43\x2e\x05" - "\x25\x8b\xfb\xf1\xae\x4c\xde\x18" - "\x0b\xf8\xd0\x9d\xaa\xd4\x56\x04", - .klen = 32, - .iv = "\xcd\xa8\x61\x89\x8d\xbb\x72\xb6" - "\x1e\xfe\x03\x34\x54\x88\x23\xe2", - .ptext = "\x66\x42\x60\x24\xf3\xe4\xe9\x7e" - "\x42\x20\xf4\x61\xce\x1c\x5e\x44" - "\x02\x26\x91\xf7\x41\xa4\xab\x34" - "\x29\x49\xdd\x78\x19\x8f\x10\x10" - "\xf0\x61\xcf\x77\x18\x17\x61\xdf" - "\xc4\xa8\x35\x0e\x75\x1b\x84\x6b" - "\xc3\x3f\x31\x59\x5a\x9c\xf4\xc3" - "\x43\xa9\xb7\xf8\x65\x40\x40\xba", - .ctext = "\xb6\x41\x55\x8f\xeb\x16\x1e\x4c" - "\x81\xa0\x85\x6c\xf0\x07\xa5\x2a" - "\x12\x0f\x1d\xb2\xaa\xba\x85\x0f" - "\xa6\x27\x1a\x91\xa6\xc5\x8c\x2a" - "\xde\x8d\x3a\xa9\x8b\xcf\x24\xf1" - "\x82\x51\x6b\xc8\x01\xd7\x7b\x89" - "\x6c\xfc\xb1\x96\x6c\xa2\xd7\x1f" - "\x4b\x7a\xd9\x8d\x34\xaa\xa0\x8a", - .len = 64, - }, { - .key = "\x4b\x4e\x11\x91\x27\xcf\x8c\x66" - "\x17\xfa\x5b\x4c\xa8\xb8\x0f\xa1" - "\x99\x5b\x07\x56\xe1\x8d\x94\x8b" - "\xf2\x86\x5a\x5f\x40\x83\xfa\x06", - .klen = 32, - .iv = "\xfd\x73\xee\x1c\x27\xf3\xb4\x38" - "\xc5\x7c\x2e\xc5\x6e\xdb\x49\x0d", - .ptext = "\x0a\xe2\xdd\x97\xdd\x5e\xd4\xb3" - "\xc1\x49\x8f\x53\xb2\x40\x85\x1c" - "\x90\x37\x2d\xbd\x21\x6b\x1f\x80" - "\x56\x98\x76\x1e\xcf\x6c\x78\xd8" - "\xa0\x3c\x79\xc3\x56\xf7\xfc\x64" - "\x35\x58\x1c\x7c\xc4\x5f\x2a\x25" - "\x8c\x01\x98\x1e\x1c\x1f\x15\x64" - "\x50\xb5\xfa\x02\xd3\x54\xe5\x29" - "\xe3\xd2\xa3\x83\x54\x40\x54\xc5" - "\xd8\x1c\xc9\x84\x7d\xc8\x31\x49", - .ctext = "\x53\x2a\xa8\xa0\x15\xaf\x2f\xc4" - "\x7d\x31\xb4\x61\x80\x5f\xd1\xb6" - "\xa4\x29\x40\x72\x1b\xb2\x96\xb7" - "\x4d\x5e\x5b\x53\x44\xa4\xf1\xe9" - "\xf0\x27\x2f\x26\x84\x66\x13\xa4" - "\xb2\x19\x55\xb1\x18\xf3\x69\xfd" - "\xb0\x2f\x08\x3f\xa5\x41\xe2\x34" - "\x5e\x63\x57\x0e\xef\x17\x78\xbc" - "\xc3\x65\x7c\xbe\x6b\xa3\xa3\xef" - "\x58\x05\x30\x5a\x08\xbd\xf7\x0e", - .len = 80, - }, { - .key = "\x77\x3b\xf5\xe7\x20\xf7\xe0\x0c" - "\x3d\x3a\x83\x17\x83\x79\xd8\x29" - "\x5a\x0a\x25\x7f\xe0\x21\x23\xff" - "\x31\xfd\x60\x10\xe6\x63\xe2\xaf", - .klen = 32, - .iv = "\xdb\x4c\x0d\xc0\x36\xdb\xc7\xa1" - "\xa4\x91\xd9\x05\xe6\xc4\x98\x00", - .ptext = "\x8d\x4d\xc6\x5e\x01\x82\xb3\x39" - "\xc8\x64\xa7\xcb\x05\x19\x84\x80" - "\x3f\x9c\xa8\x4f\x64\xb3\x11\x4b" - "\x0e\x21\xc4\x75\x04\x1d\x6f\xd5" - "\x04\x04\x4d\xc9\xc0\x4b\x4a\x9c" - "\x26\xb7\x68\x5a\xe4\xd0\x61\xe3" - "\x2c\x93\x8e\x3f\xb4\x67\x07\x31" - "\x02\x52\x0c\x0f\xe6\x6d\xa3\xd0" - "\x48\x95\x83\x67\x23\x64\x31\x50" - "\xd2\x5f\x69\x68\x8b\x71\xbf\x01" - "\x29\x99\x86\x36\x2e\xdf\xf1\x7c" - "\x08\x8c\x78\x7a\x93\x9a\x7d\x1b", - .ctext = "\x92\x90\x48\x2f\x3a\x6b\x68\x43" - "\x28\x9b\x7d\x1e\x46\x28\xd8\x58" - "\x0f\x47\x8b\xb5\x83\x35\x35\x3e" - "\xdf\x59\x3d\xb3\x47\xfc\xfc\x52" - "\x86\xeb\xb3\x58\x54\xd5\x0a\xb4" - "\xad\xbd\x5c\x09\xfc\x08\xc2\x01" - "\x5e\x9b\x30\x11\xc4\x40\x2e\x32" - "\x9c\xa0\xf1\xfd\xae\xd4\x75\x5e" - "\x52\xd9\x19\x4d\xc1\xd4\xb6\x19" - "\x88\xfb\x29\x17\x15\xbb\x60\xd6" - "\x5a\xe9\x82\x89\xaf\x30\x4e\xd4" - "\x47\xde\x86\x88\x95\x4c\x13\x59", - .len = 96, - }, { - .key = "\xe0\x6a\x30\xe1\x35\xb5\xb0\x7c" - "\x54\xc5\x73\x9b\x00\xe5\xe7\x02" - "\xbe\x16\x59\xdc\xd9\x03\x17\x53" - "\xa8\x37\xd1\x5f\x13\x8e\x45\xdb", - .klen = 32, - .iv = "\x54\xe9\x1c\xde\xfb\x26\x0e\x48" - "\x35\x50\x4d\x9b\x4d\x12\x21\x0d", - .ptext = "\x73\x72\xcf\xdb\xbd\xbc\xc0\xdf" - "\x6b\xbb\xdf\x65\x6f\x2f\x43\x3b" - "\x2d\x7c\x0e\x07\x7f\xa0\x95\xdd" - "\xfc\x67\xc1\x11\x7a\xe2\xb5\x4a" - "\xd1\x15\xb0\xd8\xe2\xf0\x35\x48" - "\xd8\x81\x6a\x35\xae\x67\xbf\x61" - "\xf2\x8a\xcf\x04\xc8\x09\x8b\x63" - "\x31\x74\x95\xa5\x8d\x3c\xea\xe2" - "\x5f\x67\xc4\x7e\x51\x88\xbf\xb5" - "\x78\xef\x3a\x76\xd8\x1d\x00\x75" - "\x2b\x7b\x28\x7c\xde\x4b\x39\x01" - "\x5d\xde\x92\xfe\x90\x07\x09\xfd" - "\xa5\xd1\xd3\x72\x11\x6d\xa4\x4e" - "\xd1\x6e\x16\xd1\xf6\x39\x4f\xa0", - .ctext = "\x3b\xc5\xee\xfc\x05\xaf\xa6\xb7" - "\xfe\x12\x24\x79\x31\xad\x32\xb5" - "\x64\x5a\x17\xc9\xbf\x1f\xdc\xce" - "\x8d\x73\x00\x71\xd9\xfb\xd2\xe6" - "\xc3\x54\xb4\xf3\x36\xe8\x89\x12" - "\x5a\x32\x0b\xa6\xec\x5f\x89\xe7" - "\xe8\x34\x92\xa6\xce\xde\x8f\xf9" - "\x4f\xda\xed\x61\x8e\xb2\x81\xbe" - "\xf2\x15\x85\xbe\xa1\x5f\x19\x85" - "\x71\x7e\xda\x46\x59\xed\x5d\xb0" - "\xd9\x68\x97\xe0\xcd\x1d\x1b\x65" - "\xf5\xc9\x44\xe2\xb4\x42\x17\x7c" - "\xe7\x58\xf3\x2f\xcf\xbe\x5c\x66" - "\xaa\xd3\x61\xa5\x9a\x79\xbb\xa0", - .len = 112, - }, { - .key = "\x60\xb6\xde\x17\xca\x4c\xe7\xe0" - "\x07\x0d\x80\xc5\x8a\x2d\x5a\xc2" - "\x2c\xb9\xa4\x5f\x2a\x85\x2c\x3d" - "\x6d\x67\xc8\xee\x0f\xa2\xf4\x09", - .klen = 32, - .iv = "\x1a\xa5\xbc\x7e\x93\xf6\xdd\x28" - "\xb7\x69\x27\xa1\x84\x95\x25\x5a", - .ptext = "\x7b\x88\x00\xeb\xa5\xba\xa1\xa7" - "\xd4\x40\x16\x74\x2b\x42\x37\xda" - "\xe0\xaf\x89\x59\x41\x2f\x62\x00" - "\xf5\x5a\x4e\x3b\x85\x27\xb2\xed" - "\x1b\xa7\xaf\xbe\x89\xf3\x49\xb7" - "\x8c\x63\xc9\x0c\x52\x00\x5f\x38" - "\x3b\x3c\x0c\x4f\xdd\xe1\xbf\x90" - "\x4a\x48\xbf\x3a\x95\xcb\x48\xa2" - "\x92\x7c\x79\x81\xde\x18\x6e\x92" - "\x1f\x36\xa9\x5d\x8d\xc4\xb6\x4d" - "\xb2\xb4\x0e\x09\x6d\xf3\x3d\x01" - "\x3d\x9b\x40\x47\xbc\x69\x31\xa1" - "\x6a\x71\x26\xdc\xac\x10\x56\x63" - "\x15\x23\x7d\x10\xe3\x76\x82\x41" - "\xcd\x80\x57\x2f\xfc\x4d\x22\x7b" - "\x57\xbb\x9a\x0a\x03\xe9\xb3\x13", - .ctext = "\x37\x0d\x47\x21\xbc\x28\x0b\xf7" - "\x85\x5f\x60\x57\xf2\x7f\x92\x20" - "\x53\x1a\xbf\xd1\x7f\x8c\x39\x29" - "\x0e\x18\xab\x0c\x00\x92\xd3\x68" - "\x60\x56\x3b\x00\xef\xf8\x02\xfa" - "\xcb\x92\x1a\x91\xe1\xf0\x4f\x8a" - "\xc6\x4f\x65\x16\x71\x8b\x5d\xd5" - "\x79\xa9\x6d\x68\x1b\x59\xe7\x2a" - "\x1c\xd0\x5d\xfb\x06\x3b\x15\x72" - "\xa8\xd1\x59\x9a\xb2\x6c\xf2\xd5" - "\x19\xef\xde\x03\x4c\x75\x65\x38" - "\x5b\xda\xc9\xf0\x44\x99\xb2\x6e" - "\x78\xfb\x85\x5a\x92\x91\x1a\x0a" - "\x13\x0c\x1b\x1c\xbe\xbe\x46\x6e" - "\x73\xff\xc2\x6e\xb9\x06\x16\x7e" - "\xf6\xc0\x01\x30\x34\x56\x46\x55", - .len = 128, - }, { - .key = "\x2a\xed\x7d\x76\xfc\xc5\x49\x50" - "\xf4\x90\x0f\xcc\x5d\xff\x0c\x3c" - "\x14\x06\xaf\x68\x8f\xd7\xb6\x25" - "\x1e\x10\x95\x2a\x71\x33\x17\x20", - .klen = 32, - .iv = "\x5b\x58\x47\xf8\xd5\x1e\x91\x81" - "\x46\xe7\x25\x3a\x02\x45\x9c\x65", - .ptext = "\x10\xaf\xde\x5c\x30\x79\x43\x28" - "\x1c\x03\xf8\x50\x0f\x30\xa5\xef" - "\x84\x19\x4c\x09\x40\x03\x75\x1f" - "\x92\x8f\x88\x01\xda\x31\x7a\xe4" - "\x48\xe3\xab\xb4\xe6\x1b\x0f\xac" - "\xd9\xfa\x8d\x23\xe4\xc6\xa4\xa9" - "\x2d\x9a\x54\x52\x44\x5c\x3c\x52" - "\x61\xf0\x00\xca\xed\xab\xed\xe2" - "\x44\x0b\xe0\x18\xba\xa5\x63\xd8" - "\xdc\x5e\x1a\x4c\xf8\xde\x5e\x75" - "\xdf\x42\x27\x7b\xe9\x11\x2f\x41" - "\x3a\x72\x54\x3d\x44\x9c\x3e\x87" - "\x8d\x8d\x43\x2f\xb2\xff\x87\xd4" - "\xad\x98\x68\x72\x53\x61\x19\x7c" - "\x20\x79\x8c\x2b\x37\x0b\x96\x15" - "\xa5\x7d\x4e\x01\xe6\xea\xb6\xfa" - "\xaa\xd3\x9d\xa2\xd9\x11\xc3\xc9" - "\xd4\x0e\x3f\x3e\xfe\x35\x1e\xe5", - .ctext = "\xb0\x2b\x75\x5f\x33\x1b\x05\x49" - "\x06\xf1\x43\x91\xc2\x85\xfa\xac" - "\x74\xd5\x8c\xc9\x47\x6e\x5a\xf6" - "\x69\x33\x4c\xcb\x2f\x36\x4b\x41" - "\xec\x05\x69\xab\x7f\x42\xc9\xd2" - "\x26\x64\x51\x9e\x3d\x65\x35\xf0" - "\x8d\x5e\x8a\xb1\xee\xdf\x1a\x98" - "\x36\xd2\x37\x49\x5b\xe2\x57\x00" - "\x1d\x72\x7e\xe8\x38\x11\x83\x15" - "\xc7\x4e\x65\xa4\x2c\x9e\x6a\x3e" - "\xb4\x78\x3f\xe9\x91\x5d\x06\xa9" - "\xf1\xfc\x6b\x08\xe5\x2b\x2a\x99" - "\x65\xa7\x2e\x47\xf9\xc2\xb1\x8b" - "\x88\x2f\xb7\x62\x84\x63\x94\x00" - "\x49\xa7\xd0\x2b\x54\x7a\x69\xb3" - "\x04\x66\xfc\x97\x40\x92\xd1\xb8" - "\xb4\x2a\x9e\xdb\x31\xcd\x48\x84" - "\x29\x3b\x02\xac\xb8\x54\x95\xb4", - .len = 144, - }, { - .key = "\x7b\xa7\x4d\x0a\x37\x30\xb9\xf5" - "\x2a\x79\xb4\xbf\xdb\x7f\x9b\x64" - "\x23\x43\xb5\x18\x34\xc4\x5f\xdf" - "\xd9\x2a\x66\x58\x00\x44\xb5\xd9", - .klen = 32, - .iv = "\x75\x34\x30\xc1\xf0\x69\xdf\x0a" - "\x52\xce\x4f\x1e\x2c\x41\x35\xec", - .ptext = "\x81\x47\x55\x3a\xcd\xfe\xa2\x3d" - "\x45\x53\xa7\x67\x61\x74\x25\x80" - "\x98\x89\xfe\xf8\x6a\x9f\x51\x7c" - "\xa4\xe4\xe7\xc7\xe0\x1a\xce\xbb" - "\x4b\x46\x43\xb0\xab\xa8\xd6\x0c" - "\xa0\xf0\xc8\x13\x29\xaf\xb8\x01" - "\x6b\x0c\x7e\x56\xae\xb8\x58\x72" - "\xa9\x24\x44\x61\xff\xf1\xac\xf8" - "\x09\xa8\x48\x21\xd6\xab\x41\x73" - "\x70\x6b\x92\x06\x61\xdc\xb4\x85" - "\x76\x26\x7a\x84\xc3\x9e\x3a\x14" - "\xe7\xf4\x2d\x95\x92\xad\x18\xcc" - "\x44\xd4\x2c\x36\x57\xed\x2b\x9b" - "\x3f\x2b\xcd\xe5\x11\xe3\x62\x33" - "\x42\x3f\xb8\x2a\xb1\x37\x3f\x8b" - "\xe8\xbd\x6b\x0b\x9f\x38\x5a\x5f" - "\x82\x34\xb7\x96\x35\x58\xde\xab" - "\x94\x98\x41\x5b\x3f\xac\x0a\x34" - "\x56\xc0\x02\xef\x81\x6d\xb1\xff" - "\x34\xe8\xc7\x6a\x31\x79\xba\xd8", - .ctext = "\x4e\x00\x7c\x52\x45\x76\xf9\x3d" - "\x1a\xd1\x72\xbc\xb9\x0f\xa9\xfb" - "\x0a\xf5\xe8\x11\x66\x8b\xad\x68" - "\x5a\x2e\xbf\x09\x33\x9d\xb6\x67" - "\xe5\xcb\x0a\xe0\xac\xed\x73\x4b" - "\xbb\x15\xde\xd8\xab\x33\x28\x5f" - "\x96\x07\x3c\x28\x79\x88\x84\xc7" - "\x13\xf7\x0d\xa5\x97\x3b\xd9\xb1" - "\xf2\x65\xb0\xac\xbb\x8a\x97\xd1" - "\x70\x3a\x91\x65\xc8\x39\x04\xe7" - "\x1a\x9c\x80\x65\x2b\x69\x4b\xdc" - "\xdc\xc7\xf1\x31\xda\xab\xb4\xd7" - "\x46\x2e\x1d\xc9\x2e\xe9\x46\xec" - "\xa4\xa1\x91\x6b\x4a\x09\xf9\x39" - "\x7b\x7d\x6d\xf5\x43\x7f\xcc\x74" - "\x96\xfa\x48\xd0\xe1\x74\x24\xd0" - "\x19\x22\x24\x84\x2b\x12\x10\x46" - "\x90\xbd\xa9\x93\xb7\xf7\x36\xd4" - "\x48\xc7\x32\x83\x8c\xa9\xcd\x5a" - "\x2f\x05\x33\xc1\x5b\x50\x70\xc4", - .len = 160, - } -}; - static const struct aead_testvec aria_gcm_tv_template[] = { { .key = "\xe9\x1e\x5e\x75\xda\x65\x55\x4a" diff --git a/drivers/char/hw_random/atmel-rng.c b/drivers/char/hw_random/atmel-rng.c index a37367ebcbac..e9157255f851 100644 --- a/drivers/char/hw_random/atmel-rng.c +++ b/drivers/char/hw_random/atmel-rng.c @@ -161,15 +161,13 @@ static int atmel_trng_probe(struct platform_device *pdev) return ret; } -static int atmel_trng_remove(struct platform_device *pdev) +static void atmel_trng_remove(struct platform_device *pdev) { struct atmel_trng *trng = platform_get_drvdata(pdev); atmel_trng_cleanup(trng); pm_runtime_disable(&pdev->dev); pm_runtime_set_suspended(&pdev->dev); - - return 0; } static int __maybe_unused atmel_trng_runtime_suspend(struct device *dev) @@ -218,7 +216,7 @@ MODULE_DEVICE_TABLE(of, atmel_trng_dt_ids); static struct platform_driver atmel_trng_driver = { .probe = atmel_trng_probe, - .remove = atmel_trng_remove, + .remove_new = atmel_trng_remove, .driver = { .name = "atmel-trng", .pm = pm_ptr(&atmel_trng_pm_ops), diff --git a/drivers/char/hw_random/cctrng.c b/drivers/char/hw_random/cctrng.c index 1abbff04a015..c0d2f824769f 100644 --- a/drivers/char/hw_random/cctrng.c +++ b/drivers/char/hw_random/cctrng.c @@ -560,7 +560,7 @@ post_pm_err: return rc; } -static int cctrng_remove(struct platform_device *pdev) +static void cctrng_remove(struct platform_device *pdev) { struct cctrng_drvdata *drvdata = platform_get_drvdata(pdev); struct device *dev = &pdev->dev; @@ -570,8 +570,6 @@ static int cctrng_remove(struct platform_device *pdev) cc_trng_pm_fini(drvdata); dev_info(dev, "ARM cctrng device terminated\n"); - - return 0; } static int __maybe_unused cctrng_suspend(struct device *dev) @@ -654,7 +652,7 @@ static struct platform_driver cctrng_driver = { .pm = &cctrng_pm, }, .probe = cctrng_probe, - .remove = cctrng_remove, + .remove_new = cctrng_remove, }; module_platform_driver(cctrng_driver); diff --git a/drivers/char/hw_random/core.c b/drivers/char/hw_random/core.c index 420f155d251f..a3bbdd6e60fc 100644 --- a/drivers/char/hw_random/core.c +++ b/drivers/char/hw_random/core.c @@ -23,10 +23,13 @@ #include <linux/sched.h> #include <linux/sched/signal.h> #include <linux/slab.h> +#include <linux/string.h> #include <linux/uaccess.h> #define RNG_MODULE_NAME "hw_random" +#define RNG_BUFFER_SIZE (SMP_CACHE_BYTES < 32 ? 32 : SMP_CACHE_BYTES) + static struct hwrng *current_rng; /* the current rng has been explicitly chosen by user via sysfs */ static int cur_rng_set_by_user; @@ -58,7 +61,7 @@ static inline int rng_get_data(struct hwrng *rng, u8 *buffer, size_t size, static size_t rng_buffer_size(void) { - return SMP_CACHE_BYTES < 32 ? 32 : SMP_CACHE_BYTES; + return RNG_BUFFER_SIZE; } static void add_early_randomness(struct hwrng *rng) @@ -209,6 +212,7 @@ static inline int rng_get_data(struct hwrng *rng, u8 *buffer, size_t size, static ssize_t rng_dev_read(struct file *filp, char __user *buf, size_t size, loff_t *offp) { + u8 buffer[RNG_BUFFER_SIZE]; ssize_t ret = 0; int err = 0; int bytes_read, len; @@ -236,34 +240,37 @@ static ssize_t rng_dev_read(struct file *filp, char __user *buf, if (bytes_read < 0) { err = bytes_read; goto out_unlock_reading; + } else if (bytes_read == 0 && + (filp->f_flags & O_NONBLOCK)) { + err = -EAGAIN; + goto out_unlock_reading; } + data_avail = bytes_read; } - if (!data_avail) { - if (filp->f_flags & O_NONBLOCK) { - err = -EAGAIN; - goto out_unlock_reading; - } - } else { - len = data_avail; + len = data_avail; + if (len) { if (len > size) len = size; data_avail -= len; - if (copy_to_user(buf + ret, rng_buffer + data_avail, - len)) { + memcpy(buffer, rng_buffer + data_avail, len); + } + mutex_unlock(&reading_mutex); + put_rng(rng); + + if (len) { + if (copy_to_user(buf + ret, buffer, len)) { err = -EFAULT; - goto out_unlock_reading; + goto out; } size -= len; ret += len; } - mutex_unlock(&reading_mutex); - put_rng(rng); if (need_resched()) schedule_timeout_interruptible(1); @@ -274,6 +281,7 @@ static ssize_t rng_dev_read(struct file *filp, char __user *buf, } } out: + memzero_explicit(buffer, sizeof(buffer)); return ret ? : err; out_unlock_reading: diff --git a/drivers/char/hw_random/exynos-trng.c b/drivers/char/hw_random/exynos-trng.c index 30207b7ac5f4..0ed5d22fe667 100644 --- a/drivers/char/hw_random/exynos-trng.c +++ b/drivers/char/hw_random/exynos-trng.c @@ -173,7 +173,7 @@ err_pm_get: return ret; } -static int exynos_trng_remove(struct platform_device *pdev) +static void exynos_trng_remove(struct platform_device *pdev) { struct exynos_trng_dev *trng = platform_get_drvdata(pdev); @@ -181,8 +181,6 @@ static int exynos_trng_remove(struct platform_device *pdev) pm_runtime_put_sync(&pdev->dev); pm_runtime_disable(&pdev->dev); - - return 0; } static int exynos_trng_suspend(struct device *dev) @@ -223,7 +221,7 @@ static struct platform_driver exynos_trng_driver = { .of_match_table = exynos_trng_dt_match, }, .probe = exynos_trng_probe, - .remove = exynos_trng_remove, + .remove_new = exynos_trng_remove, }; module_platform_driver(exynos_trng_driver); diff --git a/drivers/char/hw_random/ingenic-rng.c b/drivers/char/hw_random/ingenic-rng.c index 4f18c3fa5427..2f9b6483c4a1 100644 --- a/drivers/char/hw_random/ingenic-rng.c +++ b/drivers/char/hw_random/ingenic-rng.c @@ -11,7 +11,7 @@ #include <linux/io.h> #include <linux/iopoll.h> #include <linux/module.h> -#include <linux/of_device.h> +#include <linux/of.h> #include <linux/platform_device.h> #include <linux/slab.h> @@ -114,15 +114,13 @@ static int ingenic_rng_probe(struct platform_device *pdev) return 0; } -static int ingenic_rng_remove(struct platform_device *pdev) +static void ingenic_rng_remove(struct platform_device *pdev) { struct ingenic_rng *priv = platform_get_drvdata(pdev); hwrng_unregister(&priv->rng); writel(0, priv->base + RNG_REG_ERNG_OFFSET); - - return 0; } static const struct of_device_id ingenic_rng_of_match[] = { @@ -134,7 +132,7 @@ MODULE_DEVICE_TABLE(of, ingenic_rng_of_match); static struct platform_driver ingenic_rng_driver = { .probe = ingenic_rng_probe, - .remove = ingenic_rng_remove, + .remove_new = ingenic_rng_remove, .driver = { .name = "ingenic-rng", .of_match_table = ingenic_rng_of_match, diff --git a/drivers/char/hw_random/jh7110-trng.c b/drivers/char/hw_random/jh7110-trng.c index 38474d48a25e..9776f4daa044 100644 --- a/drivers/char/hw_random/jh7110-trng.c +++ b/drivers/char/hw_random/jh7110-trng.c @@ -300,7 +300,7 @@ static int starfive_trng_probe(struct platform_device *pdev) ret = devm_request_irq(&pdev->dev, irq, starfive_trng_irq, 0, pdev->name, (void *)trng); if (ret) - return dev_err_probe(&pdev->dev, irq, + return dev_err_probe(&pdev->dev, ret, "Failed to register interrupt handler\n"); trng->hclk = devm_clk_get(&pdev->dev, "hclk"); @@ -369,8 +369,12 @@ static int __maybe_unused starfive_trng_resume(struct device *dev) return 0; } -static DEFINE_SIMPLE_DEV_PM_OPS(starfive_trng_pm_ops, starfive_trng_suspend, - starfive_trng_resume); +static const struct dev_pm_ops starfive_trng_pm_ops = { + SET_SYSTEM_SLEEP_PM_OPS(starfive_trng_suspend, + starfive_trng_resume) + SET_RUNTIME_PM_OPS(starfive_trng_suspend, + starfive_trng_resume, NULL) +}; static const struct of_device_id trng_dt_ids[] __maybe_unused = { { .compatible = "starfive,jh7110-trng" }, diff --git a/drivers/char/hw_random/ks-sa-rng.c b/drivers/char/hw_random/ks-sa-rng.c index dff7b9db7044..36c34252b4f6 100644 --- a/drivers/char/hw_random/ks-sa-rng.c +++ b/drivers/char/hw_random/ks-sa-rng.c @@ -241,12 +241,10 @@ static int ks_sa_rng_probe(struct platform_device *pdev) return devm_hwrng_register(&pdev->dev, &ks_sa_rng->rng); } -static int ks_sa_rng_remove(struct platform_device *pdev) +static void ks_sa_rng_remove(struct platform_device *pdev) { pm_runtime_put_sync(&pdev->dev); pm_runtime_disable(&pdev->dev); - - return 0; } static const struct of_device_id ks_sa_rng_dt_match[] = { @@ -263,7 +261,7 @@ static struct platform_driver ks_sa_rng_driver = { .of_match_table = ks_sa_rng_dt_match, }, .probe = ks_sa_rng_probe, - .remove = ks_sa_rng_remove, + .remove_new = ks_sa_rng_remove, }; module_platform_driver(ks_sa_rng_driver); diff --git a/drivers/char/hw_random/mxc-rnga.c b/drivers/char/hw_random/mxc-rnga.c index 008763c988ed..07ec000e4cd7 100644 --- a/drivers/char/hw_random/mxc-rnga.c +++ b/drivers/char/hw_random/mxc-rnga.c @@ -176,15 +176,13 @@ err_ioremap: return err; } -static int __exit mxc_rnga_remove(struct platform_device *pdev) +static void __exit mxc_rnga_remove(struct platform_device *pdev) { struct mxc_rng *mxc_rng = platform_get_drvdata(pdev); hwrng_unregister(&mxc_rng->rng); clk_disable_unprepare(mxc_rng->clk); - - return 0; } static const struct of_device_id mxc_rnga_of_match[] = { @@ -199,7 +197,7 @@ static struct platform_driver mxc_rnga_driver = { .name = "mxc_rnga", .of_match_table = mxc_rnga_of_match, }, - .remove = __exit_p(mxc_rnga_remove), + .remove_new = __exit_p(mxc_rnga_remove), }; module_platform_driver_probe(mxc_rnga_driver, mxc_rnga_probe); diff --git a/drivers/char/hw_random/n2-drv.c b/drivers/char/hw_random/n2-drv.c index aaae16b98475..2e669e7c14d3 100644 --- a/drivers/char/hw_random/n2-drv.c +++ b/drivers/char/hw_random/n2-drv.c @@ -781,7 +781,7 @@ out: return err; } -static int n2rng_remove(struct platform_device *op) +static void n2rng_remove(struct platform_device *op) { struct n2rng *np = platform_get_drvdata(op); @@ -790,8 +790,6 @@ static int n2rng_remove(struct platform_device *op) cancel_delayed_work_sync(&np->work); sun4v_hvapi_unregister(HV_GRP_RNG); - - return 0; } static struct n2rng_template n2_template = { @@ -860,7 +858,7 @@ static struct platform_driver n2rng_driver = { .of_match_table = n2rng_match, }, .probe = n2rng_probe, - .remove = n2rng_remove, + .remove_new = n2rng_remove, }; module_platform_driver(n2rng_driver); diff --git a/drivers/char/hw_random/npcm-rng.c b/drivers/char/hw_random/npcm-rng.c index 8a304b754217..bce8c4829a1f 100644 --- a/drivers/char/hw_random/npcm-rng.c +++ b/drivers/char/hw_random/npcm-rng.c @@ -126,15 +126,13 @@ static int npcm_rng_probe(struct platform_device *pdev) return 0; } -static int npcm_rng_remove(struct platform_device *pdev) +static void npcm_rng_remove(struct platform_device *pdev) { struct npcm_rng *priv = platform_get_drvdata(pdev); devm_hwrng_unregister(&pdev->dev, &priv->rng); pm_runtime_disable(&pdev->dev); pm_runtime_set_suspended(&pdev->dev); - - return 0; } #ifdef CONFIG_PM @@ -178,7 +176,7 @@ static struct platform_driver npcm_rng_driver = { .of_match_table = of_match_ptr(rng_dt_id), }, .probe = npcm_rng_probe, - .remove = npcm_rng_remove, + .remove_new = npcm_rng_remove, }; module_platform_driver(npcm_rng_driver); diff --git a/drivers/char/hw_random/omap-rng.c b/drivers/char/hw_random/omap-rng.c index be03f76a2a80..d4c02e900466 100644 --- a/drivers/char/hw_random/omap-rng.c +++ b/drivers/char/hw_random/omap-rng.c @@ -509,7 +509,7 @@ err_ioremap: return ret; } -static int omap_rng_remove(struct platform_device *pdev) +static void omap_rng_remove(struct platform_device *pdev) { struct omap_rng_dev *priv = platform_get_drvdata(pdev); @@ -521,8 +521,6 @@ static int omap_rng_remove(struct platform_device *pdev) clk_disable_unprepare(priv->clk); clk_disable_unprepare(priv->clk_reg); - - return 0; } static int __maybe_unused omap_rng_suspend(struct device *dev) @@ -560,7 +558,7 @@ static struct platform_driver omap_rng_driver = { .of_match_table = of_match_ptr(omap_rng_of_match), }, .probe = omap_rng_probe, - .remove = omap_rng_remove, + .remove_new = omap_rng_remove, }; module_platform_driver(omap_rng_driver); diff --git a/drivers/char/hw_random/stm32-rng.c b/drivers/char/hw_random/stm32-rng.c index 41e1dbea5d2e..379bc245c520 100644 --- a/drivers/char/hw_random/stm32-rng.c +++ b/drivers/char/hw_random/stm32-rng.c @@ -325,6 +325,7 @@ static int stm32_rng_init(struct hwrng *rng) (!(reg & RNG_CR_CONDRST)), 10, 50000); if (err) { + clk_disable_unprepare(priv->clk); dev_err((struct device *)priv->rng.priv, "%s: timeout %x!\n", __func__, reg); return -EINVAL; @@ -362,11 +363,9 @@ static int stm32_rng_init(struct hwrng *rng) return 0; } -static int stm32_rng_remove(struct platform_device *ofdev) +static void stm32_rng_remove(struct platform_device *ofdev) { pm_runtime_disable(&ofdev->dev); - - return 0; } static int __maybe_unused stm32_rng_runtime_suspend(struct device *dev) @@ -557,7 +556,7 @@ static struct platform_driver stm32_rng_driver = { .of_match_table = stm32_rng_match, }, .probe = stm32_rng_probe, - .remove = stm32_rng_remove, + .remove_new = stm32_rng_remove, }; module_platform_driver(stm32_rng_driver); diff --git a/drivers/char/hw_random/timeriomem-rng.c b/drivers/char/hw_random/timeriomem-rng.c index 3db9d868efb1..65b8260339f5 100644 --- a/drivers/char/hw_random/timeriomem-rng.c +++ b/drivers/char/hw_random/timeriomem-rng.c @@ -174,13 +174,11 @@ static int timeriomem_rng_probe(struct platform_device *pdev) return 0; } -static int timeriomem_rng_remove(struct platform_device *pdev) +static void timeriomem_rng_remove(struct platform_device *pdev) { struct timeriomem_rng_private *priv = platform_get_drvdata(pdev); hrtimer_cancel(&priv->timer); - - return 0; } static const struct of_device_id timeriomem_rng_match[] = { @@ -195,7 +193,7 @@ static struct platform_driver timeriomem_rng_driver = { .of_match_table = timeriomem_rng_match, }, .probe = timeriomem_rng_probe, - .remove = timeriomem_rng_remove, + .remove_new = timeriomem_rng_remove, }; module_platform_driver(timeriomem_rng_driver); diff --git a/drivers/char/hw_random/virtio-rng.c b/drivers/char/hw_random/virtio-rng.c index e41a84e6b4b5..7a4b45393acb 100644 --- a/drivers/char/hw_random/virtio-rng.c +++ b/drivers/char/hw_random/virtio-rng.c @@ -135,7 +135,7 @@ static int probe_common(struct virtio_device *vdev) if (!vi) return -ENOMEM; - vi->index = index = ida_simple_get(&rng_index_ida, 0, 0, GFP_KERNEL); + vi->index = index = ida_alloc(&rng_index_ida, GFP_KERNEL); if (index < 0) { err = index; goto err_ida; @@ -166,7 +166,7 @@ static int probe_common(struct virtio_device *vdev) return 0; err_find: - ida_simple_remove(&rng_index_ida, index); + ida_free(&rng_index_ida, index); err_ida: kfree(vi); return err; @@ -184,7 +184,7 @@ static void remove_common(struct virtio_device *vdev) hwrng_unregister(&vi->hwrng); virtio_reset_device(vdev); vdev->config->del_vqs(vdev); - ida_simple_remove(&rng_index_ida, vi->index); + ida_free(&rng_index_ida, vi->index); kfree(vi); } @@ -208,7 +208,6 @@ static void virtrng_scan(struct virtio_device *vdev) vi->hwrng_register_done = true; } -#ifdef CONFIG_PM_SLEEP static int virtrng_freeze(struct virtio_device *vdev) { remove_common(vdev); @@ -238,7 +237,6 @@ static int virtrng_restore(struct virtio_device *vdev) return err; } -#endif static const struct virtio_device_id id_table[] = { { VIRTIO_ID_RNG, VIRTIO_DEV_ANY_ID }, @@ -252,10 +250,8 @@ static struct virtio_driver virtio_rng_driver = { .probe = virtrng_probe, .remove = virtrng_remove, .scan = virtrng_scan, -#ifdef CONFIG_PM_SLEEP - .freeze = virtrng_freeze, - .restore = virtrng_restore, -#endif + .freeze = pm_sleep_ptr(virtrng_freeze), + .restore = pm_sleep_ptr(virtrng_restore), }; module_virtio_driver(virtio_rng_driver); diff --git a/drivers/char/hw_random/xgene-rng.c b/drivers/char/hw_random/xgene-rng.c index 7382724bf501..642d13519464 100644 --- a/drivers/char/hw_random/xgene-rng.c +++ b/drivers/char/hw_random/xgene-rng.c @@ -357,15 +357,13 @@ static int xgene_rng_probe(struct platform_device *pdev) return 0; } -static int xgene_rng_remove(struct platform_device *pdev) +static void xgene_rng_remove(struct platform_device *pdev) { int rc; rc = device_init_wakeup(&pdev->dev, 0); if (rc) dev_err(&pdev->dev, "RNG init wakeup failed error %d\n", rc); - - return 0; } static const struct of_device_id xgene_rng_of_match[] = { @@ -377,7 +375,7 @@ MODULE_DEVICE_TABLE(of, xgene_rng_of_match); static struct platform_driver xgene_rng_driver = { .probe = xgene_rng_probe, - .remove = xgene_rng_remove, + .remove_new = xgene_rng_remove, .driver = { .name = "xgene-rng", .of_match_table = xgene_rng_of_match, diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig index 79c3bb9c99c3..0991f026cb07 100644 --- a/drivers/crypto/Kconfig +++ b/drivers/crypto/Kconfig @@ -306,6 +306,7 @@ config CRYPTO_DEV_SAHARA select CRYPTO_SKCIPHER select CRYPTO_AES select CRYPTO_ECB + select CRYPTO_ENGINE help This option enables support for the SAHARA HW crypto accelerator found in some Freescale i.MX chips. diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-cipher.c b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-cipher.c index 8d4c42863a62..1262a7773ef3 100644 --- a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-cipher.c +++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-cipher.c @@ -431,8 +431,8 @@ int sun8i_ce_cipher_init(struct crypto_tfm *tfm) return PTR_ERR(op->fallback_tfm); } - sktfm->reqsize = sizeof(struct sun8i_cipher_req_ctx) + - crypto_skcipher_reqsize(op->fallback_tfm); + crypto_skcipher_set_reqsize(sktfm, sizeof(struct sun8i_cipher_req_ctx) + + crypto_skcipher_reqsize(op->fallback_tfm)); memcpy(algt->fbname, crypto_tfm_alg_driver_name(crypto_skcipher_tfm(op->fallback_tfm)), diff --git a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-cipher.c b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-cipher.c index 7fa359725ec7..9b9605ce8ee6 100644 --- a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-cipher.c +++ b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-cipher.c @@ -405,9 +405,8 @@ int sun8i_ss_cipher_init(struct crypto_tfm *tfm) return PTR_ERR(op->fallback_tfm); } - sktfm->reqsize = sizeof(struct sun8i_cipher_req_ctx) + - crypto_skcipher_reqsize(op->fallback_tfm); - + crypto_skcipher_set_reqsize(sktfm, sizeof(struct sun8i_cipher_req_ctx) + + crypto_skcipher_reqsize(op->fallback_tfm)); memcpy(algt->fbname, crypto_tfm_alg_driver_name(crypto_skcipher_tfm(op->fallback_tfm)), diff --git a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c index d70b105dcfa1..753f67a36dc5 100644 --- a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c +++ b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c @@ -30,33 +30,16 @@ static int sun8i_ss_hashkey(struct sun8i_ss_hash_tfm_ctx *tfmctx, const u8 *key, unsigned int keylen) { struct crypto_shash *xtfm; - struct shash_desc *sdesc; - size_t len; - int ret = 0; + int ret; xtfm = crypto_alloc_shash("sha1", 0, CRYPTO_ALG_NEED_FALLBACK); if (IS_ERR(xtfm)) return PTR_ERR(xtfm); - len = sizeof(*sdesc) + crypto_shash_descsize(xtfm); - sdesc = kmalloc(len, GFP_KERNEL); - if (!sdesc) { - ret = -ENOMEM; - goto err_hashkey_sdesc; - } - sdesc->tfm = xtfm; - - ret = crypto_shash_init(sdesc); - if (ret) { - dev_err(tfmctx->ss->dev, "shash init error ret=%d\n", ret); - goto err_hashkey; - } - ret = crypto_shash_finup(sdesc, key, keylen, tfmctx->key); + ret = crypto_shash_tfm_digest(xtfm, key, keylen, tfmctx->key); if (ret) - dev_err(tfmctx->ss->dev, "shash finup error\n"); -err_hashkey: - kfree(sdesc); -err_hashkey_sdesc: + dev_err(tfmctx->ss->dev, "shash digest error ret=%d\n", ret); + crypto_free_shash(xtfm); return ret; } diff --git a/drivers/crypto/amcc/crypto4xx_alg.c b/drivers/crypto/amcc/crypto4xx_alg.c index ded732242732..e0af611a95d8 100644 --- a/drivers/crypto/amcc/crypto4xx_alg.c +++ b/drivers/crypto/amcc/crypto4xx_alg.c @@ -181,13 +181,6 @@ int crypto4xx_setkey_aes_cbc(struct crypto_skcipher *cipher, CRYPTO_FEEDBACK_MODE_NO_FB); } -int crypto4xx_setkey_aes_cfb(struct crypto_skcipher *cipher, - const u8 *key, unsigned int keylen) -{ - return crypto4xx_setkey_aes(cipher, key, keylen, CRYPTO_MODE_CFB, - CRYPTO_FEEDBACK_MODE_128BIT_CFB); -} - int crypto4xx_setkey_aes_ecb(struct crypto_skcipher *cipher, const u8 *key, unsigned int keylen) { @@ -195,13 +188,6 @@ int crypto4xx_setkey_aes_ecb(struct crypto_skcipher *cipher, CRYPTO_FEEDBACK_MODE_NO_FB); } -int crypto4xx_setkey_aes_ofb(struct crypto_skcipher *cipher, - const u8 *key, unsigned int keylen) -{ - return crypto4xx_setkey_aes(cipher, key, keylen, CRYPTO_MODE_OFB, - CRYPTO_FEEDBACK_MODE_64BIT_OFB); -} - int crypto4xx_setkey_rfc3686(struct crypto_skcipher *cipher, const u8 *key, unsigned int keylen) { diff --git a/drivers/crypto/amcc/crypto4xx_core.c b/drivers/crypto/amcc/crypto4xx_core.c index 8d53372245ad..6006703fb6d7 100644 --- a/drivers/crypto/amcc/crypto4xx_core.c +++ b/drivers/crypto/amcc/crypto4xx_core.c @@ -1211,26 +1211,6 @@ static struct crypto4xx_alg_common crypto4xx_alg[] = { } }, { .type = CRYPTO_ALG_TYPE_SKCIPHER, .u.cipher = { .base = { - .cra_name = "cfb(aes)", - .cra_driver_name = "cfb-aes-ppc4xx", - .cra_priority = CRYPTO4XX_CRYPTO_PRIORITY, - .cra_flags = CRYPTO_ALG_ASYNC | - CRYPTO_ALG_KERN_DRIVER_ONLY, - .cra_blocksize = 1, - .cra_ctxsize = sizeof(struct crypto4xx_ctx), - .cra_module = THIS_MODULE, - }, - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .ivsize = AES_IV_SIZE, - .setkey = crypto4xx_setkey_aes_cfb, - .encrypt = crypto4xx_encrypt_iv_stream, - .decrypt = crypto4xx_decrypt_iv_stream, - .init = crypto4xx_sk_init, - .exit = crypto4xx_sk_exit, - } }, - { .type = CRYPTO_ALG_TYPE_SKCIPHER, .u.cipher = { - .base = { .cra_name = "ctr(aes)", .cra_driver_name = "ctr-aes-ppc4xx", .cra_priority = CRYPTO4XX_CRYPTO_PRIORITY, @@ -1289,26 +1269,6 @@ static struct crypto4xx_alg_common crypto4xx_alg[] = { .init = crypto4xx_sk_init, .exit = crypto4xx_sk_exit, } }, - { .type = CRYPTO_ALG_TYPE_SKCIPHER, .u.cipher = { - .base = { - .cra_name = "ofb(aes)", - .cra_driver_name = "ofb-aes-ppc4xx", - .cra_priority = CRYPTO4XX_CRYPTO_PRIORITY, - .cra_flags = CRYPTO_ALG_ASYNC | - CRYPTO_ALG_KERN_DRIVER_ONLY, - .cra_blocksize = 1, - .cra_ctxsize = sizeof(struct crypto4xx_ctx), - .cra_module = THIS_MODULE, - }, - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .ivsize = AES_IV_SIZE, - .setkey = crypto4xx_setkey_aes_ofb, - .encrypt = crypto4xx_encrypt_iv_stream, - .decrypt = crypto4xx_decrypt_iv_stream, - .init = crypto4xx_sk_init, - .exit = crypto4xx_sk_exit, - } }, /* AEAD */ { .type = CRYPTO_ALG_TYPE_AEAD, .u.aead = { diff --git a/drivers/crypto/amcc/crypto4xx_core.h b/drivers/crypto/amcc/crypto4xx_core.h index 56c10668c0ab..96355d463b04 100644 --- a/drivers/crypto/amcc/crypto4xx_core.h +++ b/drivers/crypto/amcc/crypto4xx_core.h @@ -162,14 +162,10 @@ int crypto4xx_build_pd(struct crypto_async_request *req, struct scatterlist *dst_tmp); int crypto4xx_setkey_aes_cbc(struct crypto_skcipher *cipher, const u8 *key, unsigned int keylen); -int crypto4xx_setkey_aes_cfb(struct crypto_skcipher *cipher, - const u8 *key, unsigned int keylen); int crypto4xx_setkey_aes_ctr(struct crypto_skcipher *cipher, const u8 *key, unsigned int keylen); int crypto4xx_setkey_aes_ecb(struct crypto_skcipher *cipher, const u8 *key, unsigned int keylen); -int crypto4xx_setkey_aes_ofb(struct crypto_skcipher *cipher, - const u8 *key, unsigned int keylen); int crypto4xx_setkey_rfc3686(struct crypto_skcipher *cipher, const u8 *key, unsigned int keylen); int crypto4xx_encrypt_ctr(struct skcipher_request *req); diff --git a/drivers/crypto/amlogic/amlogic-gxl-cipher.c b/drivers/crypto/amlogic/amlogic-gxl-cipher.c index 3308406612fc..29048da6f50a 100644 --- a/drivers/crypto/amlogic/amlogic-gxl-cipher.c +++ b/drivers/crypto/amlogic/amlogic-gxl-cipher.c @@ -327,8 +327,8 @@ int meson_cipher_init(struct crypto_tfm *tfm) return PTR_ERR(op->fallback_tfm); } - sktfm->reqsize = sizeof(struct meson_cipher_req_ctx) + - crypto_skcipher_reqsize(op->fallback_tfm); + crypto_skcipher_set_reqsize(sktfm, sizeof(struct meson_cipher_req_ctx) + + crypto_skcipher_reqsize(op->fallback_tfm)); return 0; } diff --git a/drivers/crypto/aspeed/Kconfig b/drivers/crypto/aspeed/Kconfig index db6c5b4cdc40..e93f2f82b418 100644 --- a/drivers/crypto/aspeed/Kconfig +++ b/drivers/crypto/aspeed/Kconfig @@ -38,14 +38,12 @@ config CRYPTO_DEV_ASPEED_HACE_CRYPTO select CRYPTO_DES select CRYPTO_ECB select CRYPTO_CBC - select CRYPTO_CFB - select CRYPTO_OFB select CRYPTO_CTR help Select here to enable Aspeed Hash & Crypto Engine (HACE) crypto driver. Supports AES/DES symmetric-key encryption and decryption - with ECB/CBC/CFB/OFB/CTR options. + with ECB/CBC/CTR options. config CRYPTO_DEV_ASPEED_ACRY bool "Enable Aspeed ACRY RSA Engine" diff --git a/drivers/crypto/aspeed/aspeed-hace-crypto.c b/drivers/crypto/aspeed/aspeed-hace-crypto.c index f0eddb7854e5..a72dfebc53ff 100644 --- a/drivers/crypto/aspeed/aspeed-hace-crypto.c +++ b/drivers/crypto/aspeed/aspeed-hace-crypto.c @@ -473,30 +473,6 @@ static int aspeed_tdes_ctr_encrypt(struct skcipher_request *req) HACE_CMD_TRIPLE_DES); } -static int aspeed_tdes_ofb_decrypt(struct skcipher_request *req) -{ - return aspeed_des_crypt(req, HACE_CMD_DECRYPT | HACE_CMD_OFB | - HACE_CMD_TRIPLE_DES); -} - -static int aspeed_tdes_ofb_encrypt(struct skcipher_request *req) -{ - return aspeed_des_crypt(req, HACE_CMD_ENCRYPT | HACE_CMD_OFB | - HACE_CMD_TRIPLE_DES); -} - -static int aspeed_tdes_cfb_decrypt(struct skcipher_request *req) -{ - return aspeed_des_crypt(req, HACE_CMD_DECRYPT | HACE_CMD_CFB | - HACE_CMD_TRIPLE_DES); -} - -static int aspeed_tdes_cfb_encrypt(struct skcipher_request *req) -{ - return aspeed_des_crypt(req, HACE_CMD_ENCRYPT | HACE_CMD_CFB | - HACE_CMD_TRIPLE_DES); -} - static int aspeed_tdes_cbc_decrypt(struct skcipher_request *req) { return aspeed_des_crypt(req, HACE_CMD_DECRYPT | HACE_CMD_CBC | @@ -533,30 +509,6 @@ static int aspeed_des_ctr_encrypt(struct skcipher_request *req) HACE_CMD_SINGLE_DES); } -static int aspeed_des_ofb_decrypt(struct skcipher_request *req) -{ - return aspeed_des_crypt(req, HACE_CMD_DECRYPT | HACE_CMD_OFB | - HACE_CMD_SINGLE_DES); -} - -static int aspeed_des_ofb_encrypt(struct skcipher_request *req) -{ - return aspeed_des_crypt(req, HACE_CMD_ENCRYPT | HACE_CMD_OFB | - HACE_CMD_SINGLE_DES); -} - -static int aspeed_des_cfb_decrypt(struct skcipher_request *req) -{ - return aspeed_des_crypt(req, HACE_CMD_DECRYPT | HACE_CMD_CFB | - HACE_CMD_SINGLE_DES); -} - -static int aspeed_des_cfb_encrypt(struct skcipher_request *req) -{ - return aspeed_des_crypt(req, HACE_CMD_ENCRYPT | HACE_CMD_CFB | - HACE_CMD_SINGLE_DES); -} - static int aspeed_des_cbc_decrypt(struct skcipher_request *req) { return aspeed_des_crypt(req, HACE_CMD_DECRYPT | HACE_CMD_CBC | @@ -659,26 +611,6 @@ static int aspeed_aes_ctr_encrypt(struct skcipher_request *req) return aspeed_aes_crypt(req, HACE_CMD_ENCRYPT | HACE_CMD_CTR); } -static int aspeed_aes_ofb_decrypt(struct skcipher_request *req) -{ - return aspeed_aes_crypt(req, HACE_CMD_DECRYPT | HACE_CMD_OFB); -} - -static int aspeed_aes_ofb_encrypt(struct skcipher_request *req) -{ - return aspeed_aes_crypt(req, HACE_CMD_ENCRYPT | HACE_CMD_OFB); -} - -static int aspeed_aes_cfb_decrypt(struct skcipher_request *req) -{ - return aspeed_aes_crypt(req, HACE_CMD_DECRYPT | HACE_CMD_CFB); -} - -static int aspeed_aes_cfb_encrypt(struct skcipher_request *req) -{ - return aspeed_aes_crypt(req, HACE_CMD_ENCRYPT | HACE_CMD_CFB); -} - static int aspeed_aes_cbc_decrypt(struct skcipher_request *req) { return aspeed_aes_crypt(req, HACE_CMD_DECRYPT | HACE_CMD_CBC); @@ -792,60 +724,6 @@ static struct aspeed_hace_alg aspeed_crypto_algs[] = { }, { .alg.skcipher.base = { - .ivsize = AES_BLOCK_SIZE, - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .setkey = aspeed_aes_setkey, - .encrypt = aspeed_aes_cfb_encrypt, - .decrypt = aspeed_aes_cfb_decrypt, - .init = aspeed_crypto_cra_init, - .exit = aspeed_crypto_cra_exit, - .base = { - .cra_name = "cfb(aes)", - .cra_driver_name = "aspeed-cfb-aes", - .cra_priority = 300, - .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | - CRYPTO_ALG_ASYNC | - CRYPTO_ALG_NEED_FALLBACK, - .cra_blocksize = 1, - .cra_ctxsize = sizeof(struct aspeed_cipher_ctx), - .cra_alignmask = 0x0f, - .cra_module = THIS_MODULE, - } - }, - .alg.skcipher.op = { - .do_one_request = aspeed_crypto_do_request, - }, - }, - { - .alg.skcipher.base = { - .ivsize = AES_BLOCK_SIZE, - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .setkey = aspeed_aes_setkey, - .encrypt = aspeed_aes_ofb_encrypt, - .decrypt = aspeed_aes_ofb_decrypt, - .init = aspeed_crypto_cra_init, - .exit = aspeed_crypto_cra_exit, - .base = { - .cra_name = "ofb(aes)", - .cra_driver_name = "aspeed-ofb-aes", - .cra_priority = 300, - .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | - CRYPTO_ALG_ASYNC | - CRYPTO_ALG_NEED_FALLBACK, - .cra_blocksize = 1, - .cra_ctxsize = sizeof(struct aspeed_cipher_ctx), - .cra_alignmask = 0x0f, - .cra_module = THIS_MODULE, - } - }, - .alg.skcipher.op = { - .do_one_request = aspeed_crypto_do_request, - }, - }, - { - .alg.skcipher.base = { .min_keysize = DES_KEY_SIZE, .max_keysize = DES_KEY_SIZE, .setkey = aspeed_des_setkey, @@ -899,60 +777,6 @@ static struct aspeed_hace_alg aspeed_crypto_algs[] = { }, { .alg.skcipher.base = { - .ivsize = DES_BLOCK_SIZE, - .min_keysize = DES_KEY_SIZE, - .max_keysize = DES_KEY_SIZE, - .setkey = aspeed_des_setkey, - .encrypt = aspeed_des_cfb_encrypt, - .decrypt = aspeed_des_cfb_decrypt, - .init = aspeed_crypto_cra_init, - .exit = aspeed_crypto_cra_exit, - .base = { - .cra_name = "cfb(des)", - .cra_driver_name = "aspeed-cfb-des", - .cra_priority = 300, - .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | - CRYPTO_ALG_ASYNC | - CRYPTO_ALG_NEED_FALLBACK, - .cra_blocksize = DES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct aspeed_cipher_ctx), - .cra_alignmask = 0x0f, - .cra_module = THIS_MODULE, - } - }, - .alg.skcipher.op = { - .do_one_request = aspeed_crypto_do_request, - }, - }, - { - .alg.skcipher.base = { - .ivsize = DES_BLOCK_SIZE, - .min_keysize = DES_KEY_SIZE, - .max_keysize = DES_KEY_SIZE, - .setkey = aspeed_des_setkey, - .encrypt = aspeed_des_ofb_encrypt, - .decrypt = aspeed_des_ofb_decrypt, - .init = aspeed_crypto_cra_init, - .exit = aspeed_crypto_cra_exit, - .base = { - .cra_name = "ofb(des)", - .cra_driver_name = "aspeed-ofb-des", - .cra_priority = 300, - .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | - CRYPTO_ALG_ASYNC | - CRYPTO_ALG_NEED_FALLBACK, - .cra_blocksize = DES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct aspeed_cipher_ctx), - .cra_alignmask = 0x0f, - .cra_module = THIS_MODULE, - } - }, - .alg.skcipher.op = { - .do_one_request = aspeed_crypto_do_request, - }, - }, - { - .alg.skcipher.base = { .min_keysize = DES3_EDE_KEY_SIZE, .max_keysize = DES3_EDE_KEY_SIZE, .setkey = aspeed_des_setkey, @@ -1004,60 +828,6 @@ static struct aspeed_hace_alg aspeed_crypto_algs[] = { .do_one_request = aspeed_crypto_do_request, }, }, - { - .alg.skcipher.base = { - .ivsize = DES_BLOCK_SIZE, - .min_keysize = DES3_EDE_KEY_SIZE, - .max_keysize = DES3_EDE_KEY_SIZE, - .setkey = aspeed_des_setkey, - .encrypt = aspeed_tdes_cfb_encrypt, - .decrypt = aspeed_tdes_cfb_decrypt, - .init = aspeed_crypto_cra_init, - .exit = aspeed_crypto_cra_exit, - .base = { - .cra_name = "cfb(des3_ede)", - .cra_driver_name = "aspeed-cfb-tdes", - .cra_priority = 300, - .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | - CRYPTO_ALG_ASYNC | - CRYPTO_ALG_NEED_FALLBACK, - .cra_blocksize = DES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct aspeed_cipher_ctx), - .cra_alignmask = 0x0f, - .cra_module = THIS_MODULE, - } - }, - .alg.skcipher.op = { - .do_one_request = aspeed_crypto_do_request, - }, - }, - { - .alg.skcipher.base = { - .ivsize = DES_BLOCK_SIZE, - .min_keysize = DES3_EDE_KEY_SIZE, - .max_keysize = DES3_EDE_KEY_SIZE, - .setkey = aspeed_des_setkey, - .encrypt = aspeed_tdes_ofb_encrypt, - .decrypt = aspeed_tdes_ofb_decrypt, - .init = aspeed_crypto_cra_init, - .exit = aspeed_crypto_cra_exit, - .base = { - .cra_name = "ofb(des3_ede)", - .cra_driver_name = "aspeed-ofb-tdes", - .cra_priority = 300, - .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | - CRYPTO_ALG_ASYNC | - CRYPTO_ALG_NEED_FALLBACK, - .cra_blocksize = DES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct aspeed_cipher_ctx), - .cra_alignmask = 0x0f, - .cra_module = THIS_MODULE, - } - }, - .alg.skcipher.op = { - .do_one_request = aspeed_crypto_do_request, - }, - }, }; static struct aspeed_hace_alg aspeed_crypto_algs_g6[] = { diff --git a/drivers/crypto/atmel-aes.c b/drivers/crypto/atmel-aes.c index d1d93e897892..8bd64fc37e75 100644 --- a/drivers/crypto/atmel-aes.c +++ b/drivers/crypto/atmel-aes.c @@ -46,11 +46,6 @@ #define ATMEL_AES_BUFFER_ORDER 2 #define ATMEL_AES_BUFFER_SIZE (PAGE_SIZE << ATMEL_AES_BUFFER_ORDER) -#define CFB8_BLOCK_SIZE 1 -#define CFB16_BLOCK_SIZE 2 -#define CFB32_BLOCK_SIZE 4 -#define CFB64_BLOCK_SIZE 8 - #define SIZE_IN_WORDS(x) ((x) >> 2) /* AES flags */ @@ -60,12 +55,6 @@ #define AES_FLAGS_OPMODE_MASK (AES_MR_OPMOD_MASK | AES_MR_CFBS_MASK) #define AES_FLAGS_ECB AES_MR_OPMOD_ECB #define AES_FLAGS_CBC AES_MR_OPMOD_CBC -#define AES_FLAGS_OFB AES_MR_OPMOD_OFB -#define AES_FLAGS_CFB128 (AES_MR_OPMOD_CFB | AES_MR_CFBS_128b) -#define AES_FLAGS_CFB64 (AES_MR_OPMOD_CFB | AES_MR_CFBS_64b) -#define AES_FLAGS_CFB32 (AES_MR_OPMOD_CFB | AES_MR_CFBS_32b) -#define AES_FLAGS_CFB16 (AES_MR_OPMOD_CFB | AES_MR_CFBS_16b) -#define AES_FLAGS_CFB8 (AES_MR_OPMOD_CFB | AES_MR_CFBS_8b) #define AES_FLAGS_CTR AES_MR_OPMOD_CTR #define AES_FLAGS_GCM AES_MR_OPMOD_GCM #define AES_FLAGS_XTS AES_MR_OPMOD_XTS @@ -87,7 +76,6 @@ struct atmel_aes_caps { bool has_dualbuff; - bool has_cfb64; bool has_gcm; bool has_xts; bool has_authenc; @@ -860,22 +848,6 @@ static int atmel_aes_dma_start(struct atmel_aes_dev *dd, int err; switch (dd->ctx->block_size) { - case CFB8_BLOCK_SIZE: - addr_width = DMA_SLAVE_BUSWIDTH_1_BYTE; - maxburst = 1; - break; - - case CFB16_BLOCK_SIZE: - addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES; - maxburst = 1; - break; - - case CFB32_BLOCK_SIZE: - case CFB64_BLOCK_SIZE: - addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; - maxburst = 1; - break; - case AES_BLOCK_SIZE: addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; maxburst = dd->caps.max_burst_size; @@ -1103,7 +1075,7 @@ static int atmel_aes_crypt(struct skcipher_request *req, unsigned long mode) } /* - * ECB, CBC, CFB, OFB or CTR mode require the plaintext and ciphertext + * ECB, CBC or CTR mode require the plaintext and ciphertext * to have a positve integer length. */ if (!req->cryptlen && opmode != AES_FLAGS_XTS) @@ -1113,27 +1085,7 @@ static int atmel_aes_crypt(struct skcipher_request *req, unsigned long mode) !IS_ALIGNED(req->cryptlen, crypto_skcipher_blocksize(skcipher))) return -EINVAL; - switch (mode & AES_FLAGS_OPMODE_MASK) { - case AES_FLAGS_CFB8: - ctx->block_size = CFB8_BLOCK_SIZE; - break; - - case AES_FLAGS_CFB16: - ctx->block_size = CFB16_BLOCK_SIZE; - break; - - case AES_FLAGS_CFB32: - ctx->block_size = CFB32_BLOCK_SIZE; - break; - - case AES_FLAGS_CFB64: - ctx->block_size = CFB64_BLOCK_SIZE; - break; - - default: - ctx->block_size = AES_BLOCK_SIZE; - break; - } + ctx->block_size = AES_BLOCK_SIZE; ctx->is_aead = false; rctx = skcipher_request_ctx(req); @@ -1188,66 +1140,6 @@ static int atmel_aes_cbc_decrypt(struct skcipher_request *req) return atmel_aes_crypt(req, AES_FLAGS_CBC); } -static int atmel_aes_ofb_encrypt(struct skcipher_request *req) -{ - return atmel_aes_crypt(req, AES_FLAGS_OFB | AES_FLAGS_ENCRYPT); -} - -static int atmel_aes_ofb_decrypt(struct skcipher_request *req) -{ - return atmel_aes_crypt(req, AES_FLAGS_OFB); -} - -static int atmel_aes_cfb_encrypt(struct skcipher_request *req) -{ - return atmel_aes_crypt(req, AES_FLAGS_CFB128 | AES_FLAGS_ENCRYPT); -} - -static int atmel_aes_cfb_decrypt(struct skcipher_request *req) -{ - return atmel_aes_crypt(req, AES_FLAGS_CFB128); -} - -static int atmel_aes_cfb64_encrypt(struct skcipher_request *req) -{ - return atmel_aes_crypt(req, AES_FLAGS_CFB64 | AES_FLAGS_ENCRYPT); -} - -static int atmel_aes_cfb64_decrypt(struct skcipher_request *req) -{ - return atmel_aes_crypt(req, AES_FLAGS_CFB64); -} - -static int atmel_aes_cfb32_encrypt(struct skcipher_request *req) -{ - return atmel_aes_crypt(req, AES_FLAGS_CFB32 | AES_FLAGS_ENCRYPT); -} - -static int atmel_aes_cfb32_decrypt(struct skcipher_request *req) -{ - return atmel_aes_crypt(req, AES_FLAGS_CFB32); -} - -static int atmel_aes_cfb16_encrypt(struct skcipher_request *req) -{ - return atmel_aes_crypt(req, AES_FLAGS_CFB16 | AES_FLAGS_ENCRYPT); -} - -static int atmel_aes_cfb16_decrypt(struct skcipher_request *req) -{ - return atmel_aes_crypt(req, AES_FLAGS_CFB16); -} - -static int atmel_aes_cfb8_encrypt(struct skcipher_request *req) -{ - return atmel_aes_crypt(req, AES_FLAGS_CFB8 | AES_FLAGS_ENCRYPT); -} - -static int atmel_aes_cfb8_decrypt(struct skcipher_request *req) -{ - return atmel_aes_crypt(req, AES_FLAGS_CFB8); -} - static int atmel_aes_ctr_encrypt(struct skcipher_request *req) { return atmel_aes_crypt(req, AES_FLAGS_CTR | AES_FLAGS_ENCRYPT); @@ -1319,76 +1211,6 @@ static struct skcipher_alg aes_algs[] = { .ivsize = AES_BLOCK_SIZE, }, { - .base.cra_name = "ofb(aes)", - .base.cra_driver_name = "atmel-ofb-aes", - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct atmel_aes_ctx), - - .init = atmel_aes_init_tfm, - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .setkey = atmel_aes_setkey, - .encrypt = atmel_aes_ofb_encrypt, - .decrypt = atmel_aes_ofb_decrypt, - .ivsize = AES_BLOCK_SIZE, -}, -{ - .base.cra_name = "cfb(aes)", - .base.cra_driver_name = "atmel-cfb-aes", - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct atmel_aes_ctx), - - .init = atmel_aes_init_tfm, - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .setkey = atmel_aes_setkey, - .encrypt = atmel_aes_cfb_encrypt, - .decrypt = atmel_aes_cfb_decrypt, - .ivsize = AES_BLOCK_SIZE, -}, -{ - .base.cra_name = "cfb32(aes)", - .base.cra_driver_name = "atmel-cfb32-aes", - .base.cra_blocksize = CFB32_BLOCK_SIZE, - .base.cra_ctxsize = sizeof(struct atmel_aes_ctx), - - .init = atmel_aes_init_tfm, - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .setkey = atmel_aes_setkey, - .encrypt = atmel_aes_cfb32_encrypt, - .decrypt = atmel_aes_cfb32_decrypt, - .ivsize = AES_BLOCK_SIZE, -}, -{ - .base.cra_name = "cfb16(aes)", - .base.cra_driver_name = "atmel-cfb16-aes", - .base.cra_blocksize = CFB16_BLOCK_SIZE, - .base.cra_ctxsize = sizeof(struct atmel_aes_ctx), - - .init = atmel_aes_init_tfm, - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .setkey = atmel_aes_setkey, - .encrypt = atmel_aes_cfb16_encrypt, - .decrypt = atmel_aes_cfb16_decrypt, - .ivsize = AES_BLOCK_SIZE, -}, -{ - .base.cra_name = "cfb8(aes)", - .base.cra_driver_name = "atmel-cfb8-aes", - .base.cra_blocksize = CFB8_BLOCK_SIZE, - .base.cra_ctxsize = sizeof(struct atmel_aes_ctx), - - .init = atmel_aes_init_tfm, - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .setkey = atmel_aes_setkey, - .encrypt = atmel_aes_cfb8_encrypt, - .decrypt = atmel_aes_cfb8_decrypt, - .ivsize = AES_BLOCK_SIZE, -}, -{ .base.cra_name = "ctr(aes)", .base.cra_driver_name = "atmel-ctr-aes", .base.cra_blocksize = 1, @@ -1404,21 +1226,6 @@ static struct skcipher_alg aes_algs[] = { }, }; -static struct skcipher_alg aes_cfb64_alg = { - .base.cra_name = "cfb64(aes)", - .base.cra_driver_name = "atmel-cfb64-aes", - .base.cra_blocksize = CFB64_BLOCK_SIZE, - .base.cra_ctxsize = sizeof(struct atmel_aes_ctx), - - .init = atmel_aes_init_tfm, - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .setkey = atmel_aes_setkey, - .encrypt = atmel_aes_cfb64_encrypt, - .decrypt = atmel_aes_cfb64_decrypt, - .ivsize = AES_BLOCK_SIZE, -}; - /* gcm aead functions */ @@ -2407,9 +2214,6 @@ static void atmel_aes_unregister_algs(struct atmel_aes_dev *dd) if (dd->caps.has_gcm) crypto_unregister_aead(&aes_gcm_alg); - if (dd->caps.has_cfb64) - crypto_unregister_skcipher(&aes_cfb64_alg); - for (i = 0; i < ARRAY_SIZE(aes_algs); i++) crypto_unregister_skcipher(&aes_algs[i]); } @@ -2434,14 +2238,6 @@ static int atmel_aes_register_algs(struct atmel_aes_dev *dd) goto err_aes_algs; } - if (dd->caps.has_cfb64) { - atmel_aes_crypto_alg_init(&aes_cfb64_alg.base); - - err = crypto_register_skcipher(&aes_cfb64_alg); - if (err) - goto err_aes_cfb64_alg; - } - if (dd->caps.has_gcm) { atmel_aes_crypto_alg_init(&aes_gcm_alg.base); @@ -2482,8 +2278,6 @@ err_aes_authenc_alg: err_aes_xts_alg: crypto_unregister_aead(&aes_gcm_alg); err_aes_gcm_alg: - crypto_unregister_skcipher(&aes_cfb64_alg); -err_aes_cfb64_alg: i = ARRAY_SIZE(aes_algs); err_aes_algs: for (j = 0; j < i; j++) @@ -2495,7 +2289,6 @@ err_aes_algs: static void atmel_aes_get_cap(struct atmel_aes_dev *dd) { dd->caps.has_dualbuff = 0; - dd->caps.has_cfb64 = 0; dd->caps.has_gcm = 0; dd->caps.has_xts = 0; dd->caps.has_authenc = 0; @@ -2507,7 +2300,6 @@ static void atmel_aes_get_cap(struct atmel_aes_dev *dd) case 0x600: case 0x500: dd->caps.has_dualbuff = 1; - dd->caps.has_cfb64 = 1; dd->caps.has_gcm = 1; dd->caps.has_xts = 1; dd->caps.has_authenc = 1; @@ -2515,13 +2307,11 @@ static void atmel_aes_get_cap(struct atmel_aes_dev *dd) break; case 0x200: dd->caps.has_dualbuff = 1; - dd->caps.has_cfb64 = 1; dd->caps.has_gcm = 1; dd->caps.max_burst_size = 4; break; case 0x130: dd->caps.has_dualbuff = 1; - dd->caps.has_cfb64 = 1; dd->caps.max_burst_size = 4; break; case 0x120: diff --git a/drivers/crypto/atmel-tdes.c b/drivers/crypto/atmel-tdes.c index 27b7000e25bc..dcc2380a5889 100644 --- a/drivers/crypto/atmel-tdes.c +++ b/drivers/crypto/atmel-tdes.c @@ -45,11 +45,6 @@ #define TDES_FLAGS_OPMODE_MASK (TDES_MR_OPMOD_MASK | TDES_MR_CFBS_MASK) #define TDES_FLAGS_ECB TDES_MR_OPMOD_ECB #define TDES_FLAGS_CBC TDES_MR_OPMOD_CBC -#define TDES_FLAGS_OFB TDES_MR_OPMOD_OFB -#define TDES_FLAGS_CFB64 (TDES_MR_OPMOD_CFB | TDES_MR_CFBS_64b) -#define TDES_FLAGS_CFB32 (TDES_MR_OPMOD_CFB | TDES_MR_CFBS_32b) -#define TDES_FLAGS_CFB16 (TDES_MR_OPMOD_CFB | TDES_MR_CFBS_16b) -#define TDES_FLAGS_CFB8 (TDES_MR_OPMOD_CFB | TDES_MR_CFBS_8b) #define TDES_FLAGS_MODE_MASK (TDES_FLAGS_OPMODE_MASK | TDES_FLAGS_ENCRYPT) @@ -60,13 +55,8 @@ #define ATMEL_TDES_QUEUE_LENGTH 50 -#define CFB8_BLOCK_SIZE 1 -#define CFB16_BLOCK_SIZE 2 -#define CFB32_BLOCK_SIZE 4 - struct atmel_tdes_caps { bool has_dma; - u32 has_cfb_3keys; }; struct atmel_tdes_dev; @@ -376,7 +366,6 @@ static int atmel_tdes_crypt_pdc(struct atmel_tdes_dev *dd, dma_addr_t dma_addr_in, dma_addr_t dma_addr_out, int length) { - struct atmel_tdes_reqctx *rctx = skcipher_request_ctx(dd->req); int len32; dd->dma_size = length; @@ -386,19 +375,7 @@ static int atmel_tdes_crypt_pdc(struct atmel_tdes_dev *dd, DMA_TO_DEVICE); } - switch (rctx->mode & TDES_FLAGS_OPMODE_MASK) { - case TDES_FLAGS_CFB8: - len32 = DIV_ROUND_UP(length, sizeof(u8)); - break; - - case TDES_FLAGS_CFB16: - len32 = DIV_ROUND_UP(length, sizeof(u16)); - break; - - default: - len32 = DIV_ROUND_UP(length, sizeof(u32)); - break; - } + len32 = DIV_ROUND_UP(length, sizeof(u32)); atmel_tdes_write(dd, TDES_PTCR, TDES_PTCR_TXTDIS|TDES_PTCR_RXTDIS); atmel_tdes_write(dd, TDES_TPR, dma_addr_in); @@ -419,7 +396,6 @@ static int atmel_tdes_crypt_dma(struct atmel_tdes_dev *dd, dma_addr_t dma_addr_in, dma_addr_t dma_addr_out, int length) { - struct atmel_tdes_reqctx *rctx = skcipher_request_ctx(dd->req); struct scatterlist sg[2]; struct dma_async_tx_descriptor *in_desc, *out_desc; enum dma_slave_buswidth addr_width; @@ -431,19 +407,7 @@ static int atmel_tdes_crypt_dma(struct atmel_tdes_dev *dd, DMA_TO_DEVICE); } - switch (rctx->mode & TDES_FLAGS_OPMODE_MASK) { - case TDES_FLAGS_CFB8: - addr_width = DMA_SLAVE_BUSWIDTH_1_BYTE; - break; - - case TDES_FLAGS_CFB16: - addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES; - break; - - default: - addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; - break; - } + addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; dd->dma_lch_in.dma_conf.dst_addr_width = addr_width; dd->dma_lch_out.dma_conf.src_addr_width = addr_width; @@ -680,39 +644,11 @@ static int atmel_tdes_crypt(struct skcipher_request *req, unsigned long mode) if (!req->cryptlen) return 0; - switch (mode & TDES_FLAGS_OPMODE_MASK) { - case TDES_FLAGS_CFB8: - if (!IS_ALIGNED(req->cryptlen, CFB8_BLOCK_SIZE)) { - dev_dbg(dev, "request size is not exact amount of CFB8 blocks\n"); - return -EINVAL; - } - ctx->block_size = CFB8_BLOCK_SIZE; - break; - - case TDES_FLAGS_CFB16: - if (!IS_ALIGNED(req->cryptlen, CFB16_BLOCK_SIZE)) { - dev_dbg(dev, "request size is not exact amount of CFB16 blocks\n"); - return -EINVAL; - } - ctx->block_size = CFB16_BLOCK_SIZE; - break; - - case TDES_FLAGS_CFB32: - if (!IS_ALIGNED(req->cryptlen, CFB32_BLOCK_SIZE)) { - dev_dbg(dev, "request size is not exact amount of CFB32 blocks\n"); - return -EINVAL; - } - ctx->block_size = CFB32_BLOCK_SIZE; - break; - - default: - if (!IS_ALIGNED(req->cryptlen, DES_BLOCK_SIZE)) { - dev_dbg(dev, "request size is not exact amount of DES blocks\n"); - return -EINVAL; - } - ctx->block_size = DES_BLOCK_SIZE; - break; + if (!IS_ALIGNED(req->cryptlen, DES_BLOCK_SIZE)) { + dev_dbg(dev, "request size is not exact amount of DES blocks\n"); + return -EINVAL; } + ctx->block_size = DES_BLOCK_SIZE; rctx->mode = mode; @@ -832,55 +768,6 @@ static int atmel_tdes_cbc_decrypt(struct skcipher_request *req) { return atmel_tdes_crypt(req, TDES_FLAGS_CBC); } -static int atmel_tdes_cfb_encrypt(struct skcipher_request *req) -{ - return atmel_tdes_crypt(req, TDES_FLAGS_CFB64 | TDES_FLAGS_ENCRYPT); -} - -static int atmel_tdes_cfb_decrypt(struct skcipher_request *req) -{ - return atmel_tdes_crypt(req, TDES_FLAGS_CFB64); -} - -static int atmel_tdes_cfb8_encrypt(struct skcipher_request *req) -{ - return atmel_tdes_crypt(req, TDES_FLAGS_CFB8 | TDES_FLAGS_ENCRYPT); -} - -static int atmel_tdes_cfb8_decrypt(struct skcipher_request *req) -{ - return atmel_tdes_crypt(req, TDES_FLAGS_CFB8); -} - -static int atmel_tdes_cfb16_encrypt(struct skcipher_request *req) -{ - return atmel_tdes_crypt(req, TDES_FLAGS_CFB16 | TDES_FLAGS_ENCRYPT); -} - -static int atmel_tdes_cfb16_decrypt(struct skcipher_request *req) -{ - return atmel_tdes_crypt(req, TDES_FLAGS_CFB16); -} - -static int atmel_tdes_cfb32_encrypt(struct skcipher_request *req) -{ - return atmel_tdes_crypt(req, TDES_FLAGS_CFB32 | TDES_FLAGS_ENCRYPT); -} - -static int atmel_tdes_cfb32_decrypt(struct skcipher_request *req) -{ - return atmel_tdes_crypt(req, TDES_FLAGS_CFB32); -} - -static int atmel_tdes_ofb_encrypt(struct skcipher_request *req) -{ - return atmel_tdes_crypt(req, TDES_FLAGS_OFB | TDES_FLAGS_ENCRYPT); -} - -static int atmel_tdes_ofb_decrypt(struct skcipher_request *req) -{ - return atmel_tdes_crypt(req, TDES_FLAGS_OFB); -} static int atmel_tdes_init_tfm(struct crypto_skcipher *tfm) { @@ -932,71 +819,6 @@ static struct skcipher_alg tdes_algs[] = { .decrypt = atmel_tdes_cbc_decrypt, }, { - .base.cra_name = "cfb(des)", - .base.cra_driver_name = "atmel-cfb-des", - .base.cra_blocksize = DES_BLOCK_SIZE, - .base.cra_alignmask = 0x7, - - .min_keysize = DES_KEY_SIZE, - .max_keysize = DES_KEY_SIZE, - .ivsize = DES_BLOCK_SIZE, - .setkey = atmel_des_setkey, - .encrypt = atmel_tdes_cfb_encrypt, - .decrypt = atmel_tdes_cfb_decrypt, -}, -{ - .base.cra_name = "cfb8(des)", - .base.cra_driver_name = "atmel-cfb8-des", - .base.cra_blocksize = CFB8_BLOCK_SIZE, - .base.cra_alignmask = 0, - - .min_keysize = DES_KEY_SIZE, - .max_keysize = DES_KEY_SIZE, - .ivsize = DES_BLOCK_SIZE, - .setkey = atmel_des_setkey, - .encrypt = atmel_tdes_cfb8_encrypt, - .decrypt = atmel_tdes_cfb8_decrypt, -}, -{ - .base.cra_name = "cfb16(des)", - .base.cra_driver_name = "atmel-cfb16-des", - .base.cra_blocksize = CFB16_BLOCK_SIZE, - .base.cra_alignmask = 0x1, - - .min_keysize = DES_KEY_SIZE, - .max_keysize = DES_KEY_SIZE, - .ivsize = DES_BLOCK_SIZE, - .setkey = atmel_des_setkey, - .encrypt = atmel_tdes_cfb16_encrypt, - .decrypt = atmel_tdes_cfb16_decrypt, -}, -{ - .base.cra_name = "cfb32(des)", - .base.cra_driver_name = "atmel-cfb32-des", - .base.cra_blocksize = CFB32_BLOCK_SIZE, - .base.cra_alignmask = 0x3, - - .min_keysize = DES_KEY_SIZE, - .max_keysize = DES_KEY_SIZE, - .ivsize = DES_BLOCK_SIZE, - .setkey = atmel_des_setkey, - .encrypt = atmel_tdes_cfb32_encrypt, - .decrypt = atmel_tdes_cfb32_decrypt, -}, -{ - .base.cra_name = "ofb(des)", - .base.cra_driver_name = "atmel-ofb-des", - .base.cra_blocksize = 1, - .base.cra_alignmask = 0x7, - - .min_keysize = DES_KEY_SIZE, - .max_keysize = DES_KEY_SIZE, - .ivsize = DES_BLOCK_SIZE, - .setkey = atmel_des_setkey, - .encrypt = atmel_tdes_ofb_encrypt, - .decrypt = atmel_tdes_ofb_decrypt, -}, -{ .base.cra_name = "ecb(des3_ede)", .base.cra_driver_name = "atmel-ecb-tdes", .base.cra_blocksize = DES_BLOCK_SIZE, @@ -1021,19 +843,6 @@ static struct skcipher_alg tdes_algs[] = { .decrypt = atmel_tdes_cbc_decrypt, .ivsize = DES_BLOCK_SIZE, }, -{ - .base.cra_name = "ofb(des3_ede)", - .base.cra_driver_name = "atmel-ofb-tdes", - .base.cra_blocksize = DES_BLOCK_SIZE, - .base.cra_alignmask = 0x7, - - .min_keysize = DES3_EDE_KEY_SIZE, - .max_keysize = DES3_EDE_KEY_SIZE, - .setkey = atmel_tdes_setkey, - .encrypt = atmel_tdes_ofb_encrypt, - .decrypt = atmel_tdes_ofb_decrypt, - .ivsize = DES_BLOCK_SIZE, -}, }; static void atmel_tdes_queue_task(unsigned long data) @@ -1121,14 +930,12 @@ static void atmel_tdes_get_cap(struct atmel_tdes_dev *dd) { dd->caps.has_dma = 0; - dd->caps.has_cfb_3keys = 0; /* keep only major version number */ switch (dd->hw_version & 0xf00) { case 0x800: case 0x700: dd->caps.has_dma = 1; - dd->caps.has_cfb_3keys = 1; break; case 0x600: break; diff --git a/drivers/crypto/axis/artpec6_crypto.c b/drivers/crypto/axis/artpec6_crypto.c index ef9fe13ffa59..dbc1d483f2af 100644 --- a/drivers/crypto/axis/artpec6_crypto.c +++ b/drivers/crypto/axis/artpec6_crypto.c @@ -1535,7 +1535,8 @@ static int artpec6_crypto_aes_ecb_init(struct crypto_skcipher *tfm) { struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm); - tfm->reqsize = sizeof(struct artpec6_crypto_request_context); + crypto_skcipher_set_reqsize(tfm, + sizeof(struct artpec6_crypto_request_context)); ctx->crypto_type = ARTPEC6_CRYPTO_CIPHER_AES_ECB; return 0; @@ -1551,7 +1552,8 @@ static int artpec6_crypto_aes_ctr_init(struct crypto_skcipher *tfm) if (IS_ERR(ctx->fallback)) return PTR_ERR(ctx->fallback); - tfm->reqsize = sizeof(struct artpec6_crypto_request_context); + crypto_skcipher_set_reqsize(tfm, + sizeof(struct artpec6_crypto_request_context)); ctx->crypto_type = ARTPEC6_CRYPTO_CIPHER_AES_CTR; return 0; @@ -1561,7 +1563,8 @@ static int artpec6_crypto_aes_cbc_init(struct crypto_skcipher *tfm) { struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm); - tfm->reqsize = sizeof(struct artpec6_crypto_request_context); + crypto_skcipher_set_reqsize(tfm, + sizeof(struct artpec6_crypto_request_context)); ctx->crypto_type = ARTPEC6_CRYPTO_CIPHER_AES_CBC; return 0; @@ -1571,7 +1574,8 @@ static int artpec6_crypto_aes_xts_init(struct crypto_skcipher *tfm) { struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm); - tfm->reqsize = sizeof(struct artpec6_crypto_request_context); + crypto_skcipher_set_reqsize(tfm, + sizeof(struct artpec6_crypto_request_context)); ctx->crypto_type = ARTPEC6_CRYPTO_CIPHER_AES_XTS; return 0; diff --git a/drivers/crypto/bcm/cipher.c b/drivers/crypto/bcm/cipher.c index 10968ddb146b..1a3ecd44cbaf 100644 --- a/drivers/crypto/bcm/cipher.c +++ b/drivers/crypto/bcm/cipher.c @@ -3517,25 +3517,6 @@ static struct iproc_alg_s driver_algs[] = { { .type = CRYPTO_ALG_TYPE_SKCIPHER, .alg.skcipher = { - .base.cra_name = "ofb(des)", - .base.cra_driver_name = "ofb-des-iproc", - .base.cra_blocksize = DES_BLOCK_SIZE, - .min_keysize = DES_KEY_SIZE, - .max_keysize = DES_KEY_SIZE, - .ivsize = DES_BLOCK_SIZE, - }, - .cipher_info = { - .alg = CIPHER_ALG_DES, - .mode = CIPHER_MODE_OFB, - }, - .auth_info = { - .alg = HASH_ALG_NONE, - .mode = HASH_MODE_NONE, - }, - }, - { - .type = CRYPTO_ALG_TYPE_SKCIPHER, - .alg.skcipher = { .base.cra_name = "cbc(des)", .base.cra_driver_name = "cbc-des-iproc", .base.cra_blocksize = DES_BLOCK_SIZE, @@ -3574,25 +3555,6 @@ static struct iproc_alg_s driver_algs[] = { { .type = CRYPTO_ALG_TYPE_SKCIPHER, .alg.skcipher = { - .base.cra_name = "ofb(des3_ede)", - .base.cra_driver_name = "ofb-des3-iproc", - .base.cra_blocksize = DES3_EDE_BLOCK_SIZE, - .min_keysize = DES3_EDE_KEY_SIZE, - .max_keysize = DES3_EDE_KEY_SIZE, - .ivsize = DES3_EDE_BLOCK_SIZE, - }, - .cipher_info = { - .alg = CIPHER_ALG_3DES, - .mode = CIPHER_MODE_OFB, - }, - .auth_info = { - .alg = HASH_ALG_NONE, - .mode = HASH_MODE_NONE, - }, - }, - { - .type = CRYPTO_ALG_TYPE_SKCIPHER, - .alg.skcipher = { .base.cra_name = "cbc(des3_ede)", .base.cra_driver_name = "cbc-des3-iproc", .base.cra_blocksize = DES3_EDE_BLOCK_SIZE, @@ -3631,25 +3593,6 @@ static struct iproc_alg_s driver_algs[] = { { .type = CRYPTO_ALG_TYPE_SKCIPHER, .alg.skcipher = { - .base.cra_name = "ofb(aes)", - .base.cra_driver_name = "ofb-aes-iproc", - .base.cra_blocksize = AES_BLOCK_SIZE, - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .ivsize = AES_BLOCK_SIZE, - }, - .cipher_info = { - .alg = CIPHER_ALG_AES, - .mode = CIPHER_MODE_OFB, - }, - .auth_info = { - .alg = HASH_ALG_NONE, - .mode = HASH_MODE_NONE, - }, - }, - { - .type = CRYPTO_ALG_TYPE_SKCIPHER, - .alg.skcipher = { .base.cra_name = "cbc(aes)", .base.cra_driver_name = "cbc-aes-iproc", .base.cra_blocksize = AES_BLOCK_SIZE, diff --git a/drivers/crypto/cavium/cpt/cptvf_algs.c b/drivers/crypto/cavium/cpt/cptvf_algs.c index ee476c6c7f82..219fe9be7606 100644 --- a/drivers/crypto/cavium/cpt/cptvf_algs.c +++ b/drivers/crypto/cavium/cpt/cptvf_algs.c @@ -311,12 +311,6 @@ static int cvm_ecb_aes_setkey(struct crypto_skcipher *cipher, const u8 *key, return cvm_setkey(cipher, key, keylen, AES_ECB); } -static int cvm_cfb_aes_setkey(struct crypto_skcipher *cipher, const u8 *key, - u32 keylen) -{ - return cvm_setkey(cipher, key, keylen, AES_CFB); -} - static int cvm_cbc_des3_setkey(struct crypto_skcipher *cipher, const u8 *key, u32 keylen) { @@ -394,24 +388,6 @@ static struct skcipher_alg algs[] = { { }, { .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, - .base.cra_blocksize = AES_BLOCK_SIZE, - .base.cra_ctxsize = sizeof(struct cvm_enc_ctx), - .base.cra_alignmask = 7, - .base.cra_priority = 4001, - .base.cra_name = "cfb(aes)", - .base.cra_driver_name = "cavium-cfb-aes", - .base.cra_module = THIS_MODULE, - - .ivsize = AES_BLOCK_SIZE, - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .setkey = cvm_cfb_aes_setkey, - .encrypt = cvm_encrypt, - .decrypt = cvm_decrypt, - .init = cvm_enc_dec_init, -}, { - .base.cra_flags = CRYPTO_ALG_ASYNC | - CRYPTO_ALG_ALLOCATES_MEMORY, .base.cra_blocksize = DES3_EDE_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct cvm_des3_ctx), .base.cra_alignmask = 7, diff --git a/drivers/crypto/cavium/nitrox/nitrox_skcipher.c b/drivers/crypto/cavium/nitrox/nitrox_skcipher.c index 138261dcd032..6e5e667bab75 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_skcipher.c +++ b/drivers/crypto/cavium/nitrox/nitrox_skcipher.c @@ -421,25 +421,6 @@ static struct skcipher_alg nitrox_skciphers[] = { { .exit = nitrox_skcipher_exit, }, { .base = { - .cra_name = "cfb(aes)", - .cra_driver_name = "n5_cfb(aes)", - .cra_priority = PRIO, - .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, - .cra_blocksize = AES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct nitrox_crypto_ctx), - .cra_alignmask = 0, - .cra_module = THIS_MODULE, - }, - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .ivsize = AES_BLOCK_SIZE, - .setkey = nitrox_aes_setkey, - .encrypt = nitrox_aes_encrypt, - .decrypt = nitrox_aes_decrypt, - .init = nitrox_skcipher_init, - .exit = nitrox_skcipher_exit, -}, { - .base = { .cra_name = "xts(aes)", .cra_driver_name = "n5_xts(aes)", .cra_priority = PRIO, diff --git a/drivers/crypto/ccp/ccp-crypto-aes.c b/drivers/crypto/ccp/ccp-crypto-aes.c index 918e223f21b6..d11daaf47f06 100644 --- a/drivers/crypto/ccp/ccp-crypto-aes.c +++ b/drivers/crypto/ccp/ccp-crypto-aes.c @@ -267,24 +267,6 @@ static struct ccp_aes_def aes_algs[] = { .alg_defaults = &ccp_aes_defaults, }, { - .mode = CCP_AES_MODE_CFB, - .version = CCP_VERSION(3, 0), - .name = "cfb(aes)", - .driver_name = "cfb-aes-ccp", - .blocksize = 1, - .ivsize = AES_BLOCK_SIZE, - .alg_defaults = &ccp_aes_defaults, - }, - { - .mode = CCP_AES_MODE_OFB, - .version = CCP_VERSION(3, 0), - .name = "ofb(aes)", - .driver_name = "ofb-aes-ccp", - .blocksize = 1, - .ivsize = AES_BLOCK_SIZE, - .alg_defaults = &ccp_aes_defaults, - }, - { .mode = CCP_AES_MODE_CTR, .version = CCP_VERSION(3, 0), .name = "ctr(aes)", diff --git a/drivers/crypto/ccp/ccp-ops.c b/drivers/crypto/ccp/ccp-ops.c index aa4e1a500691..cb8e99936abb 100644 --- a/drivers/crypto/ccp/ccp-ops.c +++ b/drivers/crypto/ccp/ccp-ops.c @@ -179,8 +179,11 @@ static int ccp_init_dm_workarea(struct ccp_dm_workarea *wa, wa->dma.address = dma_map_single(wa->dev, wa->address, len, dir); - if (dma_mapping_error(wa->dev, wa->dma.address)) + if (dma_mapping_error(wa->dev, wa->dma.address)) { + kfree(wa->address); + wa->address = NULL; return -ENOMEM; + } wa->dma.length = len; } diff --git a/drivers/crypto/ccree/cc_aead.c b/drivers/crypto/ccree/cc_aead.c index 109ffb375fc6..5ef39d682389 100644 --- a/drivers/crypto/ccree/cc_aead.c +++ b/drivers/crypto/ccree/cc_aead.c @@ -2569,9 +2569,13 @@ static struct cc_crypto_alg *cc_create_aead_alg(struct cc_alg_template *tmpl, alg = &tmpl->template_aead; - snprintf(alg->base.cra_name, CRYPTO_MAX_ALG_NAME, "%s", tmpl->name); - snprintf(alg->base.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", - tmpl->driver_name); + if (snprintf(alg->base.cra_name, CRYPTO_MAX_ALG_NAME, "%s", + tmpl->name) >= CRYPTO_MAX_ALG_NAME) + return ERR_PTR(-EINVAL); + if (snprintf(alg->base.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", + tmpl->driver_name) >= CRYPTO_MAX_ALG_NAME) + return ERR_PTR(-EINVAL); + alg->base.cra_module = THIS_MODULE; alg->base.cra_priority = CC_CRA_PRIO; diff --git a/drivers/crypto/ccree/cc_cipher.c b/drivers/crypto/ccree/cc_cipher.c index 2cd44d7457a4..cd66a580e8b6 100644 --- a/drivers/crypto/ccree/cc_cipher.c +++ b/drivers/crypto/ccree/cc_cipher.c @@ -1080,24 +1080,6 @@ static const struct cc_alg_template skcipher_algs[] = { .sec_func = true, }, { - .name = "ofb(paes)", - .driver_name = "ofb-paes-ccree", - .blocksize = AES_BLOCK_SIZE, - .template_skcipher = { - .setkey = cc_cipher_sethkey, - .encrypt = cc_cipher_encrypt, - .decrypt = cc_cipher_decrypt, - .min_keysize = CC_HW_KEY_SIZE, - .max_keysize = CC_HW_KEY_SIZE, - .ivsize = AES_BLOCK_SIZE, - }, - .cipher_mode = DRV_CIPHER_OFB, - .flow_mode = S_DIN_to_AES, - .min_hw_rev = CC_HW_REV_712, - .std_body = CC_STD_NIST, - .sec_func = true, - }, - { .name = "cts(cbc(paes))", .driver_name = "cts-cbc-paes-ccree", .blocksize = AES_BLOCK_SIZE, @@ -1206,23 +1188,6 @@ static const struct cc_alg_template skcipher_algs[] = { .std_body = CC_STD_NIST, }, { - .name = "ofb(aes)", - .driver_name = "ofb-aes-ccree", - .blocksize = 1, - .template_skcipher = { - .setkey = cc_cipher_setkey, - .encrypt = cc_cipher_encrypt, - .decrypt = cc_cipher_decrypt, - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .ivsize = AES_BLOCK_SIZE, - }, - .cipher_mode = DRV_CIPHER_OFB, - .flow_mode = S_DIN_to_AES, - .min_hw_rev = CC_HW_REV_630, - .std_body = CC_STD_NIST, - }, - { .name = "cts(cbc(aes))", .driver_name = "cts-cbc-aes-ccree", .blocksize = AES_BLOCK_SIZE, @@ -1427,9 +1392,13 @@ static struct cc_crypto_alg *cc_create_alg(const struct cc_alg_template *tmpl, memcpy(alg, &tmpl->template_skcipher, sizeof(*alg)); - snprintf(alg->base.cra_name, CRYPTO_MAX_ALG_NAME, "%s", tmpl->name); - snprintf(alg->base.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", - tmpl->driver_name); + if (snprintf(alg->base.cra_name, CRYPTO_MAX_ALG_NAME, "%s", + tmpl->name) >= CRYPTO_MAX_ALG_NAME) + return ERR_PTR(-EINVAL); + if (snprintf(alg->base.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", + tmpl->driver_name) >= CRYPTO_MAX_ALG_NAME) + return ERR_PTR(-EINVAL); + alg->base.cra_module = THIS_MODULE; alg->base.cra_priority = CC_CRA_PRIO; alg->base.cra_blocksize = tmpl->blocksize; diff --git a/drivers/crypto/gemini/sl3516-ce-cipher.c b/drivers/crypto/gemini/sl3516-ce-cipher.c index 49dce9e0a834..583010b2d007 100644 --- a/drivers/crypto/gemini/sl3516-ce-cipher.c +++ b/drivers/crypto/gemini/sl3516-ce-cipher.c @@ -332,8 +332,8 @@ int sl3516_ce_cipher_init(struct crypto_tfm *tfm) return PTR_ERR(op->fallback_tfm); } - sktfm->reqsize = sizeof(struct sl3516_ce_cipher_req_ctx) + - crypto_skcipher_reqsize(op->fallback_tfm); + crypto_skcipher_set_reqsize(sktfm, sizeof(struct sl3516_ce_cipher_req_ctx) + + crypto_skcipher_reqsize(op->fallback_tfm)); dev_info(op->ce->dev, "Fallback for %s is %s\n", crypto_tfm_alg_driver_name(&sktfm->base), diff --git a/drivers/crypto/hifn_795x.c b/drivers/crypto/hifn_795x.c index 7bddc3c786c1..b4a4ec35bce0 100644 --- a/drivers/crypto/hifn_795x.c +++ b/drivers/crypto/hifn_795x.c @@ -2096,16 +2096,6 @@ static inline int hifn_encrypt_aes_cbc(struct skcipher_request *req) return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT, ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_CBC); } -static inline int hifn_encrypt_aes_cfb(struct skcipher_request *req) -{ - return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT, - ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_CFB); -} -static inline int hifn_encrypt_aes_ofb(struct skcipher_request *req) -{ - return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT, - ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_OFB); -} /* * AES decryption functions. @@ -2120,16 +2110,6 @@ static inline int hifn_decrypt_aes_cbc(struct skcipher_request *req) return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT, ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_CBC); } -static inline int hifn_decrypt_aes_cfb(struct skcipher_request *req) -{ - return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT, - ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_CFB); -} -static inline int hifn_decrypt_aes_ofb(struct skcipher_request *req) -{ - return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT, - ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_OFB); -} /* * DES ecryption functions. @@ -2144,16 +2124,6 @@ static inline int hifn_encrypt_des_cbc(struct skcipher_request *req) return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT, ACRYPTO_TYPE_DES, ACRYPTO_MODE_CBC); } -static inline int hifn_encrypt_des_cfb(struct skcipher_request *req) -{ - return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT, - ACRYPTO_TYPE_DES, ACRYPTO_MODE_CFB); -} -static inline int hifn_encrypt_des_ofb(struct skcipher_request *req) -{ - return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT, - ACRYPTO_TYPE_DES, ACRYPTO_MODE_OFB); -} /* * DES decryption functions. @@ -2168,16 +2138,6 @@ static inline int hifn_decrypt_des_cbc(struct skcipher_request *req) return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT, ACRYPTO_TYPE_DES, ACRYPTO_MODE_CBC); } -static inline int hifn_decrypt_des_cfb(struct skcipher_request *req) -{ - return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT, - ACRYPTO_TYPE_DES, ACRYPTO_MODE_CFB); -} -static inline int hifn_decrypt_des_ofb(struct skcipher_request *req) -{ - return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT, - ACRYPTO_TYPE_DES, ACRYPTO_MODE_OFB); -} /* * 3DES ecryption functions. @@ -2192,16 +2152,6 @@ static inline int hifn_encrypt_3des_cbc(struct skcipher_request *req) return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT, ACRYPTO_TYPE_3DES, ACRYPTO_MODE_CBC); } -static inline int hifn_encrypt_3des_cfb(struct skcipher_request *req) -{ - return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT, - ACRYPTO_TYPE_3DES, ACRYPTO_MODE_CFB); -} -static inline int hifn_encrypt_3des_ofb(struct skcipher_request *req) -{ - return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT, - ACRYPTO_TYPE_3DES, ACRYPTO_MODE_OFB); -} /* 3DES decryption functions. */ static inline int hifn_decrypt_3des_ecb(struct skcipher_request *req) @@ -2214,16 +2164,6 @@ static inline int hifn_decrypt_3des_cbc(struct skcipher_request *req) return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT, ACRYPTO_TYPE_3DES, ACRYPTO_MODE_CBC); } -static inline int hifn_decrypt_3des_cfb(struct skcipher_request *req) -{ - return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT, - ACRYPTO_TYPE_3DES, ACRYPTO_MODE_CFB); -} -static inline int hifn_decrypt_3des_ofb(struct skcipher_request *req) -{ - return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT, - ACRYPTO_TYPE_3DES, ACRYPTO_MODE_OFB); -} struct hifn_alg_template { char name[CRYPTO_MAX_ALG_NAME]; @@ -2234,29 +2174,9 @@ struct hifn_alg_template { static const struct hifn_alg_template hifn_alg_templates[] = { /* - * 3DES ECB, CBC, CFB and OFB modes. + * 3DES ECB and CBC modes. */ { - .name = "cfb(des3_ede)", .drv_name = "cfb-3des", .bsize = 8, - .skcipher = { - .min_keysize = HIFN_3DES_KEY_LENGTH, - .max_keysize = HIFN_3DES_KEY_LENGTH, - .setkey = hifn_des3_setkey, - .encrypt = hifn_encrypt_3des_cfb, - .decrypt = hifn_decrypt_3des_cfb, - }, - }, - { - .name = "ofb(des3_ede)", .drv_name = "ofb-3des", .bsize = 8, - .skcipher = { - .min_keysize = HIFN_3DES_KEY_LENGTH, - .max_keysize = HIFN_3DES_KEY_LENGTH, - .setkey = hifn_des3_setkey, - .encrypt = hifn_encrypt_3des_ofb, - .decrypt = hifn_decrypt_3des_ofb, - }, - }, - { .name = "cbc(des3_ede)", .drv_name = "cbc-3des", .bsize = 8, .skcipher = { .ivsize = HIFN_IV_LENGTH, @@ -2279,29 +2199,9 @@ static const struct hifn_alg_template hifn_alg_templates[] = { }, /* - * DES ECB, CBC, CFB and OFB modes. + * DES ECB and CBC modes. */ { - .name = "cfb(des)", .drv_name = "cfb-des", .bsize = 8, - .skcipher = { - .min_keysize = HIFN_DES_KEY_LENGTH, - .max_keysize = HIFN_DES_KEY_LENGTH, - .setkey = hifn_setkey, - .encrypt = hifn_encrypt_des_cfb, - .decrypt = hifn_decrypt_des_cfb, - }, - }, - { - .name = "ofb(des)", .drv_name = "ofb-des", .bsize = 8, - .skcipher = { - .min_keysize = HIFN_DES_KEY_LENGTH, - .max_keysize = HIFN_DES_KEY_LENGTH, - .setkey = hifn_setkey, - .encrypt = hifn_encrypt_des_ofb, - .decrypt = hifn_decrypt_des_ofb, - }, - }, - { .name = "cbc(des)", .drv_name = "cbc-des", .bsize = 8, .skcipher = { .ivsize = HIFN_IV_LENGTH, @@ -2324,7 +2224,7 @@ static const struct hifn_alg_template hifn_alg_templates[] = { }, /* - * AES ECB, CBC, CFB and OFB modes. + * AES ECB and CBC modes. */ { .name = "ecb(aes)", .drv_name = "ecb-aes", .bsize = 16, @@ -2347,26 +2247,6 @@ static const struct hifn_alg_template hifn_alg_templates[] = { .decrypt = hifn_decrypt_aes_cbc, }, }, - { - .name = "cfb(aes)", .drv_name = "cfb-aes", .bsize = 16, - .skcipher = { - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .setkey = hifn_setkey, - .encrypt = hifn_encrypt_aes_cfb, - .decrypt = hifn_decrypt_aes_cfb, - }, - }, - { - .name = "ofb(aes)", .drv_name = "ofb-aes", .bsize = 16, - .skcipher = { - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .setkey = hifn_setkey, - .encrypt = hifn_encrypt_aes_ofb, - .decrypt = hifn_decrypt_aes_ofb, - }, - }, }; static int hifn_init_tfm(struct crypto_skcipher *tfm) diff --git a/drivers/crypto/hisilicon/debugfs.c b/drivers/crypto/hisilicon/debugfs.c index 7e8186fe0512..80ed4b2d209c 100644 --- a/drivers/crypto/hisilicon/debugfs.c +++ b/drivers/crypto/hisilicon/debugfs.c @@ -31,6 +31,10 @@ static const char * const qm_debug_file_name[] = { [CLEAR_ENABLE] = "clear_enable", }; +static const char * const qm_s[] = { + "work", "stop", +}; + struct qm_dfx_item { const char *name; u32 offset; @@ -53,34 +57,34 @@ static struct qm_dfx_item qm_dfx_files[] = { #define CNT_CYC_REGS_NUM 10 static const struct debugfs_reg32 qm_dfx_regs[] = { /* XXX_CNT are reading clear register */ - {"QM_ECC_1BIT_CNT ", 0x104000ull}, - {"QM_ECC_MBIT_CNT ", 0x104008ull}, - {"QM_DFX_MB_CNT ", 0x104018ull}, - {"QM_DFX_DB_CNT ", 0x104028ull}, - {"QM_DFX_SQE_CNT ", 0x104038ull}, - {"QM_DFX_CQE_CNT ", 0x104048ull}, - {"QM_DFX_SEND_SQE_TO_ACC_CNT ", 0x104050ull}, - {"QM_DFX_WB_SQE_FROM_ACC_CNT ", 0x104058ull}, - {"QM_DFX_ACC_FINISH_CNT ", 0x104060ull}, - {"QM_DFX_CQE_ERR_CNT ", 0x1040b4ull}, - {"QM_DFX_FUNS_ACTIVE_ST ", 0x200ull}, - {"QM_ECC_1BIT_INF ", 0x104004ull}, - {"QM_ECC_MBIT_INF ", 0x10400cull}, - {"QM_DFX_ACC_RDY_VLD0 ", 0x1040a0ull}, - {"QM_DFX_ACC_RDY_VLD1 ", 0x1040a4ull}, - {"QM_DFX_AXI_RDY_VLD ", 0x1040a8ull}, - {"QM_DFX_FF_ST0 ", 0x1040c8ull}, - {"QM_DFX_FF_ST1 ", 0x1040ccull}, - {"QM_DFX_FF_ST2 ", 0x1040d0ull}, - {"QM_DFX_FF_ST3 ", 0x1040d4ull}, - {"QM_DFX_FF_ST4 ", 0x1040d8ull}, - {"QM_DFX_FF_ST5 ", 0x1040dcull}, - {"QM_DFX_FF_ST6 ", 0x1040e0ull}, - {"QM_IN_IDLE_ST ", 0x1040e4ull}, + {"QM_ECC_1BIT_CNT ", 0x104000}, + {"QM_ECC_MBIT_CNT ", 0x104008}, + {"QM_DFX_MB_CNT ", 0x104018}, + {"QM_DFX_DB_CNT ", 0x104028}, + {"QM_DFX_SQE_CNT ", 0x104038}, + {"QM_DFX_CQE_CNT ", 0x104048}, + {"QM_DFX_SEND_SQE_TO_ACC_CNT ", 0x104050}, + {"QM_DFX_WB_SQE_FROM_ACC_CNT ", 0x104058}, + {"QM_DFX_ACC_FINISH_CNT ", 0x104060}, + {"QM_DFX_CQE_ERR_CNT ", 0x1040b4}, + {"QM_DFX_FUNS_ACTIVE_ST ", 0x200}, + {"QM_ECC_1BIT_INF ", 0x104004}, + {"QM_ECC_MBIT_INF ", 0x10400c}, + {"QM_DFX_ACC_RDY_VLD0 ", 0x1040a0}, + {"QM_DFX_ACC_RDY_VLD1 ", 0x1040a4}, + {"QM_DFX_AXI_RDY_VLD ", 0x1040a8}, + {"QM_DFX_FF_ST0 ", 0x1040c8}, + {"QM_DFX_FF_ST1 ", 0x1040cc}, + {"QM_DFX_FF_ST2 ", 0x1040d0}, + {"QM_DFX_FF_ST3 ", 0x1040d4}, + {"QM_DFX_FF_ST4 ", 0x1040d8}, + {"QM_DFX_FF_ST5 ", 0x1040dc}, + {"QM_DFX_FF_ST6 ", 0x1040e0}, + {"QM_IN_IDLE_ST ", 0x1040e4}, }; static const struct debugfs_reg32 qm_vf_dfx_regs[] = { - {"QM_DFX_FUNS_ACTIVE_ST ", 0x200ull}, + {"QM_DFX_FUNS_ACTIVE_ST ", 0x200}, }; /* define the QM's dfx regs region and region length */ diff --git a/drivers/crypto/hisilicon/hpre/hpre_main.c b/drivers/crypto/hisilicon/hpre/hpre_main.c index 56777099ef69..3255b2a070c7 100644 --- a/drivers/crypto/hisilicon/hpre/hpre_main.c +++ b/drivers/crypto/hisilicon/hpre/hpre_main.c @@ -118,8 +118,6 @@ #define HPRE_DFX_COMMON2_LEN 0xE #define HPRE_DFX_CORE_LEN 0x43 -#define HPRE_DEV_ALG_MAX_LEN 256 - static const char hpre_name[] = "hisi_hpre"; static struct dentry *hpre_debugfs_root; static const struct pci_device_id hpre_dev_ids[] = { @@ -135,12 +133,7 @@ struct hpre_hw_error { const char *msg; }; -struct hpre_dev_alg { - u32 alg_msk; - const char *alg; -}; - -static const struct hpre_dev_alg hpre_dev_algs[] = { +static const struct qm_dev_alg hpre_dev_algs[] = { { .alg_msk = BIT(0), .alg = "rsa\n" @@ -233,6 +226,20 @@ static const struct hisi_qm_cap_info hpre_basic_info[] = { {HPRE_CORE10_ALG_BITMAP_CAP, 0x3170, 0, GENMASK(31, 0), 0x0, 0x10, 0x10} }; +enum hpre_pre_store_cap_idx { + HPRE_CLUSTER_NUM_CAP_IDX = 0x0, + HPRE_CORE_ENABLE_BITMAP_CAP_IDX, + HPRE_DRV_ALG_BITMAP_CAP_IDX, + HPRE_DEV_ALG_BITMAP_CAP_IDX, +}; + +static const u32 hpre_pre_store_caps[] = { + HPRE_CLUSTER_NUM_CAP, + HPRE_CORE_ENABLE_BITMAP_CAP, + HPRE_DRV_ALG_BITMAP_CAP, + HPRE_DEV_ALG_BITMAP_CAP, +}; + static const struct hpre_hw_error hpre_hw_errors[] = { { .int_msk = BIT(0), @@ -355,42 +362,13 @@ bool hpre_check_alg_support(struct hisi_qm *qm, u32 alg) { u32 cap_val; - cap_val = hisi_qm_get_hw_info(qm, hpre_basic_info, HPRE_DRV_ALG_BITMAP_CAP, qm->cap_ver); + cap_val = qm->cap_tables.dev_cap_table[HPRE_DRV_ALG_BITMAP_CAP_IDX].cap_val; if (alg & cap_val) return true; return false; } -static int hpre_set_qm_algs(struct hisi_qm *qm) -{ - struct device *dev = &qm->pdev->dev; - char *algs, *ptr; - u32 alg_msk; - int i; - - if (!qm->use_sva) - return 0; - - algs = devm_kzalloc(dev, HPRE_DEV_ALG_MAX_LEN * sizeof(char), GFP_KERNEL); - if (!algs) - return -ENOMEM; - - alg_msk = hisi_qm_get_hw_info(qm, hpre_basic_info, HPRE_DEV_ALG_BITMAP_CAP, qm->cap_ver); - - for (i = 0; i < ARRAY_SIZE(hpre_dev_algs); i++) - if (alg_msk & hpre_dev_algs[i].alg_msk) - strcat(algs, hpre_dev_algs[i].alg); - - ptr = strrchr(algs, '\n'); - if (ptr) - *ptr = '\0'; - - qm->uacce->algs = algs; - - return 0; -} - static int hpre_diff_regs_show(struct seq_file *s, void *unused) { struct hisi_qm *qm = s->private; @@ -460,16 +438,6 @@ static u32 vfs_num; module_param_cb(vfs_num, &vfs_num_ops, &vfs_num, 0444); MODULE_PARM_DESC(vfs_num, "Number of VFs to enable(1-63), 0(default)"); -static inline int hpre_cluster_num(struct hisi_qm *qm) -{ - return hisi_qm_get_hw_info(qm, hpre_basic_info, HPRE_CLUSTER_NUM_CAP, qm->cap_ver); -} - -static inline int hpre_cluster_core_mask(struct hisi_qm *qm) -{ - return hisi_qm_get_hw_info(qm, hpre_basic_info, HPRE_CORE_ENABLE_BITMAP_CAP, qm->cap_ver); -} - struct hisi_qp *hpre_create_qp(u8 type) { int node = cpu_to_node(smp_processor_id()); @@ -536,13 +504,15 @@ static int hpre_cfg_by_dsm(struct hisi_qm *qm) static int hpre_set_cluster(struct hisi_qm *qm) { - u32 cluster_core_mask = hpre_cluster_core_mask(qm); - u8 clusters_num = hpre_cluster_num(qm); struct device *dev = &qm->pdev->dev; unsigned long offset; + u32 cluster_core_mask; + u8 clusters_num; u32 val = 0; int ret, i; + cluster_core_mask = qm->cap_tables.dev_cap_table[HPRE_CORE_ENABLE_BITMAP_CAP_IDX].cap_val; + clusters_num = qm->cap_tables.dev_cap_table[HPRE_CLUSTER_NUM_CAP_IDX].cap_val; for (i = 0; i < clusters_num; i++) { offset = i * HPRE_CLSTR_ADDR_INTRVL; @@ -737,11 +707,12 @@ static int hpre_set_user_domain_and_cache(struct hisi_qm *qm) static void hpre_cnt_regs_clear(struct hisi_qm *qm) { - u8 clusters_num = hpre_cluster_num(qm); unsigned long offset; + u8 clusters_num; int i; /* clear clusterX/cluster_ctrl */ + clusters_num = qm->cap_tables.dev_cap_table[HPRE_CLUSTER_NUM_CAP_IDX].cap_val; for (i = 0; i < clusters_num; i++) { offset = HPRE_CLSTR_BASE + i * HPRE_CLSTR_ADDR_INTRVL; writel(0x0, qm->io_base + offset + HPRE_CLUSTER_INQURY); @@ -1028,13 +999,14 @@ static int hpre_pf_comm_regs_debugfs_init(struct hisi_qm *qm) static int hpre_cluster_debugfs_init(struct hisi_qm *qm) { - u8 clusters_num = hpre_cluster_num(qm); struct device *dev = &qm->pdev->dev; char buf[HPRE_DBGFS_VAL_MAX_LEN]; struct debugfs_regset32 *regset; struct dentry *tmp_d; + u8 clusters_num; int i, ret; + clusters_num = qm->cap_tables.dev_cap_table[HPRE_CLUSTER_NUM_CAP_IDX].cap_val; for (i = 0; i < clusters_num; i++) { ret = snprintf(buf, HPRE_DBGFS_VAL_MAX_LEN, "cluster%d", i); if (ret >= HPRE_DBGFS_VAL_MAX_LEN) @@ -1139,8 +1111,37 @@ static void hpre_debugfs_exit(struct hisi_qm *qm) debugfs_remove_recursive(qm->debug.debug_root); } +static int hpre_pre_store_cap_reg(struct hisi_qm *qm) +{ + struct hisi_qm_cap_record *hpre_cap; + struct device *dev = &qm->pdev->dev; + size_t i, size; + + size = ARRAY_SIZE(hpre_pre_store_caps); + hpre_cap = devm_kzalloc(dev, sizeof(*hpre_cap) * size, GFP_KERNEL); + if (!hpre_cap) + return -ENOMEM; + + for (i = 0; i < size; i++) { + hpre_cap[i].type = hpre_pre_store_caps[i]; + hpre_cap[i].cap_val = hisi_qm_get_hw_info(qm, hpre_basic_info, + hpre_pre_store_caps[i], qm->cap_ver); + } + + if (hpre_cap[HPRE_CLUSTER_NUM_CAP_IDX].cap_val > HPRE_CLUSTERS_NUM_MAX) { + dev_err(dev, "Device cluster num %u is out of range for driver supports %d!\n", + hpre_cap[HPRE_CLUSTER_NUM_CAP_IDX].cap_val, HPRE_CLUSTERS_NUM_MAX); + return -EINVAL; + } + + qm->cap_tables.dev_cap_table = hpre_cap; + + return 0; +} + static int hpre_qm_init(struct hisi_qm *qm, struct pci_dev *pdev) { + u64 alg_msk; int ret; if (pdev->revision == QM_HW_V1) { @@ -1171,7 +1172,16 @@ static int hpre_qm_init(struct hisi_qm *qm, struct pci_dev *pdev) return ret; } - ret = hpre_set_qm_algs(qm); + /* Fetch and save the value of capability registers */ + ret = hpre_pre_store_cap_reg(qm); + if (ret) { + pci_err(pdev, "Failed to pre-store capability registers!\n"); + hisi_qm_uninit(qm); + return ret; + } + + alg_msk = qm->cap_tables.dev_cap_table[HPRE_DEV_ALG_BITMAP_CAP_IDX].cap_val; + ret = hisi_qm_set_algs(qm, alg_msk, hpre_dev_algs, ARRAY_SIZE(hpre_dev_algs)); if (ret) { pci_err(pdev, "Failed to set hpre algs!\n"); hisi_qm_uninit(qm); @@ -1184,11 +1194,12 @@ static int hpre_show_last_regs_init(struct hisi_qm *qm) { int cluster_dfx_regs_num = ARRAY_SIZE(hpre_cluster_dfx_regs); int com_dfx_regs_num = ARRAY_SIZE(hpre_com_dfx_regs); - u8 clusters_num = hpre_cluster_num(qm); struct qm_debug *debug = &qm->debug; void __iomem *io_base; + u8 clusters_num; int i, j, idx; + clusters_num = qm->cap_tables.dev_cap_table[HPRE_CLUSTER_NUM_CAP_IDX].cap_val; debug->last_words = kcalloc(cluster_dfx_regs_num * clusters_num + com_dfx_regs_num, sizeof(unsigned int), GFP_KERNEL); if (!debug->last_words) @@ -1225,10 +1236,10 @@ static void hpre_show_last_dfx_regs(struct hisi_qm *qm) { int cluster_dfx_regs_num = ARRAY_SIZE(hpre_cluster_dfx_regs); int com_dfx_regs_num = ARRAY_SIZE(hpre_com_dfx_regs); - u8 clusters_num = hpre_cluster_num(qm); struct qm_debug *debug = &qm->debug; struct pci_dev *pdev = qm->pdev; void __iomem *io_base; + u8 clusters_num; int i, j, idx; u32 val; @@ -1243,6 +1254,7 @@ static void hpre_show_last_dfx_regs(struct hisi_qm *qm) hpre_com_dfx_regs[i].name, debug->last_words[i], val); } + clusters_num = qm->cap_tables.dev_cap_table[HPRE_CLUSTER_NUM_CAP_IDX].cap_val; for (i = 0; i < clusters_num; i++) { io_base = qm->io_base + hpre_cluster_offsets[i]; for (j = 0; j < cluster_dfx_regs_num; j++) { diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index 18599f3634c3..4b20b94e6371 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -129,16 +129,21 @@ #define QM_FIFO_OVERFLOW_TYPE 0xc0 #define QM_FIFO_OVERFLOW_TYPE_SHIFT 6 #define QM_FIFO_OVERFLOW_VF 0x3f +#define QM_FIFO_OVERFLOW_QP_SHIFT 16 #define QM_ABNORMAL_INF01 0x100014 #define QM_DB_TIMEOUT_TYPE 0xc0 #define QM_DB_TIMEOUT_TYPE_SHIFT 6 #define QM_DB_TIMEOUT_VF 0x3f +#define QM_DB_TIMEOUT_QP_SHIFT 16 +#define QM_ABNORMAL_INF02 0x100018 +#define QM_AXI_POISON_ERR BIT(22) #define QM_RAS_CE_ENABLE 0x1000ec #define QM_RAS_FE_ENABLE 0x1000f0 #define QM_RAS_NFE_ENABLE 0x1000f4 #define QM_RAS_CE_THRESHOLD 0x1000f8 #define QM_RAS_CE_TIMES_PER_IRQ 1 #define QM_OOO_SHUTDOWN_SEL 0x1040f8 +#define QM_AXI_RRESP_ERR BIT(0) #define QM_ECC_MBIT BIT(2) #define QM_DB_TIMEOUT BIT(10) #define QM_OF_FIFO_OF BIT(11) @@ -229,6 +234,8 @@ #define QM_QOS_MAX_CIR_U 6 #define QM_AUTOSUSPEND_DELAY 3000 +#define QM_DEV_ALG_MAX_LEN 256 + #define QM_MK_CQC_DW3_V1(hop_num, pg_sz, buf_sz, cqe_sz) \ (((hop_num) << QM_CQ_HOP_NUM_SHIFT) | \ ((pg_sz) << QM_CQ_PAGE_SIZE_SHIFT) | \ @@ -294,6 +301,13 @@ enum qm_basic_type { QM_VF_IRQ_NUM_CAP, }; +enum qm_pre_store_cap_idx { + QM_EQ_IRQ_TYPE_CAP_IDX = 0x0, + QM_AEQ_IRQ_TYPE_CAP_IDX, + QM_ABN_IRQ_TYPE_CAP_IDX, + QM_PF2VF_IRQ_TYPE_CAP_IDX, +}; + static const struct hisi_qm_cap_info qm_cap_info_comm[] = { {QM_SUPPORT_DB_ISOLATION, 0x30, 0, BIT(0), 0x0, 0x0, 0x0}, {QM_SUPPORT_FUNC_QOS, 0x3100, 0, BIT(8), 0x0, 0x0, 0x1}, @@ -323,6 +337,13 @@ static const struct hisi_qm_cap_info qm_basic_info[] = { {QM_VF_IRQ_NUM_CAP, 0x311c, 0, GENMASK(15, 0), 0x1, 0x2, 0x3}, }; +static const u32 qm_pre_store_caps[] = { + QM_EQ_IRQ_TYPE_CAP, + QM_AEQ_IRQ_TYPE_CAP, + QM_ABN_IRQ_TYPE_CAP, + QM_PF2VF_IRQ_TYPE_CAP, +}; + struct qm_mailbox { __le16 w0; __le16 queue_num; @@ -386,7 +407,6 @@ static const struct hisi_qm_hw_error qm_hw_error[] = { { .int_msk = BIT(12), .msg = "qm_db_random_invalid" }, { .int_msk = BIT(13), .msg = "qm_mailbox_timeout" }, { .int_msk = BIT(14), .msg = "qm_flr_timeout" }, - { /* sentinel */ } }; static const char * const qm_db_timeout[] = { @@ -397,10 +417,6 @@ static const char * const qm_fifo_overflow[] = { "cq", "eq", "aeq", }; -static const char * const qp_s[] = { - "none", "init", "start", "stop", "close", -}; - struct qm_typical_qos_table { u32 start; u32 end; @@ -428,85 +444,6 @@ static struct qm_typical_qos_table shaper_cbs_s[] = { static void qm_irqs_unregister(struct hisi_qm *qm); -static bool qm_avail_state(struct hisi_qm *qm, enum qm_state new) -{ - enum qm_state curr = atomic_read(&qm->status.flags); - bool avail = false; - - switch (curr) { - case QM_INIT: - if (new == QM_START || new == QM_CLOSE) - avail = true; - break; - case QM_START: - if (new == QM_STOP) - avail = true; - break; - case QM_STOP: - if (new == QM_CLOSE || new == QM_START) - avail = true; - break; - default: - break; - } - - dev_dbg(&qm->pdev->dev, "change qm state from %s to %s\n", - qm_s[curr], qm_s[new]); - - if (!avail) - dev_warn(&qm->pdev->dev, "Can not change qm state from %s to %s\n", - qm_s[curr], qm_s[new]); - - return avail; -} - -static bool qm_qp_avail_state(struct hisi_qm *qm, struct hisi_qp *qp, - enum qp_state new) -{ - enum qm_state qm_curr = atomic_read(&qm->status.flags); - enum qp_state qp_curr = 0; - bool avail = false; - - if (qp) - qp_curr = atomic_read(&qp->qp_status.flags); - - switch (new) { - case QP_INIT: - if (qm_curr == QM_START || qm_curr == QM_INIT) - avail = true; - break; - case QP_START: - if ((qm_curr == QM_START && qp_curr == QP_INIT) || - (qm_curr == QM_START && qp_curr == QP_STOP)) - avail = true; - break; - case QP_STOP: - if ((qm_curr == QM_START && qp_curr == QP_START) || - (qp_curr == QP_INIT)) - avail = true; - break; - case QP_CLOSE: - if ((qm_curr == QM_START && qp_curr == QP_INIT) || - (qm_curr == QM_START && qp_curr == QP_STOP) || - (qm_curr == QM_STOP && qp_curr == QP_STOP) || - (qm_curr == QM_STOP && qp_curr == QP_INIT)) - avail = true; - break; - default: - break; - } - - dev_dbg(&qm->pdev->dev, "change qp state from %s to %s in QM %s\n", - qp_s[qp_curr], qp_s[new], qm_s[qm_curr]); - - if (!avail) - dev_warn(&qm->pdev->dev, - "Can not change qp state from %s to %s in QM %s\n", - qp_s[qp_curr], qp_s[new], qm_s[qm_curr]); - - return avail; -} - static u32 qm_get_hw_error_status(struct hisi_qm *qm) { return readl(qm->io_base + QM_ABNORMAL_INT_STATUS); @@ -660,9 +597,6 @@ int hisi_qm_mb(struct hisi_qm *qm, u8 cmd, dma_addr_t dma_addr, u16 queue, struct qm_mailbox mailbox; int ret; - dev_dbg(&qm->pdev->dev, "QM mailbox request to q%u: %u-%llx\n", - queue, cmd, (unsigned long long)dma_addr); - qm_mb_pre_init(&mailbox, cmd, dma_addr, queue, op); mutex_lock(&qm->mailbox_lock); @@ -828,6 +762,40 @@ static void qm_get_xqc_depth(struct hisi_qm *qm, u16 *low_bits, *high_bits = (depth >> QM_XQ_DEPTH_SHIFT) & QM_XQ_DEPTH_MASK; } +int hisi_qm_set_algs(struct hisi_qm *qm, u64 alg_msk, const struct qm_dev_alg *dev_algs, + u32 dev_algs_size) +{ + struct device *dev = &qm->pdev->dev; + char *algs, *ptr; + int i; + + if (!qm->uacce) + return 0; + + if (dev_algs_size >= QM_DEV_ALG_MAX_LEN) { + dev_err(dev, "algs size %u is equal or larger than %d.\n", + dev_algs_size, QM_DEV_ALG_MAX_LEN); + return -EINVAL; + } + + algs = devm_kzalloc(dev, QM_DEV_ALG_MAX_LEN * sizeof(char), GFP_KERNEL); + if (!algs) + return -ENOMEM; + + for (i = 0; i < dev_algs_size; i++) + if (alg_msk & dev_algs[i].alg_msk) + strcat(algs, dev_algs[i].alg); + + ptr = strrchr(algs, '\n'); + if (ptr) { + *ptr = '\0'; + qm->uacce->algs = algs; + } + + return 0; +} +EXPORT_SYMBOL_GPL(hisi_qm_set_algs); + static u32 qm_get_irq_num(struct hisi_qm *qm) { if (qm->fun_type == QM_HW_PF) @@ -1406,7 +1374,7 @@ static void qm_log_hw_error(struct hisi_qm *qm, u32 error_status) { const struct hisi_qm_hw_error *err; struct device *dev = &qm->pdev->dev; - u32 reg_val, type, vf_num; + u32 reg_val, type, vf_num, qp_id; int i; for (i = 0; i < ARRAY_SIZE(qm_hw_error); i++) { @@ -1422,19 +1390,24 @@ static void qm_log_hw_error(struct hisi_qm *qm, u32 error_status) type = (reg_val & QM_DB_TIMEOUT_TYPE) >> QM_DB_TIMEOUT_TYPE_SHIFT; vf_num = reg_val & QM_DB_TIMEOUT_VF; - dev_err(dev, "qm %s doorbell timeout in function %u\n", - qm_db_timeout[type], vf_num); + qp_id = reg_val >> QM_DB_TIMEOUT_QP_SHIFT; + dev_err(dev, "qm %s doorbell timeout in function %u qp %u\n", + qm_db_timeout[type], vf_num, qp_id); } else if (err->int_msk & QM_OF_FIFO_OF) { reg_val = readl(qm->io_base + QM_ABNORMAL_INF00); type = (reg_val & QM_FIFO_OVERFLOW_TYPE) >> QM_FIFO_OVERFLOW_TYPE_SHIFT; vf_num = reg_val & QM_FIFO_OVERFLOW_VF; - + qp_id = reg_val >> QM_FIFO_OVERFLOW_QP_SHIFT; if (type < ARRAY_SIZE(qm_fifo_overflow)) - dev_err(dev, "qm %s fifo overflow in function %u\n", - qm_fifo_overflow[type], vf_num); + dev_err(dev, "qm %s fifo overflow in function %u qp %u\n", + qm_fifo_overflow[type], vf_num, qp_id); else dev_err(dev, "unknown error type\n"); + } else if (err->int_msk & QM_AXI_RRESP_ERR) { + reg_val = readl(qm->io_base + QM_ABNORMAL_INF02); + if (reg_val & QM_AXI_POISON_ERR) + dev_err(dev, "qm axi poison error happened\n"); } } } @@ -1843,8 +1816,10 @@ static struct hisi_qp *qm_create_qp_nolock(struct hisi_qm *qm, u8 alg_type) struct hisi_qp *qp; int qp_id; - if (!qm_qp_avail_state(qm, NULL, QP_INIT)) + if (atomic_read(&qm->status.flags) == QM_STOP) { + dev_info_ratelimited(dev, "failed to create qp as qm is stop!\n"); return ERR_PTR(-EPERM); + } if (qm->qp_in_used == qm->qp_num) { dev_info_ratelimited(dev, "All %u queues of QM are busy!\n", @@ -1871,7 +1846,6 @@ static struct hisi_qp *qm_create_qp_nolock(struct hisi_qm *qm, u8 alg_type) qp->alg_type = alg_type; qp->is_in_kernel = true; qm->qp_in_used++; - atomic_set(&qp->qp_status.flags, QP_INIT); return qp; } @@ -1914,11 +1888,6 @@ static void hisi_qm_release_qp(struct hisi_qp *qp) down_write(&qm->qps_lock); - if (!qm_qp_avail_state(qm, qp, QP_CLOSE)) { - up_write(&qm->qps_lock); - return; - } - qm->qp_in_used--; idr_remove(&qm->qp_idr, qp->qp_id); @@ -1966,6 +1935,11 @@ static int qm_cq_ctx_cfg(struct hisi_qp *qp, int qp_id, u32 pasid) cqc.dw3 = cpu_to_le32(QM_MK_CQC_DW3_V2(QM_QC_CQE_SIZE, qp->cq_depth)); cqc.w8 = 0; /* rand_qc */ } + /* + * Enable request finishing interrupts defaultly. + * So, there will be some interrupts until disabling + * this. + */ cqc.dw6 = cpu_to_le32(1 << QM_CQ_PHASE_SHIFT | 1 << QM_CQ_FLAG_SHIFT); cqc.base_l = cpu_to_le32(lower_32_bits(qp->cqe_dma)); cqc.base_h = cpu_to_le32(upper_32_bits(qp->cqe_dma)); @@ -1998,8 +1972,10 @@ static int qm_start_qp_nolock(struct hisi_qp *qp, unsigned long arg) u32 pasid = arg; int ret; - if (!qm_qp_avail_state(qm, qp, QP_START)) + if (atomic_read(&qm->status.flags) == QM_STOP) { + dev_info_ratelimited(dev, "failed to start qp as qm is stop!\n"); return -EPERM; + } ret = qm_qp_ctx_cfg(qp, qp_id, pasid); if (ret) @@ -2121,21 +2097,17 @@ static int qm_stop_qp_nolock(struct hisi_qp *qp) * is_resetting flag should be set negative so that this qp will not * be restarted after reset. */ - if (atomic_read(&qp->qp_status.flags) == QP_STOP) { + if (atomic_read(&qp->qp_status.flags) != QP_START) { qp->is_resetting = false; return 0; } - if (!qm_qp_avail_state(qp->qm, qp, QP_STOP)) - return -EPERM; - atomic_set(&qp->qp_status.flags, QP_STOP); ret = qm_drain_qp(qp); if (ret) dev_err(dev, "Failed to drain out data for stopping!\n"); - flush_workqueue(qp->qm->wq); if (unlikely(qp->is_resetting && atomic_read(&qp->qp_status.used))) qp_stop_fail_cb(qp); @@ -2855,13 +2827,8 @@ void hisi_qm_uninit(struct hisi_qm *qm) { qm_cmd_uninit(qm); hisi_qm_unint_work(qm); - down_write(&qm->qps_lock); - - if (!qm_avail_state(qm, QM_CLOSE)) { - up_write(&qm->qps_lock); - return; - } + down_write(&qm->qps_lock); hisi_qm_memory_uninit(qm); hisi_qm_set_state(qm, QM_NOT_READY); up_write(&qm->qps_lock); @@ -3035,11 +3002,6 @@ int hisi_qm_start(struct hisi_qm *qm) down_write(&qm->qps_lock); - if (!qm_avail_state(qm, QM_START)) { - up_write(&qm->qps_lock); - return -EPERM; - } - dev_dbg(dev, "qm start with %u queue pairs\n", qm->qp_num); if (!qm->qp_num) { @@ -3049,10 +3011,12 @@ int hisi_qm_start(struct hisi_qm *qm) } ret = __hisi_qm_start(qm); - if (!ret) - atomic_set(&qm->status.flags, QM_START); + if (ret) + goto err_unlock; + atomic_set(&qm->status.flags, QM_WORK); hisi_qm_set_state(qm, QM_READY); + err_unlock: up_write(&qm->qps_lock); return ret; @@ -3149,10 +3113,11 @@ int hisi_qm_stop(struct hisi_qm *qm, enum qm_stop_reason r) down_write(&qm->qps_lock); qm->status.stop_reason = r; - if (!qm_avail_state(qm, QM_STOP)) { - ret = -EPERM; + if (atomic_read(&qm->status.flags) == QM_STOP) goto err_unlock; - } + + /* Stop all the request sending at first. */ + atomic_set(&qm->status.flags, QM_STOP); if (qm->status.stop_reason == QM_SOFT_RESET || qm->status.stop_reason == QM_DOWN) { @@ -3176,7 +3141,6 @@ int hisi_qm_stop(struct hisi_qm *qm, enum qm_stop_reason r) } qm_clear_queues(qm); - atomic_set(&qm->status.flags, QM_STOP); err_unlock: up_write(&qm->qps_lock); @@ -3966,6 +3930,11 @@ static int qm_set_vf_mse(struct hisi_qm *qm, bool set) int pos; int i; + /* + * Since function qm_set_vf_mse is called only after SRIOV is enabled, + * pci_find_ext_capability cannot return 0, pos does not need to be + * checked. + */ pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_SRIOV); pci_read_config_word(pdev, pos + PCI_SRIOV_CTRL, &sriov_ctrl); if (set) @@ -4816,7 +4785,7 @@ static void qm_unregister_abnormal_irq(struct hisi_qm *qm) if (qm->fun_type == QM_HW_VF) return; - val = hisi_qm_get_hw_info(qm, qm_basic_info, QM_ABN_IRQ_TYPE_CAP, qm->cap_ver); + val = qm->cap_tables.qm_cap_table[QM_ABN_IRQ_TYPE_CAP_IDX].cap_val; if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_ABN_IRQ_TYPE_MASK)) return; @@ -4833,7 +4802,7 @@ static int qm_register_abnormal_irq(struct hisi_qm *qm) if (qm->fun_type == QM_HW_VF) return 0; - val = hisi_qm_get_hw_info(qm, qm_basic_info, QM_ABN_IRQ_TYPE_CAP, qm->cap_ver); + val = qm->cap_tables.qm_cap_table[QM_ABN_IRQ_TYPE_CAP_IDX].cap_val; if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_ABN_IRQ_TYPE_MASK)) return 0; @@ -4850,7 +4819,7 @@ static void qm_unregister_mb_cmd_irq(struct hisi_qm *qm) struct pci_dev *pdev = qm->pdev; u32 irq_vector, val; - val = hisi_qm_get_hw_info(qm, qm_basic_info, QM_PF2VF_IRQ_TYPE_CAP, qm->cap_ver); + val = qm->cap_tables.qm_cap_table[QM_PF2VF_IRQ_TYPE_CAP_IDX].cap_val; if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_IRQ_TYPE_MASK)) return; @@ -4864,7 +4833,7 @@ static int qm_register_mb_cmd_irq(struct hisi_qm *qm) u32 irq_vector, val; int ret; - val = hisi_qm_get_hw_info(qm, qm_basic_info, QM_PF2VF_IRQ_TYPE_CAP, qm->cap_ver); + val = qm->cap_tables.qm_cap_table[QM_PF2VF_IRQ_TYPE_CAP_IDX].cap_val; if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_IRQ_TYPE_MASK)) return 0; @@ -4881,7 +4850,7 @@ static void qm_unregister_aeq_irq(struct hisi_qm *qm) struct pci_dev *pdev = qm->pdev; u32 irq_vector, val; - val = hisi_qm_get_hw_info(qm, qm_basic_info, QM_AEQ_IRQ_TYPE_CAP, qm->cap_ver); + val = qm->cap_tables.qm_cap_table[QM_AEQ_IRQ_TYPE_CAP_IDX].cap_val; if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_IRQ_TYPE_MASK)) return; @@ -4895,7 +4864,7 @@ static int qm_register_aeq_irq(struct hisi_qm *qm) u32 irq_vector, val; int ret; - val = hisi_qm_get_hw_info(qm, qm_basic_info, QM_AEQ_IRQ_TYPE_CAP, qm->cap_ver); + val = qm->cap_tables.qm_cap_table[QM_AEQ_IRQ_TYPE_CAP_IDX].cap_val; if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_IRQ_TYPE_MASK)) return 0; @@ -4913,7 +4882,7 @@ static void qm_unregister_eq_irq(struct hisi_qm *qm) struct pci_dev *pdev = qm->pdev; u32 irq_vector, val; - val = hisi_qm_get_hw_info(qm, qm_basic_info, QM_EQ_IRQ_TYPE_CAP, qm->cap_ver); + val = qm->cap_tables.qm_cap_table[QM_EQ_IRQ_TYPE_CAP_IDX].cap_val; if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_IRQ_TYPE_MASK)) return; @@ -4927,7 +4896,7 @@ static int qm_register_eq_irq(struct hisi_qm *qm) u32 irq_vector, val; int ret; - val = hisi_qm_get_hw_info(qm, qm_basic_info, QM_EQ_IRQ_TYPE_CAP, qm->cap_ver); + val = qm->cap_tables.qm_cap_table[QM_EQ_IRQ_TYPE_CAP_IDX].cap_val; if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_IRQ_TYPE_MASK)) return 0; @@ -5015,7 +4984,29 @@ static int qm_get_qp_num(struct hisi_qm *qm) return 0; } -static void qm_get_hw_caps(struct hisi_qm *qm) +static int qm_pre_store_irq_type_caps(struct hisi_qm *qm) +{ + struct hisi_qm_cap_record *qm_cap; + struct pci_dev *pdev = qm->pdev; + size_t i, size; + + size = ARRAY_SIZE(qm_pre_store_caps); + qm_cap = devm_kzalloc(&pdev->dev, sizeof(*qm_cap) * size, GFP_KERNEL); + if (!qm_cap) + return -ENOMEM; + + for (i = 0; i < size; i++) { + qm_cap[i].type = qm_pre_store_caps[i]; + qm_cap[i].cap_val = hisi_qm_get_hw_info(qm, qm_basic_info, + qm_pre_store_caps[i], qm->cap_ver); + } + + qm->cap_tables.qm_cap_table = qm_cap; + + return 0; +} + +static int qm_get_hw_caps(struct hisi_qm *qm) { const struct hisi_qm_cap_info *cap_info = qm->fun_type == QM_HW_PF ? qm_cap_info_pf : qm_cap_info_vf; @@ -5046,6 +5037,9 @@ static void qm_get_hw_caps(struct hisi_qm *qm) if (val) set_bit(cap_info[i].type, &qm->caps); } + + /* Fetch and save the value of irq type related capability registers */ + return qm_pre_store_irq_type_caps(qm); } static int qm_get_pci_res(struct hisi_qm *qm) @@ -5067,7 +5061,10 @@ static int qm_get_pci_res(struct hisi_qm *qm) goto err_request_mem_regions; } - qm_get_hw_caps(qm); + ret = qm_get_hw_caps(qm); + if (ret) + goto err_ioremap; + if (test_bit(QM_SUPPORT_DB_ISOLATION, &qm->caps)) { qm->db_interval = QM_QP_DB_INTERVAL; qm->db_phys_base = pci_resource_start(pdev, PCI_BAR_4); @@ -5340,7 +5337,6 @@ int hisi_qm_init(struct hisi_qm *qm) goto err_free_qm_memory; qm_cmd_init(qm); - atomic_set(&qm->status.flags, QM_INIT); return 0; diff --git a/drivers/crypto/hisilicon/qm_common.h b/drivers/crypto/hisilicon/qm_common.h index 7b0b15c83ec1..0760bf55f13e 100644 --- a/drivers/crypto/hisilicon/qm_common.h +++ b/drivers/crypto/hisilicon/qm_common.h @@ -72,10 +72,6 @@ struct qm_aeqc { __le32 dw6; }; -static const char * const qm_s[] = { - "init", "start", "close", "stop", -}; - int qm_set_and_get_xqc(struct hisi_qm *qm, u8 cmd, void *xqc, u32 qp_id, bool op); void hisi_qm_show_last_dfx_regs(struct hisi_qm *qm); void hisi_qm_set_algqos_init(struct hisi_qm *qm); diff --git a/drivers/crypto/hisilicon/sec2/sec.h b/drivers/crypto/hisilicon/sec2/sec.h index 3e57fc04b377..410c83712e28 100644 --- a/drivers/crypto/hisilicon/sec2/sec.h +++ b/drivers/crypto/hisilicon/sec2/sec.h @@ -220,6 +220,13 @@ enum sec_cap_type { SEC_CORE4_ALG_BITMAP_HIGH, }; +enum sec_cap_reg_record_idx { + SEC_DRV_ALG_BITMAP_LOW_IDX = 0x0, + SEC_DRV_ALG_BITMAP_HIGH_IDX, + SEC_DEV_ALG_BITMAP_LOW_IDX, + SEC_DEV_ALG_BITMAP_HIGH_IDX, +}; + void sec_destroy_qps(struct hisi_qp **qps, int qp_num); struct hisi_qp **sec_create_qps(void); int sec_register_to_crypto(struct hisi_qm *qm); diff --git a/drivers/crypto/hisilicon/sec2/sec_crypto.c b/drivers/crypto/hisilicon/sec2/sec_crypto.c index 6fcabbc87860..f028dcfd0ead 100644 --- a/drivers/crypto/hisilicon/sec2/sec_crypto.c +++ b/drivers/crypto/hisilicon/sec2/sec_crypto.c @@ -850,6 +850,7 @@ static int sec_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key, ret = sec_skcipher_aes_sm4_setkey(c_ctx, keylen, c_mode); break; default: + dev_err(dev, "sec c_alg err!\n"); return -EINVAL; } @@ -879,15 +880,11 @@ static int sec_setkey_##name(struct crypto_skcipher *tfm, const u8 *key,\ GEN_SEC_SETKEY_FUNC(aes_ecb, SEC_CALG_AES, SEC_CMODE_ECB) GEN_SEC_SETKEY_FUNC(aes_cbc, SEC_CALG_AES, SEC_CMODE_CBC) GEN_SEC_SETKEY_FUNC(aes_xts, SEC_CALG_AES, SEC_CMODE_XTS) -GEN_SEC_SETKEY_FUNC(aes_ofb, SEC_CALG_AES, SEC_CMODE_OFB) -GEN_SEC_SETKEY_FUNC(aes_cfb, SEC_CALG_AES, SEC_CMODE_CFB) GEN_SEC_SETKEY_FUNC(aes_ctr, SEC_CALG_AES, SEC_CMODE_CTR) GEN_SEC_SETKEY_FUNC(3des_ecb, SEC_CALG_3DES, SEC_CMODE_ECB) GEN_SEC_SETKEY_FUNC(3des_cbc, SEC_CALG_3DES, SEC_CMODE_CBC) GEN_SEC_SETKEY_FUNC(sm4_xts, SEC_CALG_SM4, SEC_CMODE_XTS) GEN_SEC_SETKEY_FUNC(sm4_cbc, SEC_CALG_SM4, SEC_CMODE_CBC) -GEN_SEC_SETKEY_FUNC(sm4_ofb, SEC_CALG_SM4, SEC_CMODE_OFB) -GEN_SEC_SETKEY_FUNC(sm4_cfb, SEC_CALG_SM4, SEC_CMODE_CFB) GEN_SEC_SETKEY_FUNC(sm4_ctr, SEC_CALG_SM4, SEC_CMODE_CTR) static int sec_cipher_pbuf_map(struct sec_ctx *ctx, struct sec_req *req, @@ -1176,7 +1173,8 @@ static int sec_aead_setkey(struct crypto_aead *tfm, const u8 *key, return 0; } - if (crypto_authenc_extractkeys(&keys, key, keylen)) + ret = crypto_authenc_extractkeys(&keys, key, keylen); + if (ret) goto bad_key; ret = sec_aead_aes_set_key(c_ctx, &keys); @@ -1193,6 +1191,7 @@ static int sec_aead_setkey(struct crypto_aead *tfm, const u8 *key, if ((ctx->a_ctx.mac_len & SEC_SQE_LEN_RATE_MASK) || (ctx->a_ctx.a_key_len & SEC_SQE_LEN_RATE_MASK)) { + ret = -EINVAL; dev_err(dev, "MAC or AUTH key length error!\n"); goto bad_key; } @@ -1201,7 +1200,7 @@ static int sec_aead_setkey(struct crypto_aead *tfm, const u8 *key, bad_key: memzero_explicit(&keys, sizeof(struct crypto_authenc_keys)); - return -EINVAL; + return ret; } @@ -2032,8 +2031,6 @@ static int sec_skcipher_cryptlen_check(struct sec_ctx *ctx, ret = -EINVAL; } break; - case SEC_CMODE_CFB: - case SEC_CMODE_OFB: case SEC_CMODE_CTR: if (unlikely(ctx->sec->qm.ver < QM_HW_V3)) { dev_err(dev, "skcipher HW version error!\n"); @@ -2198,16 +2195,6 @@ static struct sec_skcipher sec_skciphers[] = { SEC_XTS_MAX_KEY_SIZE, AES_BLOCK_SIZE, AES_BLOCK_SIZE), }, { - .alg_msk = BIT(4), - .alg = SEC_SKCIPHER_ALG("ofb(aes)", sec_setkey_aes_ofb, AES_MIN_KEY_SIZE, - AES_MAX_KEY_SIZE, SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE), - }, - { - .alg_msk = BIT(5), - .alg = SEC_SKCIPHER_ALG("cfb(aes)", sec_setkey_aes_cfb, AES_MIN_KEY_SIZE, - AES_MAX_KEY_SIZE, SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE), - }, - { .alg_msk = BIT(12), .alg = SEC_SKCIPHER_ALG("cbc(sm4)", sec_setkey_sm4_cbc, AES_MIN_KEY_SIZE, AES_MIN_KEY_SIZE, AES_BLOCK_SIZE, AES_BLOCK_SIZE), @@ -2223,16 +2210,6 @@ static struct sec_skcipher sec_skciphers[] = { SEC_XTS_MIN_KEY_SIZE, AES_BLOCK_SIZE, AES_BLOCK_SIZE), }, { - .alg_msk = BIT(15), - .alg = SEC_SKCIPHER_ALG("ofb(sm4)", sec_setkey_sm4_ofb, AES_MIN_KEY_SIZE, - AES_MIN_KEY_SIZE, SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE), - }, - { - .alg_msk = BIT(16), - .alg = SEC_SKCIPHER_ALG("cfb(sm4)", sec_setkey_sm4_cfb, AES_MIN_KEY_SIZE, - AES_MIN_KEY_SIZE, SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE), - }, - { .alg_msk = BIT(23), .alg = SEC_SKCIPHER_ALG("ecb(des3_ede)", sec_setkey_3des_ecb, SEC_DES3_3KEY_SIZE, SEC_DES3_3KEY_SIZE, DES3_EDE_BLOCK_SIZE, 0), @@ -2547,9 +2524,12 @@ err: int sec_register_to_crypto(struct hisi_qm *qm) { - u64 alg_mask = sec_get_alg_bitmap(qm, SEC_DRV_ALG_BITMAP_HIGH, SEC_DRV_ALG_BITMAP_LOW); + u64 alg_mask; int ret = 0; + alg_mask = sec_get_alg_bitmap(qm, SEC_DRV_ALG_BITMAP_HIGH_IDX, + SEC_DRV_ALG_BITMAP_LOW_IDX); + mutex_lock(&sec_algs_lock); if (sec_available_devs) { sec_available_devs++; @@ -2578,7 +2558,10 @@ unlock: void sec_unregister_from_crypto(struct hisi_qm *qm) { - u64 alg_mask = sec_get_alg_bitmap(qm, SEC_DRV_ALG_BITMAP_HIGH, SEC_DRV_ALG_BITMAP_LOW); + u64 alg_mask; + + alg_mask = sec_get_alg_bitmap(qm, SEC_DRV_ALG_BITMAP_HIGH_IDX, + SEC_DRV_ALG_BITMAP_LOW_IDX); mutex_lock(&sec_algs_lock); if (--sec_available_devs) diff --git a/drivers/crypto/hisilicon/sec2/sec_crypto.h b/drivers/crypto/hisilicon/sec2/sec_crypto.h index d033f63b583f..27a0ee5ad913 100644 --- a/drivers/crypto/hisilicon/sec2/sec_crypto.h +++ b/drivers/crypto/hisilicon/sec2/sec_crypto.h @@ -37,8 +37,6 @@ enum sec_mac_len { enum sec_cmode { SEC_CMODE_ECB = 0x0, SEC_CMODE_CBC = 0x1, - SEC_CMODE_CFB = 0x2, - SEC_CMODE_OFB = 0x3, SEC_CMODE_CTR = 0x4, SEC_CMODE_CCM = 0x5, SEC_CMODE_GCM = 0x6, diff --git a/drivers/crypto/hisilicon/sec2/sec_main.c b/drivers/crypto/hisilicon/sec2/sec_main.c index 0e56a47eb862..7bb99381bbdf 100644 --- a/drivers/crypto/hisilicon/sec2/sec_main.c +++ b/drivers/crypto/hisilicon/sec2/sec_main.c @@ -120,7 +120,6 @@ GENMASK_ULL(42, 25)) #define SEC_AEAD_BITMAP (GENMASK_ULL(7, 6) | GENMASK_ULL(18, 17) | \ GENMASK_ULL(45, 43)) -#define SEC_DEV_ALG_MAX_LEN 256 struct sec_hw_error { u32 int_msk; @@ -132,11 +131,6 @@ struct sec_dfx_item { u32 offset; }; -struct sec_dev_alg { - u64 alg_msk; - const char *algs; -}; - static const char sec_name[] = "hisi_sec2"; static struct dentry *sec_debugfs_root; @@ -159,7 +153,7 @@ static const struct hisi_qm_cap_info sec_basic_info[] = { {SEC_CORE_NUM_CAP, 0x313c, 8, GENMASK(7, 0), 0x4, 0x4, 0x4}, {SEC_CORES_PER_CLUSTER_NUM_CAP, 0x313c, 0, GENMASK(7, 0), 0x4, 0x4, 0x4}, {SEC_CORE_ENABLE_BITMAP, 0x3140, 32, GENMASK(31, 0), 0x17F, 0x17F, 0xF}, - {SEC_DRV_ALG_BITMAP_LOW, 0x3144, 0, GENMASK(31, 0), 0x18050CB, 0x18050CB, 0x187F0FF}, + {SEC_DRV_ALG_BITMAP_LOW, 0x3144, 0, GENMASK(31, 0), 0x18050CB, 0x18050CB, 0x18670CF}, {SEC_DRV_ALG_BITMAP_HIGH, 0x3148, 0, GENMASK(31, 0), 0x395C, 0x395C, 0x395C}, {SEC_DEV_ALG_BITMAP_LOW, 0x314c, 0, GENMASK(31, 0), 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, {SEC_DEV_ALG_BITMAP_HIGH, 0x3150, 0, GENMASK(31, 0), 0x3FFF, 0x3FFF, 0x3FFF}, @@ -173,15 +167,22 @@ static const struct hisi_qm_cap_info sec_basic_info[] = { {SEC_CORE4_ALG_BITMAP_HIGH, 0x3170, 0, GENMASK(31, 0), 0x3FFF, 0x3FFF, 0x3FFF}, }; -static const struct sec_dev_alg sec_dev_algs[] = { { +static const u32 sec_pre_store_caps[] = { + SEC_DRV_ALG_BITMAP_LOW, + SEC_DRV_ALG_BITMAP_HIGH, + SEC_DEV_ALG_BITMAP_LOW, + SEC_DEV_ALG_BITMAP_HIGH, +}; + +static const struct qm_dev_alg sec_dev_algs[] = { { .alg_msk = SEC_CIPHER_BITMAP, - .algs = "cipher\n", + .alg = "cipher\n", }, { .alg_msk = SEC_DIGEST_BITMAP, - .algs = "digest\n", + .alg = "digest\n", }, { .alg_msk = SEC_AEAD_BITMAP, - .algs = "aead\n", + .alg = "aead\n", }, }; @@ -394,8 +395,8 @@ u64 sec_get_alg_bitmap(struct hisi_qm *qm, u32 high, u32 low) { u32 cap_val_h, cap_val_l; - cap_val_h = hisi_qm_get_hw_info(qm, sec_basic_info, high, qm->cap_ver); - cap_val_l = hisi_qm_get_hw_info(qm, sec_basic_info, low, qm->cap_ver); + cap_val_h = qm->cap_tables.dev_cap_table[high].cap_val; + cap_val_l = qm->cap_tables.dev_cap_table[low].cap_val; return ((u64)cap_val_h << SEC_ALG_BITMAP_SHIFT) | (u64)cap_val_l; } @@ -1077,37 +1078,31 @@ static int sec_pf_probe_init(struct sec_dev *sec) return ret; } -static int sec_set_qm_algs(struct hisi_qm *qm) +static int sec_pre_store_cap_reg(struct hisi_qm *qm) { - struct device *dev = &qm->pdev->dev; - char *algs, *ptr; - u64 alg_mask; - int i; - - if (!qm->use_sva) - return 0; + struct hisi_qm_cap_record *sec_cap; + struct pci_dev *pdev = qm->pdev; + size_t i, size; - algs = devm_kzalloc(dev, SEC_DEV_ALG_MAX_LEN * sizeof(char), GFP_KERNEL); - if (!algs) + size = ARRAY_SIZE(sec_pre_store_caps); + sec_cap = devm_kzalloc(&pdev->dev, sizeof(*sec_cap) * size, GFP_KERNEL); + if (!sec_cap) return -ENOMEM; - alg_mask = sec_get_alg_bitmap(qm, SEC_DEV_ALG_BITMAP_HIGH, SEC_DEV_ALG_BITMAP_LOW); - - for (i = 0; i < ARRAY_SIZE(sec_dev_algs); i++) - if (alg_mask & sec_dev_algs[i].alg_msk) - strcat(algs, sec_dev_algs[i].algs); - - ptr = strrchr(algs, '\n'); - if (ptr) - *ptr = '\0'; + for (i = 0; i < size; i++) { + sec_cap[i].type = sec_pre_store_caps[i]; + sec_cap[i].cap_val = hisi_qm_get_hw_info(qm, sec_basic_info, + sec_pre_store_caps[i], qm->cap_ver); + } - qm->uacce->algs = algs; + qm->cap_tables.dev_cap_table = sec_cap; return 0; } static int sec_qm_init(struct hisi_qm *qm, struct pci_dev *pdev) { + u64 alg_msk; int ret; qm->pdev = pdev; @@ -1142,7 +1137,16 @@ static int sec_qm_init(struct hisi_qm *qm, struct pci_dev *pdev) return ret; } - ret = sec_set_qm_algs(qm); + /* Fetch and save the value of capability registers */ + ret = sec_pre_store_cap_reg(qm); + if (ret) { + pci_err(qm->pdev, "Failed to pre-store capability registers!\n"); + hisi_qm_uninit(qm); + return ret; + } + + alg_msk = sec_get_alg_bitmap(qm, SEC_DEV_ALG_BITMAP_HIGH_IDX, SEC_DEV_ALG_BITMAP_LOW_IDX); + ret = hisi_qm_set_algs(qm, alg_msk, sec_dev_algs, ARRAY_SIZE(sec_dev_algs)); if (ret) { pci_err(qm->pdev, "Failed to set sec algs!\n"); hisi_qm_uninit(qm); diff --git a/drivers/crypto/hisilicon/sgl.c b/drivers/crypto/hisilicon/sgl.c index 5c1012d7ffa9..0beca257c20b 100644 --- a/drivers/crypto/hisilicon/sgl.c +++ b/drivers/crypto/hisilicon/sgl.c @@ -121,10 +121,10 @@ struct hisi_acc_sgl_pool *hisi_acc_create_sgl_pool(struct device *dev, return pool; err_free_mem: - for (j = 0; j < i; j++) { + for (j = 0; j < i; j++) dma_free_coherent(dev, block_size, block[j].sgl, block[j].sgl_dma); - } + kfree_sensitive(pool); return ERR_PTR(-ENOMEM); } @@ -140,7 +140,7 @@ EXPORT_SYMBOL_GPL(hisi_acc_create_sgl_pool); void hisi_acc_free_sgl_pool(struct device *dev, struct hisi_acc_sgl_pool *pool) { struct mem_block *block; - int i; + u32 i; if (!dev || !pool) return; @@ -196,9 +196,10 @@ static void update_hw_sgl_sum_sge(struct hisi_acc_hw_sgl *hw_sgl, u16 sum) static void clear_hw_sgl_sge(struct hisi_acc_hw_sgl *hw_sgl) { struct acc_hw_sge *hw_sge = hw_sgl->sge_entries; + u16 entry_sum = le16_to_cpu(hw_sgl->entry_sum_in_sgl); int i; - for (i = 0; i < le16_to_cpu(hw_sgl->entry_sum_in_sgl); i++) { + for (i = 0; i < entry_sum; i++) { hw_sge[i].page_ctrl = NULL; hw_sge[i].buf = 0; hw_sge[i].len = 0; @@ -223,10 +224,11 @@ hisi_acc_sg_buf_map_to_hw_sgl(struct device *dev, u32 index, dma_addr_t *hw_sgl_dma) { struct hisi_acc_hw_sgl *curr_hw_sgl; + unsigned int i, sg_n_mapped; dma_addr_t curr_sgl_dma = 0; struct acc_hw_sge *curr_hw_sge; struct scatterlist *sg; - int i, sg_n, sg_n_mapped; + int sg_n; if (!dev || !sgl || !pool || !hw_sgl_dma) return ERR_PTR(-EINVAL); diff --git a/drivers/crypto/hisilicon/zip/zip_main.c b/drivers/crypto/hisilicon/zip/zip_main.c index db4c964cd649..479ba8a1d6b5 100644 --- a/drivers/crypto/hisilicon/zip/zip_main.c +++ b/drivers/crypto/hisilicon/zip/zip_main.c @@ -74,7 +74,6 @@ #define HZIP_AXI_SHUTDOWN_ENABLE BIT(14) #define HZIP_WR_PORT BIT(11) -#define HZIP_DEV_ALG_MAX_LEN 256 #define HZIP_ALG_ZLIB_BIT GENMASK(1, 0) #define HZIP_ALG_GZIP_BIT GENMASK(3, 2) #define HZIP_ALG_DEFLATE_BIT GENMASK(5, 4) @@ -107,6 +106,14 @@ #define HZIP_CLOCK_GATED_EN (HZIP_CORE_GATED_EN | \ HZIP_CORE_GATED_OOO_EN) +/* zip comp high performance */ +#define HZIP_HIGH_PERF_OFFSET 0x301208 + +enum { + HZIP_HIGH_COMP_RATE, + HZIP_HIGH_COMP_PERF, +}; + static const char hisi_zip_name[] = "hisi_zip"; static struct dentry *hzip_debugfs_root; @@ -120,23 +127,18 @@ struct zip_dfx_item { u32 offset; }; -struct zip_dev_alg { - u32 alg_msk; - const char *algs; -}; - -static const struct zip_dev_alg zip_dev_algs[] = { { +static const struct qm_dev_alg zip_dev_algs[] = { { .alg_msk = HZIP_ALG_ZLIB_BIT, - .algs = "zlib\n", + .alg = "zlib\n", }, { .alg_msk = HZIP_ALG_GZIP_BIT, - .algs = "gzip\n", + .alg = "gzip\n", }, { .alg_msk = HZIP_ALG_DEFLATE_BIT, - .algs = "deflate\n", + .alg = "deflate\n", }, { .alg_msk = HZIP_ALG_LZ77_BIT, - .algs = "lz77_zstd\n", + .alg = "lz77_zstd\n", }, }; @@ -247,6 +249,26 @@ static struct hisi_qm_cap_info zip_basic_cap_info[] = { {ZIP_CAP_MAX, 0x317c, 0, GENMASK(0, 0), 0x0, 0x0, 0x0} }; +enum zip_pre_store_cap_idx { + ZIP_CORE_NUM_CAP_IDX = 0x0, + ZIP_CLUSTER_COMP_NUM_CAP_IDX, + ZIP_CLUSTER_DECOMP_NUM_CAP_IDX, + ZIP_DECOMP_ENABLE_BITMAP_IDX, + ZIP_COMP_ENABLE_BITMAP_IDX, + ZIP_DRV_ALG_BITMAP_IDX, + ZIP_DEV_ALG_BITMAP_IDX, +}; + +static const u32 zip_pre_store_caps[] = { + ZIP_CORE_NUM_CAP, + ZIP_CLUSTER_COMP_NUM_CAP, + ZIP_CLUSTER_DECOMP_NUM_CAP, + ZIP_DECOMP_ENABLE_BITMAP, + ZIP_COMP_ENABLE_BITMAP, + ZIP_DRV_ALG_BITMAP, + ZIP_DEV_ALG_BITMAP, +}; + enum { HZIP_COMP_CORE0, HZIP_COMP_CORE1, @@ -270,28 +292,28 @@ static const u64 core_offsets[] = { }; static const struct debugfs_reg32 hzip_dfx_regs[] = { - {"HZIP_GET_BD_NUM ", 0x00ull}, - {"HZIP_GET_RIGHT_BD ", 0x04ull}, - {"HZIP_GET_ERROR_BD ", 0x08ull}, - {"HZIP_DONE_BD_NUM ", 0x0cull}, - {"HZIP_WORK_CYCLE ", 0x10ull}, - {"HZIP_IDLE_CYCLE ", 0x18ull}, - {"HZIP_MAX_DELAY ", 0x20ull}, - {"HZIP_MIN_DELAY ", 0x24ull}, - {"HZIP_AVG_DELAY ", 0x28ull}, - {"HZIP_MEM_VISIBLE_DATA ", 0x30ull}, - {"HZIP_MEM_VISIBLE_ADDR ", 0x34ull}, - {"HZIP_CONSUMED_BYTE ", 0x38ull}, - {"HZIP_PRODUCED_BYTE ", 0x40ull}, - {"HZIP_COMP_INF ", 0x70ull}, - {"HZIP_PRE_OUT ", 0x78ull}, - {"HZIP_BD_RD ", 0x7cull}, - {"HZIP_BD_WR ", 0x80ull}, - {"HZIP_GET_BD_AXI_ERR_NUM ", 0x84ull}, - {"HZIP_GET_BD_PARSE_ERR_NUM ", 0x88ull}, - {"HZIP_ADD_BD_AXI_ERR_NUM ", 0x8cull}, - {"HZIP_DECOMP_STF_RELOAD_CURR_ST ", 0x94ull}, - {"HZIP_DECOMP_LZ77_CURR_ST ", 0x9cull}, + {"HZIP_GET_BD_NUM ", 0x00}, + {"HZIP_GET_RIGHT_BD ", 0x04}, + {"HZIP_GET_ERROR_BD ", 0x08}, + {"HZIP_DONE_BD_NUM ", 0x0c}, + {"HZIP_WORK_CYCLE ", 0x10}, + {"HZIP_IDLE_CYCLE ", 0x18}, + {"HZIP_MAX_DELAY ", 0x20}, + {"HZIP_MIN_DELAY ", 0x24}, + {"HZIP_AVG_DELAY ", 0x28}, + {"HZIP_MEM_VISIBLE_DATA ", 0x30}, + {"HZIP_MEM_VISIBLE_ADDR ", 0x34}, + {"HZIP_CONSUMED_BYTE ", 0x38}, + {"HZIP_PRODUCED_BYTE ", 0x40}, + {"HZIP_COMP_INF ", 0x70}, + {"HZIP_PRE_OUT ", 0x78}, + {"HZIP_BD_RD ", 0x7c}, + {"HZIP_BD_WR ", 0x80}, + {"HZIP_GET_BD_AXI_ERR_NUM ", 0x84}, + {"HZIP_GET_BD_PARSE_ERR_NUM ", 0x88}, + {"HZIP_ADD_BD_AXI_ERR_NUM ", 0x8c}, + {"HZIP_DECOMP_STF_RELOAD_CURR_ST ", 0x94}, + {"HZIP_DECOMP_LZ77_CURR_ST ", 0x9c}, }; static const struct debugfs_reg32 hzip_com_dfx_regs[] = { @@ -303,11 +325,11 @@ static const struct debugfs_reg32 hzip_com_dfx_regs[] = { }; static const struct debugfs_reg32 hzip_dump_dfx_regs[] = { - {"HZIP_GET_BD_NUM ", 0x00ull}, - {"HZIP_GET_RIGHT_BD ", 0x04ull}, - {"HZIP_GET_ERROR_BD ", 0x08ull}, - {"HZIP_DONE_BD_NUM ", 0x0cull}, - {"HZIP_MAX_DELAY ", 0x20ull}, + {"HZIP_GET_BD_NUM ", 0x00}, + {"HZIP_GET_RIGHT_BD ", 0x04}, + {"HZIP_GET_ERROR_BD ", 0x08}, + {"HZIP_DONE_BD_NUM ", 0x0c}, + {"HZIP_MAX_DELAY ", 0x20}, }; /* define the ZIP's dfx regs region and region length */ @@ -352,6 +374,37 @@ static int hzip_diff_regs_show(struct seq_file *s, void *unused) return 0; } DEFINE_SHOW_ATTRIBUTE(hzip_diff_regs); + +static int perf_mode_set(const char *val, const struct kernel_param *kp) +{ + int ret; + u32 n; + + if (!val) + return -EINVAL; + + ret = kstrtou32(val, 10, &n); + if (ret != 0 || (n != HZIP_HIGH_COMP_PERF && + n != HZIP_HIGH_COMP_RATE)) + return -EINVAL; + + return param_set_int(val, kp); +} + +static const struct kernel_param_ops zip_com_perf_ops = { + .set = perf_mode_set, + .get = param_get_int, +}; + +/* + * perf_mode = 0 means enable high compression rate mode, + * perf_mode = 1 means enable high compression performance mode. + * These two modes only apply to the compression direction. + */ +static u32 perf_mode = HZIP_HIGH_COMP_RATE; +module_param_cb(perf_mode, &zip_com_perf_ops, &perf_mode, 0444); +MODULE_PARM_DESC(perf_mode, "ZIP high perf mode 0(default), 1(enable)"); + static const struct kernel_param_ops zip_uacce_mode_ops = { .set = uacce_mode_set, .get = param_get_int, @@ -410,40 +463,33 @@ bool hisi_zip_alg_support(struct hisi_qm *qm, u32 alg) { u32 cap_val; - cap_val = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_DRV_ALG_BITMAP, qm->cap_ver); + cap_val = qm->cap_tables.dev_cap_table[ZIP_DRV_ALG_BITMAP_IDX].cap_val; if ((alg & cap_val) == alg) return true; return false; } -static int hisi_zip_set_qm_algs(struct hisi_qm *qm) +static int hisi_zip_set_high_perf(struct hisi_qm *qm) { - struct device *dev = &qm->pdev->dev; - char *algs, *ptr; - u32 alg_mask; - int i; - - if (!qm->use_sva) - return 0; - - algs = devm_kzalloc(dev, HZIP_DEV_ALG_MAX_LEN * sizeof(char), GFP_KERNEL); - if (!algs) - return -ENOMEM; - - alg_mask = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_DEV_ALG_BITMAP, qm->cap_ver); - - for (i = 0; i < ARRAY_SIZE(zip_dev_algs); i++) - if (alg_mask & zip_dev_algs[i].alg_msk) - strcat(algs, zip_dev_algs[i].algs); - - ptr = strrchr(algs, '\n'); - if (ptr) - *ptr = '\0'; + u32 val; + int ret; - qm->uacce->algs = algs; + val = readl_relaxed(qm->io_base + HZIP_HIGH_PERF_OFFSET); + if (perf_mode == HZIP_HIGH_COMP_PERF) + val |= HZIP_HIGH_COMP_PERF; + else + val &= ~HZIP_HIGH_COMP_PERF; + + /* Set perf mode */ + writel(val, qm->io_base + HZIP_HIGH_PERF_OFFSET); + ret = readl_relaxed_poll_timeout(qm->io_base + HZIP_HIGH_PERF_OFFSET, + val, val == perf_mode, HZIP_DELAY_1_US, + HZIP_POLL_TIMEOUT_US); + if (ret) + pci_err(qm->pdev, "failed to set perf mode\n"); - return 0; + return ret; } static void hisi_zip_open_sva_prefetch(struct hisi_qm *qm) @@ -542,10 +588,8 @@ static int hisi_zip_set_user_domain_and_cache(struct hisi_qm *qm) } /* let's open all compression/decompression cores */ - dcomp_bm = hisi_qm_get_hw_info(qm, zip_basic_cap_info, - ZIP_DECOMP_ENABLE_BITMAP, qm->cap_ver); - comp_bm = hisi_qm_get_hw_info(qm, zip_basic_cap_info, - ZIP_COMP_ENABLE_BITMAP, qm->cap_ver); + dcomp_bm = qm->cap_tables.dev_cap_table[ZIP_DECOMP_ENABLE_BITMAP_IDX].cap_val; + comp_bm = qm->cap_tables.dev_cap_table[ZIP_COMP_ENABLE_BITMAP_IDX].cap_val; writel(HZIP_DECOMP_CHECK_ENABLE | dcomp_bm | comp_bm, base + HZIP_CLOCK_GATE_CTRL); /* enable sqc,cqc writeback */ @@ -772,9 +816,8 @@ static int hisi_zip_core_debug_init(struct hisi_qm *qm) char buf[HZIP_BUF_SIZE]; int i; - zip_core_num = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_CORE_NUM_CAP, qm->cap_ver); - zip_comp_core_num = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_CLUSTER_COMP_NUM_CAP, - qm->cap_ver); + zip_core_num = qm->cap_tables.dev_cap_table[ZIP_CORE_NUM_CAP_IDX].cap_val; + zip_comp_core_num = qm->cap_tables.dev_cap_table[ZIP_CLUSTER_COMP_NUM_CAP_IDX].cap_val; for (i = 0; i < zip_core_num; i++) { if (i < zip_comp_core_num) @@ -916,7 +959,7 @@ static int hisi_zip_show_last_regs_init(struct hisi_qm *qm) u32 zip_core_num; int i, j, idx; - zip_core_num = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_CORE_NUM_CAP, qm->cap_ver); + zip_core_num = qm->cap_tables.dev_cap_table[ZIP_CORE_NUM_CAP_IDX].cap_val; debug->last_words = kcalloc(core_dfx_regs_num * zip_core_num + com_dfx_regs_num, sizeof(unsigned int), GFP_KERNEL); @@ -972,9 +1015,9 @@ static void hisi_zip_show_last_dfx_regs(struct hisi_qm *qm) hzip_com_dfx_regs[i].name, debug->last_words[i], val); } - zip_core_num = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_CORE_NUM_CAP, qm->cap_ver); - zip_comp_core_num = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_CLUSTER_COMP_NUM_CAP, - qm->cap_ver); + zip_core_num = qm->cap_tables.dev_cap_table[ZIP_CORE_NUM_CAP_IDX].cap_val; + zip_comp_core_num = qm->cap_tables.dev_cap_table[ZIP_CLUSTER_COMP_NUM_CAP_IDX].cap_val; + for (i = 0; i < zip_core_num; i++) { if (i < zip_comp_core_num) scnprintf(buf, sizeof(buf), "Comp_core-%d", i); @@ -1115,6 +1158,10 @@ static int hisi_zip_pf_probe_init(struct hisi_zip *hisi_zip) if (ret) return ret; + ret = hisi_zip_set_high_perf(qm); + if (ret) + return ret; + hisi_zip_open_sva_prefetch(qm); hisi_qm_dev_err_init(qm); hisi_zip_debug_regs_clear(qm); @@ -1126,8 +1173,31 @@ static int hisi_zip_pf_probe_init(struct hisi_zip *hisi_zip) return ret; } +static int zip_pre_store_cap_reg(struct hisi_qm *qm) +{ + struct hisi_qm_cap_record *zip_cap; + struct pci_dev *pdev = qm->pdev; + size_t i, size; + + size = ARRAY_SIZE(zip_pre_store_caps); + zip_cap = devm_kzalloc(&pdev->dev, sizeof(*zip_cap) * size, GFP_KERNEL); + if (!zip_cap) + return -ENOMEM; + + for (i = 0; i < size; i++) { + zip_cap[i].type = zip_pre_store_caps[i]; + zip_cap[i].cap_val = hisi_qm_get_hw_info(qm, zip_basic_cap_info, + zip_pre_store_caps[i], qm->cap_ver); + } + + qm->cap_tables.dev_cap_table = zip_cap; + + return 0; +} + static int hisi_zip_qm_init(struct hisi_qm *qm, struct pci_dev *pdev) { + u64 alg_msk; int ret; qm->pdev = pdev; @@ -1163,7 +1233,16 @@ static int hisi_zip_qm_init(struct hisi_qm *qm, struct pci_dev *pdev) return ret; } - ret = hisi_zip_set_qm_algs(qm); + /* Fetch and save the value of capability registers */ + ret = zip_pre_store_cap_reg(qm); + if (ret) { + pci_err(qm->pdev, "Failed to pre-store capability registers!\n"); + hisi_qm_uninit(qm); + return ret; + } + + alg_msk = qm->cap_tables.dev_cap_table[ZIP_DEV_ALG_BITMAP_IDX].cap_val; + ret = hisi_qm_set_algs(qm, alg_msk, zip_dev_algs, ARRAY_SIZE(zip_dev_algs)); if (ret) { pci_err(qm->pdev, "Failed to set zip algs!\n"); hisi_qm_uninit(qm); diff --git a/drivers/crypto/inside-secure/safexcel.c b/drivers/crypto/inside-secure/safexcel.c index 76da14af74b5..f5c1912aa564 100644 --- a/drivers/crypto/inside-secure/safexcel.c +++ b/drivers/crypto/inside-secure/safexcel.c @@ -1191,8 +1191,6 @@ static struct safexcel_alg_template *safexcel_algs[] = { &safexcel_alg_cbc_des3_ede, &safexcel_alg_ecb_aes, &safexcel_alg_cbc_aes, - &safexcel_alg_cfb_aes, - &safexcel_alg_ofb_aes, &safexcel_alg_ctr_aes, &safexcel_alg_md5, &safexcel_alg_sha1, @@ -1231,8 +1229,6 @@ static struct safexcel_alg_template *safexcel_algs[] = { &safexcel_alg_hmac_sm3, &safexcel_alg_ecb_sm4, &safexcel_alg_cbc_sm4, - &safexcel_alg_ofb_sm4, - &safexcel_alg_cfb_sm4, &safexcel_alg_ctr_sm4, &safexcel_alg_authenc_hmac_sha1_cbc_sm4, &safexcel_alg_authenc_hmac_sm3_cbc_sm4, diff --git a/drivers/crypto/inside-secure/safexcel.h b/drivers/crypto/inside-secure/safexcel.h index 47ef6c7cd02c..d0059ce954dd 100644 --- a/drivers/crypto/inside-secure/safexcel.h +++ b/drivers/crypto/inside-secure/safexcel.h @@ -933,8 +933,6 @@ extern struct safexcel_alg_template safexcel_alg_ecb_des3_ede; extern struct safexcel_alg_template safexcel_alg_cbc_des3_ede; extern struct safexcel_alg_template safexcel_alg_ecb_aes; extern struct safexcel_alg_template safexcel_alg_cbc_aes; -extern struct safexcel_alg_template safexcel_alg_cfb_aes; -extern struct safexcel_alg_template safexcel_alg_ofb_aes; extern struct safexcel_alg_template safexcel_alg_ctr_aes; extern struct safexcel_alg_template safexcel_alg_md5; extern struct safexcel_alg_template safexcel_alg_sha1; @@ -973,8 +971,6 @@ extern struct safexcel_alg_template safexcel_alg_sm3; extern struct safexcel_alg_template safexcel_alg_hmac_sm3; extern struct safexcel_alg_template safexcel_alg_ecb_sm4; extern struct safexcel_alg_template safexcel_alg_cbc_sm4; -extern struct safexcel_alg_template safexcel_alg_ofb_sm4; -extern struct safexcel_alg_template safexcel_alg_cfb_sm4; extern struct safexcel_alg_template safexcel_alg_ctr_sm4; extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_cbc_sm4; extern struct safexcel_alg_template safexcel_alg_authenc_hmac_sm3_cbc_sm4; diff --git a/drivers/crypto/inside-secure/safexcel_cipher.c b/drivers/crypto/inside-secure/safexcel_cipher.c index 272c28b5a088..42677f7458b7 100644 --- a/drivers/crypto/inside-secure/safexcel_cipher.c +++ b/drivers/crypto/inside-secure/safexcel_cipher.c @@ -742,9 +742,9 @@ static int safexcel_send_req(struct crypto_async_request *base, int ring, max(totlen_src, totlen_dst)); return -EINVAL; } - if (sreq->nr_src > 0) - dma_map_sg(priv->dev, src, sreq->nr_src, - DMA_BIDIRECTIONAL); + if (sreq->nr_src > 0 && + !dma_map_sg(priv->dev, src, sreq->nr_src, DMA_BIDIRECTIONAL)) + return -EIO; } else { if (unlikely(totlen_src && (sreq->nr_src <= 0))) { dev_err(priv->dev, "Source buffer not large enough (need %d bytes)!", @@ -752,8 +752,9 @@ static int safexcel_send_req(struct crypto_async_request *base, int ring, return -EINVAL; } - if (sreq->nr_src > 0) - dma_map_sg(priv->dev, src, sreq->nr_src, DMA_TO_DEVICE); + if (sreq->nr_src > 0 && + !dma_map_sg(priv->dev, src, sreq->nr_src, DMA_TO_DEVICE)) + return -EIO; if (unlikely(totlen_dst && (sreq->nr_dst <= 0))) { dev_err(priv->dev, "Dest buffer not large enough (need %d bytes)!", @@ -762,9 +763,11 @@ static int safexcel_send_req(struct crypto_async_request *base, int ring, goto unmap; } - if (sreq->nr_dst > 0) - dma_map_sg(priv->dev, dst, sreq->nr_dst, - DMA_FROM_DEVICE); + if (sreq->nr_dst > 0 && + !dma_map_sg(priv->dev, dst, sreq->nr_dst, DMA_FROM_DEVICE)) { + ret = -EIO; + goto unmap; + } } memcpy(ctx->base.ctxr->data, ctx->key, ctx->key_len); @@ -1349,82 +1352,6 @@ struct safexcel_alg_template safexcel_alg_cbc_aes = { }, }; -static int safexcel_skcipher_aes_cfb_cra_init(struct crypto_tfm *tfm) -{ - struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); - - safexcel_skcipher_cra_init(tfm); - ctx->alg = SAFEXCEL_AES; - ctx->blocksz = AES_BLOCK_SIZE; - ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CFB; - return 0; -} - -struct safexcel_alg_template safexcel_alg_cfb_aes = { - .type = SAFEXCEL_ALG_TYPE_SKCIPHER, - .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_AES_XFB, - .alg.skcipher = { - .setkey = safexcel_skcipher_aes_setkey, - .encrypt = safexcel_encrypt, - .decrypt = safexcel_decrypt, - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .ivsize = AES_BLOCK_SIZE, - .base = { - .cra_name = "cfb(aes)", - .cra_driver_name = "safexcel-cfb-aes", - .cra_priority = SAFEXCEL_CRA_PRIORITY, - .cra_flags = CRYPTO_ALG_ASYNC | - CRYPTO_ALG_ALLOCATES_MEMORY | - CRYPTO_ALG_KERN_DRIVER_ONLY, - .cra_blocksize = 1, - .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), - .cra_alignmask = 0, - .cra_init = safexcel_skcipher_aes_cfb_cra_init, - .cra_exit = safexcel_skcipher_cra_exit, - .cra_module = THIS_MODULE, - }, - }, -}; - -static int safexcel_skcipher_aes_ofb_cra_init(struct crypto_tfm *tfm) -{ - struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); - - safexcel_skcipher_cra_init(tfm); - ctx->alg = SAFEXCEL_AES; - ctx->blocksz = AES_BLOCK_SIZE; - ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_OFB; - return 0; -} - -struct safexcel_alg_template safexcel_alg_ofb_aes = { - .type = SAFEXCEL_ALG_TYPE_SKCIPHER, - .algo_mask = SAFEXCEL_ALG_AES | SAFEXCEL_ALG_AES_XFB, - .alg.skcipher = { - .setkey = safexcel_skcipher_aes_setkey, - .encrypt = safexcel_encrypt, - .decrypt = safexcel_decrypt, - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .ivsize = AES_BLOCK_SIZE, - .base = { - .cra_name = "ofb(aes)", - .cra_driver_name = "safexcel-ofb-aes", - .cra_priority = SAFEXCEL_CRA_PRIORITY, - .cra_flags = CRYPTO_ALG_ASYNC | - CRYPTO_ALG_ALLOCATES_MEMORY | - CRYPTO_ALG_KERN_DRIVER_ONLY, - .cra_blocksize = 1, - .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), - .cra_alignmask = 0, - .cra_init = safexcel_skcipher_aes_ofb_cra_init, - .cra_exit = safexcel_skcipher_cra_exit, - .cra_module = THIS_MODULE, - }, - }, -}; - static int safexcel_skcipher_aesctr_setkey(struct crypto_skcipher *ctfm, const u8 *key, unsigned int len) { @@ -3183,82 +3110,6 @@ struct safexcel_alg_template safexcel_alg_cbc_sm4 = { }, }; -static int safexcel_skcipher_sm4_ofb_cra_init(struct crypto_tfm *tfm) -{ - struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); - - safexcel_skcipher_cra_init(tfm); - ctx->alg = SAFEXCEL_SM4; - ctx->blocksz = SM4_BLOCK_SIZE; - ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_OFB; - return 0; -} - -struct safexcel_alg_template safexcel_alg_ofb_sm4 = { - .type = SAFEXCEL_ALG_TYPE_SKCIPHER, - .algo_mask = SAFEXCEL_ALG_SM4 | SAFEXCEL_ALG_AES_XFB, - .alg.skcipher = { - .setkey = safexcel_skcipher_sm4_setkey, - .encrypt = safexcel_encrypt, - .decrypt = safexcel_decrypt, - .min_keysize = SM4_KEY_SIZE, - .max_keysize = SM4_KEY_SIZE, - .ivsize = SM4_BLOCK_SIZE, - .base = { - .cra_name = "ofb(sm4)", - .cra_driver_name = "safexcel-ofb-sm4", - .cra_priority = SAFEXCEL_CRA_PRIORITY, - .cra_flags = CRYPTO_ALG_ASYNC | - CRYPTO_ALG_ALLOCATES_MEMORY | - CRYPTO_ALG_KERN_DRIVER_ONLY, - .cra_blocksize = 1, - .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), - .cra_alignmask = 0, - .cra_init = safexcel_skcipher_sm4_ofb_cra_init, - .cra_exit = safexcel_skcipher_cra_exit, - .cra_module = THIS_MODULE, - }, - }, -}; - -static int safexcel_skcipher_sm4_cfb_cra_init(struct crypto_tfm *tfm) -{ - struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); - - safexcel_skcipher_cra_init(tfm); - ctx->alg = SAFEXCEL_SM4; - ctx->blocksz = SM4_BLOCK_SIZE; - ctx->mode = CONTEXT_CONTROL_CRYPTO_MODE_CFB; - return 0; -} - -struct safexcel_alg_template safexcel_alg_cfb_sm4 = { - .type = SAFEXCEL_ALG_TYPE_SKCIPHER, - .algo_mask = SAFEXCEL_ALG_SM4 | SAFEXCEL_ALG_AES_XFB, - .alg.skcipher = { - .setkey = safexcel_skcipher_sm4_setkey, - .encrypt = safexcel_encrypt, - .decrypt = safexcel_decrypt, - .min_keysize = SM4_KEY_SIZE, - .max_keysize = SM4_KEY_SIZE, - .ivsize = SM4_BLOCK_SIZE, - .base = { - .cra_name = "cfb(sm4)", - .cra_driver_name = "safexcel-cfb-sm4", - .cra_priority = SAFEXCEL_CRA_PRIORITY, - .cra_flags = CRYPTO_ALG_ASYNC | - CRYPTO_ALG_ALLOCATES_MEMORY | - CRYPTO_ALG_KERN_DRIVER_ONLY, - .cra_blocksize = 1, - .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), - .cra_alignmask = 0, - .cra_init = safexcel_skcipher_sm4_cfb_cra_init, - .cra_exit = safexcel_skcipher_cra_exit, - .cra_module = THIS_MODULE, - }, - }, -}; - static int safexcel_skcipher_sm4ctr_setkey(struct crypto_skcipher *ctfm, const u8 *key, unsigned int len) { diff --git a/drivers/crypto/intel/Kconfig b/drivers/crypto/intel/Kconfig index 3d90c87d4094..f38cd62a3f67 100644 --- a/drivers/crypto/intel/Kconfig +++ b/drivers/crypto/intel/Kconfig @@ -3,3 +3,4 @@ source "drivers/crypto/intel/keembay/Kconfig" source "drivers/crypto/intel/ixp4xx/Kconfig" source "drivers/crypto/intel/qat/Kconfig" +source "drivers/crypto/intel/iaa/Kconfig" diff --git a/drivers/crypto/intel/Makefile b/drivers/crypto/intel/Makefile index b3d0352ae188..2f56f6d34cf0 100644 --- a/drivers/crypto/intel/Makefile +++ b/drivers/crypto/intel/Makefile @@ -3,3 +3,4 @@ obj-y += keembay/ obj-y += ixp4xx/ obj-$(CONFIG_CRYPTO_DEV_QAT) += qat/ +obj-$(CONFIG_CRYPTO_DEV_IAA_CRYPTO) += iaa/ diff --git a/drivers/crypto/intel/iaa/Kconfig b/drivers/crypto/intel/iaa/Kconfig new file mode 100644 index 000000000000..d53f4b1d494f --- /dev/null +++ b/drivers/crypto/intel/iaa/Kconfig @@ -0,0 +1,19 @@ +config CRYPTO_DEV_IAA_CRYPTO + tristate "Support for Intel(R) IAA Compression Accelerator" + depends on CRYPTO_DEFLATE + depends on INTEL_IDXD + default n + help + This driver supports acceleration for compression and + decompression with the Intel Analytics Accelerator (IAA) + hardware using the cryptographic API. If you choose 'M' + here, the module will be called iaa_crypto. + +config CRYPTO_DEV_IAA_CRYPTO_STATS + bool "Enable Intel(R) IAA Compression Accelerator Statistics" + depends on CRYPTO_DEV_IAA_CRYPTO + default n + help + Enable statistics for the IAA compression accelerator. + These include per-device and per-workqueue statistics in + addition to global driver statistics. diff --git a/drivers/crypto/intel/iaa/Makefile b/drivers/crypto/intel/iaa/Makefile new file mode 100644 index 000000000000..b64b208d2344 --- /dev/null +++ b/drivers/crypto/intel/iaa/Makefile @@ -0,0 +1,12 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Makefile for IAA crypto device drivers +# + +ccflags-y += -I $(srctree)/drivers/dma/idxd -DDEFAULT_SYMBOL_NAMESPACE=IDXD + +obj-$(CONFIG_CRYPTO_DEV_IAA_CRYPTO) := iaa_crypto.o + +iaa_crypto-y := iaa_crypto_main.o iaa_crypto_comp_fixed.o + +iaa_crypto-$(CONFIG_CRYPTO_DEV_IAA_CRYPTO_STATS) += iaa_crypto_stats.o diff --git a/drivers/crypto/intel/iaa/iaa_crypto.h b/drivers/crypto/intel/iaa/iaa_crypto.h new file mode 100644 index 000000000000..014420f7beb0 --- /dev/null +++ b/drivers/crypto/intel/iaa/iaa_crypto.h @@ -0,0 +1,173 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2021 Intel Corporation. All rights rsvd. */ + +#ifndef __IAA_CRYPTO_H__ +#define __IAA_CRYPTO_H__ + +#include <linux/crypto.h> +#include <linux/idxd.h> +#include <uapi/linux/idxd.h> + +#define IDXD_SUBDRIVER_NAME "crypto" + +#define IAA_DECOMP_ENABLE BIT(0) +#define IAA_DECOMP_FLUSH_OUTPUT BIT(1) +#define IAA_DECOMP_CHECK_FOR_EOB BIT(2) +#define IAA_DECOMP_STOP_ON_EOB BIT(3) +#define IAA_DECOMP_SUPPRESS_OUTPUT BIT(9) + +#define IAA_COMP_FLUSH_OUTPUT BIT(1) +#define IAA_COMP_APPEND_EOB BIT(2) + +#define IAA_COMPLETION_TIMEOUT 1000000 + +#define IAA_ANALYTICS_ERROR 0x0a +#define IAA_ERROR_DECOMP_BUF_OVERFLOW 0x0b +#define IAA_ERROR_COMP_BUF_OVERFLOW 0x19 +#define IAA_ERROR_WATCHDOG_EXPIRED 0x24 + +#define IAA_COMP_MODES_MAX 2 + +#define FIXED_HDR 0x2 +#define FIXED_HDR_SIZE 3 + +#define IAA_COMP_FLAGS (IAA_COMP_FLUSH_OUTPUT | \ + IAA_COMP_APPEND_EOB) + +#define IAA_DECOMP_FLAGS (IAA_DECOMP_ENABLE | \ + IAA_DECOMP_FLUSH_OUTPUT | \ + IAA_DECOMP_CHECK_FOR_EOB | \ + IAA_DECOMP_STOP_ON_EOB) + +/* Representation of IAA workqueue */ +struct iaa_wq { + struct list_head list; + + struct idxd_wq *wq; + int ref; + bool remove; + + struct iaa_device *iaa_device; + + u64 comp_calls; + u64 comp_bytes; + u64 decomp_calls; + u64 decomp_bytes; +}; + +struct iaa_device_compression_mode { + const char *name; + + struct aecs_comp_table_record *aecs_comp_table; + struct aecs_decomp_table_record *aecs_decomp_table; + + dma_addr_t aecs_comp_table_dma_addr; + dma_addr_t aecs_decomp_table_dma_addr; +}; + +/* Representation of IAA device with wqs, populated by probe */ +struct iaa_device { + struct list_head list; + struct idxd_device *idxd; + + struct iaa_device_compression_mode *compression_modes[IAA_COMP_MODES_MAX]; + + int n_wq; + struct list_head wqs; + + u64 comp_calls; + u64 comp_bytes; + u64 decomp_calls; + u64 decomp_bytes; +}; + +struct wq_table_entry { + struct idxd_wq **wqs; + int max_wqs; + int n_wqs; + int cur_wq; +}; + +#define IAA_AECS_ALIGN 32 + +/* + * Analytics Engine Configuration and State (AECS) contains parameters and + * internal state of the analytics engine. + */ +struct aecs_comp_table_record { + u32 crc; + u32 xor_checksum; + u32 reserved0[5]; + u32 num_output_accum_bits; + u8 output_accum[256]; + u32 ll_sym[286]; + u32 reserved1; + u32 reserved2; + u32 d_sym[30]; + u32 reserved_padding[2]; +} __packed; + +/* AECS for decompress */ +struct aecs_decomp_table_record { + u32 crc; + u32 xor_checksum; + u32 low_filter_param; + u32 high_filter_param; + u32 output_mod_idx; + u32 drop_init_decomp_out_bytes; + u32 reserved[36]; + u32 output_accum_data[2]; + u32 out_bits_valid; + u32 bit_off_indexing; + u32 input_accum_data[64]; + u8 size_qw[32]; + u32 decomp_state[1220]; +} __packed; + +int iaa_aecs_init_fixed(void); +void iaa_aecs_cleanup_fixed(void); + +typedef int (*iaa_dev_comp_init_fn_t) (struct iaa_device_compression_mode *mode); +typedef int (*iaa_dev_comp_free_fn_t) (struct iaa_device_compression_mode *mode); + +struct iaa_compression_mode { + const char *name; + u32 *ll_table; + int ll_table_size; + u32 *d_table; + int d_table_size; + u32 *header_table; + int header_table_size; + u16 gen_decomp_table_flags; + iaa_dev_comp_init_fn_t init; + iaa_dev_comp_free_fn_t free; +}; + +int add_iaa_compression_mode(const char *name, + const u32 *ll_table, + int ll_table_size, + const u32 *d_table, + int d_table_size, + const u8 *header_table, + int header_table_size, + u16 gen_decomp_table_flags, + iaa_dev_comp_init_fn_t init, + iaa_dev_comp_free_fn_t free); + +void remove_iaa_compression_mode(const char *name); + +enum iaa_mode { + IAA_MODE_FIXED, +}; + +struct iaa_compression_ctx { + enum iaa_mode mode; + bool verify_compress; + bool async_mode; + bool use_irq; +}; + +extern struct list_head iaa_devices; +extern struct mutex iaa_devices_lock; + +#endif diff --git a/drivers/crypto/intel/iaa/iaa_crypto_comp_fixed.c b/drivers/crypto/intel/iaa/iaa_crypto_comp_fixed.c new file mode 100644 index 000000000000..45cf5d74f0fb --- /dev/null +++ b/drivers/crypto/intel/iaa/iaa_crypto_comp_fixed.c @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright(c) 2021 Intel Corporation. All rights rsvd. */ + +#include "idxd.h" +#include "iaa_crypto.h" + +/* + * Fixed Huffman tables the IAA hardware requires to implement RFC-1951. + */ +static const u32 fixed_ll_sym[286] = { + 0x40030, 0x40031, 0x40032, 0x40033, 0x40034, 0x40035, 0x40036, 0x40037, + 0x40038, 0x40039, 0x4003A, 0x4003B, 0x4003C, 0x4003D, 0x4003E, 0x4003F, + 0x40040, 0x40041, 0x40042, 0x40043, 0x40044, 0x40045, 0x40046, 0x40047, + 0x40048, 0x40049, 0x4004A, 0x4004B, 0x4004C, 0x4004D, 0x4004E, 0x4004F, + 0x40050, 0x40051, 0x40052, 0x40053, 0x40054, 0x40055, 0x40056, 0x40057, + 0x40058, 0x40059, 0x4005A, 0x4005B, 0x4005C, 0x4005D, 0x4005E, 0x4005F, + 0x40060, 0x40061, 0x40062, 0x40063, 0x40064, 0x40065, 0x40066, 0x40067, + 0x40068, 0x40069, 0x4006A, 0x4006B, 0x4006C, 0x4006D, 0x4006E, 0x4006F, + 0x40070, 0x40071, 0x40072, 0x40073, 0x40074, 0x40075, 0x40076, 0x40077, + 0x40078, 0x40079, 0x4007A, 0x4007B, 0x4007C, 0x4007D, 0x4007E, 0x4007F, + 0x40080, 0x40081, 0x40082, 0x40083, 0x40084, 0x40085, 0x40086, 0x40087, + 0x40088, 0x40089, 0x4008A, 0x4008B, 0x4008C, 0x4008D, 0x4008E, 0x4008F, + 0x40090, 0x40091, 0x40092, 0x40093, 0x40094, 0x40095, 0x40096, 0x40097, + 0x40098, 0x40099, 0x4009A, 0x4009B, 0x4009C, 0x4009D, 0x4009E, 0x4009F, + 0x400A0, 0x400A1, 0x400A2, 0x400A3, 0x400A4, 0x400A5, 0x400A6, 0x400A7, + 0x400A8, 0x400A9, 0x400AA, 0x400AB, 0x400AC, 0x400AD, 0x400AE, 0x400AF, + 0x400B0, 0x400B1, 0x400B2, 0x400B3, 0x400B4, 0x400B5, 0x400B6, 0x400B7, + 0x400B8, 0x400B9, 0x400BA, 0x400BB, 0x400BC, 0x400BD, 0x400BE, 0x400BF, + 0x48190, 0x48191, 0x48192, 0x48193, 0x48194, 0x48195, 0x48196, 0x48197, + 0x48198, 0x48199, 0x4819A, 0x4819B, 0x4819C, 0x4819D, 0x4819E, 0x4819F, + 0x481A0, 0x481A1, 0x481A2, 0x481A3, 0x481A4, 0x481A5, 0x481A6, 0x481A7, + 0x481A8, 0x481A9, 0x481AA, 0x481AB, 0x481AC, 0x481AD, 0x481AE, 0x481AF, + 0x481B0, 0x481B1, 0x481B2, 0x481B3, 0x481B4, 0x481B5, 0x481B6, 0x481B7, + 0x481B8, 0x481B9, 0x481BA, 0x481BB, 0x481BC, 0x481BD, 0x481BE, 0x481BF, + 0x481C0, 0x481C1, 0x481C2, 0x481C3, 0x481C4, 0x481C5, 0x481C6, 0x481C7, + 0x481C8, 0x481C9, 0x481CA, 0x481CB, 0x481CC, 0x481CD, 0x481CE, 0x481CF, + 0x481D0, 0x481D1, 0x481D2, 0x481D3, 0x481D4, 0x481D5, 0x481D6, 0x481D7, + 0x481D8, 0x481D9, 0x481DA, 0x481DB, 0x481DC, 0x481DD, 0x481DE, 0x481DF, + 0x481E0, 0x481E1, 0x481E2, 0x481E3, 0x481E4, 0x481E5, 0x481E6, 0x481E7, + 0x481E8, 0x481E9, 0x481EA, 0x481EB, 0x481EC, 0x481ED, 0x481EE, 0x481EF, + 0x481F0, 0x481F1, 0x481F2, 0x481F3, 0x481F4, 0x481F5, 0x481F6, 0x481F7, + 0x481F8, 0x481F9, 0x481FA, 0x481FB, 0x481FC, 0x481FD, 0x481FE, 0x481FF, + 0x38000, 0x38001, 0x38002, 0x38003, 0x38004, 0x38005, 0x38006, 0x38007, + 0x38008, 0x38009, 0x3800A, 0x3800B, 0x3800C, 0x3800D, 0x3800E, 0x3800F, + 0x38010, 0x38011, 0x38012, 0x38013, 0x38014, 0x38015, 0x38016, 0x38017, + 0x400C0, 0x400C1, 0x400C2, 0x400C3, 0x400C4, 0x400C5 +}; + +static const u32 fixed_d_sym[30] = { + 0x28000, 0x28001, 0x28002, 0x28003, 0x28004, 0x28005, 0x28006, 0x28007, + 0x28008, 0x28009, 0x2800A, 0x2800B, 0x2800C, 0x2800D, 0x2800E, 0x2800F, + 0x28010, 0x28011, 0x28012, 0x28013, 0x28014, 0x28015, 0x28016, 0x28017, + 0x28018, 0x28019, 0x2801A, 0x2801B, 0x2801C, 0x2801D +}; + +static int init_fixed_mode(struct iaa_device_compression_mode *mode) +{ + struct aecs_comp_table_record *comp_table = mode->aecs_comp_table; + u32 bfinal = 1; + u32 offset; + + /* Configure aecs table using fixed Huffman table */ + comp_table->crc = 0; + comp_table->xor_checksum = 0; + offset = comp_table->num_output_accum_bits / 8; + comp_table->output_accum[offset] = FIXED_HDR | bfinal; + comp_table->num_output_accum_bits = FIXED_HDR_SIZE; + + return 0; +} + +int iaa_aecs_init_fixed(void) +{ + int ret; + + ret = add_iaa_compression_mode("fixed", + fixed_ll_sym, + sizeof(fixed_ll_sym), + fixed_d_sym, + sizeof(fixed_d_sym), + NULL, 0, 0, + init_fixed_mode, NULL); + if (!ret) + pr_debug("IAA fixed compression mode initialized\n"); + + return ret; +} + +void iaa_aecs_cleanup_fixed(void) +{ + remove_iaa_compression_mode("fixed"); +} diff --git a/drivers/crypto/intel/iaa/iaa_crypto_main.c b/drivers/crypto/intel/iaa/iaa_crypto_main.c new file mode 100644 index 000000000000..dfd3baf0a8d8 --- /dev/null +++ b/drivers/crypto/intel/iaa/iaa_crypto_main.c @@ -0,0 +1,2193 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright(c) 2021 Intel Corporation. All rights rsvd. */ + +#include <linux/init.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/pci.h> +#include <linux/device.h> +#include <linux/iommu.h> +#include <uapi/linux/idxd.h> +#include <linux/highmem.h> +#include <linux/sched/smt.h> +#include <crypto/internal/acompress.h> + +#include "idxd.h" +#include "iaa_crypto.h" +#include "iaa_crypto_stats.h" + +#ifdef pr_fmt +#undef pr_fmt +#endif + +#define pr_fmt(fmt) "idxd: " IDXD_SUBDRIVER_NAME ": " fmt + +#define IAA_ALG_PRIORITY 300 + +/* number of iaa instances probed */ +static unsigned int nr_iaa; +static unsigned int nr_cpus; +static unsigned int nr_nodes; +static unsigned int nr_cpus_per_node; + +/* Number of physical cpus sharing each iaa instance */ +static unsigned int cpus_per_iaa; + +static struct crypto_comp *deflate_generic_tfm; + +/* Per-cpu lookup table for balanced wqs */ +static struct wq_table_entry __percpu *wq_table; + +static struct idxd_wq *wq_table_next_wq(int cpu) +{ + struct wq_table_entry *entry = per_cpu_ptr(wq_table, cpu); + + if (++entry->cur_wq >= entry->n_wqs) + entry->cur_wq = 0; + + if (!entry->wqs[entry->cur_wq]) + return NULL; + + pr_debug("%s: returning wq at idx %d (iaa wq %d.%d) from cpu %d\n", __func__, + entry->cur_wq, entry->wqs[entry->cur_wq]->idxd->id, + entry->wqs[entry->cur_wq]->id, cpu); + + return entry->wqs[entry->cur_wq]; +} + +static void wq_table_add(int cpu, struct idxd_wq *wq) +{ + struct wq_table_entry *entry = per_cpu_ptr(wq_table, cpu); + + if (WARN_ON(entry->n_wqs == entry->max_wqs)) + return; + + entry->wqs[entry->n_wqs++] = wq; + + pr_debug("%s: added iaa wq %d.%d to idx %d of cpu %d\n", __func__, + entry->wqs[entry->n_wqs - 1]->idxd->id, + entry->wqs[entry->n_wqs - 1]->id, entry->n_wqs - 1, cpu); +} + +static void wq_table_free_entry(int cpu) +{ + struct wq_table_entry *entry = per_cpu_ptr(wq_table, cpu); + + kfree(entry->wqs); + memset(entry, 0, sizeof(*entry)); +} + +static void wq_table_clear_entry(int cpu) +{ + struct wq_table_entry *entry = per_cpu_ptr(wq_table, cpu); + + entry->n_wqs = 0; + entry->cur_wq = 0; + memset(entry->wqs, 0, entry->max_wqs * sizeof(struct idxd_wq *)); +} + +LIST_HEAD(iaa_devices); +DEFINE_MUTEX(iaa_devices_lock); + +/* If enabled, IAA hw crypto algos are registered, unavailable otherwise */ +static bool iaa_crypto_enabled; +static bool iaa_crypto_registered; + +/* Verify results of IAA compress or not */ +static bool iaa_verify_compress = true; + +static ssize_t verify_compress_show(struct device_driver *driver, char *buf) +{ + return sprintf(buf, "%d\n", iaa_verify_compress); +} + +static ssize_t verify_compress_store(struct device_driver *driver, + const char *buf, size_t count) +{ + int ret = -EBUSY; + + mutex_lock(&iaa_devices_lock); + + if (iaa_crypto_enabled) + goto out; + + ret = kstrtobool(buf, &iaa_verify_compress); + if (ret) + goto out; + + ret = count; +out: + mutex_unlock(&iaa_devices_lock); + + return ret; +} +static DRIVER_ATTR_RW(verify_compress); + +/* + * The iaa crypto driver supports three 'sync' methods determining how + * compressions and decompressions are performed: + * + * - sync: the compression or decompression completes before + * returning. This is the mode used by the async crypto + * interface when the sync mode is set to 'sync' and by + * the sync crypto interface regardless of setting. + * + * - async: the compression or decompression is submitted and returns + * immediately. Completion interrupts are not used so + * the caller is responsible for polling the descriptor + * for completion. This mode is applicable to only the + * async crypto interface and is ignored for anything + * else. + * + * - async_irq: the compression or decompression is submitted and + * returns immediately. Completion interrupts are + * enabled so the caller can wait for the completion and + * yield to other threads. When the compression or + * decompression completes, the completion is signaled + * and the caller awakened. This mode is applicable to + * only the async crypto interface and is ignored for + * anything else. + * + * These modes can be set using the iaa_crypto sync_mode driver + * attribute. + */ + +/* Use async mode */ +static bool async_mode; +/* Use interrupts */ +static bool use_irq; + +/** + * set_iaa_sync_mode - Set IAA sync mode + * @name: The name of the sync mode + * + * Make the IAA sync mode named @name the current sync mode used by + * compression/decompression. + */ + +static int set_iaa_sync_mode(const char *name) +{ + int ret = 0; + + if (sysfs_streq(name, "sync")) { + async_mode = false; + use_irq = false; + } else if (sysfs_streq(name, "async")) { + async_mode = true; + use_irq = false; + } else if (sysfs_streq(name, "async_irq")) { + async_mode = true; + use_irq = true; + } else { + ret = -EINVAL; + } + + return ret; +} + +static ssize_t sync_mode_show(struct device_driver *driver, char *buf) +{ + int ret = 0; + + if (!async_mode && !use_irq) + ret = sprintf(buf, "%s\n", "sync"); + else if (async_mode && !use_irq) + ret = sprintf(buf, "%s\n", "async"); + else if (async_mode && use_irq) + ret = sprintf(buf, "%s\n", "async_irq"); + + return ret; +} + +static ssize_t sync_mode_store(struct device_driver *driver, + const char *buf, size_t count) +{ + int ret = -EBUSY; + + mutex_lock(&iaa_devices_lock); + + if (iaa_crypto_enabled) + goto out; + + ret = set_iaa_sync_mode(buf); + if (ret == 0) + ret = count; +out: + mutex_unlock(&iaa_devices_lock); + + return ret; +} +static DRIVER_ATTR_RW(sync_mode); + +static struct iaa_compression_mode *iaa_compression_modes[IAA_COMP_MODES_MAX]; + +static int find_empty_iaa_compression_mode(void) +{ + int i = -EINVAL; + + for (i = 0; i < IAA_COMP_MODES_MAX; i++) { + if (iaa_compression_modes[i]) + continue; + break; + } + + return i; +} + +static struct iaa_compression_mode *find_iaa_compression_mode(const char *name, int *idx) +{ + struct iaa_compression_mode *mode; + int i; + + for (i = 0; i < IAA_COMP_MODES_MAX; i++) { + mode = iaa_compression_modes[i]; + if (!mode) + continue; + + if (!strcmp(mode->name, name)) { + *idx = i; + return iaa_compression_modes[i]; + } + } + + return NULL; +} + +static void free_iaa_compression_mode(struct iaa_compression_mode *mode) +{ + kfree(mode->name); + kfree(mode->ll_table); + kfree(mode->d_table); + kfree(mode->header_table); + + kfree(mode); +} + +/* + * IAA Compression modes are defined by an ll_table, a d_table, and an + * optional header_table. These tables are typically generated and + * captured using statistics collected from running actual + * compress/decompress workloads. + * + * A module or other kernel code can add and remove compression modes + * with a given name using the exported @add_iaa_compression_mode() + * and @remove_iaa_compression_mode functions. + * + * When a new compression mode is added, the tables are saved in a + * global compression mode list. When IAA devices are added, a + * per-IAA device dma mapping is created for each IAA device, for each + * compression mode. These are the tables used to do the actual + * compression/deccompression and are unmapped if/when the devices are + * removed. Currently, compression modes must be added before any + * device is added, and removed after all devices have been removed. + */ + +/** + * remove_iaa_compression_mode - Remove an IAA compression mode + * @name: The name the compression mode will be known as + * + * Remove the IAA compression mode named @name. + */ +void remove_iaa_compression_mode(const char *name) +{ + struct iaa_compression_mode *mode; + int idx; + + mutex_lock(&iaa_devices_lock); + + if (!list_empty(&iaa_devices)) + goto out; + + mode = find_iaa_compression_mode(name, &idx); + if (mode) { + free_iaa_compression_mode(mode); + iaa_compression_modes[idx] = NULL; + } +out: + mutex_unlock(&iaa_devices_lock); +} +EXPORT_SYMBOL_GPL(remove_iaa_compression_mode); + +/** + * add_iaa_compression_mode - Add an IAA compression mode + * @name: The name the compression mode will be known as + * @ll_table: The ll table + * @ll_table_size: The ll table size in bytes + * @d_table: The d table + * @d_table_size: The d table size in bytes + * @header_table: Optional header table + * @header_table_size: Optional header table size in bytes + * @gen_decomp_table_flags: Otional flags used to generate the decomp table + * @init: Optional callback function to init the compression mode data + * @free: Optional callback function to free the compression mode data + * + * Add a new IAA compression mode named @name. + * + * Returns 0 if successful, errcode otherwise. + */ +int add_iaa_compression_mode(const char *name, + const u32 *ll_table, + int ll_table_size, + const u32 *d_table, + int d_table_size, + const u8 *header_table, + int header_table_size, + u16 gen_decomp_table_flags, + iaa_dev_comp_init_fn_t init, + iaa_dev_comp_free_fn_t free) +{ + struct iaa_compression_mode *mode; + int idx, ret = -ENOMEM; + + mutex_lock(&iaa_devices_lock); + + if (!list_empty(&iaa_devices)) { + ret = -EBUSY; + goto out; + } + + mode = kzalloc(sizeof(*mode), GFP_KERNEL); + if (!mode) + goto out; + + mode->name = kstrdup(name, GFP_KERNEL); + if (!mode->name) + goto free; + + if (ll_table) { + mode->ll_table = kzalloc(ll_table_size, GFP_KERNEL); + if (!mode->ll_table) + goto free; + memcpy(mode->ll_table, ll_table, ll_table_size); + mode->ll_table_size = ll_table_size; + } + + if (d_table) { + mode->d_table = kzalloc(d_table_size, GFP_KERNEL); + if (!mode->d_table) + goto free; + memcpy(mode->d_table, d_table, d_table_size); + mode->d_table_size = d_table_size; + } + + if (header_table) { + mode->header_table = kzalloc(header_table_size, GFP_KERNEL); + if (!mode->header_table) + goto free; + memcpy(mode->header_table, header_table, header_table_size); + mode->header_table_size = header_table_size; + } + + mode->gen_decomp_table_flags = gen_decomp_table_flags; + + mode->init = init; + mode->free = free; + + idx = find_empty_iaa_compression_mode(); + if (idx < 0) + goto free; + + pr_debug("IAA compression mode %s added at idx %d\n", + mode->name, idx); + + iaa_compression_modes[idx] = mode; + + ret = 0; +out: + mutex_unlock(&iaa_devices_lock); + + return ret; +free: + free_iaa_compression_mode(mode); + goto out; +} +EXPORT_SYMBOL_GPL(add_iaa_compression_mode); + +static struct iaa_device_compression_mode * +get_iaa_device_compression_mode(struct iaa_device *iaa_device, int idx) +{ + return iaa_device->compression_modes[idx]; +} + +static void free_device_compression_mode(struct iaa_device *iaa_device, + struct iaa_device_compression_mode *device_mode) +{ + size_t size = sizeof(struct aecs_comp_table_record) + IAA_AECS_ALIGN; + struct device *dev = &iaa_device->idxd->pdev->dev; + + kfree(device_mode->name); + + if (device_mode->aecs_comp_table) + dma_free_coherent(dev, size, device_mode->aecs_comp_table, + device_mode->aecs_comp_table_dma_addr); + if (device_mode->aecs_decomp_table) + dma_free_coherent(dev, size, device_mode->aecs_decomp_table, + device_mode->aecs_decomp_table_dma_addr); + + kfree(device_mode); +} + +#define IDXD_OP_FLAG_AECS_RW_TGLS 0x400000 +#define IAX_AECS_DEFAULT_FLAG (IDXD_OP_FLAG_CRAV | IDXD_OP_FLAG_RCR | IDXD_OP_FLAG_CC) +#define IAX_AECS_COMPRESS_FLAG (IAX_AECS_DEFAULT_FLAG | IDXD_OP_FLAG_RD_SRC2_AECS) +#define IAX_AECS_DECOMPRESS_FLAG (IAX_AECS_DEFAULT_FLAG | IDXD_OP_FLAG_RD_SRC2_AECS) +#define IAX_AECS_GEN_FLAG (IAX_AECS_DEFAULT_FLAG | \ + IDXD_OP_FLAG_WR_SRC2_AECS_COMP | \ + IDXD_OP_FLAG_AECS_RW_TGLS) + +static int check_completion(struct device *dev, + struct iax_completion_record *comp, + bool compress, + bool only_once); + +static int decompress_header(struct iaa_device_compression_mode *device_mode, + struct iaa_compression_mode *mode, + struct idxd_wq *wq) +{ + dma_addr_t src_addr, src2_addr; + struct idxd_desc *idxd_desc; + struct iax_hw_desc *desc; + struct device *dev; + int ret = 0; + + idxd_desc = idxd_alloc_desc(wq, IDXD_OP_BLOCK); + if (IS_ERR(idxd_desc)) + return PTR_ERR(idxd_desc); + + desc = idxd_desc->iax_hw; + + dev = &wq->idxd->pdev->dev; + + src_addr = dma_map_single(dev, (void *)mode->header_table, + mode->header_table_size, DMA_TO_DEVICE); + dev_dbg(dev, "%s: mode->name %s, src_addr %llx, dev %p, src %p, slen %d\n", + __func__, mode->name, src_addr, dev, + mode->header_table, mode->header_table_size); + if (unlikely(dma_mapping_error(dev, src_addr))) { + dev_dbg(dev, "dma_map_single err, exiting\n"); + ret = -ENOMEM; + return ret; + } + + desc->flags = IAX_AECS_GEN_FLAG; + desc->opcode = IAX_OPCODE_DECOMPRESS; + + desc->src1_addr = (u64)src_addr; + desc->src1_size = mode->header_table_size; + + src2_addr = device_mode->aecs_decomp_table_dma_addr; + desc->src2_addr = (u64)src2_addr; + desc->src2_size = 1088; + dev_dbg(dev, "%s: mode->name %s, src2_addr %llx, dev %p, src2_size %d\n", + __func__, mode->name, desc->src2_addr, dev, desc->src2_size); + desc->max_dst_size = 0; // suppressed output + + desc->decompr_flags = mode->gen_decomp_table_flags; + + desc->priv = 0; + + desc->completion_addr = idxd_desc->compl_dma; + + ret = idxd_submit_desc(wq, idxd_desc); + if (ret) { + pr_err("%s: submit_desc failed ret=0x%x\n", __func__, ret); + goto out; + } + + ret = check_completion(dev, idxd_desc->iax_completion, false, false); + if (ret) + dev_dbg(dev, "%s: mode->name %s check_completion failed ret=%d\n", + __func__, mode->name, ret); + else + dev_dbg(dev, "%s: mode->name %s succeeded\n", __func__, + mode->name); +out: + dma_unmap_single(dev, src_addr, 1088, DMA_TO_DEVICE); + + return ret; +} + +static int init_device_compression_mode(struct iaa_device *iaa_device, + struct iaa_compression_mode *mode, + int idx, struct idxd_wq *wq) +{ + size_t size = sizeof(struct aecs_comp_table_record) + IAA_AECS_ALIGN; + struct device *dev = &iaa_device->idxd->pdev->dev; + struct iaa_device_compression_mode *device_mode; + int ret = -ENOMEM; + + device_mode = kzalloc(sizeof(*device_mode), GFP_KERNEL); + if (!device_mode) + return -ENOMEM; + + device_mode->name = kstrdup(mode->name, GFP_KERNEL); + if (!device_mode->name) + goto free; + + device_mode->aecs_comp_table = dma_alloc_coherent(dev, size, + &device_mode->aecs_comp_table_dma_addr, GFP_KERNEL); + if (!device_mode->aecs_comp_table) + goto free; + + device_mode->aecs_decomp_table = dma_alloc_coherent(dev, size, + &device_mode->aecs_decomp_table_dma_addr, GFP_KERNEL); + if (!device_mode->aecs_decomp_table) + goto free; + + /* Add Huffman table to aecs */ + memset(device_mode->aecs_comp_table, 0, sizeof(*device_mode->aecs_comp_table)); + memcpy(device_mode->aecs_comp_table->ll_sym, mode->ll_table, mode->ll_table_size); + memcpy(device_mode->aecs_comp_table->d_sym, mode->d_table, mode->d_table_size); + + if (mode->header_table) { + ret = decompress_header(device_mode, mode, wq); + if (ret) { + pr_debug("iaa header decompression failed: ret=%d\n", ret); + goto free; + } + } + + if (mode->init) { + ret = mode->init(device_mode); + if (ret) + goto free; + } + + /* mode index should match iaa_compression_modes idx */ + iaa_device->compression_modes[idx] = device_mode; + + pr_debug("IAA %s compression mode initialized for iaa device %d\n", + mode->name, iaa_device->idxd->id); + + ret = 0; +out: + return ret; +free: + pr_debug("IAA %s compression mode initialization failed for iaa device %d\n", + mode->name, iaa_device->idxd->id); + + free_device_compression_mode(iaa_device, device_mode); + goto out; +} + +static int init_device_compression_modes(struct iaa_device *iaa_device, + struct idxd_wq *wq) +{ + struct iaa_compression_mode *mode; + int i, ret = 0; + + for (i = 0; i < IAA_COMP_MODES_MAX; i++) { + mode = iaa_compression_modes[i]; + if (!mode) + continue; + + ret = init_device_compression_mode(iaa_device, mode, i, wq); + if (ret) + break; + } + + return ret; +} + +static void remove_device_compression_modes(struct iaa_device *iaa_device) +{ + struct iaa_device_compression_mode *device_mode; + int i; + + for (i = 0; i < IAA_COMP_MODES_MAX; i++) { + device_mode = iaa_device->compression_modes[i]; + if (!device_mode) + continue; + + free_device_compression_mode(iaa_device, device_mode); + iaa_device->compression_modes[i] = NULL; + if (iaa_compression_modes[i]->free) + iaa_compression_modes[i]->free(device_mode); + } +} + +static struct iaa_device *iaa_device_alloc(void) +{ + struct iaa_device *iaa_device; + + iaa_device = kzalloc(sizeof(*iaa_device), GFP_KERNEL); + if (!iaa_device) + return NULL; + + INIT_LIST_HEAD(&iaa_device->wqs); + + return iaa_device; +} + +static bool iaa_has_wq(struct iaa_device *iaa_device, struct idxd_wq *wq) +{ + struct iaa_wq *iaa_wq; + + list_for_each_entry(iaa_wq, &iaa_device->wqs, list) { + if (iaa_wq->wq == wq) + return true; + } + + return false; +} + +static struct iaa_device *add_iaa_device(struct idxd_device *idxd) +{ + struct iaa_device *iaa_device; + + iaa_device = iaa_device_alloc(); + if (!iaa_device) + return NULL; + + iaa_device->idxd = idxd; + + list_add_tail(&iaa_device->list, &iaa_devices); + + nr_iaa++; + + return iaa_device; +} + +static int init_iaa_device(struct iaa_device *iaa_device, struct iaa_wq *iaa_wq) +{ + int ret = 0; + + ret = init_device_compression_modes(iaa_device, iaa_wq->wq); + if (ret) + return ret; + + return ret; +} + +static void del_iaa_device(struct iaa_device *iaa_device) +{ + list_del(&iaa_device->list); + + nr_iaa--; +} + +static int add_iaa_wq(struct iaa_device *iaa_device, struct idxd_wq *wq, + struct iaa_wq **new_wq) +{ + struct idxd_device *idxd = iaa_device->idxd; + struct pci_dev *pdev = idxd->pdev; + struct device *dev = &pdev->dev; + struct iaa_wq *iaa_wq; + + iaa_wq = kzalloc(sizeof(*iaa_wq), GFP_KERNEL); + if (!iaa_wq) + return -ENOMEM; + + iaa_wq->wq = wq; + iaa_wq->iaa_device = iaa_device; + idxd_wq_set_private(wq, iaa_wq); + + list_add_tail(&iaa_wq->list, &iaa_device->wqs); + + iaa_device->n_wq++; + + if (new_wq) + *new_wq = iaa_wq; + + dev_dbg(dev, "added wq %d to iaa device %d, n_wq %d\n", + wq->id, iaa_device->idxd->id, iaa_device->n_wq); + + return 0; +} + +static void del_iaa_wq(struct iaa_device *iaa_device, struct idxd_wq *wq) +{ + struct idxd_device *idxd = iaa_device->idxd; + struct pci_dev *pdev = idxd->pdev; + struct device *dev = &pdev->dev; + struct iaa_wq *iaa_wq; + + list_for_each_entry(iaa_wq, &iaa_device->wqs, list) { + if (iaa_wq->wq == wq) { + list_del(&iaa_wq->list); + iaa_device->n_wq--; + + dev_dbg(dev, "removed wq %d from iaa_device %d, n_wq %d, nr_iaa %d\n", + wq->id, iaa_device->idxd->id, + iaa_device->n_wq, nr_iaa); + + if (iaa_device->n_wq == 0) + del_iaa_device(iaa_device); + break; + } + } +} + +static void clear_wq_table(void) +{ + int cpu; + + for (cpu = 0; cpu < nr_cpus; cpu++) + wq_table_clear_entry(cpu); + + pr_debug("cleared wq table\n"); +} + +static void free_iaa_device(struct iaa_device *iaa_device) +{ + if (!iaa_device) + return; + + remove_device_compression_modes(iaa_device); + kfree(iaa_device); +} + +static void __free_iaa_wq(struct iaa_wq *iaa_wq) +{ + struct iaa_device *iaa_device; + + if (!iaa_wq) + return; + + iaa_device = iaa_wq->iaa_device; + if (iaa_device->n_wq == 0) + free_iaa_device(iaa_wq->iaa_device); +} + +static void free_iaa_wq(struct iaa_wq *iaa_wq) +{ + struct idxd_wq *wq; + + __free_iaa_wq(iaa_wq); + + wq = iaa_wq->wq; + + kfree(iaa_wq); + idxd_wq_set_private(wq, NULL); +} + +static int iaa_wq_get(struct idxd_wq *wq) +{ + struct idxd_device *idxd = wq->idxd; + struct iaa_wq *iaa_wq; + int ret = 0; + + spin_lock(&idxd->dev_lock); + iaa_wq = idxd_wq_get_private(wq); + if (iaa_wq && !iaa_wq->remove) { + iaa_wq->ref++; + idxd_wq_get(wq); + } else { + ret = -ENODEV; + } + spin_unlock(&idxd->dev_lock); + + return ret; +} + +static int iaa_wq_put(struct idxd_wq *wq) +{ + struct idxd_device *idxd = wq->idxd; + struct iaa_wq *iaa_wq; + bool free = false; + int ret = 0; + + spin_lock(&idxd->dev_lock); + iaa_wq = idxd_wq_get_private(wq); + if (iaa_wq) { + iaa_wq->ref--; + if (iaa_wq->ref == 0 && iaa_wq->remove) { + idxd_wq_set_private(wq, NULL); + free = true; + } + idxd_wq_put(wq); + } else { + ret = -ENODEV; + } + spin_unlock(&idxd->dev_lock); + if (free) { + __free_iaa_wq(iaa_wq); + kfree(iaa_wq); + } + + return ret; +} + +static void free_wq_table(void) +{ + int cpu; + + for (cpu = 0; cpu < nr_cpus; cpu++) + wq_table_free_entry(cpu); + + free_percpu(wq_table); + + pr_debug("freed wq table\n"); +} + +static int alloc_wq_table(int max_wqs) +{ + struct wq_table_entry *entry; + int cpu; + + wq_table = alloc_percpu(struct wq_table_entry); + if (!wq_table) + return -ENOMEM; + + for (cpu = 0; cpu < nr_cpus; cpu++) { + entry = per_cpu_ptr(wq_table, cpu); + entry->wqs = kcalloc(max_wqs, sizeof(struct wq *), GFP_KERNEL); + if (!entry->wqs) { + free_wq_table(); + return -ENOMEM; + } + + entry->max_wqs = max_wqs; + } + + pr_debug("initialized wq table\n"); + + return 0; +} + +static int save_iaa_wq(struct idxd_wq *wq) +{ + struct iaa_device *iaa_device, *found = NULL; + struct idxd_device *idxd; + struct pci_dev *pdev; + struct device *dev; + int ret = 0; + + list_for_each_entry(iaa_device, &iaa_devices, list) { + if (iaa_device->idxd == wq->idxd) { + idxd = iaa_device->idxd; + pdev = idxd->pdev; + dev = &pdev->dev; + /* + * Check to see that we don't already have this wq. + * Shouldn't happen but we don't control probing. + */ + if (iaa_has_wq(iaa_device, wq)) { + dev_dbg(dev, "same wq probed multiple times for iaa_device %p\n", + iaa_device); + goto out; + } + + found = iaa_device; + + ret = add_iaa_wq(iaa_device, wq, NULL); + if (ret) + goto out; + + break; + } + } + + if (!found) { + struct iaa_device *new_device; + struct iaa_wq *new_wq; + + new_device = add_iaa_device(wq->idxd); + if (!new_device) { + ret = -ENOMEM; + goto out; + } + + ret = add_iaa_wq(new_device, wq, &new_wq); + if (ret) { + del_iaa_device(new_device); + free_iaa_device(new_device); + goto out; + } + + ret = init_iaa_device(new_device, new_wq); + if (ret) { + del_iaa_wq(new_device, new_wq->wq); + del_iaa_device(new_device); + free_iaa_wq(new_wq); + goto out; + } + } + + if (WARN_ON(nr_iaa == 0)) + return -EINVAL; + + cpus_per_iaa = (nr_nodes * nr_cpus_per_node) / nr_iaa; +out: + return 0; +} + +static void remove_iaa_wq(struct idxd_wq *wq) +{ + struct iaa_device *iaa_device; + + list_for_each_entry(iaa_device, &iaa_devices, list) { + if (iaa_has_wq(iaa_device, wq)) { + del_iaa_wq(iaa_device, wq); + break; + } + } + + if (nr_iaa) + cpus_per_iaa = (nr_nodes * nr_cpus_per_node) / nr_iaa; + else + cpus_per_iaa = 0; +} + +static int wq_table_add_wqs(int iaa, int cpu) +{ + struct iaa_device *iaa_device, *found_device = NULL; + int ret = 0, cur_iaa = 0, n_wqs_added = 0; + struct idxd_device *idxd; + struct iaa_wq *iaa_wq; + struct pci_dev *pdev; + struct device *dev; + + list_for_each_entry(iaa_device, &iaa_devices, list) { + idxd = iaa_device->idxd; + pdev = idxd->pdev; + dev = &pdev->dev; + + if (cur_iaa != iaa) { + cur_iaa++; + continue; + } + + found_device = iaa_device; + dev_dbg(dev, "getting wq from iaa_device %d, cur_iaa %d\n", + found_device->idxd->id, cur_iaa); + break; + } + + if (!found_device) { + found_device = list_first_entry_or_null(&iaa_devices, + struct iaa_device, list); + if (!found_device) { + pr_debug("couldn't find any iaa devices with wqs!\n"); + ret = -EINVAL; + goto out; + } + cur_iaa = 0; + + idxd = found_device->idxd; + pdev = idxd->pdev; + dev = &pdev->dev; + dev_dbg(dev, "getting wq from only iaa_device %d, cur_iaa %d\n", + found_device->idxd->id, cur_iaa); + } + + list_for_each_entry(iaa_wq, &found_device->wqs, list) { + wq_table_add(cpu, iaa_wq->wq); + pr_debug("rebalance: added wq for cpu=%d: iaa wq %d.%d\n", + cpu, iaa_wq->wq->idxd->id, iaa_wq->wq->id); + n_wqs_added++; + } + + if (!n_wqs_added) { + pr_debug("couldn't find any iaa wqs!\n"); + ret = -EINVAL; + goto out; + } +out: + return ret; +} + +/* + * Rebalance the wq table so that given a cpu, it's easy to find the + * closest IAA instance. The idea is to try to choose the most + * appropriate IAA instance for a caller and spread available + * workqueues around to clients. + */ +static void rebalance_wq_table(void) +{ + const struct cpumask *node_cpus; + int node, cpu, iaa = -1; + + if (nr_iaa == 0) + return; + + pr_debug("rebalance: nr_nodes=%d, nr_cpus %d, nr_iaa %d, cpus_per_iaa %d\n", + nr_nodes, nr_cpus, nr_iaa, cpus_per_iaa); + + clear_wq_table(); + + if (nr_iaa == 1) { + for (cpu = 0; cpu < nr_cpus; cpu++) { + if (WARN_ON(wq_table_add_wqs(0, cpu))) { + pr_debug("could not add any wqs for iaa 0 to cpu %d!\n", cpu); + return; + } + } + + return; + } + + for_each_node_with_cpus(node) { + node_cpus = cpumask_of_node(node); + + for (cpu = 0; cpu < nr_cpus_per_node; cpu++) { + int node_cpu = cpumask_nth(cpu, node_cpus); + + if (WARN_ON(node_cpu >= nr_cpu_ids)) { + pr_debug("node_cpu %d doesn't exist!\n", node_cpu); + return; + } + + if ((cpu % cpus_per_iaa) == 0) + iaa++; + + if (WARN_ON(wq_table_add_wqs(iaa, node_cpu))) { + pr_debug("could not add any wqs for iaa %d to cpu %d!\n", iaa, cpu); + return; + } + } + } +} + +static inline int check_completion(struct device *dev, + struct iax_completion_record *comp, + bool compress, + bool only_once) +{ + char *op_str = compress ? "compress" : "decompress"; + int ret = 0; + + while (!comp->status) { + if (only_once) + return -EAGAIN; + cpu_relax(); + } + + if (comp->status != IAX_COMP_SUCCESS) { + if (comp->status == IAA_ERROR_WATCHDOG_EXPIRED) { + ret = -ETIMEDOUT; + dev_dbg(dev, "%s timed out, size=0x%x\n", + op_str, comp->output_size); + update_completion_timeout_errs(); + goto out; + } + + if (comp->status == IAA_ANALYTICS_ERROR && + comp->error_code == IAA_ERROR_COMP_BUF_OVERFLOW && compress) { + ret = -E2BIG; + dev_dbg(dev, "compressed > uncompressed size," + " not compressing, size=0x%x\n", + comp->output_size); + update_completion_comp_buf_overflow_errs(); + goto out; + } + + if (comp->status == IAA_ERROR_DECOMP_BUF_OVERFLOW) { + ret = -EOVERFLOW; + goto out; + } + + ret = -EINVAL; + dev_dbg(dev, "iaa %s status=0x%x, error=0x%x, size=0x%x\n", + op_str, comp->status, comp->error_code, comp->output_size); + print_hex_dump(KERN_INFO, "cmp-rec: ", DUMP_PREFIX_OFFSET, 8, 1, comp, 64, 0); + update_completion_einval_errs(); + + goto out; + } +out: + return ret; +} + +static int deflate_generic_decompress(struct acomp_req *req) +{ + void *src, *dst; + int ret; + + src = kmap_local_page(sg_page(req->src)) + req->src->offset; + dst = kmap_local_page(sg_page(req->dst)) + req->dst->offset; + + ret = crypto_comp_decompress(deflate_generic_tfm, + src, req->slen, dst, &req->dlen); + + kunmap_local(src); + kunmap_local(dst); + + update_total_sw_decomp_calls(); + + return ret; +} + +static int iaa_remap_for_verify(struct device *dev, struct iaa_wq *iaa_wq, + struct acomp_req *req, + dma_addr_t *src_addr, dma_addr_t *dst_addr); + +static int iaa_compress_verify(struct crypto_tfm *tfm, struct acomp_req *req, + struct idxd_wq *wq, + dma_addr_t src_addr, unsigned int slen, + dma_addr_t dst_addr, unsigned int *dlen, + u32 compression_crc); + +static void iaa_desc_complete(struct idxd_desc *idxd_desc, + enum idxd_complete_type comp_type, + bool free_desc, void *__ctx, + u32 *status) +{ + struct iaa_device_compression_mode *active_compression_mode; + struct iaa_compression_ctx *compression_ctx; + struct crypto_ctx *ctx = __ctx; + struct iaa_device *iaa_device; + struct idxd_device *idxd; + struct iaa_wq *iaa_wq; + struct pci_dev *pdev; + struct device *dev; + int ret, err = 0; + + compression_ctx = crypto_tfm_ctx(ctx->tfm); + + iaa_wq = idxd_wq_get_private(idxd_desc->wq); + iaa_device = iaa_wq->iaa_device; + idxd = iaa_device->idxd; + pdev = idxd->pdev; + dev = &pdev->dev; + + active_compression_mode = get_iaa_device_compression_mode(iaa_device, + compression_ctx->mode); + dev_dbg(dev, "%s: compression mode %s," + " ctx->src_addr %llx, ctx->dst_addr %llx\n", __func__, + active_compression_mode->name, + ctx->src_addr, ctx->dst_addr); + + ret = check_completion(dev, idxd_desc->iax_completion, + ctx->compress, false); + if (ret) { + dev_dbg(dev, "%s: check_completion failed ret=%d\n", __func__, ret); + if (!ctx->compress && + idxd_desc->iax_completion->status == IAA_ANALYTICS_ERROR) { + pr_warn("%s: falling back to deflate-generic decompress, " + "analytics error code %x\n", __func__, + idxd_desc->iax_completion->error_code); + ret = deflate_generic_decompress(ctx->req); + if (ret) { + dev_dbg(dev, "%s: deflate-generic failed ret=%d\n", + __func__, ret); + err = -EIO; + goto err; + } + } else { + err = -EIO; + goto err; + } + } else { + ctx->req->dlen = idxd_desc->iax_completion->output_size; + } + + /* Update stats */ + if (ctx->compress) { + update_total_comp_bytes_out(ctx->req->dlen); + update_wq_comp_bytes(iaa_wq->wq, ctx->req->dlen); + } else { + update_total_decomp_bytes_in(ctx->req->dlen); + update_wq_decomp_bytes(iaa_wq->wq, ctx->req->dlen); + } + + if (ctx->compress && compression_ctx->verify_compress) { + dma_addr_t src_addr, dst_addr; + u32 compression_crc; + + compression_crc = idxd_desc->iax_completion->crc; + + ret = iaa_remap_for_verify(dev, iaa_wq, ctx->req, &src_addr, &dst_addr); + if (ret) { + dev_dbg(dev, "%s: compress verify remap failed ret=%d\n", __func__, ret); + err = -EIO; + goto out; + } + + ret = iaa_compress_verify(ctx->tfm, ctx->req, iaa_wq->wq, src_addr, + ctx->req->slen, dst_addr, &ctx->req->dlen, + compression_crc); + if (ret) { + dev_dbg(dev, "%s: compress verify failed ret=%d\n", __func__, ret); + err = -EIO; + } + + dma_unmap_sg(dev, ctx->req->dst, sg_nents(ctx->req->dst), DMA_TO_DEVICE); + dma_unmap_sg(dev, ctx->req->src, sg_nents(ctx->req->src), DMA_FROM_DEVICE); + + goto out; + } +err: + dma_unmap_sg(dev, ctx->req->dst, sg_nents(ctx->req->dst), DMA_FROM_DEVICE); + dma_unmap_sg(dev, ctx->req->src, sg_nents(ctx->req->src), DMA_TO_DEVICE); +out: + if (ret != 0) + dev_dbg(dev, "asynchronous compress failed ret=%d\n", ret); + + if (ctx->req->base.complete) + acomp_request_complete(ctx->req, err); + + if (free_desc) + idxd_free_desc(idxd_desc->wq, idxd_desc); + iaa_wq_put(idxd_desc->wq); +} + +static int iaa_compress(struct crypto_tfm *tfm, struct acomp_req *req, + struct idxd_wq *wq, + dma_addr_t src_addr, unsigned int slen, + dma_addr_t dst_addr, unsigned int *dlen, + u32 *compression_crc, + bool disable_async) +{ + struct iaa_device_compression_mode *active_compression_mode; + struct iaa_compression_ctx *ctx = crypto_tfm_ctx(tfm); + struct iaa_device *iaa_device; + struct idxd_desc *idxd_desc; + struct iax_hw_desc *desc; + struct idxd_device *idxd; + struct iaa_wq *iaa_wq; + struct pci_dev *pdev; + struct device *dev; + int ret = 0; + + iaa_wq = idxd_wq_get_private(wq); + iaa_device = iaa_wq->iaa_device; + idxd = iaa_device->idxd; + pdev = idxd->pdev; + dev = &pdev->dev; + + active_compression_mode = get_iaa_device_compression_mode(iaa_device, ctx->mode); + + idxd_desc = idxd_alloc_desc(wq, IDXD_OP_BLOCK); + if (IS_ERR(idxd_desc)) { + dev_dbg(dev, "idxd descriptor allocation failed\n"); + dev_dbg(dev, "iaa compress failed: ret=%ld\n", PTR_ERR(idxd_desc)); + return PTR_ERR(idxd_desc); + } + desc = idxd_desc->iax_hw; + + desc->flags = IDXD_OP_FLAG_CRAV | IDXD_OP_FLAG_RCR | + IDXD_OP_FLAG_RD_SRC2_AECS | IDXD_OP_FLAG_CC; + desc->opcode = IAX_OPCODE_COMPRESS; + desc->compr_flags = IAA_COMP_FLAGS; + desc->priv = 0; + + desc->src1_addr = (u64)src_addr; + desc->src1_size = slen; + desc->dst_addr = (u64)dst_addr; + desc->max_dst_size = *dlen; + desc->src2_addr = active_compression_mode->aecs_comp_table_dma_addr; + desc->src2_size = sizeof(struct aecs_comp_table_record); + desc->completion_addr = idxd_desc->compl_dma; + + if (ctx->use_irq && !disable_async) { + desc->flags |= IDXD_OP_FLAG_RCI; + + idxd_desc->crypto.req = req; + idxd_desc->crypto.tfm = tfm; + idxd_desc->crypto.src_addr = src_addr; + idxd_desc->crypto.dst_addr = dst_addr; + idxd_desc->crypto.compress = true; + + dev_dbg(dev, "%s use_async_irq: compression mode %s," + " src_addr %llx, dst_addr %llx\n", __func__, + active_compression_mode->name, + src_addr, dst_addr); + } else if (ctx->async_mode && !disable_async) + req->base.data = idxd_desc; + + dev_dbg(dev, "%s: compression mode %s," + " desc->src1_addr %llx, desc->src1_size %d," + " desc->dst_addr %llx, desc->max_dst_size %d," + " desc->src2_addr %llx, desc->src2_size %d\n", __func__, + active_compression_mode->name, + desc->src1_addr, desc->src1_size, desc->dst_addr, + desc->max_dst_size, desc->src2_addr, desc->src2_size); + + ret = idxd_submit_desc(wq, idxd_desc); + if (ret) { + dev_dbg(dev, "submit_desc failed ret=%d\n", ret); + goto err; + } + + /* Update stats */ + update_total_comp_calls(); + update_wq_comp_calls(wq); + + if (ctx->async_mode && !disable_async) { + ret = -EINPROGRESS; + dev_dbg(dev, "%s: returning -EINPROGRESS\n", __func__); + goto out; + } + + ret = check_completion(dev, idxd_desc->iax_completion, true, false); + if (ret) { + dev_dbg(dev, "check_completion failed ret=%d\n", ret); + goto err; + } + + *dlen = idxd_desc->iax_completion->output_size; + + /* Update stats */ + update_total_comp_bytes_out(*dlen); + update_wq_comp_bytes(wq, *dlen); + + *compression_crc = idxd_desc->iax_completion->crc; + + if (!ctx->async_mode) + idxd_free_desc(wq, idxd_desc); +out: + return ret; +err: + idxd_free_desc(wq, idxd_desc); + dev_dbg(dev, "iaa compress failed: ret=%d\n", ret); + + goto out; +} + +static int iaa_remap_for_verify(struct device *dev, struct iaa_wq *iaa_wq, + struct acomp_req *req, + dma_addr_t *src_addr, dma_addr_t *dst_addr) +{ + int ret = 0; + int nr_sgs; + + dma_unmap_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE); + dma_unmap_sg(dev, req->src, sg_nents(req->src), DMA_TO_DEVICE); + + nr_sgs = dma_map_sg(dev, req->src, sg_nents(req->src), DMA_FROM_DEVICE); + if (nr_sgs <= 0 || nr_sgs > 1) { + dev_dbg(dev, "verify: couldn't map src sg for iaa device %d," + " wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id, + iaa_wq->wq->id, ret); + ret = -EIO; + goto out; + } + *src_addr = sg_dma_address(req->src); + dev_dbg(dev, "verify: dma_map_sg, src_addr %llx, nr_sgs %d, req->src %p," + " req->slen %d, sg_dma_len(sg) %d\n", *src_addr, nr_sgs, + req->src, req->slen, sg_dma_len(req->src)); + + nr_sgs = dma_map_sg(dev, req->dst, sg_nents(req->dst), DMA_TO_DEVICE); + if (nr_sgs <= 0 || nr_sgs > 1) { + dev_dbg(dev, "verify: couldn't map dst sg for iaa device %d," + " wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id, + iaa_wq->wq->id, ret); + ret = -EIO; + dma_unmap_sg(dev, req->src, sg_nents(req->src), DMA_FROM_DEVICE); + goto out; + } + *dst_addr = sg_dma_address(req->dst); + dev_dbg(dev, "verify: dma_map_sg, dst_addr %llx, nr_sgs %d, req->dst %p," + " req->dlen %d, sg_dma_len(sg) %d\n", *dst_addr, nr_sgs, + req->dst, req->dlen, sg_dma_len(req->dst)); +out: + return ret; +} + +static int iaa_compress_verify(struct crypto_tfm *tfm, struct acomp_req *req, + struct idxd_wq *wq, + dma_addr_t src_addr, unsigned int slen, + dma_addr_t dst_addr, unsigned int *dlen, + u32 compression_crc) +{ + struct iaa_device_compression_mode *active_compression_mode; + struct iaa_compression_ctx *ctx = crypto_tfm_ctx(tfm); + struct iaa_device *iaa_device; + struct idxd_desc *idxd_desc; + struct iax_hw_desc *desc; + struct idxd_device *idxd; + struct iaa_wq *iaa_wq; + struct pci_dev *pdev; + struct device *dev; + int ret = 0; + + iaa_wq = idxd_wq_get_private(wq); + iaa_device = iaa_wq->iaa_device; + idxd = iaa_device->idxd; + pdev = idxd->pdev; + dev = &pdev->dev; + + active_compression_mode = get_iaa_device_compression_mode(iaa_device, ctx->mode); + + idxd_desc = idxd_alloc_desc(wq, IDXD_OP_BLOCK); + if (IS_ERR(idxd_desc)) { + dev_dbg(dev, "idxd descriptor allocation failed\n"); + dev_dbg(dev, "iaa compress failed: ret=%ld\n", + PTR_ERR(idxd_desc)); + return PTR_ERR(idxd_desc); + } + desc = idxd_desc->iax_hw; + + /* Verify (optional) - decompress and check crc, suppress dest write */ + + desc->flags = IDXD_OP_FLAG_CRAV | IDXD_OP_FLAG_RCR | IDXD_OP_FLAG_CC; + desc->opcode = IAX_OPCODE_DECOMPRESS; + desc->decompr_flags = IAA_DECOMP_FLAGS | IAA_DECOMP_SUPPRESS_OUTPUT; + desc->priv = 0; + + desc->src1_addr = (u64)dst_addr; + desc->src1_size = *dlen; + desc->dst_addr = (u64)src_addr; + desc->max_dst_size = slen; + desc->completion_addr = idxd_desc->compl_dma; + + dev_dbg(dev, "(verify) compression mode %s," + " desc->src1_addr %llx, desc->src1_size %d," + " desc->dst_addr %llx, desc->max_dst_size %d," + " desc->src2_addr %llx, desc->src2_size %d\n", + active_compression_mode->name, + desc->src1_addr, desc->src1_size, desc->dst_addr, + desc->max_dst_size, desc->src2_addr, desc->src2_size); + + ret = idxd_submit_desc(wq, idxd_desc); + if (ret) { + dev_dbg(dev, "submit_desc (verify) failed ret=%d\n", ret); + goto err; + } + + ret = check_completion(dev, idxd_desc->iax_completion, false, false); + if (ret) { + dev_dbg(dev, "(verify) check_completion failed ret=%d\n", ret); + goto err; + } + + if (compression_crc != idxd_desc->iax_completion->crc) { + ret = -EINVAL; + dev_dbg(dev, "(verify) iaa comp/decomp crc mismatch:" + " comp=0x%x, decomp=0x%x\n", compression_crc, + idxd_desc->iax_completion->crc); + print_hex_dump(KERN_INFO, "cmp-rec: ", DUMP_PREFIX_OFFSET, + 8, 1, idxd_desc->iax_completion, 64, 0); + goto err; + } + + idxd_free_desc(wq, idxd_desc); +out: + return ret; +err: + idxd_free_desc(wq, idxd_desc); + dev_dbg(dev, "iaa compress failed: ret=%d\n", ret); + + goto out; +} + +static int iaa_decompress(struct crypto_tfm *tfm, struct acomp_req *req, + struct idxd_wq *wq, + dma_addr_t src_addr, unsigned int slen, + dma_addr_t dst_addr, unsigned int *dlen, + bool disable_async) +{ + struct iaa_device_compression_mode *active_compression_mode; + struct iaa_compression_ctx *ctx = crypto_tfm_ctx(tfm); + struct iaa_device *iaa_device; + struct idxd_desc *idxd_desc; + struct iax_hw_desc *desc; + struct idxd_device *idxd; + struct iaa_wq *iaa_wq; + struct pci_dev *pdev; + struct device *dev; + int ret = 0; + + iaa_wq = idxd_wq_get_private(wq); + iaa_device = iaa_wq->iaa_device; + idxd = iaa_device->idxd; + pdev = idxd->pdev; + dev = &pdev->dev; + + active_compression_mode = get_iaa_device_compression_mode(iaa_device, ctx->mode); + + idxd_desc = idxd_alloc_desc(wq, IDXD_OP_BLOCK); + if (IS_ERR(idxd_desc)) { + dev_dbg(dev, "idxd descriptor allocation failed\n"); + dev_dbg(dev, "iaa decompress failed: ret=%ld\n", + PTR_ERR(idxd_desc)); + return PTR_ERR(idxd_desc); + } + desc = idxd_desc->iax_hw; + + desc->flags = IDXD_OP_FLAG_CRAV | IDXD_OP_FLAG_RCR | IDXD_OP_FLAG_CC; + desc->opcode = IAX_OPCODE_DECOMPRESS; + desc->max_dst_size = PAGE_SIZE; + desc->decompr_flags = IAA_DECOMP_FLAGS; + desc->priv = 0; + + desc->src1_addr = (u64)src_addr; + desc->dst_addr = (u64)dst_addr; + desc->max_dst_size = *dlen; + desc->src1_size = slen; + desc->completion_addr = idxd_desc->compl_dma; + + if (ctx->use_irq && !disable_async) { + desc->flags |= IDXD_OP_FLAG_RCI; + + idxd_desc->crypto.req = req; + idxd_desc->crypto.tfm = tfm; + idxd_desc->crypto.src_addr = src_addr; + idxd_desc->crypto.dst_addr = dst_addr; + idxd_desc->crypto.compress = false; + + dev_dbg(dev, "%s: use_async_irq compression mode %s," + " src_addr %llx, dst_addr %llx\n", __func__, + active_compression_mode->name, + src_addr, dst_addr); + } else if (ctx->async_mode && !disable_async) + req->base.data = idxd_desc; + + dev_dbg(dev, "%s: decompression mode %s," + " desc->src1_addr %llx, desc->src1_size %d," + " desc->dst_addr %llx, desc->max_dst_size %d," + " desc->src2_addr %llx, desc->src2_size %d\n", __func__, + active_compression_mode->name, + desc->src1_addr, desc->src1_size, desc->dst_addr, + desc->max_dst_size, desc->src2_addr, desc->src2_size); + + ret = idxd_submit_desc(wq, idxd_desc); + if (ret) { + dev_dbg(dev, "submit_desc failed ret=%d\n", ret); + goto err; + } + + /* Update stats */ + update_total_decomp_calls(); + update_wq_decomp_calls(wq); + + if (ctx->async_mode && !disable_async) { + ret = -EINPROGRESS; + dev_dbg(dev, "%s: returning -EINPROGRESS\n", __func__); + goto out; + } + + ret = check_completion(dev, idxd_desc->iax_completion, false, false); + if (ret) { + dev_dbg(dev, "%s: check_completion failed ret=%d\n", __func__, ret); + if (idxd_desc->iax_completion->status == IAA_ANALYTICS_ERROR) { + pr_warn("%s: falling back to deflate-generic decompress, " + "analytics error code %x\n", __func__, + idxd_desc->iax_completion->error_code); + ret = deflate_generic_decompress(req); + if (ret) { + dev_dbg(dev, "%s: deflate-generic failed ret=%d\n", + __func__, ret); + goto err; + } + } else { + goto err; + } + } else { + req->dlen = idxd_desc->iax_completion->output_size; + } + + *dlen = req->dlen; + + if (!ctx->async_mode) + idxd_free_desc(wq, idxd_desc); + + /* Update stats */ + update_total_decomp_bytes_in(slen); + update_wq_decomp_bytes(wq, slen); +out: + return ret; +err: + idxd_free_desc(wq, idxd_desc); + dev_dbg(dev, "iaa decompress failed: ret=%d\n", ret); + + goto out; +} + +static int iaa_comp_acompress(struct acomp_req *req) +{ + struct iaa_compression_ctx *compression_ctx; + struct crypto_tfm *tfm = req->base.tfm; + dma_addr_t src_addr, dst_addr; + bool disable_async = false; + int nr_sgs, cpu, ret = 0; + struct iaa_wq *iaa_wq; + u32 compression_crc; + struct idxd_wq *wq; + struct device *dev; + int order = -1; + + compression_ctx = crypto_tfm_ctx(tfm); + + if (!iaa_crypto_enabled) { + pr_debug("iaa_crypto disabled, not compressing\n"); + return -ENODEV; + } + + if (!req->src || !req->slen) { + pr_debug("invalid src, not compressing\n"); + return -EINVAL; + } + + cpu = get_cpu(); + wq = wq_table_next_wq(cpu); + put_cpu(); + if (!wq) { + pr_debug("no wq configured for cpu=%d\n", cpu); + return -ENODEV; + } + + ret = iaa_wq_get(wq); + if (ret) { + pr_debug("no wq available for cpu=%d\n", cpu); + return -ENODEV; + } + + iaa_wq = idxd_wq_get_private(wq); + + if (!req->dst) { + gfp_t flags = req->flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : GFP_ATOMIC; + + /* incompressible data will always be < 2 * slen */ + req->dlen = 2 * req->slen; + order = order_base_2(round_up(req->dlen, PAGE_SIZE) / PAGE_SIZE); + req->dst = sgl_alloc_order(req->dlen, order, false, flags, NULL); + if (!req->dst) { + ret = -ENOMEM; + order = -1; + goto out; + } + disable_async = true; + } + + dev = &wq->idxd->pdev->dev; + + nr_sgs = dma_map_sg(dev, req->src, sg_nents(req->src), DMA_TO_DEVICE); + if (nr_sgs <= 0 || nr_sgs > 1) { + dev_dbg(dev, "couldn't map src sg for iaa device %d," + " wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id, + iaa_wq->wq->id, ret); + ret = -EIO; + goto out; + } + src_addr = sg_dma_address(req->src); + dev_dbg(dev, "dma_map_sg, src_addr %llx, nr_sgs %d, req->src %p," + " req->slen %d, sg_dma_len(sg) %d\n", src_addr, nr_sgs, + req->src, req->slen, sg_dma_len(req->src)); + + nr_sgs = dma_map_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE); + if (nr_sgs <= 0 || nr_sgs > 1) { + dev_dbg(dev, "couldn't map dst sg for iaa device %d," + " wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id, + iaa_wq->wq->id, ret); + ret = -EIO; + goto err_map_dst; + } + dst_addr = sg_dma_address(req->dst); + dev_dbg(dev, "dma_map_sg, dst_addr %llx, nr_sgs %d, req->dst %p," + " req->dlen %d, sg_dma_len(sg) %d\n", dst_addr, nr_sgs, + req->dst, req->dlen, sg_dma_len(req->dst)); + + ret = iaa_compress(tfm, req, wq, src_addr, req->slen, dst_addr, + &req->dlen, &compression_crc, disable_async); + if (ret == -EINPROGRESS) + return ret; + + if (!ret && compression_ctx->verify_compress) { + ret = iaa_remap_for_verify(dev, iaa_wq, req, &src_addr, &dst_addr); + if (ret) { + dev_dbg(dev, "%s: compress verify remap failed ret=%d\n", __func__, ret); + goto out; + } + + ret = iaa_compress_verify(tfm, req, wq, src_addr, req->slen, + dst_addr, &req->dlen, compression_crc); + if (ret) + dev_dbg(dev, "asynchronous compress verification failed ret=%d\n", ret); + + dma_unmap_sg(dev, req->dst, sg_nents(req->dst), DMA_TO_DEVICE); + dma_unmap_sg(dev, req->src, sg_nents(req->src), DMA_FROM_DEVICE); + + goto out; + } + + if (ret) + dev_dbg(dev, "asynchronous compress failed ret=%d\n", ret); + + dma_unmap_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE); +err_map_dst: + dma_unmap_sg(dev, req->src, sg_nents(req->src), DMA_TO_DEVICE); +out: + iaa_wq_put(wq); + + if (order >= 0) + sgl_free_order(req->dst, order); + + return ret; +} + +static int iaa_comp_adecompress_alloc_dest(struct acomp_req *req) +{ + gfp_t flags = req->flags & CRYPTO_TFM_REQ_MAY_SLEEP ? + GFP_KERNEL : GFP_ATOMIC; + struct crypto_tfm *tfm = req->base.tfm; + dma_addr_t src_addr, dst_addr; + int nr_sgs, cpu, ret = 0; + struct iaa_wq *iaa_wq; + struct device *dev; + struct idxd_wq *wq; + int order = -1; + + cpu = get_cpu(); + wq = wq_table_next_wq(cpu); + put_cpu(); + if (!wq) { + pr_debug("no wq configured for cpu=%d\n", cpu); + return -ENODEV; + } + + ret = iaa_wq_get(wq); + if (ret) { + pr_debug("no wq available for cpu=%d\n", cpu); + return -ENODEV; + } + + iaa_wq = idxd_wq_get_private(wq); + + dev = &wq->idxd->pdev->dev; + + nr_sgs = dma_map_sg(dev, req->src, sg_nents(req->src), DMA_TO_DEVICE); + if (nr_sgs <= 0 || nr_sgs > 1) { + dev_dbg(dev, "couldn't map src sg for iaa device %d," + " wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id, + iaa_wq->wq->id, ret); + ret = -EIO; + goto out; + } + src_addr = sg_dma_address(req->src); + dev_dbg(dev, "dma_map_sg, src_addr %llx, nr_sgs %d, req->src %p," + " req->slen %d, sg_dma_len(sg) %d\n", src_addr, nr_sgs, + req->src, req->slen, sg_dma_len(req->src)); + + req->dlen = 4 * req->slen; /* start with ~avg comp rato */ +alloc_dest: + order = order_base_2(round_up(req->dlen, PAGE_SIZE) / PAGE_SIZE); + req->dst = sgl_alloc_order(req->dlen, order, false, flags, NULL); + if (!req->dst) { + ret = -ENOMEM; + order = -1; + goto out; + } + + nr_sgs = dma_map_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE); + if (nr_sgs <= 0 || nr_sgs > 1) { + dev_dbg(dev, "couldn't map dst sg for iaa device %d," + " wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id, + iaa_wq->wq->id, ret); + ret = -EIO; + goto err_map_dst; + } + + dst_addr = sg_dma_address(req->dst); + dev_dbg(dev, "dma_map_sg, dst_addr %llx, nr_sgs %d, req->dst %p," + " req->dlen %d, sg_dma_len(sg) %d\n", dst_addr, nr_sgs, + req->dst, req->dlen, sg_dma_len(req->dst)); + ret = iaa_decompress(tfm, req, wq, src_addr, req->slen, + dst_addr, &req->dlen, true); + if (ret == -EOVERFLOW) { + dma_unmap_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE); + req->dlen *= 2; + if (req->dlen > CRYPTO_ACOMP_DST_MAX) + goto err_map_dst; + goto alloc_dest; + } + + if (ret != 0) + dev_dbg(dev, "asynchronous decompress failed ret=%d\n", ret); + + dma_unmap_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE); +err_map_dst: + dma_unmap_sg(dev, req->src, sg_nents(req->src), DMA_TO_DEVICE); +out: + iaa_wq_put(wq); + + if (order >= 0) + sgl_free_order(req->dst, order); + + return ret; +} + +static int iaa_comp_adecompress(struct acomp_req *req) +{ + struct crypto_tfm *tfm = req->base.tfm; + dma_addr_t src_addr, dst_addr; + int nr_sgs, cpu, ret = 0; + struct iaa_wq *iaa_wq; + struct device *dev; + struct idxd_wq *wq; + + if (!iaa_crypto_enabled) { + pr_debug("iaa_crypto disabled, not decompressing\n"); + return -ENODEV; + } + + if (!req->src || !req->slen) { + pr_debug("invalid src, not decompressing\n"); + return -EINVAL; + } + + if (!req->dst) + return iaa_comp_adecompress_alloc_dest(req); + + cpu = get_cpu(); + wq = wq_table_next_wq(cpu); + put_cpu(); + if (!wq) { + pr_debug("no wq configured for cpu=%d\n", cpu); + return -ENODEV; + } + + ret = iaa_wq_get(wq); + if (ret) { + pr_debug("no wq available for cpu=%d\n", cpu); + return -ENODEV; + } + + iaa_wq = idxd_wq_get_private(wq); + + dev = &wq->idxd->pdev->dev; + + nr_sgs = dma_map_sg(dev, req->src, sg_nents(req->src), DMA_TO_DEVICE); + if (nr_sgs <= 0 || nr_sgs > 1) { + dev_dbg(dev, "couldn't map src sg for iaa device %d," + " wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id, + iaa_wq->wq->id, ret); + ret = -EIO; + goto out; + } + src_addr = sg_dma_address(req->src); + dev_dbg(dev, "dma_map_sg, src_addr %llx, nr_sgs %d, req->src %p," + " req->slen %d, sg_dma_len(sg) %d\n", src_addr, nr_sgs, + req->src, req->slen, sg_dma_len(req->src)); + + nr_sgs = dma_map_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE); + if (nr_sgs <= 0 || nr_sgs > 1) { + dev_dbg(dev, "couldn't map dst sg for iaa device %d," + " wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id, + iaa_wq->wq->id, ret); + ret = -EIO; + goto err_map_dst; + } + dst_addr = sg_dma_address(req->dst); + dev_dbg(dev, "dma_map_sg, dst_addr %llx, nr_sgs %d, req->dst %p," + " req->dlen %d, sg_dma_len(sg) %d\n", dst_addr, nr_sgs, + req->dst, req->dlen, sg_dma_len(req->dst)); + + ret = iaa_decompress(tfm, req, wq, src_addr, req->slen, + dst_addr, &req->dlen, false); + if (ret == -EINPROGRESS) + return ret; + + if (ret != 0) + dev_dbg(dev, "asynchronous decompress failed ret=%d\n", ret); + + dma_unmap_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE); +err_map_dst: + dma_unmap_sg(dev, req->src, sg_nents(req->src), DMA_TO_DEVICE); +out: + iaa_wq_put(wq); + + return ret; +} + +static void compression_ctx_init(struct iaa_compression_ctx *ctx) +{ + ctx->verify_compress = iaa_verify_compress; + ctx->async_mode = async_mode; + ctx->use_irq = use_irq; +} + +static int iaa_comp_init_fixed(struct crypto_acomp *acomp_tfm) +{ + struct crypto_tfm *tfm = crypto_acomp_tfm(acomp_tfm); + struct iaa_compression_ctx *ctx = crypto_tfm_ctx(tfm); + + compression_ctx_init(ctx); + + ctx->mode = IAA_MODE_FIXED; + + return 0; +} + +static void dst_free(struct scatterlist *sgl) +{ + /* + * Called for req->dst = NULL cases but we free elsewhere + * using sgl_free_order(). + */ +} + +static struct acomp_alg iaa_acomp_fixed_deflate = { + .init = iaa_comp_init_fixed, + .compress = iaa_comp_acompress, + .decompress = iaa_comp_adecompress, + .dst_free = dst_free, + .base = { + .cra_name = "deflate", + .cra_driver_name = "deflate-iaa", + .cra_ctxsize = sizeof(struct iaa_compression_ctx), + .cra_module = THIS_MODULE, + .cra_priority = IAA_ALG_PRIORITY, + } +}; + +static int iaa_register_compression_device(void) +{ + int ret; + + ret = crypto_register_acomp(&iaa_acomp_fixed_deflate); + if (ret) { + pr_err("deflate algorithm acomp fixed registration failed (%d)\n", ret); + goto out; + } + + iaa_crypto_registered = true; +out: + return ret; +} + +static int iaa_unregister_compression_device(void) +{ + if (iaa_crypto_registered) + crypto_unregister_acomp(&iaa_acomp_fixed_deflate); + + return 0; +} + +static int iaa_crypto_probe(struct idxd_dev *idxd_dev) +{ + struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev); + struct idxd_device *idxd = wq->idxd; + struct idxd_driver_data *data = idxd->data; + struct device *dev = &idxd_dev->conf_dev; + bool first_wq = false; + int ret = 0; + + if (idxd->state != IDXD_DEV_ENABLED) + return -ENXIO; + + if (data->type != IDXD_TYPE_IAX) + return -ENODEV; + + mutex_lock(&wq->wq_lock); + + if (idxd_wq_get_private(wq)) { + mutex_unlock(&wq->wq_lock); + return -EBUSY; + } + + if (!idxd_wq_driver_name_match(wq, dev)) { + dev_dbg(dev, "wq %d.%d driver_name match failed: wq driver_name %s, dev driver name %s\n", + idxd->id, wq->id, wq->driver_name, dev->driver->name); + idxd->cmd_status = IDXD_SCMD_WQ_NO_DRV_NAME; + ret = -ENODEV; + goto err; + } + + wq->type = IDXD_WQT_KERNEL; + + ret = idxd_drv_enable_wq(wq); + if (ret < 0) { + dev_dbg(dev, "enable wq %d.%d failed: %d\n", + idxd->id, wq->id, ret); + ret = -ENXIO; + goto err; + } + + mutex_lock(&iaa_devices_lock); + + if (list_empty(&iaa_devices)) { + ret = alloc_wq_table(wq->idxd->max_wqs); + if (ret) + goto err_alloc; + first_wq = true; + } + + ret = save_iaa_wq(wq); + if (ret) + goto err_save; + + rebalance_wq_table(); + + if (first_wq) { + iaa_crypto_enabled = true; + ret = iaa_register_compression_device(); + if (ret != 0) { + iaa_crypto_enabled = false; + dev_dbg(dev, "IAA compression device registration failed\n"); + goto err_register; + } + try_module_get(THIS_MODULE); + + pr_info("iaa_crypto now ENABLED\n"); + } + + mutex_unlock(&iaa_devices_lock); +out: + mutex_unlock(&wq->wq_lock); + + return ret; + +err_register: + remove_iaa_wq(wq); + free_iaa_wq(idxd_wq_get_private(wq)); +err_save: + if (first_wq) + free_wq_table(); +err_alloc: + mutex_unlock(&iaa_devices_lock); + idxd_drv_disable_wq(wq); +err: + wq->type = IDXD_WQT_NONE; + + goto out; +} + +static void iaa_crypto_remove(struct idxd_dev *idxd_dev) +{ + struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev); + struct idxd_device *idxd = wq->idxd; + struct iaa_wq *iaa_wq; + bool free = false; + + idxd_wq_quiesce(wq); + + mutex_lock(&wq->wq_lock); + mutex_lock(&iaa_devices_lock); + + remove_iaa_wq(wq); + + spin_lock(&idxd->dev_lock); + iaa_wq = idxd_wq_get_private(wq); + if (!iaa_wq) { + spin_unlock(&idxd->dev_lock); + pr_err("%s: no iaa_wq available to remove\n", __func__); + goto out; + } + + if (iaa_wq->ref) { + iaa_wq->remove = true; + } else { + wq = iaa_wq->wq; + idxd_wq_set_private(wq, NULL); + free = true; + } + spin_unlock(&idxd->dev_lock); + if (free) { + __free_iaa_wq(iaa_wq); + kfree(iaa_wq); + } + + idxd_drv_disable_wq(wq); + rebalance_wq_table(); + + if (nr_iaa == 0) { + iaa_crypto_enabled = false; + free_wq_table(); + module_put(THIS_MODULE); + + pr_info("iaa_crypto now DISABLED\n"); + } +out: + mutex_unlock(&iaa_devices_lock); + mutex_unlock(&wq->wq_lock); +} + +static enum idxd_dev_type dev_types[] = { + IDXD_DEV_WQ, + IDXD_DEV_NONE, +}; + +static struct idxd_device_driver iaa_crypto_driver = { + .probe = iaa_crypto_probe, + .remove = iaa_crypto_remove, + .name = IDXD_SUBDRIVER_NAME, + .type = dev_types, + .desc_complete = iaa_desc_complete, +}; + +static int __init iaa_crypto_init_module(void) +{ + int ret = 0; + int node; + + nr_cpus = num_online_cpus(); + for_each_node_with_cpus(node) + nr_nodes++; + if (!nr_nodes) { + pr_err("IAA couldn't find any nodes with cpus\n"); + return -ENODEV; + } + nr_cpus_per_node = nr_cpus / nr_nodes; + + if (crypto_has_comp("deflate-generic", 0, 0)) + deflate_generic_tfm = crypto_alloc_comp("deflate-generic", 0, 0); + + if (IS_ERR_OR_NULL(deflate_generic_tfm)) { + pr_err("IAA could not alloc %s tfm: errcode = %ld\n", + "deflate-generic", PTR_ERR(deflate_generic_tfm)); + return -ENOMEM; + } + + ret = iaa_aecs_init_fixed(); + if (ret < 0) { + pr_debug("IAA fixed compression mode init failed\n"); + goto err_aecs_init; + } + + ret = idxd_driver_register(&iaa_crypto_driver); + if (ret) { + pr_debug("IAA wq sub-driver registration failed\n"); + goto err_driver_reg; + } + + ret = driver_create_file(&iaa_crypto_driver.drv, + &driver_attr_verify_compress); + if (ret) { + pr_debug("IAA verify_compress attr creation failed\n"); + goto err_verify_attr_create; + } + + ret = driver_create_file(&iaa_crypto_driver.drv, + &driver_attr_sync_mode); + if (ret) { + pr_debug("IAA sync mode attr creation failed\n"); + goto err_sync_attr_create; + } + + if (iaa_crypto_debugfs_init()) + pr_warn("debugfs init failed, stats not available\n"); + + pr_debug("initialized\n"); +out: + return ret; + +err_sync_attr_create: + driver_remove_file(&iaa_crypto_driver.drv, + &driver_attr_verify_compress); +err_verify_attr_create: + idxd_driver_unregister(&iaa_crypto_driver); +err_driver_reg: + iaa_aecs_cleanup_fixed(); +err_aecs_init: + crypto_free_comp(deflate_generic_tfm); + + goto out; +} + +static void __exit iaa_crypto_cleanup_module(void) +{ + if (iaa_unregister_compression_device()) + pr_debug("IAA compression device unregister failed\n"); + + iaa_crypto_debugfs_cleanup(); + driver_remove_file(&iaa_crypto_driver.drv, + &driver_attr_sync_mode); + driver_remove_file(&iaa_crypto_driver.drv, + &driver_attr_verify_compress); + idxd_driver_unregister(&iaa_crypto_driver); + iaa_aecs_cleanup_fixed(); + crypto_free_comp(deflate_generic_tfm); + + pr_debug("cleaned up\n"); +} + +MODULE_IMPORT_NS(IDXD); +MODULE_LICENSE("GPL"); +MODULE_ALIAS_IDXD_DEVICE(0); +MODULE_AUTHOR("Intel Corporation"); +MODULE_DESCRIPTION("IAA Compression Accelerator Crypto Driver"); + +module_init(iaa_crypto_init_module); +module_exit(iaa_crypto_cleanup_module); diff --git a/drivers/crypto/intel/iaa/iaa_crypto_stats.c b/drivers/crypto/intel/iaa/iaa_crypto_stats.c new file mode 100644 index 000000000000..2e3b7b73af20 --- /dev/null +++ b/drivers/crypto/intel/iaa/iaa_crypto_stats.c @@ -0,0 +1,312 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright(c) 2021 Intel Corporation. All rights rsvd. */ + +#include <linux/module.h> +#include <linux/kernel.h> +#include <linux/highmem.h> +#include <linux/mm.h> +#include <linux/slab.h> +#include <linux/delay.h> +#include <linux/smp.h> +#include <uapi/linux/idxd.h> +#include <linux/idxd.h> +#include <linux/dmaengine.h> +#include "../../dma/idxd/idxd.h" +#include <linux/debugfs.h> +#include <crypto/internal/acompress.h> +#include "iaa_crypto.h" +#include "iaa_crypto_stats.h" + +static u64 total_comp_calls; +static u64 total_decomp_calls; +static u64 total_sw_decomp_calls; +static u64 max_comp_delay_ns; +static u64 max_decomp_delay_ns; +static u64 max_acomp_delay_ns; +static u64 max_adecomp_delay_ns; +static u64 total_comp_bytes_out; +static u64 total_decomp_bytes_in; +static u64 total_completion_einval_errors; +static u64 total_completion_timeout_errors; +static u64 total_completion_comp_buf_overflow_errors; + +static struct dentry *iaa_crypto_debugfs_root; + +void update_total_comp_calls(void) +{ + total_comp_calls++; +} + +void update_total_comp_bytes_out(int n) +{ + total_comp_bytes_out += n; +} + +void update_total_decomp_calls(void) +{ + total_decomp_calls++; +} + +void update_total_sw_decomp_calls(void) +{ + total_sw_decomp_calls++; +} + +void update_total_decomp_bytes_in(int n) +{ + total_decomp_bytes_in += n; +} + +void update_completion_einval_errs(void) +{ + total_completion_einval_errors++; +} + +void update_completion_timeout_errs(void) +{ + total_completion_timeout_errors++; +} + +void update_completion_comp_buf_overflow_errs(void) +{ + total_completion_comp_buf_overflow_errors++; +} + +void update_max_comp_delay_ns(u64 start_time_ns) +{ + u64 time_diff; + + time_diff = ktime_get_ns() - start_time_ns; + + if (time_diff > max_comp_delay_ns) + max_comp_delay_ns = time_diff; +} + +void update_max_decomp_delay_ns(u64 start_time_ns) +{ + u64 time_diff; + + time_diff = ktime_get_ns() - start_time_ns; + + if (time_diff > max_decomp_delay_ns) + max_decomp_delay_ns = time_diff; +} + +void update_max_acomp_delay_ns(u64 start_time_ns) +{ + u64 time_diff; + + time_diff = ktime_get_ns() - start_time_ns; + + if (time_diff > max_acomp_delay_ns) + max_acomp_delay_ns = time_diff; +} + +void update_max_adecomp_delay_ns(u64 start_time_ns) +{ + u64 time_diff; + + time_diff = ktime_get_ns() - start_time_ns; + + if (time_diff > max_adecomp_delay_ns) + max_adecomp_delay_ns = time_diff; +} + +void update_wq_comp_calls(struct idxd_wq *idxd_wq) +{ + struct iaa_wq *wq = idxd_wq_get_private(idxd_wq); + + wq->comp_calls++; + wq->iaa_device->comp_calls++; +} + +void update_wq_comp_bytes(struct idxd_wq *idxd_wq, int n) +{ + struct iaa_wq *wq = idxd_wq_get_private(idxd_wq); + + wq->comp_bytes += n; + wq->iaa_device->comp_bytes += n; +} + +void update_wq_decomp_calls(struct idxd_wq *idxd_wq) +{ + struct iaa_wq *wq = idxd_wq_get_private(idxd_wq); + + wq->decomp_calls++; + wq->iaa_device->decomp_calls++; +} + +void update_wq_decomp_bytes(struct idxd_wq *idxd_wq, int n) +{ + struct iaa_wq *wq = idxd_wq_get_private(idxd_wq); + + wq->decomp_bytes += n; + wq->iaa_device->decomp_bytes += n; +} + +static void reset_iaa_crypto_stats(void) +{ + total_comp_calls = 0; + total_decomp_calls = 0; + total_sw_decomp_calls = 0; + max_comp_delay_ns = 0; + max_decomp_delay_ns = 0; + max_acomp_delay_ns = 0; + max_adecomp_delay_ns = 0; + total_comp_bytes_out = 0; + total_decomp_bytes_in = 0; + total_completion_einval_errors = 0; + total_completion_timeout_errors = 0; + total_completion_comp_buf_overflow_errors = 0; +} + +static void reset_wq_stats(struct iaa_wq *wq) +{ + wq->comp_calls = 0; + wq->comp_bytes = 0; + wq->decomp_calls = 0; + wq->decomp_bytes = 0; +} + +static void reset_device_stats(struct iaa_device *iaa_device) +{ + struct iaa_wq *iaa_wq; + + iaa_device->comp_calls = 0; + iaa_device->comp_bytes = 0; + iaa_device->decomp_calls = 0; + iaa_device->decomp_bytes = 0; + + list_for_each_entry(iaa_wq, &iaa_device->wqs, list) + reset_wq_stats(iaa_wq); +} + +static void wq_show(struct seq_file *m, struct iaa_wq *iaa_wq) +{ + seq_printf(m, " name: %s\n", iaa_wq->wq->name); + seq_printf(m, " comp_calls: %llu\n", iaa_wq->comp_calls); + seq_printf(m, " comp_bytes: %llu\n", iaa_wq->comp_bytes); + seq_printf(m, " decomp_calls: %llu\n", iaa_wq->decomp_calls); + seq_printf(m, " decomp_bytes: %llu\n\n", iaa_wq->decomp_bytes); +} + +static void device_stats_show(struct seq_file *m, struct iaa_device *iaa_device) +{ + struct iaa_wq *iaa_wq; + + seq_puts(m, "iaa device:\n"); + seq_printf(m, " id: %d\n", iaa_device->idxd->id); + seq_printf(m, " n_wqs: %d\n", iaa_device->n_wq); + seq_printf(m, " comp_calls: %llu\n", iaa_device->comp_calls); + seq_printf(m, " comp_bytes: %llu\n", iaa_device->comp_bytes); + seq_printf(m, " decomp_calls: %llu\n", iaa_device->decomp_calls); + seq_printf(m, " decomp_bytes: %llu\n", iaa_device->decomp_bytes); + seq_puts(m, " wqs:\n"); + + list_for_each_entry(iaa_wq, &iaa_device->wqs, list) + wq_show(m, iaa_wq); +} + +static void global_stats_show(struct seq_file *m) +{ + seq_puts(m, "global stats:\n"); + seq_printf(m, " total_comp_calls: %llu\n", total_comp_calls); + seq_printf(m, " total_decomp_calls: %llu\n", total_decomp_calls); + seq_printf(m, " total_sw_decomp_calls: %llu\n", total_sw_decomp_calls); + seq_printf(m, " total_comp_bytes_out: %llu\n", total_comp_bytes_out); + seq_printf(m, " total_decomp_bytes_in: %llu\n", total_decomp_bytes_in); + seq_printf(m, " total_completion_einval_errors: %llu\n", + total_completion_einval_errors); + seq_printf(m, " total_completion_timeout_errors: %llu\n", + total_completion_timeout_errors); + seq_printf(m, " total_completion_comp_buf_overflow_errors: %llu\n\n", + total_completion_comp_buf_overflow_errors); +} + +static int wq_stats_show(struct seq_file *m, void *v) +{ + struct iaa_device *iaa_device; + + mutex_lock(&iaa_devices_lock); + + global_stats_show(m); + + list_for_each_entry(iaa_device, &iaa_devices, list) + device_stats_show(m, iaa_device); + + mutex_unlock(&iaa_devices_lock); + + return 0; +} + +static int iaa_crypto_stats_reset(void *data, u64 value) +{ + struct iaa_device *iaa_device; + + reset_iaa_crypto_stats(); + + mutex_lock(&iaa_devices_lock); + + list_for_each_entry(iaa_device, &iaa_devices, list) + reset_device_stats(iaa_device); + + mutex_unlock(&iaa_devices_lock); + + return 0; +} + +static int wq_stats_open(struct inode *inode, struct file *file) +{ + return single_open(file, wq_stats_show, file); +} + +static const struct file_operations wq_stats_fops = { + .open = wq_stats_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +DEFINE_DEBUGFS_ATTRIBUTE(wq_stats_reset_fops, NULL, iaa_crypto_stats_reset, "%llu\n"); + +int __init iaa_crypto_debugfs_init(void) +{ + if (!debugfs_initialized()) + return -ENODEV; + + iaa_crypto_debugfs_root = debugfs_create_dir("iaa_crypto", NULL); + if (!iaa_crypto_debugfs_root) + return -ENOMEM; + + debugfs_create_u64("max_comp_delay_ns", 0644, + iaa_crypto_debugfs_root, &max_comp_delay_ns); + debugfs_create_u64("max_decomp_delay_ns", 0644, + iaa_crypto_debugfs_root, &max_decomp_delay_ns); + debugfs_create_u64("max_acomp_delay_ns", 0644, + iaa_crypto_debugfs_root, &max_comp_delay_ns); + debugfs_create_u64("max_adecomp_delay_ns", 0644, + iaa_crypto_debugfs_root, &max_decomp_delay_ns); + debugfs_create_u64("total_comp_calls", 0644, + iaa_crypto_debugfs_root, &total_comp_calls); + debugfs_create_u64("total_decomp_calls", 0644, + iaa_crypto_debugfs_root, &total_decomp_calls); + debugfs_create_u64("total_sw_decomp_calls", 0644, + iaa_crypto_debugfs_root, &total_sw_decomp_calls); + debugfs_create_u64("total_comp_bytes_out", 0644, + iaa_crypto_debugfs_root, &total_comp_bytes_out); + debugfs_create_u64("total_decomp_bytes_in", 0644, + iaa_crypto_debugfs_root, &total_decomp_bytes_in); + debugfs_create_file("wq_stats", 0644, iaa_crypto_debugfs_root, NULL, + &wq_stats_fops); + debugfs_create_file("stats_reset", 0644, iaa_crypto_debugfs_root, NULL, + &wq_stats_reset_fops); + + return 0; +} + +void __exit iaa_crypto_debugfs_cleanup(void) +{ + debugfs_remove_recursive(iaa_crypto_debugfs_root); +} + +MODULE_LICENSE("GPL"); diff --git a/drivers/crypto/intel/iaa/iaa_crypto_stats.h b/drivers/crypto/intel/iaa/iaa_crypto_stats.h new file mode 100644 index 000000000000..c10b87b86fa4 --- /dev/null +++ b/drivers/crypto/intel/iaa/iaa_crypto_stats.h @@ -0,0 +1,53 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright(c) 2021 Intel Corporation. All rights rsvd. */ + +#ifndef __CRYPTO_DEV_IAA_CRYPTO_STATS_H__ +#define __CRYPTO_DEV_IAA_CRYPTO_STATS_H__ + +#if defined(CONFIG_CRYPTO_DEV_IAA_CRYPTO_STATS) +int iaa_crypto_debugfs_init(void); +void iaa_crypto_debugfs_cleanup(void); + +void update_total_comp_calls(void); +void update_total_comp_bytes_out(int n); +void update_total_decomp_calls(void); +void update_total_sw_decomp_calls(void); +void update_total_decomp_bytes_in(int n); +void update_max_comp_delay_ns(u64 start_time_ns); +void update_max_decomp_delay_ns(u64 start_time_ns); +void update_max_acomp_delay_ns(u64 start_time_ns); +void update_max_adecomp_delay_ns(u64 start_time_ns); +void update_completion_einval_errs(void); +void update_completion_timeout_errs(void); +void update_completion_comp_buf_overflow_errs(void); + +void update_wq_comp_calls(struct idxd_wq *idxd_wq); +void update_wq_comp_bytes(struct idxd_wq *idxd_wq, int n); +void update_wq_decomp_calls(struct idxd_wq *idxd_wq); +void update_wq_decomp_bytes(struct idxd_wq *idxd_wq, int n); + +#else +static inline int iaa_crypto_debugfs_init(void) { return 0; } +static inline void iaa_crypto_debugfs_cleanup(void) {} + +static inline void update_total_comp_calls(void) {} +static inline void update_total_comp_bytes_out(int n) {} +static inline void update_total_decomp_calls(void) {} +static inline void update_total_sw_decomp_calls(void) {} +static inline void update_total_decomp_bytes_in(int n) {} +static inline void update_max_comp_delay_ns(u64 start_time_ns) {} +static inline void update_max_decomp_delay_ns(u64 start_time_ns) {} +static inline void update_max_acomp_delay_ns(u64 start_time_ns) {} +static inline void update_max_adecomp_delay_ns(u64 start_time_ns) {} +static inline void update_completion_einval_errs(void) {} +static inline void update_completion_timeout_errs(void) {} +static inline void update_completion_comp_buf_overflow_errs(void) {} + +static inline void update_wq_comp_calls(struct idxd_wq *idxd_wq) {} +static inline void update_wq_comp_bytes(struct idxd_wq *idxd_wq, int n) {} +static inline void update_wq_decomp_calls(struct idxd_wq *idxd_wq) {} +static inline void update_wq_decomp_bytes(struct idxd_wq *idxd_wq, int n) {} + +#endif // CONFIG_CRYPTO_DEV_IAA_CRYPTO_STATS + +#endif diff --git a/drivers/crypto/intel/qat/Kconfig b/drivers/crypto/intel/qat/Kconfig index 1220cc86f910..c120f6715a09 100644 --- a/drivers/crypto/intel/qat/Kconfig +++ b/drivers/crypto/intel/qat/Kconfig @@ -59,6 +59,17 @@ config CRYPTO_DEV_QAT_4XXX To compile this as a module, choose M here: the module will be called qat_4xxx. +config CRYPTO_DEV_QAT_420XX + tristate "Support for Intel(R) QAT_420XX" + depends on PCI && (!CPU_BIG_ENDIAN || COMPILE_TEST) + select CRYPTO_DEV_QAT + help + Support for Intel(R) QuickAssist Technology QAT_420xx + for accelerating crypto and compression workloads. + + To compile this as a module, choose M here: the module + will be called qat_420xx. + config CRYPTO_DEV_QAT_DH895xCCVF tristate "Support for Intel(R) DH895xCC Virtual Function" depends on PCI && (!CPU_BIG_ENDIAN || COMPILE_TEST) diff --git a/drivers/crypto/intel/qat/Makefile b/drivers/crypto/intel/qat/Makefile index 258c8a626ce0..235b69f4f3f7 100644 --- a/drivers/crypto/intel/qat/Makefile +++ b/drivers/crypto/intel/qat/Makefile @@ -4,6 +4,7 @@ obj-$(CONFIG_CRYPTO_DEV_QAT_DH895xCC) += qat_dh895xcc/ obj-$(CONFIG_CRYPTO_DEV_QAT_C3XXX) += qat_c3xxx/ obj-$(CONFIG_CRYPTO_DEV_QAT_C62X) += qat_c62x/ obj-$(CONFIG_CRYPTO_DEV_QAT_4XXX) += qat_4xxx/ +obj-$(CONFIG_CRYPTO_DEV_QAT_420XX) += qat_420xx/ obj-$(CONFIG_CRYPTO_DEV_QAT_DH895xCCVF) += qat_dh895xccvf/ obj-$(CONFIG_CRYPTO_DEV_QAT_C3XXXVF) += qat_c3xxxvf/ obj-$(CONFIG_CRYPTO_DEV_QAT_C62XVF) += qat_c62xvf/ diff --git a/drivers/crypto/intel/qat/qat_420xx/Makefile b/drivers/crypto/intel/qat/qat_420xx/Makefile new file mode 100644 index 000000000000..a90fbe00b3c8 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_420xx/Makefile @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0-only +ccflags-y := -I $(srctree)/$(src)/../qat_common +obj-$(CONFIG_CRYPTO_DEV_QAT_420XX) += qat_420xx.o +qat_420xx-objs := adf_drv.o adf_420xx_hw_data.o diff --git a/drivers/crypto/intel/qat/qat_420xx/adf_420xx_hw_data.c b/drivers/crypto/intel/qat/qat_420xx/adf_420xx_hw_data.c new file mode 100644 index 000000000000..a87d29ae724f --- /dev/null +++ b/drivers/crypto/intel/qat/qat_420xx/adf_420xx_hw_data.c @@ -0,0 +1,528 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2023 Intel Corporation */ +#include <linux/iopoll.h> +#include <adf_accel_devices.h> +#include <adf_admin.h> +#include <adf_cfg.h> +#include <adf_cfg_services.h> +#include <adf_clock.h> +#include <adf_common_drv.h> +#include <adf_fw_config.h> +#include <adf_gen4_config.h> +#include <adf_gen4_dc.h> +#include <adf_gen4_hw_data.h> +#include <adf_gen4_pfvf.h> +#include <adf_gen4_pm.h> +#include <adf_gen4_ras.h> +#include <adf_gen4_timer.h> +#include <adf_gen4_tl.h> +#include "adf_420xx_hw_data.h" +#include "icp_qat_hw.h" + +#define ADF_AE_GROUP_0 GENMASK(3, 0) +#define ADF_AE_GROUP_1 GENMASK(7, 4) +#define ADF_AE_GROUP_2 GENMASK(11, 8) +#define ADF_AE_GROUP_3 GENMASK(15, 12) +#define ADF_AE_GROUP_4 BIT(16) + +#define ENA_THD_MASK_ASYM GENMASK(1, 0) +#define ENA_THD_MASK_SYM GENMASK(3, 0) +#define ENA_THD_MASK_DC GENMASK(1, 0) + +static const char * const adf_420xx_fw_objs[] = { + [ADF_FW_SYM_OBJ] = ADF_420XX_SYM_OBJ, + [ADF_FW_ASYM_OBJ] = ADF_420XX_ASYM_OBJ, + [ADF_FW_DC_OBJ] = ADF_420XX_DC_OBJ, + [ADF_FW_ADMIN_OBJ] = ADF_420XX_ADMIN_OBJ, +}; + +static const struct adf_fw_config adf_fw_cy_config[] = { + {ADF_AE_GROUP_3, ADF_FW_SYM_OBJ}, + {ADF_AE_GROUP_2, ADF_FW_ASYM_OBJ}, + {ADF_AE_GROUP_1, ADF_FW_SYM_OBJ}, + {ADF_AE_GROUP_0, ADF_FW_ASYM_OBJ}, + {ADF_AE_GROUP_4, ADF_FW_ADMIN_OBJ}, +}; + +static const struct adf_fw_config adf_fw_dc_config[] = { + {ADF_AE_GROUP_1, ADF_FW_DC_OBJ}, + {ADF_AE_GROUP_0, ADF_FW_DC_OBJ}, + {ADF_AE_GROUP_4, ADF_FW_ADMIN_OBJ}, +}; + +static const struct adf_fw_config adf_fw_sym_config[] = { + {ADF_AE_GROUP_3, ADF_FW_SYM_OBJ}, + {ADF_AE_GROUP_2, ADF_FW_SYM_OBJ}, + {ADF_AE_GROUP_1, ADF_FW_SYM_OBJ}, + {ADF_AE_GROUP_0, ADF_FW_SYM_OBJ}, + {ADF_AE_GROUP_4, ADF_FW_ADMIN_OBJ}, +}; + +static const struct adf_fw_config adf_fw_asym_config[] = { + {ADF_AE_GROUP_3, ADF_FW_ASYM_OBJ}, + {ADF_AE_GROUP_2, ADF_FW_ASYM_OBJ}, + {ADF_AE_GROUP_1, ADF_FW_ASYM_OBJ}, + {ADF_AE_GROUP_0, ADF_FW_ASYM_OBJ}, + {ADF_AE_GROUP_4, ADF_FW_ADMIN_OBJ}, +}; + +static const struct adf_fw_config adf_fw_asym_dc_config[] = { + {ADF_AE_GROUP_3, ADF_FW_ASYM_OBJ}, + {ADF_AE_GROUP_2, ADF_FW_ASYM_OBJ}, + {ADF_AE_GROUP_1, ADF_FW_ASYM_OBJ}, + {ADF_AE_GROUP_0, ADF_FW_DC_OBJ}, + {ADF_AE_GROUP_4, ADF_FW_ADMIN_OBJ}, +}; + +static const struct adf_fw_config adf_fw_sym_dc_config[] = { + {ADF_AE_GROUP_2, ADF_FW_SYM_OBJ}, + {ADF_AE_GROUP_1, ADF_FW_SYM_OBJ}, + {ADF_AE_GROUP_0, ADF_FW_DC_OBJ}, + {ADF_AE_GROUP_4, ADF_FW_ADMIN_OBJ}, +}; + +static const struct adf_fw_config adf_fw_dcc_config[] = { + {ADF_AE_GROUP_1, ADF_FW_DC_OBJ}, + {ADF_AE_GROUP_0, ADF_FW_SYM_OBJ}, + {ADF_AE_GROUP_4, ADF_FW_ADMIN_OBJ}, +}; + + +static struct adf_hw_device_class adf_420xx_class = { + .name = ADF_420XX_DEVICE_NAME, + .type = DEV_420XX, + .instances = 0, +}; + +static u32 get_ae_mask(struct adf_hw_device_data *self) +{ + u32 me_disable = self->fuses; + + return ~me_disable & ADF_420XX_ACCELENGINES_MASK; +} + +static u32 uof_get_num_objs(struct adf_accel_dev *accel_dev) +{ + switch (adf_get_service_enabled(accel_dev)) { + case SVC_CY: + case SVC_CY2: + return ARRAY_SIZE(adf_fw_cy_config); + case SVC_DC: + return ARRAY_SIZE(adf_fw_dc_config); + case SVC_DCC: + return ARRAY_SIZE(adf_fw_dcc_config); + case SVC_SYM: + return ARRAY_SIZE(adf_fw_sym_config); + case SVC_ASYM: + return ARRAY_SIZE(adf_fw_asym_config); + case SVC_ASYM_DC: + case SVC_DC_ASYM: + return ARRAY_SIZE(adf_fw_asym_dc_config); + case SVC_SYM_DC: + case SVC_DC_SYM: + return ARRAY_SIZE(adf_fw_sym_dc_config); + default: + return 0; + } +} + +static const struct adf_fw_config *get_fw_config(struct adf_accel_dev *accel_dev) +{ + switch (adf_get_service_enabled(accel_dev)) { + case SVC_CY: + case SVC_CY2: + return adf_fw_cy_config; + case SVC_DC: + return adf_fw_dc_config; + case SVC_DCC: + return adf_fw_dcc_config; + case SVC_SYM: + return adf_fw_sym_config; + case SVC_ASYM: + return adf_fw_asym_config; + case SVC_ASYM_DC: + case SVC_DC_ASYM: + return adf_fw_asym_dc_config; + case SVC_SYM_DC: + case SVC_DC_SYM: + return adf_fw_sym_dc_config; + default: + return NULL; + } +} + +static void update_ae_mask(struct adf_accel_dev *accel_dev) +{ + struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev); + const struct adf_fw_config *fw_config; + u32 config_ae_mask = 0; + u32 ae_mask, num_objs; + int i; + + ae_mask = get_ae_mask(hw_data); + + /* Modify the AE mask based on the firmware configuration loaded */ + fw_config = get_fw_config(accel_dev); + num_objs = uof_get_num_objs(accel_dev); + + config_ae_mask |= ADF_420XX_ADMIN_AE_MASK; + for (i = 0; i < num_objs; i++) + config_ae_mask |= fw_config[i].ae_mask; + + hw_data->ae_mask = ae_mask & config_ae_mask; +} + +static u32 get_accel_cap(struct adf_accel_dev *accel_dev) +{ + u32 capabilities_sym, capabilities_asym, capabilities_dc; + struct pci_dev *pdev = accel_dev->accel_pci_dev.pci_dev; + u32 capabilities_dcc; + u32 fusectl1; + + /* As a side effect, update ae_mask based on configuration */ + update_ae_mask(accel_dev); + + /* Read accelerator capabilities mask */ + pci_read_config_dword(pdev, ADF_GEN4_FUSECTL1_OFFSET, &fusectl1); + + capabilities_sym = ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC | + ICP_ACCEL_CAPABILITIES_CIPHER | + ICP_ACCEL_CAPABILITIES_AUTHENTICATION | + ICP_ACCEL_CAPABILITIES_SHA3 | + ICP_ACCEL_CAPABILITIES_SHA3_EXT | + ICP_ACCEL_CAPABILITIES_HKDF | + ICP_ACCEL_CAPABILITIES_CHACHA_POLY | + ICP_ACCEL_CAPABILITIES_AESGCM_SPC | + ICP_ACCEL_CAPABILITIES_SM3 | + ICP_ACCEL_CAPABILITIES_SM4 | + ICP_ACCEL_CAPABILITIES_AES_V2 | + ICP_ACCEL_CAPABILITIES_ZUC | + ICP_ACCEL_CAPABILITIES_ZUC_256 | + ICP_ACCEL_CAPABILITIES_WIRELESS_CRYPTO_EXT | + ICP_ACCEL_CAPABILITIES_EXT_ALGCHAIN; + + /* A set bit in fusectl1 means the feature is OFF in this SKU */ + if (fusectl1 & ICP_ACCEL_GEN4_MASK_CIPHER_SLICE) { + capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC; + capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_HKDF; + capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_CIPHER; + } + + if (fusectl1 & ICP_ACCEL_GEN4_MASK_UCS_SLICE) { + capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_CHACHA_POLY; + capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_AESGCM_SPC; + capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_AES_V2; + capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_CIPHER; + } + + if (fusectl1 & ICP_ACCEL_GEN4_MASK_AUTH_SLICE) { + capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_AUTHENTICATION; + capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_SHA3; + capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_SHA3_EXT; + capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_CIPHER; + } + + if (fusectl1 & ICP_ACCEL_GEN4_MASK_SMX_SLICE) { + capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_SM3; + capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_SM4; + } + + if (fusectl1 & ICP_ACCEL_GEN4_MASK_WCP_WAT_SLICE) { + capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_ZUC; + capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_ZUC_256; + capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_WIRELESS_CRYPTO_EXT; + } + + if (fusectl1 & ICP_ACCEL_GEN4_MASK_EIA3_SLICE) { + capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_ZUC; + capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_ZUC_256; + } + + if (fusectl1 & ICP_ACCEL_GEN4_MASK_ZUC_256_SLICE) + capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_ZUC_256; + + capabilities_asym = ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC | + ICP_ACCEL_CAPABILITIES_SM2 | + ICP_ACCEL_CAPABILITIES_ECEDMONT; + + if (fusectl1 & ICP_ACCEL_GEN4_MASK_PKE_SLICE) { + capabilities_asym &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC; + capabilities_asym &= ~ICP_ACCEL_CAPABILITIES_SM2; + capabilities_asym &= ~ICP_ACCEL_CAPABILITIES_ECEDMONT; + } + + capabilities_dc = ICP_ACCEL_CAPABILITIES_COMPRESSION | + ICP_ACCEL_CAPABILITIES_LZ4_COMPRESSION | + ICP_ACCEL_CAPABILITIES_LZ4S_COMPRESSION | + ICP_ACCEL_CAPABILITIES_CNV_INTEGRITY64; + + if (fusectl1 & ICP_ACCEL_GEN4_MASK_COMPRESS_SLICE) { + capabilities_dc &= ~ICP_ACCEL_CAPABILITIES_COMPRESSION; + capabilities_dc &= ~ICP_ACCEL_CAPABILITIES_LZ4_COMPRESSION; + capabilities_dc &= ~ICP_ACCEL_CAPABILITIES_LZ4S_COMPRESSION; + capabilities_dc &= ~ICP_ACCEL_CAPABILITIES_CNV_INTEGRITY64; + } + + switch (adf_get_service_enabled(accel_dev)) { + case SVC_CY: + case SVC_CY2: + return capabilities_sym | capabilities_asym; + case SVC_DC: + return capabilities_dc; + case SVC_DCC: + /* + * Sym capabilities are available for chaining operations, + * but sym crypto instances cannot be supported + */ + capabilities_dcc = capabilities_dc | capabilities_sym; + capabilities_dcc &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC; + return capabilities_dcc; + case SVC_SYM: + return capabilities_sym; + case SVC_ASYM: + return capabilities_asym; + case SVC_ASYM_DC: + case SVC_DC_ASYM: + return capabilities_asym | capabilities_dc; + case SVC_SYM_DC: + case SVC_DC_SYM: + return capabilities_sym | capabilities_dc; + default: + return 0; + } +} + +static const u32 *adf_get_arbiter_mapping(struct adf_accel_dev *accel_dev) +{ + if (adf_gen4_init_thd2arb_map(accel_dev)) + dev_warn(&GET_DEV(accel_dev), + "Generate of the thread to arbiter map failed"); + + return GET_HW_DATA(accel_dev)->thd_to_arb_map; +} + +static void adf_init_rl_data(struct adf_rl_hw_data *rl_data) +{ + rl_data->pciout_tb_offset = ADF_GEN4_RL_TOKEN_PCIEOUT_BUCKET_OFFSET; + rl_data->pciin_tb_offset = ADF_GEN4_RL_TOKEN_PCIEIN_BUCKET_OFFSET; + rl_data->r2l_offset = ADF_GEN4_RL_R2L_OFFSET; + rl_data->l2c_offset = ADF_GEN4_RL_L2C_OFFSET; + rl_data->c2s_offset = ADF_GEN4_RL_C2S_OFFSET; + + rl_data->pcie_scale_div = ADF_420XX_RL_PCIE_SCALE_FACTOR_DIV; + rl_data->pcie_scale_mul = ADF_420XX_RL_PCIE_SCALE_FACTOR_MUL; + rl_data->dcpr_correction = ADF_420XX_RL_DCPR_CORRECTION; + rl_data->max_tp[ADF_SVC_ASYM] = ADF_420XX_RL_MAX_TP_ASYM; + rl_data->max_tp[ADF_SVC_SYM] = ADF_420XX_RL_MAX_TP_SYM; + rl_data->max_tp[ADF_SVC_DC] = ADF_420XX_RL_MAX_TP_DC; + rl_data->scan_interval = ADF_420XX_RL_SCANS_PER_SEC; + rl_data->scale_ref = ADF_420XX_RL_SLICE_REF; +} + +static int get_rp_group(struct adf_accel_dev *accel_dev, u32 ae_mask) +{ + switch (ae_mask) { + case ADF_AE_GROUP_0: + return RP_GROUP_0; + case ADF_AE_GROUP_1: + case ADF_AE_GROUP_3: + return RP_GROUP_1; + case ADF_AE_GROUP_2: + if (get_fw_config(accel_dev) == adf_fw_cy_config) + return RP_GROUP_0; + else + return RP_GROUP_1; + default: + dev_dbg(&GET_DEV(accel_dev), "ae_mask not recognized"); + return -EINVAL; + } +} + +static u32 get_ena_thd_mask(struct adf_accel_dev *accel_dev, u32 obj_num) +{ + const struct adf_fw_config *fw_config; + + if (obj_num >= uof_get_num_objs(accel_dev)) + return ADF_GEN4_ENA_THD_MASK_ERROR; + + fw_config = get_fw_config(accel_dev); + if (!fw_config) + return ADF_GEN4_ENA_THD_MASK_ERROR; + + switch (fw_config[obj_num].obj) { + case ADF_FW_ASYM_OBJ: + return ENA_THD_MASK_ASYM; + case ADF_FW_SYM_OBJ: + return ENA_THD_MASK_SYM; + case ADF_FW_DC_OBJ: + return ENA_THD_MASK_DC; + default: + return ADF_GEN4_ENA_THD_MASK_ERROR; + } +} + +static u16 get_ring_to_svc_map(struct adf_accel_dev *accel_dev) +{ + enum adf_cfg_service_type rps[RP_GROUP_COUNT] = { }; + const struct adf_fw_config *fw_config; + u16 ring_to_svc_map; + int i, j; + + fw_config = get_fw_config(accel_dev); + if (!fw_config) + return 0; + + for (i = 0; i < RP_GROUP_COUNT; i++) { + switch (fw_config[i].ae_mask) { + case ADF_AE_GROUP_0: + j = RP_GROUP_0; + break; + case ADF_AE_GROUP_1: + j = RP_GROUP_1; + break; + default: + return 0; + } + + switch (fw_config[i].obj) { + case ADF_FW_SYM_OBJ: + rps[j] = SYM; + break; + case ADF_FW_ASYM_OBJ: + rps[j] = ASYM; + break; + case ADF_FW_DC_OBJ: + rps[j] = COMP; + break; + default: + rps[j] = 0; + break; + } + } + + ring_to_svc_map = rps[RP_GROUP_0] << ADF_CFG_SERV_RING_PAIR_0_SHIFT | + rps[RP_GROUP_1] << ADF_CFG_SERV_RING_PAIR_1_SHIFT | + rps[RP_GROUP_0] << ADF_CFG_SERV_RING_PAIR_2_SHIFT | + rps[RP_GROUP_1] << ADF_CFG_SERV_RING_PAIR_3_SHIFT; + + return ring_to_svc_map; +} + +static const char *uof_get_name(struct adf_accel_dev *accel_dev, u32 obj_num, + const char * const fw_objs[], int num_objs) +{ + const struct adf_fw_config *fw_config; + int id; + + fw_config = get_fw_config(accel_dev); + if (fw_config) + id = fw_config[obj_num].obj; + else + id = -EINVAL; + + if (id < 0 || id > num_objs) + return NULL; + + return fw_objs[id]; +} + +static const char *uof_get_name_420xx(struct adf_accel_dev *accel_dev, u32 obj_num) +{ + int num_fw_objs = ARRAY_SIZE(adf_420xx_fw_objs); + + return uof_get_name(accel_dev, obj_num, adf_420xx_fw_objs, num_fw_objs); +} + +static u32 uof_get_ae_mask(struct adf_accel_dev *accel_dev, u32 obj_num) +{ + const struct adf_fw_config *fw_config; + + fw_config = get_fw_config(accel_dev); + if (!fw_config) + return 0; + + return fw_config[obj_num].ae_mask; +} + +static void adf_gen4_set_err_mask(struct adf_dev_err_mask *dev_err_mask) +{ + dev_err_mask->cppagentcmdpar_mask = ADF_420XX_HICPPAGENTCMDPARERRLOG_MASK; + dev_err_mask->parerr_ath_cph_mask = ADF_420XX_PARITYERRORMASK_ATH_CPH_MASK; + dev_err_mask->parerr_cpr_xlt_mask = ADF_420XX_PARITYERRORMASK_CPR_XLT_MASK; + dev_err_mask->parerr_dcpr_ucs_mask = ADF_420XX_PARITYERRORMASK_DCPR_UCS_MASK; + dev_err_mask->parerr_pke_mask = ADF_420XX_PARITYERRORMASK_PKE_MASK; + dev_err_mask->ssmfeatren_mask = ADF_420XX_SSMFEATREN_MASK; +} + +void adf_init_hw_data_420xx(struct adf_hw_device_data *hw_data, u32 dev_id) +{ + hw_data->dev_class = &adf_420xx_class; + hw_data->instance_id = adf_420xx_class.instances++; + hw_data->num_banks = ADF_GEN4_ETR_MAX_BANKS; + hw_data->num_banks_per_vf = ADF_GEN4_NUM_BANKS_PER_VF; + hw_data->num_rings_per_bank = ADF_GEN4_NUM_RINGS_PER_BANK; + hw_data->num_accel = ADF_GEN4_MAX_ACCELERATORS; + hw_data->num_engines = ADF_420XX_MAX_ACCELENGINES; + hw_data->num_logical_accel = 1; + hw_data->tx_rx_gap = ADF_GEN4_RX_RINGS_OFFSET; + hw_data->tx_rings_mask = ADF_GEN4_TX_RINGS_MASK; + hw_data->ring_to_svc_map = ADF_GEN4_DEFAULT_RING_TO_SRV_MAP; + hw_data->alloc_irq = adf_isr_resource_alloc; + hw_data->free_irq = adf_isr_resource_free; + hw_data->enable_error_correction = adf_gen4_enable_error_correction; + hw_data->get_accel_mask = adf_gen4_get_accel_mask; + hw_data->get_ae_mask = get_ae_mask; + hw_data->get_num_accels = adf_gen4_get_num_accels; + hw_data->get_num_aes = adf_gen4_get_num_aes; + hw_data->get_sram_bar_id = adf_gen4_get_sram_bar_id; + hw_data->get_etr_bar_id = adf_gen4_get_etr_bar_id; + hw_data->get_misc_bar_id = adf_gen4_get_misc_bar_id; + hw_data->get_arb_info = adf_gen4_get_arb_info; + hw_data->get_admin_info = adf_gen4_get_admin_info; + hw_data->get_accel_cap = get_accel_cap; + hw_data->get_sku = adf_gen4_get_sku; + hw_data->init_admin_comms = adf_init_admin_comms; + hw_data->exit_admin_comms = adf_exit_admin_comms; + hw_data->send_admin_init = adf_send_admin_init; + hw_data->init_arb = adf_init_arb; + hw_data->exit_arb = adf_exit_arb; + hw_data->get_arb_mapping = adf_get_arbiter_mapping; + hw_data->enable_ints = adf_gen4_enable_ints; + hw_data->init_device = adf_gen4_init_device; + hw_data->reset_device = adf_reset_flr; + hw_data->admin_ae_mask = ADF_420XX_ADMIN_AE_MASK; + hw_data->num_rps = ADF_GEN4_MAX_RPS; + hw_data->fw_name = ADF_420XX_FW; + hw_data->fw_mmp_name = ADF_420XX_MMP; + hw_data->uof_get_name = uof_get_name_420xx; + hw_data->uof_get_num_objs = uof_get_num_objs; + hw_data->uof_get_ae_mask = uof_get_ae_mask; + hw_data->get_rp_group = get_rp_group; + hw_data->get_ena_thd_mask = get_ena_thd_mask; + hw_data->set_msix_rttable = adf_gen4_set_msix_default_rttable; + hw_data->set_ssm_wdtimer = adf_gen4_set_ssm_wdtimer; + hw_data->get_ring_to_svc_map = get_ring_to_svc_map; + hw_data->disable_iov = adf_disable_sriov; + hw_data->ring_pair_reset = adf_gen4_ring_pair_reset; + hw_data->enable_pm = adf_gen4_enable_pm; + hw_data->handle_pm_interrupt = adf_gen4_handle_pm_interrupt; + hw_data->dev_config = adf_gen4_dev_config; + hw_data->start_timer = adf_gen4_timer_start; + hw_data->stop_timer = adf_gen4_timer_stop; + hw_data->get_hb_clock = adf_gen4_get_heartbeat_clock; + hw_data->num_hb_ctrs = ADF_NUM_HB_CNT_PER_AE; + hw_data->clock_frequency = ADF_420XX_AE_FREQ; + + adf_gen4_set_err_mask(&hw_data->dev_err_mask); + adf_gen4_init_hw_csr_ops(&hw_data->csr_ops); + adf_gen4_init_pf_pfvf_ops(&hw_data->pfvf_ops); + adf_gen4_init_dc_ops(&hw_data->dc_ops); + adf_gen4_init_ras_ops(&hw_data->ras_ops); + adf_gen4_init_tl_data(&hw_data->tl_data); + adf_init_rl_data(&hw_data->rl_data); +} + +void adf_clean_hw_data_420xx(struct adf_hw_device_data *hw_data) +{ + hw_data->dev_class->instances--; +} diff --git a/drivers/crypto/intel/qat/qat_420xx/adf_420xx_hw_data.h b/drivers/crypto/intel/qat/qat_420xx/adf_420xx_hw_data.h new file mode 100644 index 000000000000..99abbfc14820 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_420xx/adf_420xx_hw_data.h @@ -0,0 +1,55 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2023 Intel Corporation */ +#ifndef ADF_420XX_HW_DATA_H_ +#define ADF_420XX_HW_DATA_H_ + +#include <adf_accel_devices.h> + +#define ADF_420XX_MAX_ACCELENGINES 17 + +#define ADF_420XX_ACCELENGINES_MASK 0x1FFFF +#define ADF_420XX_ADMIN_AE_MASK 0x10000 + +#define ADF_420XX_HICPPAGENTCMDPARERRLOG_MASK (0xFF) +#define ADF_420XX_PARITYERRORMASK_ATH_CPH_MASK (0xFF00FF) +#define ADF_420XX_PARITYERRORMASK_CPR_XLT_MASK (0x10001) +#define ADF_420XX_PARITYERRORMASK_DCPR_UCS_MASK (0xF0007) +#define ADF_420XX_PARITYERRORMASK_PKE_MASK (0xFFF) +#define ADF_420XX_PARITYERRORMASK_WAT_WCP_MASK (0x3FF03FF) + +/* + * SSMFEATREN bit mask + * BIT(4) - enables parity detection on CPP + * BIT(12) - enables the logging of push/pull data errors + * in pperr register + * BIT(16) - BIT(27) - enable parity detection on SPPs + */ +#define ADF_420XX_SSMFEATREN_MASK \ + (BIT(4) | BIT(12) | BIT(16) | BIT(17) | BIT(18) | BIT(19) | BIT(20) | \ + BIT(21) | BIT(22) | BIT(23) | BIT(24) | BIT(25) | BIT(26) | BIT(27)) + +/* Firmware Binaries */ +#define ADF_420XX_FW "qat_420xx.bin" +#define ADF_420XX_MMP "qat_420xx_mmp.bin" +#define ADF_420XX_SYM_OBJ "qat_420xx_sym.bin" +#define ADF_420XX_DC_OBJ "qat_420xx_dc.bin" +#define ADF_420XX_ASYM_OBJ "qat_420xx_asym.bin" +#define ADF_420XX_ADMIN_OBJ "qat_420xx_admin.bin" + +/* RL constants */ +#define ADF_420XX_RL_PCIE_SCALE_FACTOR_DIV 100 +#define ADF_420XX_RL_PCIE_SCALE_FACTOR_MUL 102 +#define ADF_420XX_RL_DCPR_CORRECTION 1 +#define ADF_420XX_RL_SCANS_PER_SEC 954 +#define ADF_420XX_RL_MAX_TP_ASYM 173750UL +#define ADF_420XX_RL_MAX_TP_SYM 95000UL +#define ADF_420XX_RL_MAX_TP_DC 40000UL +#define ADF_420XX_RL_SLICE_REF 1000UL + +/* Clocks frequency */ +#define ADF_420XX_AE_FREQ (1000 * HZ_PER_MHZ) + +void adf_init_hw_data_420xx(struct adf_hw_device_data *hw_data, u32 dev_id); +void adf_clean_hw_data_420xx(struct adf_hw_device_data *hw_data); + +#endif diff --git a/drivers/crypto/intel/qat/qat_420xx/adf_drv.c b/drivers/crypto/intel/qat/qat_420xx/adf_drv.c new file mode 100644 index 000000000000..2a3598409eeb --- /dev/null +++ b/drivers/crypto/intel/qat/qat_420xx/adf_drv.c @@ -0,0 +1,202 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2023 Intel Corporation */ +#include <linux/device.h> +#include <linux/module.h> +#include <linux/pci.h> + +#include <adf_accel_devices.h> +#include <adf_gen4_hw_data.h> +#include <adf_gen4_config.h> +#include <adf_cfg.h> +#include <adf_common_drv.h> +#include <adf_dbgfs.h> + +#include "adf_420xx_hw_data.h" + +static const struct pci_device_id adf_pci_tbl[] = { + { PCI_VDEVICE(INTEL, ADF_420XX_PCI_DEVICE_ID), }, + { } +}; +MODULE_DEVICE_TABLE(pci, adf_pci_tbl); + +static void adf_cleanup_accel(struct adf_accel_dev *accel_dev) +{ + if (accel_dev->hw_device) { + adf_clean_hw_data_420xx(accel_dev->hw_device); + accel_dev->hw_device = NULL; + } + adf_dbgfs_exit(accel_dev); + adf_cfg_dev_remove(accel_dev); + adf_devmgr_rm_dev(accel_dev, NULL); +} + +static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) +{ + struct adf_accel_dev *accel_dev; + struct adf_accel_pci *accel_pci_dev; + struct adf_hw_device_data *hw_data; + unsigned int i, bar_nr; + unsigned long bar_mask; + struct adf_bar *bar; + int ret; + + if (num_possible_nodes() > 1 && dev_to_node(&pdev->dev) < 0) { + /* + * If the accelerator is connected to a node with no memory + * there is no point in using the accelerator since the remote + * memory transaction will be very slow. + */ + dev_err(&pdev->dev, "Invalid NUMA configuration.\n"); + return -EINVAL; + } + + accel_dev = devm_kzalloc(&pdev->dev, sizeof(*accel_dev), GFP_KERNEL); + if (!accel_dev) + return -ENOMEM; + + INIT_LIST_HEAD(&accel_dev->crypto_list); + accel_pci_dev = &accel_dev->accel_pci_dev; + accel_pci_dev->pci_dev = pdev; + + /* + * Add accel device to accel table + * This should be called before adf_cleanup_accel is called + */ + if (adf_devmgr_add_dev(accel_dev, NULL)) { + dev_err(&pdev->dev, "Failed to add new accelerator device.\n"); + return -EFAULT; + } + + accel_dev->owner = THIS_MODULE; + /* Allocate and initialise device hardware meta-data structure */ + hw_data = devm_kzalloc(&pdev->dev, sizeof(*hw_data), GFP_KERNEL); + if (!hw_data) { + ret = -ENOMEM; + goto out_err; + } + + accel_dev->hw_device = hw_data; + adf_init_hw_data_420xx(accel_dev->hw_device, ent->device); + + pci_read_config_byte(pdev, PCI_REVISION_ID, &accel_pci_dev->revid); + pci_read_config_dword(pdev, ADF_GEN4_FUSECTL4_OFFSET, &hw_data->fuses); + + /* Get Accelerators and Accelerators Engines masks */ + hw_data->accel_mask = hw_data->get_accel_mask(hw_data); + hw_data->ae_mask = hw_data->get_ae_mask(hw_data); + accel_pci_dev->sku = hw_data->get_sku(hw_data); + /* If the device has no acceleration engines then ignore it */ + if (!hw_data->accel_mask || !hw_data->ae_mask || + (~hw_data->ae_mask & 0x01)) { + dev_err(&pdev->dev, "No acceleration units found.\n"); + ret = -EFAULT; + goto out_err; + } + + /* Create device configuration table */ + ret = adf_cfg_dev_add(accel_dev); + if (ret) + goto out_err; + + /* Enable PCI device */ + ret = pcim_enable_device(pdev); + if (ret) { + dev_err(&pdev->dev, "Can't enable PCI device.\n"); + goto out_err; + } + + /* Set DMA identifier */ + ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); + if (ret) { + dev_err(&pdev->dev, "No usable DMA configuration.\n"); + goto out_err; + } + + ret = adf_gen4_cfg_dev_init(accel_dev); + if (ret) { + dev_err(&pdev->dev, "Failed to initialize configuration.\n"); + goto out_err; + } + + /* Get accelerator capabilities mask */ + hw_data->accel_capabilities_mask = hw_data->get_accel_cap(accel_dev); + if (!hw_data->accel_capabilities_mask) { + dev_err(&pdev->dev, "Failed to get capabilities mask.\n"); + ret = -EINVAL; + goto out_err; + } + + /* Find and map all the device's BARS */ + bar_mask = pci_select_bars(pdev, IORESOURCE_MEM) & ADF_GEN4_BAR_MASK; + + ret = pcim_iomap_regions_request_all(pdev, bar_mask, pci_name(pdev)); + if (ret) { + dev_err(&pdev->dev, "Failed to map pci regions.\n"); + goto out_err; + } + + i = 0; + for_each_set_bit(bar_nr, &bar_mask, PCI_STD_NUM_BARS) { + bar = &accel_pci_dev->pci_bars[i++]; + bar->virt_addr = pcim_iomap_table(pdev)[bar_nr]; + } + + pci_set_master(pdev); + + if (pci_save_state(pdev)) { + dev_err(&pdev->dev, "Failed to save pci state.\n"); + ret = -ENOMEM; + goto out_err; + } + + accel_dev->ras_errors.enabled = true; + adf_dbgfs_init(accel_dev); + + ret = adf_dev_up(accel_dev, true); + if (ret) + goto out_err_dev_stop; + + ret = adf_sysfs_init(accel_dev); + if (ret) + goto out_err_dev_stop; + + return ret; + +out_err_dev_stop: + adf_dev_down(accel_dev, false); +out_err: + adf_cleanup_accel(accel_dev); + return ret; +} + +static void adf_remove(struct pci_dev *pdev) +{ + struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev); + + if (!accel_dev) { + pr_err("QAT: Driver removal failed\n"); + return; + } + adf_dev_down(accel_dev, false); + adf_cleanup_accel(accel_dev); +} + +static struct pci_driver adf_driver = { + .id_table = adf_pci_tbl, + .name = ADF_420XX_DEVICE_NAME, + .probe = adf_probe, + .remove = adf_remove, + .sriov_configure = adf_sriov_configure, + .err_handler = &adf_err_handler, +}; + +module_pci_driver(adf_driver); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Intel"); +MODULE_FIRMWARE(ADF_420XX_FW); +MODULE_FIRMWARE(ADF_420XX_MMP); +MODULE_DESCRIPTION("Intel(R) QuickAssist Technology"); +MODULE_VERSION(ADF_DRV_VERSION); +MODULE_SOFTDEP("pre: crypto-intel_qat"); +MODULE_IMPORT_NS(CRYPTO_QAT); diff --git a/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.c b/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.c index 0faedb5b2eb5..479062aa5e6b 100644 --- a/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.c +++ b/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.c @@ -7,12 +7,15 @@ #include <adf_cfg_services.h> #include <adf_clock.h> #include <adf_common_drv.h> +#include <adf_fw_config.h> +#include <adf_gen4_config.h> #include <adf_gen4_dc.h> #include <adf_gen4_hw_data.h> #include <adf_gen4_pfvf.h> #include <adf_gen4_pm.h> #include "adf_gen4_ras.h" #include <adf_gen4_timer.h> +#include <adf_gen4_tl.h> #include "adf_4xxx_hw_data.h" #include "icp_qat_hw.h" @@ -20,12 +23,10 @@ #define ADF_AE_GROUP_1 GENMASK(7, 4) #define ADF_AE_GROUP_2 BIT(8) -enum adf_fw_objs { - ADF_FW_SYM_OBJ, - ADF_FW_ASYM_OBJ, - ADF_FW_DC_OBJ, - ADF_FW_ADMIN_OBJ, -}; +#define ENA_THD_MASK_ASYM GENMASK(1, 0) +#define ENA_THD_MASK_ASYM_401XX GENMASK(5, 0) +#define ENA_THD_MASK_SYM GENMASK(6, 0) +#define ENA_THD_MASK_DC GENMASK(1, 0) static const char * const adf_4xxx_fw_objs[] = { [ADF_FW_SYM_OBJ] = ADF_4XXX_SYM_OBJ, @@ -41,11 +42,6 @@ static const char * const adf_402xx_fw_objs[] = { [ADF_FW_ADMIN_OBJ] = ADF_402XX_ADMIN_OBJ, }; -struct adf_fw_config { - u32 ae_mask; - enum adf_fw_objs obj; -}; - static const struct adf_fw_config adf_fw_cy_config[] = { {ADF_AE_GROUP_1, ADF_FW_SYM_OBJ}, {ADF_AE_GROUP_0, ADF_FW_ASYM_OBJ}, @@ -95,59 +91,12 @@ static_assert(ARRAY_SIZE(adf_fw_cy_config) == ARRAY_SIZE(adf_fw_asym_dc_config)) static_assert(ARRAY_SIZE(adf_fw_cy_config) == ARRAY_SIZE(adf_fw_sym_dc_config)); static_assert(ARRAY_SIZE(adf_fw_cy_config) == ARRAY_SIZE(adf_fw_dcc_config)); -/* Worker thread to service arbiter mappings */ -static const u32 default_thrd_to_arb_map[ADF_4XXX_MAX_ACCELENGINES] = { - 0x5555555, 0x5555555, 0x5555555, 0x5555555, - 0xAAAAAAA, 0xAAAAAAA, 0xAAAAAAA, 0xAAAAAAA, - 0x0 -}; - -static const u32 thrd_to_arb_map_dc[ADF_4XXX_MAX_ACCELENGINES] = { - 0x000000FF, 0x000000FF, 0x000000FF, 0x000000FF, - 0x000000FF, 0x000000FF, 0x000000FF, 0x000000FF, - 0x0 -}; - -static const u32 thrd_to_arb_map_dcc[ADF_4XXX_MAX_ACCELENGINES] = { - 0x00000000, 0x00000000, 0x00000000, 0x00000000, - 0x0000FFFF, 0x0000FFFF, 0x0000FFFF, 0x0000FFFF, - 0x0 -}; - static struct adf_hw_device_class adf_4xxx_class = { .name = ADF_4XXX_DEVICE_NAME, .type = DEV_4XXX, .instances = 0, }; -static int get_service_enabled(struct adf_accel_dev *accel_dev) -{ - char services[ADF_CFG_MAX_VAL_LEN_IN_BYTES] = {0}; - int ret; - - ret = adf_cfg_get_param_value(accel_dev, ADF_GENERAL_SEC, - ADF_SERVICES_ENABLED, services); - if (ret) { - dev_err(&GET_DEV(accel_dev), - ADF_SERVICES_ENABLED " param not found\n"); - return ret; - } - - ret = match_string(adf_cfg_services, ARRAY_SIZE(adf_cfg_services), - services); - if (ret < 0) - dev_err(&GET_DEV(accel_dev), - "Invalid value of " ADF_SERVICES_ENABLED " param: %s\n", - services); - - return ret; -} - -static u32 get_accel_mask(struct adf_hw_device_data *self) -{ - return ADF_4XXX_ACCELERATORS_MASK; -} - static u32 get_ae_mask(struct adf_hw_device_data *self) { u32 me_disable = self->fuses; @@ -155,55 +104,6 @@ static u32 get_ae_mask(struct adf_hw_device_data *self) return ~me_disable & ADF_4XXX_ACCELENGINES_MASK; } -static u32 get_num_accels(struct adf_hw_device_data *self) -{ - return ADF_4XXX_MAX_ACCELERATORS; -} - -static u32 get_num_aes(struct adf_hw_device_data *self) -{ - if (!self || !self->ae_mask) - return 0; - - return hweight32(self->ae_mask); -} - -static u32 get_misc_bar_id(struct adf_hw_device_data *self) -{ - return ADF_4XXX_PMISC_BAR; -} - -static u32 get_etr_bar_id(struct adf_hw_device_data *self) -{ - return ADF_4XXX_ETR_BAR; -} - -static u32 get_sram_bar_id(struct adf_hw_device_data *self) -{ - return ADF_4XXX_SRAM_BAR; -} - -/* - * The vector routing table is used to select the MSI-X entry to use for each - * interrupt source. - * The first ADF_4XXX_ETR_MAX_BANKS entries correspond to ring interrupts. - * The final entry corresponds to VF2PF or error interrupts. - * This vector table could be used to configure one MSI-X entry to be shared - * between multiple interrupt sources. - * - * The default routing is set to have a one to one correspondence between the - * interrupt source and the MSI-X entry used. - */ -static void set_msix_default_rttable(struct adf_accel_dev *accel_dev) -{ - void __iomem *csr; - int i; - - csr = (&GET_BARS(accel_dev)[ADF_4XXX_PMISC_BAR])->virt_addr; - for (i = 0; i <= ADF_4XXX_ETR_MAX_BANKS; i++) - ADF_CSR_WR(csr, ADF_4XXX_MSIX_RTTABLE_OFFSET(i), i); -} - static u32 get_accel_cap(struct adf_accel_dev *accel_dev) { struct pci_dev *pdev = accel_dev->accel_pci_dev.pci_dev; @@ -212,7 +112,7 @@ static u32 get_accel_cap(struct adf_accel_dev *accel_dev) u32 fusectl1; /* Read accelerator capabilities mask */ - pci_read_config_dword(pdev, ADF_4XXX_FUSECTL1_OFFSET, &fusectl1); + pci_read_config_dword(pdev, ADF_GEN4_FUSECTL1_OFFSET, &fusectl1); capabilities_sym = ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC | ICP_ACCEL_CAPABILITIES_CIPHER | @@ -227,27 +127,27 @@ static u32 get_accel_cap(struct adf_accel_dev *accel_dev) ICP_ACCEL_CAPABILITIES_AES_V2; /* A set bit in fusectl1 means the feature is OFF in this SKU */ - if (fusectl1 & ICP_ACCEL_4XXX_MASK_CIPHER_SLICE) { + if (fusectl1 & ICP_ACCEL_GEN4_MASK_CIPHER_SLICE) { capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC; capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_HKDF; capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_CIPHER; } - if (fusectl1 & ICP_ACCEL_4XXX_MASK_UCS_SLICE) { + if (fusectl1 & ICP_ACCEL_GEN4_MASK_UCS_SLICE) { capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_CHACHA_POLY; capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_AESGCM_SPC; capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_AES_V2; capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_CIPHER; } - if (fusectl1 & ICP_ACCEL_4XXX_MASK_AUTH_SLICE) { + if (fusectl1 & ICP_ACCEL_GEN4_MASK_AUTH_SLICE) { capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_AUTHENTICATION; capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_SHA3; capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_SHA3_EXT; capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_CIPHER; } - if (fusectl1 & ICP_ACCEL_4XXX_MASK_SMX_SLICE) { + if (fusectl1 & ICP_ACCEL_GEN4_MASK_SMX_SLICE) { capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_SM3; capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_SM4; } @@ -257,7 +157,7 @@ static u32 get_accel_cap(struct adf_accel_dev *accel_dev) ICP_ACCEL_CAPABILITIES_SM2 | ICP_ACCEL_CAPABILITIES_ECEDMONT; - if (fusectl1 & ICP_ACCEL_4XXX_MASK_PKE_SLICE) { + if (fusectl1 & ICP_ACCEL_GEN4_MASK_PKE_SLICE) { capabilities_asym &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC; capabilities_asym &= ~ICP_ACCEL_CAPABILITIES_SM2; capabilities_asym &= ~ICP_ACCEL_CAPABILITIES_ECEDMONT; @@ -268,14 +168,14 @@ static u32 get_accel_cap(struct adf_accel_dev *accel_dev) ICP_ACCEL_CAPABILITIES_LZ4S_COMPRESSION | ICP_ACCEL_CAPABILITIES_CNV_INTEGRITY64; - if (fusectl1 & ICP_ACCEL_4XXX_MASK_COMPRESS_SLICE) { + if (fusectl1 & ICP_ACCEL_GEN4_MASK_COMPRESS_SLICE) { capabilities_dc &= ~ICP_ACCEL_CAPABILITIES_COMPRESSION; capabilities_dc &= ~ICP_ACCEL_CAPABILITIES_LZ4_COMPRESSION; capabilities_dc &= ~ICP_ACCEL_CAPABILITIES_LZ4S_COMPRESSION; capabilities_dc &= ~ICP_ACCEL_CAPABILITIES_CNV_INTEGRITY64; } - switch (get_service_enabled(accel_dev)) { + switch (adf_get_service_enabled(accel_dev)) { case SVC_CY: case SVC_CY2: return capabilities_sym | capabilities_asym; @@ -304,43 +204,13 @@ static u32 get_accel_cap(struct adf_accel_dev *accel_dev) } } -static enum dev_sku_info get_sku(struct adf_hw_device_data *self) -{ - return DEV_SKU_1; -} - static const u32 *adf_get_arbiter_mapping(struct adf_accel_dev *accel_dev) { - switch (get_service_enabled(accel_dev)) { - case SVC_DC: - return thrd_to_arb_map_dc; - case SVC_DCC: - return thrd_to_arb_map_dcc; - default: - return default_thrd_to_arb_map; - } -} - -static void get_arb_info(struct arb_info *arb_info) -{ - arb_info->arb_cfg = ADF_4XXX_ARB_CONFIG; - arb_info->arb_offset = ADF_4XXX_ARB_OFFSET; - arb_info->wt2sam_offset = ADF_4XXX_ARB_WRK_2_SER_MAP_OFFSET; -} + if (adf_gen4_init_thd2arb_map(accel_dev)) + dev_warn(&GET_DEV(accel_dev), + "Generate of the thread to arbiter map failed"); -static void get_admin_info(struct admin_info *admin_csrs_info) -{ - admin_csrs_info->mailbox_offset = ADF_4XXX_MAILBOX_BASE_OFFSET; - admin_csrs_info->admin_msg_ur = ADF_4XXX_ADMINMSGUR_OFFSET; - admin_csrs_info->admin_msg_lr = ADF_4XXX_ADMINMSGLR_OFFSET; -} - -static u32 get_heartbeat_clock(struct adf_hw_device_data *self) -{ - /* - * 4XXX uses KPT counter for HB - */ - return ADF_4XXX_KPT_COUNTER_FREQ; + return GET_HW_DATA(accel_dev)->thd_to_arb_map; } static void adf_init_rl_data(struct adf_rl_hw_data *rl_data) @@ -361,66 +231,14 @@ static void adf_init_rl_data(struct adf_rl_hw_data *rl_data) rl_data->scale_ref = ADF_4XXX_RL_SLICE_REF; } -static void adf_enable_error_correction(struct adf_accel_dev *accel_dev) -{ - struct adf_bar *misc_bar = &GET_BARS(accel_dev)[ADF_4XXX_PMISC_BAR]; - void __iomem *csr = misc_bar->virt_addr; - - /* Enable all in errsou3 except VFLR notification on host */ - ADF_CSR_WR(csr, ADF_GEN4_ERRMSK3, ADF_GEN4_VFLNOTIFY); -} - -static void adf_enable_ints(struct adf_accel_dev *accel_dev) -{ - void __iomem *addr; - - addr = (&GET_BARS(accel_dev)[ADF_4XXX_PMISC_BAR])->virt_addr; - - /* Enable bundle interrupts */ - ADF_CSR_WR(addr, ADF_4XXX_SMIAPF_RP_X0_MASK_OFFSET, 0); - ADF_CSR_WR(addr, ADF_4XXX_SMIAPF_RP_X1_MASK_OFFSET, 0); - - /* Enable misc interrupts */ - ADF_CSR_WR(addr, ADF_4XXX_SMIAPF_MASK_OFFSET, 0); -} - -static int adf_init_device(struct adf_accel_dev *accel_dev) -{ - void __iomem *addr; - u32 status; - u32 csr; - int ret; - - addr = (&GET_BARS(accel_dev)[ADF_4XXX_PMISC_BAR])->virt_addr; - - /* Temporarily mask PM interrupt */ - csr = ADF_CSR_RD(addr, ADF_GEN4_ERRMSK2); - csr |= ADF_GEN4_PM_SOU; - ADF_CSR_WR(addr, ADF_GEN4_ERRMSK2, csr); - - /* Set DRV_ACTIVE bit to power up the device */ - ADF_CSR_WR(addr, ADF_GEN4_PM_INTERRUPT, ADF_GEN4_PM_DRV_ACTIVE); - - /* Poll status register to make sure the device is powered up */ - ret = read_poll_timeout(ADF_CSR_RD, status, - status & ADF_GEN4_PM_INIT_STATE, - ADF_GEN4_PM_POLL_DELAY_US, - ADF_GEN4_PM_POLL_TIMEOUT_US, true, addr, - ADF_GEN4_PM_STATUS); - if (ret) - dev_err(&GET_DEV(accel_dev), "Failed to power up the device\n"); - - return ret; -} - -static u32 uof_get_num_objs(void) +static u32 uof_get_num_objs(struct adf_accel_dev *accel_dev) { return ARRAY_SIZE(adf_fw_cy_config); } static const struct adf_fw_config *get_fw_config(struct adf_accel_dev *accel_dev) { - switch (get_service_enabled(accel_dev)) { + switch (adf_get_service_enabled(accel_dev)) { case SVC_CY: case SVC_CY2: return adf_fw_cy_config; @@ -443,11 +261,64 @@ static const struct adf_fw_config *get_fw_config(struct adf_accel_dev *accel_dev } } -enum adf_rp_groups { - RP_GROUP_0 = 0, - RP_GROUP_1, - RP_GROUP_COUNT -}; +static int get_rp_group(struct adf_accel_dev *accel_dev, u32 ae_mask) +{ + switch (ae_mask) { + case ADF_AE_GROUP_0: + return RP_GROUP_0; + case ADF_AE_GROUP_1: + return RP_GROUP_1; + default: + dev_dbg(&GET_DEV(accel_dev), "ae_mask not recognized"); + return -EINVAL; + } +} + +static u32 get_ena_thd_mask(struct adf_accel_dev *accel_dev, u32 obj_num) +{ + const struct adf_fw_config *fw_config; + + if (obj_num >= uof_get_num_objs(accel_dev)) + return ADF_GEN4_ENA_THD_MASK_ERROR; + + fw_config = get_fw_config(accel_dev); + if (!fw_config) + return ADF_GEN4_ENA_THD_MASK_ERROR; + + switch (fw_config[obj_num].obj) { + case ADF_FW_ASYM_OBJ: + return ENA_THD_MASK_ASYM; + case ADF_FW_SYM_OBJ: + return ENA_THD_MASK_SYM; + case ADF_FW_DC_OBJ: + return ENA_THD_MASK_DC; + default: + return ADF_GEN4_ENA_THD_MASK_ERROR; + } +} + +static u32 get_ena_thd_mask_401xx(struct adf_accel_dev *accel_dev, u32 obj_num) +{ + const struct adf_fw_config *fw_config; + + if (obj_num >= uof_get_num_objs(accel_dev)) + return ADF_GEN4_ENA_THD_MASK_ERROR; + + fw_config = get_fw_config(accel_dev); + if (!fw_config) + return ADF_GEN4_ENA_THD_MASK_ERROR; + + switch (fw_config[obj_num].obj) { + case ADF_FW_ASYM_OBJ: + return ENA_THD_MASK_ASYM_401XX; + case ADF_FW_SYM_OBJ: + return ENA_THD_MASK_SYM; + case ADF_FW_DC_OBJ: + return ENA_THD_MASK_DC; + default: + return ADF_GEN4_ENA_THD_MASK_ERROR; + } +} static u16 get_ring_to_svc_map(struct adf_accel_dev *accel_dev) { @@ -553,54 +424,63 @@ void adf_init_hw_data_4xxx(struct adf_hw_device_data *hw_data, u32 dev_id) { hw_data->dev_class = &adf_4xxx_class; hw_data->instance_id = adf_4xxx_class.instances++; - hw_data->num_banks = ADF_4XXX_ETR_MAX_BANKS; - hw_data->num_banks_per_vf = ADF_4XXX_NUM_BANKS_PER_VF; - hw_data->num_rings_per_bank = ADF_4XXX_NUM_RINGS_PER_BANK; - hw_data->num_accel = ADF_4XXX_MAX_ACCELERATORS; + hw_data->num_banks = ADF_GEN4_ETR_MAX_BANKS; + hw_data->num_banks_per_vf = ADF_GEN4_NUM_BANKS_PER_VF; + hw_data->num_rings_per_bank = ADF_GEN4_NUM_RINGS_PER_BANK; + hw_data->num_accel = ADF_GEN4_MAX_ACCELERATORS; hw_data->num_engines = ADF_4XXX_MAX_ACCELENGINES; hw_data->num_logical_accel = 1; - hw_data->tx_rx_gap = ADF_4XXX_RX_RINGS_OFFSET; - hw_data->tx_rings_mask = ADF_4XXX_TX_RINGS_MASK; + hw_data->tx_rx_gap = ADF_GEN4_RX_RINGS_OFFSET; + hw_data->tx_rings_mask = ADF_GEN4_TX_RINGS_MASK; hw_data->ring_to_svc_map = ADF_GEN4_DEFAULT_RING_TO_SRV_MAP; hw_data->alloc_irq = adf_isr_resource_alloc; hw_data->free_irq = adf_isr_resource_free; - hw_data->enable_error_correction = adf_enable_error_correction; - hw_data->get_accel_mask = get_accel_mask; + hw_data->enable_error_correction = adf_gen4_enable_error_correction; + hw_data->get_accel_mask = adf_gen4_get_accel_mask; hw_data->get_ae_mask = get_ae_mask; - hw_data->get_num_accels = get_num_accels; - hw_data->get_num_aes = get_num_aes; - hw_data->get_sram_bar_id = get_sram_bar_id; - hw_data->get_etr_bar_id = get_etr_bar_id; - hw_data->get_misc_bar_id = get_misc_bar_id; - hw_data->get_arb_info = get_arb_info; - hw_data->get_admin_info = get_admin_info; + hw_data->get_num_accels = adf_gen4_get_num_accels; + hw_data->get_num_aes = adf_gen4_get_num_aes; + hw_data->get_sram_bar_id = adf_gen4_get_sram_bar_id; + hw_data->get_etr_bar_id = adf_gen4_get_etr_bar_id; + hw_data->get_misc_bar_id = adf_gen4_get_misc_bar_id; + hw_data->get_arb_info = adf_gen4_get_arb_info; + hw_data->get_admin_info = adf_gen4_get_admin_info; hw_data->get_accel_cap = get_accel_cap; - hw_data->get_sku = get_sku; + hw_data->get_sku = adf_gen4_get_sku; hw_data->init_admin_comms = adf_init_admin_comms; hw_data->exit_admin_comms = adf_exit_admin_comms; hw_data->send_admin_init = adf_send_admin_init; hw_data->init_arb = adf_init_arb; hw_data->exit_arb = adf_exit_arb; hw_data->get_arb_mapping = adf_get_arbiter_mapping; - hw_data->enable_ints = adf_enable_ints; - hw_data->init_device = adf_init_device; + hw_data->enable_ints = adf_gen4_enable_ints; + hw_data->init_device = adf_gen4_init_device; hw_data->reset_device = adf_reset_flr; hw_data->admin_ae_mask = ADF_4XXX_ADMIN_AE_MASK; + hw_data->num_rps = ADF_GEN4_MAX_RPS; switch (dev_id) { case ADF_402XX_PCI_DEVICE_ID: hw_data->fw_name = ADF_402XX_FW; hw_data->fw_mmp_name = ADF_402XX_MMP; hw_data->uof_get_name = uof_get_name_402xx; break; - + case ADF_401XX_PCI_DEVICE_ID: + hw_data->fw_name = ADF_4XXX_FW; + hw_data->fw_mmp_name = ADF_4XXX_MMP; + hw_data->uof_get_name = uof_get_name_4xxx; + hw_data->get_ena_thd_mask = get_ena_thd_mask_401xx; + break; default: hw_data->fw_name = ADF_4XXX_FW; hw_data->fw_mmp_name = ADF_4XXX_MMP; hw_data->uof_get_name = uof_get_name_4xxx; + hw_data->get_ena_thd_mask = get_ena_thd_mask; + break; } hw_data->uof_get_num_objs = uof_get_num_objs; hw_data->uof_get_ae_mask = uof_get_ae_mask; - hw_data->set_msix_rttable = set_msix_default_rttable; + hw_data->get_rp_group = get_rp_group; + hw_data->set_msix_rttable = adf_gen4_set_msix_default_rttable; hw_data->set_ssm_wdtimer = adf_gen4_set_ssm_wdtimer; hw_data->get_ring_to_svc_map = get_ring_to_svc_map; hw_data->disable_iov = adf_disable_sriov; @@ -610,7 +490,7 @@ void adf_init_hw_data_4xxx(struct adf_hw_device_data *hw_data, u32 dev_id) hw_data->dev_config = adf_gen4_dev_config; hw_data->start_timer = adf_gen4_timer_start; hw_data->stop_timer = adf_gen4_timer_stop; - hw_data->get_hb_clock = get_heartbeat_clock; + hw_data->get_hb_clock = adf_gen4_get_heartbeat_clock; hw_data->num_hb_ctrs = ADF_NUM_HB_CNT_PER_AE; hw_data->clock_frequency = ADF_4XXX_AE_FREQ; @@ -619,6 +499,7 @@ void adf_init_hw_data_4xxx(struct adf_hw_device_data *hw_data, u32 dev_id) adf_gen4_init_pf_pfvf_ops(&hw_data->pfvf_ops); adf_gen4_init_dc_ops(&hw_data->dc_ops); adf_gen4_init_ras_ops(&hw_data->ras_ops); + adf_gen4_init_tl_data(&hw_data->tl_data); adf_init_rl_data(&hw_data->rl_data); } diff --git a/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.h b/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.h index 33423295e90f..76388363ea87 100644 --- a/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.h +++ b/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.h @@ -6,25 +6,8 @@ #include <linux/units.h> #include <adf_accel_devices.h> -/* PCIe configuration space */ -#define ADF_4XXX_SRAM_BAR 0 -#define ADF_4XXX_PMISC_BAR 1 -#define ADF_4XXX_ETR_BAR 2 -#define ADF_4XXX_RX_RINGS_OFFSET 1 -#define ADF_4XXX_TX_RINGS_MASK 0x1 -#define ADF_4XXX_MAX_ACCELERATORS 1 #define ADF_4XXX_MAX_ACCELENGINES 9 -#define ADF_4XXX_BAR_MASK (BIT(0) | BIT(2) | BIT(4)) -/* Physical function fuses */ -#define ADF_4XXX_FUSECTL0_OFFSET (0x2C8) -#define ADF_4XXX_FUSECTL1_OFFSET (0x2CC) -#define ADF_4XXX_FUSECTL2_OFFSET (0x2D0) -#define ADF_4XXX_FUSECTL3_OFFSET (0x2D4) -#define ADF_4XXX_FUSECTL4_OFFSET (0x2D8) -#define ADF_4XXX_FUSECTL5_OFFSET (0x2DC) - -#define ADF_4XXX_ACCELERATORS_MASK (0x1) #define ADF_4XXX_ACCELENGINES_MASK (0x1FF) #define ADF_4XXX_ADMIN_AE_MASK (0x100) @@ -45,28 +28,6 @@ (BIT(4) | BIT(12) | BIT(16) | BIT(17) | BIT(18) | \ BIT(19) | BIT(20) | BIT(21) | BIT(22) | BIT(23)) -#define ADF_4XXX_ETR_MAX_BANKS 64 - -/* MSIX interrupt */ -#define ADF_4XXX_SMIAPF_RP_X0_MASK_OFFSET (0x41A040) -#define ADF_4XXX_SMIAPF_RP_X1_MASK_OFFSET (0x41A044) -#define ADF_4XXX_SMIAPF_MASK_OFFSET (0x41A084) -#define ADF_4XXX_MSIX_RTTABLE_OFFSET(i) (0x409000 + ((i) * 0x04)) - -/* Bank and ring configuration */ -#define ADF_4XXX_NUM_RINGS_PER_BANK 2 -#define ADF_4XXX_NUM_BANKS_PER_VF 4 - -/* Arbiter configuration */ -#define ADF_4XXX_ARB_CONFIG (BIT(31) | BIT(6) | BIT(0)) -#define ADF_4XXX_ARB_OFFSET (0x0) -#define ADF_4XXX_ARB_WRK_2_SER_MAP_OFFSET (0x400) - -/* Admin Interface Reg Offset */ -#define ADF_4XXX_ADMINMSGUR_OFFSET (0x500574) -#define ADF_4XXX_ADMINMSGLR_OFFSET (0x500578) -#define ADF_4XXX_MAILBOX_BASE_OFFSET (0x600970) - /* Firmware Binaries */ #define ADF_4XXX_FW "qat_4xxx.bin" #define ADF_4XXX_MMP "qat_4xxx_mmp.bin" @@ -93,22 +54,9 @@ #define ADF_4XXX_RL_SLICE_REF 1000UL /* Clocks frequency */ -#define ADF_4XXX_KPT_COUNTER_FREQ (100 * HZ_PER_MHZ) #define ADF_4XXX_AE_FREQ (1000 * HZ_PER_MHZ) -/* qat_4xxx fuse bits are different from old GENs, redefine them */ -enum icp_qat_4xxx_slice_mask { - ICP_ACCEL_4XXX_MASK_CIPHER_SLICE = BIT(0), - ICP_ACCEL_4XXX_MASK_AUTH_SLICE = BIT(1), - ICP_ACCEL_4XXX_MASK_PKE_SLICE = BIT(2), - ICP_ACCEL_4XXX_MASK_COMPRESS_SLICE = BIT(3), - ICP_ACCEL_4XXX_MASK_UCS_SLICE = BIT(4), - ICP_ACCEL_4XXX_MASK_EIA3_SLICE = BIT(5), - ICP_ACCEL_4XXX_MASK_SMX_SLICE = BIT(7), -}; - void adf_init_hw_data_4xxx(struct adf_hw_device_data *hw_data, u32 dev_id); void adf_clean_hw_data_4xxx(struct adf_hw_device_data *hw_data); -int adf_gen4_dev_config(struct adf_accel_dev *accel_dev); #endif diff --git a/drivers/crypto/intel/qat/qat_4xxx/adf_drv.c b/drivers/crypto/intel/qat/qat_4xxx/adf_drv.c index 8f483d1197dd..9762f2bf7727 100644 --- a/drivers/crypto/intel/qat/qat_4xxx/adf_drv.c +++ b/drivers/crypto/intel/qat/qat_4xxx/adf_drv.c @@ -8,13 +8,10 @@ #include <adf_cfg.h> #include <adf_common_drv.h> #include <adf_dbgfs.h> -#include <adf_heartbeat.h> +#include <adf_gen4_config.h> +#include <adf_gen4_hw_data.h> #include "adf_4xxx_hw_data.h" -#include "adf_cfg_services.h" -#include "qat_compression.h" -#include "qat_crypto.h" -#include "adf_transport_access_macros.h" static const struct pci_device_id adf_pci_tbl[] = { { PCI_VDEVICE(INTEL, ADF_4XXX_PCI_DEVICE_ID), }, @@ -35,270 +32,6 @@ static void adf_cleanup_accel(struct adf_accel_dev *accel_dev) adf_devmgr_rm_dev(accel_dev, NULL); } -static int adf_cfg_dev_init(struct adf_accel_dev *accel_dev) -{ - const char *config; - int ret; - - config = accel_dev->accel_id % 2 ? ADF_CFG_DC : ADF_CFG_CY; - - ret = adf_cfg_section_add(accel_dev, ADF_GENERAL_SEC); - if (ret) - return ret; - - /* Default configuration is crypto only for even devices - * and compression for odd devices - */ - ret = adf_cfg_add_key_value_param(accel_dev, ADF_GENERAL_SEC, - ADF_SERVICES_ENABLED, config, - ADF_STR); - if (ret) - return ret; - - adf_heartbeat_save_cfg_param(accel_dev, ADF_CFG_HB_TIMER_MIN_MS); - - return 0; -} - -static int adf_crypto_dev_config(struct adf_accel_dev *accel_dev) -{ - char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES]; - int banks = GET_MAX_BANKS(accel_dev); - int cpus = num_online_cpus(); - unsigned long bank, val; - int instances; - int ret; - int i; - - if (adf_hw_dev_has_crypto(accel_dev)) - instances = min(cpus, banks / 2); - else - instances = 0; - - for (i = 0; i < instances; i++) { - val = i; - bank = i * 2; - snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_BANK_NUM, i); - ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, &bank, ADF_DEC); - if (ret) - goto err; - - bank += 1; - snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_BANK_NUM, i); - ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, &bank, ADF_DEC); - if (ret) - goto err; - - snprintf(key, sizeof(key), ADF_CY "%d" ADF_ETRMGR_CORE_AFFINITY, - i); - ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, &val, ADF_DEC); - if (ret) - goto err; - - snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_SIZE, i); - val = 128; - ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, &val, ADF_DEC); - if (ret) - goto err; - - val = 512; - snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_SIZE, i); - ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, &val, ADF_DEC); - if (ret) - goto err; - - val = 0; - snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_TX, i); - ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, &val, ADF_DEC); - if (ret) - goto err; - - val = 0; - snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_TX, i); - ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, &val, ADF_DEC); - if (ret) - goto err; - - val = 1; - snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_RX, i); - ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, &val, ADF_DEC); - if (ret) - goto err; - - val = 1; - snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_RX, i); - ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, &val, ADF_DEC); - if (ret) - goto err; - - val = ADF_COALESCING_DEF_TIME; - snprintf(key, sizeof(key), ADF_ETRMGR_COALESCE_TIMER_FORMAT, i); - ret = adf_cfg_add_key_value_param(accel_dev, "Accelerator0", - key, &val, ADF_DEC); - if (ret) - goto err; - } - - val = i; - ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, ADF_NUM_CY, - &val, ADF_DEC); - if (ret) - goto err; - - val = 0; - ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, ADF_NUM_DC, - &val, ADF_DEC); - if (ret) - goto err; - - return 0; -err: - dev_err(&GET_DEV(accel_dev), "Failed to add configuration for crypto\n"); - return ret; -} - -static int adf_comp_dev_config(struct adf_accel_dev *accel_dev) -{ - char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES]; - int banks = GET_MAX_BANKS(accel_dev); - int cpus = num_online_cpus(); - unsigned long val; - int instances; - int ret; - int i; - - if (adf_hw_dev_has_compression(accel_dev)) - instances = min(cpus, banks); - else - instances = 0; - - for (i = 0; i < instances; i++) { - val = i; - snprintf(key, sizeof(key), ADF_DC "%d" ADF_RING_DC_BANK_NUM, i); - ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, &val, ADF_DEC); - if (ret) - goto err; - - val = 512; - snprintf(key, sizeof(key), ADF_DC "%d" ADF_RING_DC_SIZE, i); - ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, &val, ADF_DEC); - if (ret) - goto err; - - val = 0; - snprintf(key, sizeof(key), ADF_DC "%d" ADF_RING_DC_TX, i); - ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, &val, ADF_DEC); - if (ret) - goto err; - - val = 1; - snprintf(key, sizeof(key), ADF_DC "%d" ADF_RING_DC_RX, i); - ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, &val, ADF_DEC); - if (ret) - goto err; - - val = ADF_COALESCING_DEF_TIME; - snprintf(key, sizeof(key), ADF_ETRMGR_COALESCE_TIMER_FORMAT, i); - ret = adf_cfg_add_key_value_param(accel_dev, "Accelerator0", - key, &val, ADF_DEC); - if (ret) - goto err; - } - - val = i; - ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, ADF_NUM_DC, - &val, ADF_DEC); - if (ret) - goto err; - - val = 0; - ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, ADF_NUM_CY, - &val, ADF_DEC); - if (ret) - goto err; - - return 0; -err: - dev_err(&GET_DEV(accel_dev), "Failed to add configuration for compression\n"); - return ret; -} - -static int adf_no_dev_config(struct adf_accel_dev *accel_dev) -{ - unsigned long val; - int ret; - - val = 0; - ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, ADF_NUM_DC, - &val, ADF_DEC); - if (ret) - return ret; - - return adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, ADF_NUM_CY, - &val, ADF_DEC); -} - -int adf_gen4_dev_config(struct adf_accel_dev *accel_dev) -{ - char services[ADF_CFG_MAX_VAL_LEN_IN_BYTES] = {0}; - int ret; - - ret = adf_cfg_section_add(accel_dev, ADF_KERNEL_SEC); - if (ret) - goto err; - - ret = adf_cfg_section_add(accel_dev, "Accelerator0"); - if (ret) - goto err; - - ret = adf_cfg_get_param_value(accel_dev, ADF_GENERAL_SEC, - ADF_SERVICES_ENABLED, services); - if (ret) - goto err; - - ret = sysfs_match_string(adf_cfg_services, services); - if (ret < 0) - goto err; - - switch (ret) { - case SVC_CY: - case SVC_CY2: - ret = adf_crypto_dev_config(accel_dev); - break; - case SVC_DC: - case SVC_DCC: - ret = adf_comp_dev_config(accel_dev); - break; - default: - ret = adf_no_dev_config(accel_dev); - break; - } - - if (ret) - goto err; - - set_bit(ADF_STATUS_CONFIGURED, &accel_dev->status); - - return ret; - -err: - dev_err(&GET_DEV(accel_dev), "Failed to configure QAT driver\n"); - return ret; -} - static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) { struct adf_accel_dev *accel_dev; @@ -348,7 +81,7 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) adf_init_hw_data_4xxx(accel_dev->hw_device, ent->device); pci_read_config_byte(pdev, PCI_REVISION_ID, &accel_pci_dev->revid); - pci_read_config_dword(pdev, ADF_4XXX_FUSECTL4_OFFSET, &hw_data->fuses); + pci_read_config_dword(pdev, ADF_GEN4_FUSECTL4_OFFSET, &hw_data->fuses); /* Get Accelerators and Accelerators Engines masks */ hw_data->accel_mask = hw_data->get_accel_mask(hw_data); @@ -381,7 +114,7 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) goto out_err; } - ret = adf_cfg_dev_init(accel_dev); + ret = adf_gen4_cfg_dev_init(accel_dev); if (ret) { dev_err(&pdev->dev, "Failed to initialize configuration.\n"); goto out_err; @@ -396,7 +129,7 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) } /* Find and map all the device's BARS */ - bar_mask = pci_select_bars(pdev, IORESOURCE_MEM) & ADF_4XXX_BAR_MASK; + bar_mask = pci_select_bars(pdev, IORESOURCE_MEM) & ADF_GEN4_BAR_MASK; ret = pcim_iomap_regions_request_all(pdev, bar_mask, pci_name(pdev)); if (ret) { diff --git a/drivers/crypto/intel/qat/qat_common/Makefile b/drivers/crypto/intel/qat/qat_common/Makefile index 779a8aa0b8d2..6908727bff3b 100644 --- a/drivers/crypto/intel/qat/qat_common/Makefile +++ b/drivers/crypto/intel/qat/qat_common/Makefile @@ -16,6 +16,7 @@ intel_qat-objs := adf_cfg.o \ adf_sysfs_ras_counters.o \ adf_gen2_hw_data.o \ adf_gen2_config.o \ + adf_gen4_config.o \ adf_gen4_hw_data.o \ adf_gen4_pm.o \ adf_gen2_dc.o \ @@ -40,9 +41,12 @@ intel_qat-$(CONFIG_DEBUG_FS) += adf_transport_debug.o \ adf_fw_counters.o \ adf_cnv_dbgfs.o \ adf_gen4_pm_debugfs.o \ + adf_gen4_tl.o \ adf_heartbeat.o \ adf_heartbeat_dbgfs.o \ adf_pm_dbgfs.o \ + adf_telemetry.o \ + adf_tl_debugfs.o \ adf_dbgfs.o intel_qat-$(CONFIG_PCI_IOV) += adf_sriov.o adf_vf_isr.o adf_pfvf_utils.o \ diff --git a/drivers/crypto/intel/qat/qat_common/adf_accel_devices.h b/drivers/crypto/intel/qat/qat_common/adf_accel_devices.h index 4ff5729a3496..a16c7e6edc65 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/intel/qat/qat_common/adf_accel_devices.h @@ -6,11 +6,14 @@ #include <linux/module.h> #include <linux/list.h> #include <linux/io.h> +#include <linux/pci.h> #include <linux/ratelimit.h> #include <linux/types.h> #include "adf_cfg_common.h" #include "adf_rl.h" +#include "adf_telemetry.h" #include "adf_pfvf_msg.h" +#include "icp_qat_hw.h" #define ADF_DH895XCC_DEVICE_NAME "dh895xcc" #define ADF_DH895XCCVF_DEVICE_NAME "dh895xccvf" @@ -19,12 +22,15 @@ #define ADF_C3XXX_DEVICE_NAME "c3xxx" #define ADF_C3XXXVF_DEVICE_NAME "c3xxxvf" #define ADF_4XXX_DEVICE_NAME "4xxx" +#define ADF_420XX_DEVICE_NAME "420xx" #define ADF_4XXX_PCI_DEVICE_ID 0x4940 #define ADF_4XXXIOV_PCI_DEVICE_ID 0x4941 #define ADF_401XX_PCI_DEVICE_ID 0x4942 #define ADF_401XXIOV_PCI_DEVICE_ID 0x4943 #define ADF_402XX_PCI_DEVICE_ID 0x4944 #define ADF_402XXIOV_PCI_DEVICE_ID 0x4945 +#define ADF_420XX_PCI_DEVICE_ID 0x4946 +#define ADF_420XXIOV_PCI_DEVICE_ID 0x4947 #define ADF_DEVICE_FUSECTL_OFFSET 0x40 #define ADF_DEVICE_LEGFUSE_OFFSET 0x4C #define ADF_DEVICE_FUSECTL_MASK 0x80000000 @@ -92,6 +98,7 @@ enum ras_errors { struct adf_error_counters { atomic_t counter[ADF_RAS_ERRORS]; + bool sysfs_added; bool enabled; }; @@ -240,8 +247,10 @@ struct adf_hw_device_data { void (*reset_device)(struct adf_accel_dev *accel_dev); void (*set_msix_rttable)(struct adf_accel_dev *accel_dev); const char *(*uof_get_name)(struct adf_accel_dev *accel_dev, u32 obj_num); - u32 (*uof_get_num_objs)(void); + u32 (*uof_get_num_objs)(struct adf_accel_dev *accel_dev); u32 (*uof_get_ae_mask)(struct adf_accel_dev *accel_dev, u32 obj_num); + int (*get_rp_group)(struct adf_accel_dev *accel_dev, u32 ae_mask); + u32 (*get_ena_thd_mask)(struct adf_accel_dev *accel_dev, u32 obj_num); int (*dev_config)(struct adf_accel_dev *accel_dev); struct adf_pfvf_ops pfvf_ops; struct adf_hw_csr_ops csr_ops; @@ -249,6 +258,7 @@ struct adf_hw_device_data { struct adf_ras_ops ras_ops; struct adf_dev_err_mask dev_err_mask; struct adf_rl_hw_data rl_data; + struct adf_tl_hw_data tl_data; const char *fw_name; const char *fw_mmp_name; u32 fuses; @@ -263,6 +273,7 @@ struct adf_hw_device_data { u32 admin_ae_mask; u16 tx_rings_mask; u16 ring_to_svc_map; + u32 thd_to_arb_map[ICP_QAT_HW_AE_DELIMITER]; u8 tx_rx_gap; u8 num_banks; u16 num_banks_per_vf; @@ -271,6 +282,7 @@ struct adf_hw_device_data { u8 num_logical_accel; u8 num_engines; u32 num_hb_ctrs; + u8 num_rps; }; /* CSR write macro */ @@ -303,6 +315,7 @@ struct adf_hw_device_data { #define GET_CSR_OPS(accel_dev) (&(accel_dev)->hw_device->csr_ops) #define GET_PFVF_OPS(accel_dev) (&(accel_dev)->hw_device->pfvf_ops) #define GET_DC_OPS(accel_dev) (&(accel_dev)->hw_device->dc_ops) +#define GET_TL_DATA(accel_dev) GET_HW_DATA(accel_dev)->tl_data #define accel_to_pci_dev(accel_ptr) accel_ptr->accel_pci_dev.pci_dev struct adf_admin_comms; @@ -351,6 +364,7 @@ struct adf_accel_dev { struct adf_cfg_device_data *cfg; struct adf_fw_loader_data *fw_loader; struct adf_admin_comms *admin; + struct adf_telemetry *telemetry; struct adf_dc_data *dc_data; struct adf_pm power_management; struct list_head crypto_list; diff --git a/drivers/crypto/intel/qat/qat_common/adf_accel_engine.c b/drivers/crypto/intel/qat/qat_common/adf_accel_engine.c index 6be064dc64c8..4b5d0350fc2e 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_accel_engine.c +++ b/drivers/crypto/intel/qat/qat_common/adf_accel_engine.c @@ -19,7 +19,7 @@ static int adf_ae_fw_load_images(struct adf_accel_dev *accel_dev, void *fw_addr, int i; loader = loader_data->fw_loader; - num_objs = hw_device->uof_get_num_objs(); + num_objs = hw_device->uof_get_num_objs(accel_dev); for (i = 0; i < num_objs; i++) { obj_name = hw_device->uof_get_name(accel_dev, i); diff --git a/drivers/crypto/intel/qat/qat_common/adf_admin.c b/drivers/crypto/intel/qat/qat_common/adf_admin.c index 54b673ec2362..acad526eb741 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_admin.c +++ b/drivers/crypto/intel/qat/qat_common/adf_admin.c @@ -498,6 +498,43 @@ int adf_get_cnv_stats(struct adf_accel_dev *accel_dev, u16 ae, u16 *err_cnt, return ret; } +int adf_send_admin_tl_start(struct adf_accel_dev *accel_dev, + dma_addr_t tl_dma_addr, size_t layout_sz, u8 *rp_indexes, + struct icp_qat_fw_init_admin_slice_cnt *slice_count) +{ + u32 ae_mask = GET_HW_DATA(accel_dev)->admin_ae_mask; + struct icp_qat_fw_init_admin_resp resp = { }; + struct icp_qat_fw_init_admin_req req = { }; + int ret; + + req.cmd_id = ICP_QAT_FW_TL_START; + req.init_cfg_ptr = tl_dma_addr; + req.init_cfg_sz = layout_sz; + + if (rp_indexes) + memcpy(&req.rp_indexes, rp_indexes, sizeof(req.rp_indexes)); + + ret = adf_send_admin(accel_dev, &req, &resp, ae_mask); + if (ret) + return ret; + + memcpy(slice_count, &resp.slices, sizeof(*slice_count)); + + return 0; +} + +int adf_send_admin_tl_stop(struct adf_accel_dev *accel_dev) +{ + struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev); + struct icp_qat_fw_init_admin_resp resp = { }; + struct icp_qat_fw_init_admin_req req = { }; + u32 ae_mask = hw_data->admin_ae_mask; + + req.cmd_id = ICP_QAT_FW_TL_STOP; + + return adf_send_admin(accel_dev, &req, &resp, ae_mask); +} + int adf_init_admin_comms(struct adf_accel_dev *accel_dev) { struct adf_admin_comms *admin; diff --git a/drivers/crypto/intel/qat/qat_common/adf_admin.h b/drivers/crypto/intel/qat/qat_common/adf_admin.h index 55cbcbc66c9f..647c8e196752 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_admin.h +++ b/drivers/crypto/intel/qat/qat_common/adf_admin.h @@ -23,5 +23,9 @@ int adf_send_admin_rl_delete(struct adf_accel_dev *accel_dev, u16 node_id, int adf_get_fw_timestamp(struct adf_accel_dev *accel_dev, u64 *timestamp); int adf_get_pm_info(struct adf_accel_dev *accel_dev, dma_addr_t p_state_addr, size_t buff_size); int adf_get_cnv_stats(struct adf_accel_dev *accel_dev, u16 ae, u16 *err_cnt, u16 *latest_err); +int adf_send_admin_tl_start(struct adf_accel_dev *accel_dev, + dma_addr_t tl_dma_addr, size_t layout_sz, u8 *rp_indexes, + struct icp_qat_fw_init_admin_slice_cnt *slice_count); +int adf_send_admin_tl_stop(struct adf_accel_dev *accel_dev); #endif diff --git a/drivers/crypto/intel/qat/qat_common/adf_cfg_common.h b/drivers/crypto/intel/qat/qat_common/adf_cfg_common.h index 6e5de1dab97b..89df3888d7ea 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_cfg_common.h +++ b/drivers/crypto/intel/qat/qat_common/adf_cfg_common.h @@ -47,6 +47,7 @@ enum adf_device_type { DEV_C3XXX, DEV_C3XXXVF, DEV_4XXX, + DEV_420XX, }; struct adf_dev_status_info { diff --git a/drivers/crypto/intel/qat/qat_common/adf_cfg_services.c b/drivers/crypto/intel/qat/qat_common/adf_cfg_services.c index 8e13fe938959..268052294468 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_cfg_services.c +++ b/drivers/crypto/intel/qat/qat_common/adf_cfg_services.c @@ -2,6 +2,9 @@ /* Copyright(c) 2023 Intel Corporation */ #include <linux/export.h> +#include <linux/pci.h> +#include <linux/string.h> +#include "adf_cfg.h" #include "adf_cfg_services.h" #include "adf_cfg_strings.h" @@ -18,3 +21,27 @@ const char *const adf_cfg_services[] = { [SVC_SYM_DC] = ADF_CFG_SYM_DC, }; EXPORT_SYMBOL_GPL(adf_cfg_services); + +int adf_get_service_enabled(struct adf_accel_dev *accel_dev) +{ + char services[ADF_CFG_MAX_VAL_LEN_IN_BYTES] = {0}; + int ret; + + ret = adf_cfg_get_param_value(accel_dev, ADF_GENERAL_SEC, + ADF_SERVICES_ENABLED, services); + if (ret) { + dev_err(&GET_DEV(accel_dev), + ADF_SERVICES_ENABLED " param not found\n"); + return ret; + } + + ret = match_string(adf_cfg_services, ARRAY_SIZE(adf_cfg_services), + services); + if (ret < 0) + dev_err(&GET_DEV(accel_dev), + "Invalid value of " ADF_SERVICES_ENABLED " param: %s\n", + services); + + return ret; +} +EXPORT_SYMBOL_GPL(adf_get_service_enabled); diff --git a/drivers/crypto/intel/qat/qat_common/adf_cfg_services.h b/drivers/crypto/intel/qat/qat_common/adf_cfg_services.h index f78fd697b4be..c6b0328b0f5b 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_cfg_services.h +++ b/drivers/crypto/intel/qat/qat_common/adf_cfg_services.h @@ -5,6 +5,8 @@ #include "adf_cfg_strings.h" +struct adf_accel_dev; + enum adf_services { SVC_CY = 0, SVC_CY2, @@ -21,4 +23,6 @@ enum adf_services { extern const char *const adf_cfg_services[SVC_COUNT]; +int adf_get_service_enabled(struct adf_accel_dev *accel_dev); + #endif diff --git a/drivers/crypto/intel/qat/qat_common/adf_dbgfs.c b/drivers/crypto/intel/qat/qat_common/adf_dbgfs.c index 477efcc81a16..c42f5c25aabd 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_dbgfs.c +++ b/drivers/crypto/intel/qat/qat_common/adf_dbgfs.c @@ -10,6 +10,7 @@ #include "adf_fw_counters.h" #include "adf_heartbeat_dbgfs.h" #include "adf_pm_dbgfs.h" +#include "adf_tl_debugfs.h" /** * adf_dbgfs_init() - add persistent debugfs entries @@ -66,6 +67,7 @@ void adf_dbgfs_add(struct adf_accel_dev *accel_dev) adf_heartbeat_dbgfs_add(accel_dev); adf_pm_dbgfs_add(accel_dev); adf_cnv_dbgfs_add(accel_dev); + adf_tl_dbgfs_add(accel_dev); } } @@ -79,6 +81,7 @@ void adf_dbgfs_rm(struct adf_accel_dev *accel_dev) return; if (!accel_dev->is_vf) { + adf_tl_dbgfs_rm(accel_dev); adf_cnv_dbgfs_rm(accel_dev); adf_pm_dbgfs_rm(accel_dev); adf_heartbeat_dbgfs_rm(accel_dev); diff --git a/drivers/crypto/intel/qat/qat_common/adf_fw_config.h b/drivers/crypto/intel/qat/qat_common/adf_fw_config.h new file mode 100644 index 000000000000..4f86696800c9 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_fw_config.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2023 Intel Corporation */ +#ifndef ADF_FW_CONFIG_H_ +#define ADF_FW_CONFIG_H_ + +enum adf_fw_objs { + ADF_FW_SYM_OBJ, + ADF_FW_ASYM_OBJ, + ADF_FW_DC_OBJ, + ADF_FW_ADMIN_OBJ, +}; + +struct adf_fw_config { + u32 ae_mask; + enum adf_fw_objs obj; +}; + +#endif diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen4_config.c b/drivers/crypto/intel/qat/qat_common/adf_gen4_config.c new file mode 100644 index 000000000000..fe1f3d727dc5 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_gen4_config.c @@ -0,0 +1,287 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2023 Intel Corporation */ +#include "adf_accel_devices.h" +#include "adf_cfg.h" +#include "adf_cfg_services.h" +#include "adf_cfg_strings.h" +#include "adf_common_drv.h" +#include "adf_gen4_config.h" +#include "adf_heartbeat.h" +#include "adf_transport_access_macros.h" +#include "qat_compression.h" +#include "qat_crypto.h" + +static int adf_crypto_dev_config(struct adf_accel_dev *accel_dev) +{ + char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES]; + int banks = GET_MAX_BANKS(accel_dev); + int cpus = num_online_cpus(); + unsigned long bank, val; + int instances; + int ret; + int i; + + if (adf_hw_dev_has_crypto(accel_dev)) + instances = min(cpus, banks / 2); + else + instances = 0; + + for (i = 0; i < instances; i++) { + val = i; + bank = i * 2; + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_BANK_NUM, i); + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &bank, ADF_DEC); + if (ret) + goto err; + + bank += 1; + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_BANK_NUM, i); + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &bank, ADF_DEC); + if (ret) + goto err; + + snprintf(key, sizeof(key), ADF_CY "%d" ADF_ETRMGR_CORE_AFFINITY, + i); + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) + goto err; + + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_SIZE, i); + val = 128; + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) + goto err; + + val = 512; + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_SIZE, i); + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) + goto err; + + val = 0; + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_TX, i); + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) + goto err; + + val = 0; + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_TX, i); + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) + goto err; + + val = 1; + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_RX, i); + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) + goto err; + + val = 1; + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_RX, i); + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) + goto err; + + val = ADF_COALESCING_DEF_TIME; + snprintf(key, sizeof(key), ADF_ETRMGR_COALESCE_TIMER_FORMAT, i); + ret = adf_cfg_add_key_value_param(accel_dev, "Accelerator0", + key, &val, ADF_DEC); + if (ret) + goto err; + } + + val = i; + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, ADF_NUM_CY, + &val, ADF_DEC); + if (ret) + goto err; + + val = 0; + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, ADF_NUM_DC, + &val, ADF_DEC); + if (ret) + goto err; + + return 0; +err: + dev_err(&GET_DEV(accel_dev), "Failed to add configuration for crypto\n"); + return ret; +} + +static int adf_comp_dev_config(struct adf_accel_dev *accel_dev) +{ + char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES]; + int banks = GET_MAX_BANKS(accel_dev); + int cpus = num_online_cpus(); + unsigned long val; + int instances; + int ret; + int i; + + if (adf_hw_dev_has_compression(accel_dev)) + instances = min(cpus, banks); + else + instances = 0; + + for (i = 0; i < instances; i++) { + val = i; + snprintf(key, sizeof(key), ADF_DC "%d" ADF_RING_DC_BANK_NUM, i); + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) + goto err; + + val = 512; + snprintf(key, sizeof(key), ADF_DC "%d" ADF_RING_DC_SIZE, i); + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) + goto err; + + val = 0; + snprintf(key, sizeof(key), ADF_DC "%d" ADF_RING_DC_TX, i); + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) + goto err; + + val = 1; + snprintf(key, sizeof(key), ADF_DC "%d" ADF_RING_DC_RX, i); + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) + goto err; + + val = ADF_COALESCING_DEF_TIME; + snprintf(key, sizeof(key), ADF_ETRMGR_COALESCE_TIMER_FORMAT, i); + ret = adf_cfg_add_key_value_param(accel_dev, "Accelerator0", + key, &val, ADF_DEC); + if (ret) + goto err; + } + + val = i; + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, ADF_NUM_DC, + &val, ADF_DEC); + if (ret) + goto err; + + val = 0; + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, ADF_NUM_CY, + &val, ADF_DEC); + if (ret) + goto err; + + return 0; +err: + dev_err(&GET_DEV(accel_dev), "Failed to add configuration for compression\n"); + return ret; +} + +static int adf_no_dev_config(struct adf_accel_dev *accel_dev) +{ + unsigned long val; + int ret; + + val = 0; + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, ADF_NUM_DC, + &val, ADF_DEC); + if (ret) + return ret; + + return adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, ADF_NUM_CY, + &val, ADF_DEC); +} + +/** + * adf_gen4_dev_config() - create dev config required to create instances + * + * @accel_dev: Pointer to acceleration device. + * + * Function creates device configuration required to create instances + * + * Return: 0 on success, error code otherwise. + */ +int adf_gen4_dev_config(struct adf_accel_dev *accel_dev) +{ + char services[ADF_CFG_MAX_VAL_LEN_IN_BYTES] = {0}; + int ret; + + ret = adf_cfg_section_add(accel_dev, ADF_KERNEL_SEC); + if (ret) + goto err; + + ret = adf_cfg_section_add(accel_dev, "Accelerator0"); + if (ret) + goto err; + + ret = adf_cfg_get_param_value(accel_dev, ADF_GENERAL_SEC, + ADF_SERVICES_ENABLED, services); + if (ret) + goto err; + + ret = sysfs_match_string(adf_cfg_services, services); + if (ret < 0) + goto err; + + switch (ret) { + case SVC_CY: + case SVC_CY2: + ret = adf_crypto_dev_config(accel_dev); + break; + case SVC_DC: + case SVC_DCC: + ret = adf_comp_dev_config(accel_dev); + break; + default: + ret = adf_no_dev_config(accel_dev); + break; + } + + if (ret) + goto err; + + set_bit(ADF_STATUS_CONFIGURED, &accel_dev->status); + + return ret; + +err: + dev_err(&GET_DEV(accel_dev), "Failed to configure QAT driver\n"); + return ret; +} +EXPORT_SYMBOL_GPL(adf_gen4_dev_config); + +int adf_gen4_cfg_dev_init(struct adf_accel_dev *accel_dev) +{ + const char *config; + int ret; + + config = accel_dev->accel_id % 2 ? ADF_CFG_DC : ADF_CFG_CY; + + ret = adf_cfg_section_add(accel_dev, ADF_GENERAL_SEC); + if (ret) + return ret; + + /* Default configuration is crypto only for even devices + * and compression for odd devices + */ + ret = adf_cfg_add_key_value_param(accel_dev, ADF_GENERAL_SEC, + ADF_SERVICES_ENABLED, config, + ADF_STR); + if (ret) + return ret; + + adf_heartbeat_save_cfg_param(accel_dev, ADF_CFG_HB_TIMER_MIN_MS); + + return 0; +} +EXPORT_SYMBOL_GPL(adf_gen4_cfg_dev_init); diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen4_config.h b/drivers/crypto/intel/qat/qat_common/adf_gen4_config.h new file mode 100644 index 000000000000..bb87655f69a8 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_gen4_config.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2023 Intel Corporation */ +#ifndef ADF_GEN4_CONFIG_H_ +#define ADF_GEN4_CONFIG_H_ + +#include "adf_accel_devices.h" + +int adf_gen4_dev_config(struct adf_accel_dev *accel_dev); +int adf_gen4_cfg_dev_init(struct adf_accel_dev *accel_dev); + +#endif diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen4_hw_data.c b/drivers/crypto/intel/qat/qat_common/adf_gen4_hw_data.c index 3148a62938fd..9985683056d5 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_gen4_hw_data.c +++ b/drivers/crypto/intel/qat/qat_common/adf_gen4_hw_data.c @@ -2,8 +2,10 @@ /* Copyright(c) 2020 Intel Corporation */ #include <linux/iopoll.h> #include "adf_accel_devices.h" +#include "adf_cfg_services.h" #include "adf_common_drv.h" #include "adf_gen4_hw_data.h" +#include "adf_gen4_pm.h" static u64 build_csr_ring_base_addr(dma_addr_t addr, u32 size) { @@ -102,6 +104,131 @@ void adf_gen4_init_hw_csr_ops(struct adf_hw_csr_ops *csr_ops) } EXPORT_SYMBOL_GPL(adf_gen4_init_hw_csr_ops); +u32 adf_gen4_get_accel_mask(struct adf_hw_device_data *self) +{ + return ADF_GEN4_ACCELERATORS_MASK; +} +EXPORT_SYMBOL_GPL(adf_gen4_get_accel_mask); + +u32 adf_gen4_get_num_accels(struct adf_hw_device_data *self) +{ + return ADF_GEN4_MAX_ACCELERATORS; +} +EXPORT_SYMBOL_GPL(adf_gen4_get_num_accels); + +u32 adf_gen4_get_num_aes(struct adf_hw_device_data *self) +{ + if (!self || !self->ae_mask) + return 0; + + return hweight32(self->ae_mask); +} +EXPORT_SYMBOL_GPL(adf_gen4_get_num_aes); + +u32 adf_gen4_get_misc_bar_id(struct adf_hw_device_data *self) +{ + return ADF_GEN4_PMISC_BAR; +} +EXPORT_SYMBOL_GPL(adf_gen4_get_misc_bar_id); + +u32 adf_gen4_get_etr_bar_id(struct adf_hw_device_data *self) +{ + return ADF_GEN4_ETR_BAR; +} +EXPORT_SYMBOL_GPL(adf_gen4_get_etr_bar_id); + +u32 adf_gen4_get_sram_bar_id(struct adf_hw_device_data *self) +{ + return ADF_GEN4_SRAM_BAR; +} +EXPORT_SYMBOL_GPL(adf_gen4_get_sram_bar_id); + +enum dev_sku_info adf_gen4_get_sku(struct adf_hw_device_data *self) +{ + return DEV_SKU_1; +} +EXPORT_SYMBOL_GPL(adf_gen4_get_sku); + +void adf_gen4_get_arb_info(struct arb_info *arb_info) +{ + arb_info->arb_cfg = ADF_GEN4_ARB_CONFIG; + arb_info->arb_offset = ADF_GEN4_ARB_OFFSET; + arb_info->wt2sam_offset = ADF_GEN4_ARB_WRK_2_SER_MAP_OFFSET; +} +EXPORT_SYMBOL_GPL(adf_gen4_get_arb_info); + +void adf_gen4_get_admin_info(struct admin_info *admin_csrs_info) +{ + admin_csrs_info->mailbox_offset = ADF_GEN4_MAILBOX_BASE_OFFSET; + admin_csrs_info->admin_msg_ur = ADF_GEN4_ADMINMSGUR_OFFSET; + admin_csrs_info->admin_msg_lr = ADF_GEN4_ADMINMSGLR_OFFSET; +} +EXPORT_SYMBOL_GPL(adf_gen4_get_admin_info); + +u32 adf_gen4_get_heartbeat_clock(struct adf_hw_device_data *self) +{ + /* + * GEN4 uses KPT counter for HB + */ + return ADF_GEN4_KPT_COUNTER_FREQ; +} +EXPORT_SYMBOL_GPL(adf_gen4_get_heartbeat_clock); + +void adf_gen4_enable_error_correction(struct adf_accel_dev *accel_dev) +{ + struct adf_bar *misc_bar = &GET_BARS(accel_dev)[ADF_GEN4_PMISC_BAR]; + void __iomem *csr = misc_bar->virt_addr; + + /* Enable all in errsou3 except VFLR notification on host */ + ADF_CSR_WR(csr, ADF_GEN4_ERRMSK3, ADF_GEN4_VFLNOTIFY); +} +EXPORT_SYMBOL_GPL(adf_gen4_enable_error_correction); + +void adf_gen4_enable_ints(struct adf_accel_dev *accel_dev) +{ + void __iomem *addr; + + addr = (&GET_BARS(accel_dev)[ADF_GEN4_PMISC_BAR])->virt_addr; + + /* Enable bundle interrupts */ + ADF_CSR_WR(addr, ADF_GEN4_SMIAPF_RP_X0_MASK_OFFSET, 0); + ADF_CSR_WR(addr, ADF_GEN4_SMIAPF_RP_X1_MASK_OFFSET, 0); + + /* Enable misc interrupts */ + ADF_CSR_WR(addr, ADF_GEN4_SMIAPF_MASK_OFFSET, 0); +} +EXPORT_SYMBOL_GPL(adf_gen4_enable_ints); + +int adf_gen4_init_device(struct adf_accel_dev *accel_dev) +{ + void __iomem *addr; + u32 status; + u32 csr; + int ret; + + addr = (&GET_BARS(accel_dev)[ADF_GEN4_PMISC_BAR])->virt_addr; + + /* Temporarily mask PM interrupt */ + csr = ADF_CSR_RD(addr, ADF_GEN4_ERRMSK2); + csr |= ADF_GEN4_PM_SOU; + ADF_CSR_WR(addr, ADF_GEN4_ERRMSK2, csr); + + /* Set DRV_ACTIVE bit to power up the device */ + ADF_CSR_WR(addr, ADF_GEN4_PM_INTERRUPT, ADF_GEN4_PM_DRV_ACTIVE); + + /* Poll status register to make sure the device is powered up */ + ret = read_poll_timeout(ADF_CSR_RD, status, + status & ADF_GEN4_PM_INIT_STATE, + ADF_GEN4_PM_POLL_DELAY_US, + ADF_GEN4_PM_POLL_TIMEOUT_US, true, addr, + ADF_GEN4_PM_STATUS); + if (ret) + dev_err(&GET_DEV(accel_dev), "Failed to power up the device\n"); + + return ret; +} +EXPORT_SYMBOL_GPL(adf_gen4_init_device); + static inline void adf_gen4_unpack_ssm_wdtimer(u64 value, u32 *upper, u32 *lower) { @@ -135,6 +262,28 @@ void adf_gen4_set_ssm_wdtimer(struct adf_accel_dev *accel_dev) } EXPORT_SYMBOL_GPL(adf_gen4_set_ssm_wdtimer); +/* + * The vector routing table is used to select the MSI-X entry to use for each + * interrupt source. + * The first ADF_GEN4_ETR_MAX_BANKS entries correspond to ring interrupts. + * The final entry corresponds to VF2PF or error interrupts. + * This vector table could be used to configure one MSI-X entry to be shared + * between multiple interrupt sources. + * + * The default routing is set to have a one to one correspondence between the + * interrupt source and the MSI-X entry used. + */ +void adf_gen4_set_msix_default_rttable(struct adf_accel_dev *accel_dev) +{ + void __iomem *csr; + int i; + + csr = (&GET_BARS(accel_dev)[ADF_GEN4_PMISC_BAR])->virt_addr; + for (i = 0; i <= ADF_GEN4_ETR_MAX_BANKS; i++) + ADF_CSR_WR(csr, ADF_GEN4_MSIX_RTTABLE_OFFSET(i), i); +} +EXPORT_SYMBOL_GPL(adf_gen4_set_msix_default_rttable); + int adf_pfvf_comms_disabled(struct adf_accel_dev *accel_dev) { return 0; @@ -192,3 +341,92 @@ int adf_gen4_ring_pair_reset(struct adf_accel_dev *accel_dev, u32 bank_number) return ret; } EXPORT_SYMBOL_GPL(adf_gen4_ring_pair_reset); + +static const u32 thrd_to_arb_map_dcc[] = { + 0x00000000, 0x00000000, 0x00000000, 0x00000000, + 0x0000FFFF, 0x0000FFFF, 0x0000FFFF, 0x0000FFFF, + 0x00000000, 0x00000000, 0x00000000, 0x00000000, + 0x00000000, 0x00000000, 0x00000000, 0x00000000, + 0x0 +}; + +static const u16 rp_group_to_arb_mask[] = { + [RP_GROUP_0] = 0x5, + [RP_GROUP_1] = 0xA, +}; + +static bool is_single_service(int service_id) +{ + switch (service_id) { + case SVC_DC: + case SVC_SYM: + case SVC_ASYM: + return true; + case SVC_CY: + case SVC_CY2: + case SVC_DCC: + case SVC_ASYM_DC: + case SVC_DC_ASYM: + case SVC_SYM_DC: + case SVC_DC_SYM: + default: + return false; + } +} + +int adf_gen4_init_thd2arb_map(struct adf_accel_dev *accel_dev) +{ + struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev); + u32 *thd2arb_map = hw_data->thd_to_arb_map; + unsigned int ae_cnt, worker_obj_cnt, i, j; + unsigned long ae_mask, thds_mask; + int srv_id, rp_group; + u32 thd2arb_map_base; + u16 arb_mask; + + if (!hw_data->get_rp_group || !hw_data->get_ena_thd_mask || + !hw_data->get_num_aes || !hw_data->uof_get_num_objs || + !hw_data->uof_get_ae_mask) + return -EFAULT; + + srv_id = adf_get_service_enabled(accel_dev); + if (srv_id < 0) + return srv_id; + + ae_cnt = hw_data->get_num_aes(hw_data); + worker_obj_cnt = hw_data->uof_get_num_objs(accel_dev) - + ADF_GEN4_ADMIN_ACCELENGINES; + + if (srv_id == SVC_DCC) { + memcpy(thd2arb_map, thrd_to_arb_map_dcc, + array_size(sizeof(*thd2arb_map), ae_cnt)); + return 0; + } + + for (i = 0; i < worker_obj_cnt; i++) { + ae_mask = hw_data->uof_get_ae_mask(accel_dev, i); + rp_group = hw_data->get_rp_group(accel_dev, ae_mask); + thds_mask = hw_data->get_ena_thd_mask(accel_dev, i); + thd2arb_map_base = 0; + + if (rp_group >= RP_GROUP_COUNT || rp_group < RP_GROUP_0) + return -EINVAL; + + if (thds_mask == ADF_GEN4_ENA_THD_MASK_ERROR) + return -EINVAL; + + if (is_single_service(srv_id)) + arb_mask = rp_group_to_arb_mask[RP_GROUP_0] | + rp_group_to_arb_mask[RP_GROUP_1]; + else + arb_mask = rp_group_to_arb_mask[rp_group]; + + for_each_set_bit(j, &thds_mask, ADF_NUM_THREADS_PER_AE) + thd2arb_map_base |= arb_mask << (j * 4); + + for_each_set_bit(j, &ae_mask, ae_cnt) + thd2arb_map[j] = thd2arb_map_base; + } + return 0; +} +EXPORT_SYMBOL_GPL(adf_gen4_init_thd2arb_map); diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen4_hw_data.h b/drivers/crypto/intel/qat/qat_common/adf_gen4_hw_data.h index 1813fe1d5a06..7d8a774cadc8 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_gen4_hw_data.h +++ b/drivers/crypto/intel/qat/qat_common/adf_gen4_hw_data.h @@ -3,9 +3,57 @@ #ifndef ADF_GEN4_HW_CSR_DATA_H_ #define ADF_GEN4_HW_CSR_DATA_H_ +#include <linux/units.h> + #include "adf_accel_devices.h" #include "adf_cfg_common.h" +/* PCIe configuration space */ +#define ADF_GEN4_BAR_MASK (BIT(0) | BIT(2) | BIT(4)) +#define ADF_GEN4_SRAM_BAR 0 +#define ADF_GEN4_PMISC_BAR 1 +#define ADF_GEN4_ETR_BAR 2 + +/* Clocks frequency */ +#define ADF_GEN4_KPT_COUNTER_FREQ (100 * HZ_PER_MHZ) + +/* Physical function fuses */ +#define ADF_GEN4_FUSECTL0_OFFSET 0x2C8 +#define ADF_GEN4_FUSECTL1_OFFSET 0x2CC +#define ADF_GEN4_FUSECTL2_OFFSET 0x2D0 +#define ADF_GEN4_FUSECTL3_OFFSET 0x2D4 +#define ADF_GEN4_FUSECTL4_OFFSET 0x2D8 +#define ADF_GEN4_FUSECTL5_OFFSET 0x2DC + +/* Accelerators */ +#define ADF_GEN4_ACCELERATORS_MASK 0x1 +#define ADF_GEN4_MAX_ACCELERATORS 1 +#define ADF_GEN4_ADMIN_ACCELENGINES 1 + +/* MSIX interrupt */ +#define ADF_GEN4_SMIAPF_RP_X0_MASK_OFFSET 0x41A040 +#define ADF_GEN4_SMIAPF_RP_X1_MASK_OFFSET 0x41A044 +#define ADF_GEN4_SMIAPF_MASK_OFFSET 0x41A084 +#define ADF_GEN4_MSIX_RTTABLE_OFFSET(i) (0x409000 + ((i) * 0x04)) + +/* Bank and ring configuration */ +#define ADF_GEN4_MAX_RPS 64 +#define ADF_GEN4_NUM_RINGS_PER_BANK 2 +#define ADF_GEN4_NUM_BANKS_PER_VF 4 +#define ADF_GEN4_ETR_MAX_BANKS 64 +#define ADF_GEN4_RX_RINGS_OFFSET 1 +#define ADF_GEN4_TX_RINGS_MASK 0x1 + +/* Arbiter configuration */ +#define ADF_GEN4_ARB_CONFIG (BIT(31) | BIT(6) | BIT(0)) +#define ADF_GEN4_ARB_OFFSET 0x0 +#define ADF_GEN4_ARB_WRK_2_SER_MAP_OFFSET 0x400 + +/* Admin Interface Reg Offset */ +#define ADF_GEN4_ADMINMSGUR_OFFSET 0x500574 +#define ADF_GEN4_ADMINMSGLR_OFFSET 0x500578 +#define ADF_GEN4_MAILBOX_BASE_OFFSET 0x600970 + /* Transport access */ #define ADF_BANK_INT_SRC_SEL_MASK 0x44UL #define ADF_RING_CSR_RING_CONFIG 0x1000 @@ -146,7 +194,46 @@ do { \ #define ADF_GEN4_RL_TOKEN_PCIEIN_BUCKET_OFFSET 0x508800 #define ADF_GEN4_RL_TOKEN_PCIEOUT_BUCKET_OFFSET 0x508804 +/* Arbiter threads mask with error value */ +#define ADF_GEN4_ENA_THD_MASK_ERROR GENMASK(ADF_NUM_THREADS_PER_AE, 0) + void adf_gen4_set_ssm_wdtimer(struct adf_accel_dev *accel_dev); + +enum icp_qat_gen4_slice_mask { + ICP_ACCEL_GEN4_MASK_CIPHER_SLICE = BIT(0), + ICP_ACCEL_GEN4_MASK_AUTH_SLICE = BIT(1), + ICP_ACCEL_GEN4_MASK_PKE_SLICE = BIT(2), + ICP_ACCEL_GEN4_MASK_COMPRESS_SLICE = BIT(3), + ICP_ACCEL_GEN4_MASK_UCS_SLICE = BIT(4), + ICP_ACCEL_GEN4_MASK_EIA3_SLICE = BIT(5), + ICP_ACCEL_GEN4_MASK_SMX_SLICE = BIT(7), + ICP_ACCEL_GEN4_MASK_WCP_WAT_SLICE = BIT(8), + ICP_ACCEL_GEN4_MASK_ZUC_256_SLICE = BIT(9), +}; + +enum adf_gen4_rp_groups { + RP_GROUP_0, + RP_GROUP_1, + RP_GROUP_COUNT +}; + +void adf_gen4_enable_error_correction(struct adf_accel_dev *accel_dev); +void adf_gen4_enable_ints(struct adf_accel_dev *accel_dev); +u32 adf_gen4_get_accel_mask(struct adf_hw_device_data *self); +void adf_gen4_get_admin_info(struct admin_info *admin_csrs_info); +void adf_gen4_get_arb_info(struct arb_info *arb_info); +u32 adf_gen4_get_etr_bar_id(struct adf_hw_device_data *self); +u32 adf_gen4_get_heartbeat_clock(struct adf_hw_device_data *self); +u32 adf_gen4_get_misc_bar_id(struct adf_hw_device_data *self); +u32 adf_gen4_get_num_accels(struct adf_hw_device_data *self); +u32 adf_gen4_get_num_aes(struct adf_hw_device_data *self); +enum dev_sku_info adf_gen4_get_sku(struct adf_hw_device_data *self); +u32 adf_gen4_get_sram_bar_id(struct adf_hw_device_data *self); +int adf_gen4_init_device(struct adf_accel_dev *accel_dev); void adf_gen4_init_hw_csr_ops(struct adf_hw_csr_ops *csr_ops); int adf_gen4_ring_pair_reset(struct adf_accel_dev *accel_dev, u32 bank_number); +void adf_gen4_set_msix_default_rttable(struct adf_accel_dev *accel_dev); +void adf_gen4_set_ssm_wdtimer(struct adf_accel_dev *accel_dev); +int adf_gen4_init_thd2arb_map(struct adf_accel_dev *accel_dev); + #endif diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen4_tl.c b/drivers/crypto/intel/qat/qat_common/adf_gen4_tl.c new file mode 100644 index 000000000000..7fc7a77f6aed --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_gen4_tl.c @@ -0,0 +1,153 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (c) 2023 Intel Corporation. */ +#include <linux/export.h> +#include <linux/kernel.h> + +#include "adf_gen4_tl.h" +#include "adf_telemetry.h" +#include "adf_tl_debugfs.h" + +#define ADF_GEN4_TL_DEV_REG_OFF(reg) ADF_TL_DEV_REG_OFF(reg, gen4) + +#define ADF_GEN4_TL_RP_REG_OFF(reg) ADF_TL_RP_REG_OFF(reg, gen4) + +#define ADF_GEN4_TL_SL_UTIL_COUNTER(_name) \ + ADF_TL_COUNTER("util_" #_name, \ + ADF_TL_SIMPLE_COUNT, \ + ADF_TL_SLICE_REG_OFF(_name, reg_tm_slice_util, gen4)) + +#define ADF_GEN4_TL_SL_EXEC_COUNTER(_name) \ + ADF_TL_COUNTER("exec_" #_name, \ + ADF_TL_SIMPLE_COUNT, \ + ADF_TL_SLICE_REG_OFF(_name, reg_tm_slice_exec_cnt, gen4)) + +/* Device level counters. */ +static const struct adf_tl_dbg_counter dev_counters[] = { + /* PCIe partial transactions. */ + ADF_TL_COUNTER(PCI_TRANS_CNT_NAME, ADF_TL_SIMPLE_COUNT, + ADF_GEN4_TL_DEV_REG_OFF(reg_tl_pci_trans_cnt)), + /* Max read latency[ns]. */ + ADF_TL_COUNTER(MAX_RD_LAT_NAME, ADF_TL_COUNTER_NS, + ADF_GEN4_TL_DEV_REG_OFF(reg_tl_rd_lat_max)), + /* Read latency average[ns]. */ + ADF_TL_COUNTER_LATENCY(RD_LAT_ACC_NAME, ADF_TL_COUNTER_NS_AVG, + ADF_GEN4_TL_DEV_REG_OFF(reg_tl_rd_lat_acc), + ADF_GEN4_TL_DEV_REG_OFF(reg_tl_rd_cmpl_cnt)), + /* Max get to put latency[ns]. */ + ADF_TL_COUNTER(MAX_LAT_NAME, ADF_TL_COUNTER_NS, + ADF_GEN4_TL_DEV_REG_OFF(reg_tl_gp_lat_max)), + /* Get to put latency average[ns]. */ + ADF_TL_COUNTER_LATENCY(LAT_ACC_NAME, ADF_TL_COUNTER_NS_AVG, + ADF_GEN4_TL_DEV_REG_OFF(reg_tl_gp_lat_acc), + ADF_GEN4_TL_DEV_REG_OFF(reg_tl_ae_put_cnt)), + /* PCIe write bandwidth[Mbps]. */ + ADF_TL_COUNTER(BW_IN_NAME, ADF_TL_COUNTER_MBPS, + ADF_GEN4_TL_DEV_REG_OFF(reg_tl_bw_in)), + /* PCIe read bandwidth[Mbps]. */ + ADF_TL_COUNTER(BW_OUT_NAME, ADF_TL_COUNTER_MBPS, + ADF_GEN4_TL_DEV_REG_OFF(reg_tl_bw_out)), + /* Page request latency average[ns]. */ + ADF_TL_COUNTER_LATENCY(PAGE_REQ_LAT_NAME, ADF_TL_COUNTER_NS_AVG, + ADF_GEN4_TL_DEV_REG_OFF(reg_tl_at_page_req_lat_acc), + ADF_GEN4_TL_DEV_REG_OFF(reg_tl_at_page_req_cnt)), + /* Page translation latency average[ns]. */ + ADF_TL_COUNTER_LATENCY(AT_TRANS_LAT_NAME, ADF_TL_COUNTER_NS_AVG, + ADF_GEN4_TL_DEV_REG_OFF(reg_tl_at_trans_lat_acc), + ADF_GEN4_TL_DEV_REG_OFF(reg_tl_at_trans_lat_cnt)), + /* Maximum uTLB used. */ + ADF_TL_COUNTER(AT_MAX_UTLB_USED_NAME, ADF_TL_SIMPLE_COUNT, + ADF_GEN4_TL_DEV_REG_OFF(reg_tl_at_max_tlb_used)), +}; + +/* Slice utilization counters. */ +static const struct adf_tl_dbg_counter sl_util_counters[ADF_TL_SL_CNT_COUNT] = { + /* Compression slice utilization. */ + ADF_GEN4_TL_SL_UTIL_COUNTER(cpr), + /* Translator slice utilization. */ + ADF_GEN4_TL_SL_UTIL_COUNTER(xlt), + /* Decompression slice utilization. */ + ADF_GEN4_TL_SL_UTIL_COUNTER(dcpr), + /* PKE utilization. */ + ADF_GEN4_TL_SL_UTIL_COUNTER(pke), + /* Wireless Authentication slice utilization. */ + ADF_GEN4_TL_SL_UTIL_COUNTER(wat), + /* Wireless Cipher slice utilization. */ + ADF_GEN4_TL_SL_UTIL_COUNTER(wcp), + /* UCS slice utilization. */ + ADF_GEN4_TL_SL_UTIL_COUNTER(ucs), + /* Cipher slice utilization. */ + ADF_GEN4_TL_SL_UTIL_COUNTER(cph), + /* Authentication slice utilization. */ + ADF_GEN4_TL_SL_UTIL_COUNTER(ath), +}; + +/* Slice execution counters. */ +static const struct adf_tl_dbg_counter sl_exec_counters[ADF_TL_SL_CNT_COUNT] = { + /* Compression slice execution count. */ + ADF_GEN4_TL_SL_EXEC_COUNTER(cpr), + /* Translator slice execution count. */ + ADF_GEN4_TL_SL_EXEC_COUNTER(xlt), + /* Decompression slice execution count. */ + ADF_GEN4_TL_SL_EXEC_COUNTER(dcpr), + /* PKE execution count. */ + ADF_GEN4_TL_SL_EXEC_COUNTER(pke), + /* Wireless Authentication slice execution count. */ + ADF_GEN4_TL_SL_EXEC_COUNTER(wat), + /* Wireless Cipher slice execution count. */ + ADF_GEN4_TL_SL_EXEC_COUNTER(wcp), + /* UCS slice execution count. */ + ADF_GEN4_TL_SL_EXEC_COUNTER(ucs), + /* Cipher slice execution count. */ + ADF_GEN4_TL_SL_EXEC_COUNTER(cph), + /* Authentication slice execution count. */ + ADF_GEN4_TL_SL_EXEC_COUNTER(ath), +}; + +/* Ring pair counters. */ +static const struct adf_tl_dbg_counter rp_counters[] = { + /* PCIe partial transactions. */ + ADF_TL_COUNTER(PCI_TRANS_CNT_NAME, ADF_TL_SIMPLE_COUNT, + ADF_GEN4_TL_RP_REG_OFF(reg_tl_pci_trans_cnt)), + /* Get to put latency average[ns]. */ + ADF_TL_COUNTER_LATENCY(LAT_ACC_NAME, ADF_TL_COUNTER_NS_AVG, + ADF_GEN4_TL_RP_REG_OFF(reg_tl_gp_lat_acc), + ADF_GEN4_TL_RP_REG_OFF(reg_tl_ae_put_cnt)), + /* PCIe write bandwidth[Mbps]. */ + ADF_TL_COUNTER(BW_IN_NAME, ADF_TL_COUNTER_MBPS, + ADF_GEN4_TL_RP_REG_OFF(reg_tl_bw_in)), + /* PCIe read bandwidth[Mbps]. */ + ADF_TL_COUNTER(BW_OUT_NAME, ADF_TL_COUNTER_MBPS, + ADF_GEN4_TL_RP_REG_OFF(reg_tl_bw_out)), + /* Message descriptor DevTLB hit rate. */ + ADF_TL_COUNTER(AT_GLOB_DTLB_HIT_NAME, ADF_TL_SIMPLE_COUNT, + ADF_GEN4_TL_RP_REG_OFF(reg_tl_at_glob_devtlb_hit)), + /* Message descriptor DevTLB miss rate. */ + ADF_TL_COUNTER(AT_GLOB_DTLB_MISS_NAME, ADF_TL_SIMPLE_COUNT, + ADF_GEN4_TL_RP_REG_OFF(reg_tl_at_glob_devtlb_miss)), + /* Payload DevTLB hit rate. */ + ADF_TL_COUNTER(AT_PAYLD_DTLB_HIT_NAME, ADF_TL_SIMPLE_COUNT, + ADF_GEN4_TL_RP_REG_OFF(reg_tl_at_payld_devtlb_hit)), + /* Payload DevTLB miss rate. */ + ADF_TL_COUNTER(AT_PAYLD_DTLB_MISS_NAME, ADF_TL_SIMPLE_COUNT, + ADF_GEN4_TL_RP_REG_OFF(reg_tl_at_payld_devtlb_miss)), +}; + +void adf_gen4_init_tl_data(struct adf_tl_hw_data *tl_data) +{ + tl_data->layout_sz = ADF_GEN4_TL_LAYOUT_SZ; + tl_data->slice_reg_sz = ADF_GEN4_TL_SLICE_REG_SZ; + tl_data->rp_reg_sz = ADF_GEN4_TL_RP_REG_SZ; + tl_data->num_hbuff = ADF_GEN4_TL_NUM_HIST_BUFFS; + tl_data->max_rp = ADF_GEN4_TL_MAX_RP_NUM; + tl_data->msg_cnt_off = ADF_GEN4_TL_MSG_CNT_OFF; + tl_data->cpp_ns_per_cycle = ADF_GEN4_CPP_NS_PER_CYCLE; + tl_data->bw_units_to_bytes = ADF_GEN4_TL_BW_HW_UNITS_TO_BYTES; + + tl_data->dev_counters = dev_counters; + tl_data->num_dev_counters = ARRAY_SIZE(dev_counters); + tl_data->sl_util_counters = sl_util_counters; + tl_data->sl_exec_counters = sl_exec_counters; + tl_data->rp_counters = rp_counters; + tl_data->num_rp_counters = ARRAY_SIZE(rp_counters); +} +EXPORT_SYMBOL_GPL(adf_gen4_init_tl_data); diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen4_tl.h b/drivers/crypto/intel/qat/qat_common/adf_gen4_tl.h new file mode 100644 index 000000000000..32df4163beb9 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_gen4_tl.h @@ -0,0 +1,158 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (c) 2023 Intel Corporation. */ +#ifndef ADF_GEN4_TL_H +#define ADF_GEN4_TL_H + +#include <linux/stddef.h> +#include <linux/types.h> + +struct adf_tl_hw_data; + +/* Computation constants. */ +#define ADF_GEN4_CPP_NS_PER_CYCLE 2 +#define ADF_GEN4_TL_BW_HW_UNITS_TO_BYTES 64 + +/* Maximum aggregation time. Value in milliseconds. */ +#define ADF_GEN4_TL_MAX_AGGR_TIME_MS 4000 +/* Num of buffers to store historic values. */ +#define ADF_GEN4_TL_NUM_HIST_BUFFS \ + (ADF_GEN4_TL_MAX_AGGR_TIME_MS / ADF_TL_DATA_WR_INTERVAL_MS) + +/* Max number of HW resources of one type. */ +#define ADF_GEN4_TL_MAX_SLICES_PER_TYPE 24 + +/* Max number of simultaneously monitored ring pairs. */ +#define ADF_GEN4_TL_MAX_RP_NUM 4 + +/** + * struct adf_gen4_tl_slice_data_regs - HW slice data as populated by FW. + * @reg_tm_slice_exec_cnt: Slice execution count. + * @reg_tm_slice_util: Slice utilization. + */ +struct adf_gen4_tl_slice_data_regs { + __u32 reg_tm_slice_exec_cnt; + __u32 reg_tm_slice_util; +}; + +#define ADF_GEN4_TL_SLICE_REG_SZ sizeof(struct adf_gen4_tl_slice_data_regs) + +/** + * struct adf_gen4_tl_device_data_regs - This structure stores device telemetry + * counter values as are being populated periodically by device. + * @reg_tl_rd_lat_acc: read latency accumulator + * @reg_tl_gp_lat_acc: get-put latency accumulator + * @reg_tl_at_page_req_lat_acc: AT/DevTLB page request latency accumulator + * @reg_tl_at_trans_lat_acc: DevTLB transaction latency accumulator + * @reg_tl_re_acc: accumulated ring empty time + * @reg_tl_pci_trans_cnt: PCIe partial transactions + * @reg_tl_rd_lat_max: maximum logged read latency + * @reg_tl_rd_cmpl_cnt: read requests completed count + * @reg_tl_gp_lat_max: maximum logged get to put latency + * @reg_tl_ae_put_cnt: Accelerator Engine put counts across all rings + * @reg_tl_bw_in: PCIe write bandwidth + * @reg_tl_bw_out: PCIe read bandwidth + * @reg_tl_at_page_req_cnt: DevTLB page requests count + * @reg_tl_at_trans_lat_cnt: DevTLB transaction latency samples count + * @reg_tl_at_max_tlb_used: maximum uTLB used + * @reg_tl_re_cnt: ring empty time samples count + * @reserved: reserved + * @ath_slices: array of Authentication slices utilization registers + * @cph_slices: array of Cipher slices utilization registers + * @cpr_slices: array of Compression slices utilization registers + * @xlt_slices: array of Translator slices utilization registers + * @dcpr_slices: array of Decompression slices utilization registers + * @pke_slices: array of PKE slices utilization registers + * @ucs_slices: array of UCS slices utilization registers + * @wat_slices: array of Wireless Authentication slices utilization registers + * @wcp_slices: array of Wireless Cipher slices utilization registers + */ +struct adf_gen4_tl_device_data_regs { + __u64 reg_tl_rd_lat_acc; + __u64 reg_tl_gp_lat_acc; + __u64 reg_tl_at_page_req_lat_acc; + __u64 reg_tl_at_trans_lat_acc; + __u64 reg_tl_re_acc; + __u32 reg_tl_pci_trans_cnt; + __u32 reg_tl_rd_lat_max; + __u32 reg_tl_rd_cmpl_cnt; + __u32 reg_tl_gp_lat_max; + __u32 reg_tl_ae_put_cnt; + __u32 reg_tl_bw_in; + __u32 reg_tl_bw_out; + __u32 reg_tl_at_page_req_cnt; + __u32 reg_tl_at_trans_lat_cnt; + __u32 reg_tl_at_max_tlb_used; + __u32 reg_tl_re_cnt; + __u32 reserved; + struct adf_gen4_tl_slice_data_regs ath_slices[ADF_GEN4_TL_MAX_SLICES_PER_TYPE]; + struct adf_gen4_tl_slice_data_regs cph_slices[ADF_GEN4_TL_MAX_SLICES_PER_TYPE]; + struct adf_gen4_tl_slice_data_regs cpr_slices[ADF_GEN4_TL_MAX_SLICES_PER_TYPE]; + struct adf_gen4_tl_slice_data_regs xlt_slices[ADF_GEN4_TL_MAX_SLICES_PER_TYPE]; + struct adf_gen4_tl_slice_data_regs dcpr_slices[ADF_GEN4_TL_MAX_SLICES_PER_TYPE]; + struct adf_gen4_tl_slice_data_regs pke_slices[ADF_GEN4_TL_MAX_SLICES_PER_TYPE]; + struct adf_gen4_tl_slice_data_regs ucs_slices[ADF_GEN4_TL_MAX_SLICES_PER_TYPE]; + struct adf_gen4_tl_slice_data_regs wat_slices[ADF_GEN4_TL_MAX_SLICES_PER_TYPE]; + struct adf_gen4_tl_slice_data_regs wcp_slices[ADF_GEN4_TL_MAX_SLICES_PER_TYPE]; +}; + +/** + * struct adf_gen4_tl_ring_pair_data_regs - This structure stores Ring Pair + * telemetry counter values as are being populated periodically by device. + * @reg_tl_gp_lat_acc: get-put latency accumulator + * @reserved: reserved + * @reg_tl_pci_trans_cnt: PCIe partial transactions + * @reg_tl_ae_put_cnt: Accelerator Engine put counts across all rings + * @reg_tl_bw_in: PCIe write bandwidth + * @reg_tl_bw_out: PCIe read bandwidth + * @reg_tl_at_glob_devtlb_hit: Message descriptor DevTLB hit rate + * @reg_tl_at_glob_devtlb_miss: Message descriptor DevTLB miss rate + * @reg_tl_at_payld_devtlb_hit: Payload DevTLB hit rate + * @reg_tl_at_payld_devtlb_miss: Payload DevTLB miss rate + * @reg_tl_re_cnt: ring empty time samples count + * @reserved1: reserved + */ +struct adf_gen4_tl_ring_pair_data_regs { + __u64 reg_tl_gp_lat_acc; + __u64 reserved; + __u32 reg_tl_pci_trans_cnt; + __u32 reg_tl_ae_put_cnt; + __u32 reg_tl_bw_in; + __u32 reg_tl_bw_out; + __u32 reg_tl_at_glob_devtlb_hit; + __u32 reg_tl_at_glob_devtlb_miss; + __u32 reg_tl_at_payld_devtlb_hit; + __u32 reg_tl_at_payld_devtlb_miss; + __u32 reg_tl_re_cnt; + __u32 reserved1; +}; + +#define ADF_GEN4_TL_RP_REG_SZ sizeof(struct adf_gen4_tl_ring_pair_data_regs) + +/** + * struct adf_gen4_tl_layout - This structure represents entire telemetry + * counters data: Device + 4 Ring Pairs as are being populated periodically + * by device. + * @tl_device_data_regs: structure of device telemetry registers + * @tl_ring_pairs_data_regs: array of ring pairs telemetry registers + * @reg_tl_msg_cnt: telemetry messages counter + * @reserved: reserved + */ +struct adf_gen4_tl_layout { + struct adf_gen4_tl_device_data_regs tl_device_data_regs; + struct adf_gen4_tl_ring_pair_data_regs + tl_ring_pairs_data_regs[ADF_GEN4_TL_MAX_RP_NUM]; + __u32 reg_tl_msg_cnt; + __u32 reserved; +}; + +#define ADF_GEN4_TL_LAYOUT_SZ sizeof(struct adf_gen4_tl_layout) +#define ADF_GEN4_TL_MSG_CNT_OFF offsetof(struct adf_gen4_tl_layout, reg_tl_msg_cnt) + +#ifdef CONFIG_DEBUG_FS +void adf_gen4_init_tl_data(struct adf_tl_hw_data *tl_data); +#else +static inline void adf_gen4_init_tl_data(struct adf_tl_hw_data *tl_data) +{ +} +#endif /* CONFIG_DEBUG_FS */ +#endif /* ADF_GEN4_TL_H */ diff --git a/drivers/crypto/intel/qat/qat_common/adf_init.c b/drivers/crypto/intel/qat/qat_common/adf_init.c index 81c39f3d07e1..f43ae9111553 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_init.c +++ b/drivers/crypto/intel/qat/qat_common/adf_init.c @@ -11,6 +11,7 @@ #include "adf_heartbeat.h" #include "adf_rl.h" #include "adf_sysfs_ras_counters.h" +#include "adf_telemetry.h" static LIST_HEAD(service_table); static DEFINE_MUTEX(service_lock); @@ -142,6 +143,10 @@ static int adf_dev_init(struct adf_accel_dev *accel_dev) if (ret && ret != -EOPNOTSUPP) return ret; + ret = adf_tl_init(accel_dev); + if (ret && ret != -EOPNOTSUPP) + return ret; + /* * Subservice initialisation is divided into two stages: init and start. * This is to facilitate any ordering dependencies between services @@ -220,6 +225,10 @@ static int adf_dev_start(struct adf_accel_dev *accel_dev) if (ret && ret != -EOPNOTSUPP) return ret; + ret = adf_tl_start(accel_dev); + if (ret && ret != -EOPNOTSUPP) + return ret; + list_for_each_entry(service, &service_table, list) { if (service->event_hld(accel_dev, ADF_EVENT_START)) { dev_err(&GET_DEV(accel_dev), @@ -279,6 +288,7 @@ static void adf_dev_stop(struct adf_accel_dev *accel_dev) !test_bit(ADF_STATUS_STARTING, &accel_dev->status)) return; + adf_tl_stop(accel_dev); adf_rl_stop(accel_dev); adf_dbgfs_rm(accel_dev); adf_sysfs_stop_ras(accel_dev); @@ -374,6 +384,8 @@ static void adf_dev_shutdown(struct adf_accel_dev *accel_dev) adf_heartbeat_shutdown(accel_dev); + adf_tl_shutdown(accel_dev); + hw_data->disable_iov(accel_dev); if (test_bit(ADF_STATUS_IRQ_ALLOCATED, &accel_dev->status)) { diff --git a/drivers/crypto/intel/qat/qat_common/adf_rl.c b/drivers/crypto/intel/qat/qat_common/adf_rl.c index 86e3e2152b1b..de1b214dba1f 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_rl.c +++ b/drivers/crypto/intel/qat/qat_common/adf_rl.c @@ -812,17 +812,16 @@ static int add_update_sla(struct adf_accel_dev *accel_dev, if (!sla_in) { dev_warn(&GET_DEV(accel_dev), "SLA input data pointer is missing\n"); - ret = -EFAULT; - goto ret_err; + return -EFAULT; } + mutex_lock(&rl_data->rl_lock); + /* Input validation */ ret = validate_user_input(accel_dev, sla_in, is_update); if (ret) goto ret_err; - mutex_lock(&rl_data->rl_lock); - if (is_update) { ret = validate_sla_id(accel_dev, sla_in->sla_id); if (ret) diff --git a/drivers/crypto/intel/qat/qat_common/adf_rl.h b/drivers/crypto/intel/qat/qat_common/adf_rl.h index eb5a330f8543..269c6656fb90 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_rl.h +++ b/drivers/crypto/intel/qat/qat_common/adf_rl.h @@ -79,6 +79,7 @@ struct adf_rl_interface_data { struct adf_rl_sla_input_data input; enum adf_base_services cap_rem_srv; struct rw_semaphore lock; + bool sysfs_added; }; struct adf_rl_hw_data { diff --git a/drivers/crypto/intel/qat/qat_common/adf_sysfs.c b/drivers/crypto/intel/qat/qat_common/adf_sysfs.c index ddffc98119c6..d450dad32c9e 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_sysfs.c +++ b/drivers/crypto/intel/qat/qat_common/adf_sysfs.c @@ -215,6 +215,9 @@ static ssize_t rp2srv_show(struct device *dev, struct device_attribute *attr, enum adf_cfg_service_type svc; accel_dev = adf_devmgr_pci_to_accel_dev(to_pci_dev(dev)); + if (!accel_dev) + return -EINVAL; + hw_data = GET_HW_DATA(accel_dev); if (accel_dev->sysfs.ring_num == UNSET_RING_NUM) @@ -242,7 +245,8 @@ static ssize_t rp2srv_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { struct adf_accel_dev *accel_dev; - int ring, num_rings, ret; + int num_rings, ret; + unsigned int ring; accel_dev = adf_devmgr_pci_to_accel_dev(to_pci_dev(dev)); if (!accel_dev) diff --git a/drivers/crypto/intel/qat/qat_common/adf_sysfs_ras_counters.c b/drivers/crypto/intel/qat/qat_common/adf_sysfs_ras_counters.c index cffe2d722995..e97c67c87b3c 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_sysfs_ras_counters.c +++ b/drivers/crypto/intel/qat/qat_common/adf_sysfs_ras_counters.c @@ -99,6 +99,8 @@ void adf_sysfs_start_ras(struct adf_accel_dev *accel_dev) if (device_add_group(&GET_DEV(accel_dev), &qat_ras_group)) dev_err(&GET_DEV(accel_dev), "Failed to create qat_ras attribute group.\n"); + + accel_dev->ras_errors.sysfs_added = true; } void adf_sysfs_stop_ras(struct adf_accel_dev *accel_dev) @@ -106,7 +108,10 @@ void adf_sysfs_stop_ras(struct adf_accel_dev *accel_dev) if (!accel_dev->ras_errors.enabled) return; - device_remove_group(&GET_DEV(accel_dev), &qat_ras_group); + if (accel_dev->ras_errors.sysfs_added) { + device_remove_group(&GET_DEV(accel_dev), &qat_ras_group); + accel_dev->ras_errors.sysfs_added = false; + } ADF_RAS_ERR_CTR_CLEAR(accel_dev->ras_errors); } diff --git a/drivers/crypto/intel/qat/qat_common/adf_sysfs_rl.c b/drivers/crypto/intel/qat/qat_common/adf_sysfs_rl.c index abf9c52474ec..bedb514d4e30 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_sysfs_rl.c +++ b/drivers/crypto/intel/qat/qat_common/adf_sysfs_rl.c @@ -441,11 +441,19 @@ int adf_sysfs_rl_add(struct adf_accel_dev *accel_dev) data->cap_rem_srv = ADF_SVC_NONE; data->input.srv = ADF_SVC_NONE; + data->sysfs_added = true; return ret; } void adf_sysfs_rl_rm(struct adf_accel_dev *accel_dev) { + struct adf_rl_interface_data *data; + + data = &GET_RL_STRUCT(accel_dev); + if (!data->sysfs_added) + return; + device_remove_group(&GET_DEV(accel_dev), &qat_rl_group); + data->sysfs_added = false; } diff --git a/drivers/crypto/intel/qat/qat_common/adf_telemetry.c b/drivers/crypto/intel/qat/qat_common/adf_telemetry.c new file mode 100644 index 000000000000..2ff714d11bd2 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_telemetry.c @@ -0,0 +1,288 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (c) 2023 Intel Corporation. */ +#define dev_fmt(fmt) "Telemetry: " fmt + +#include <asm/errno.h> +#include <linux/atomic.h> +#include <linux/device.h> +#include <linux/dev_printk.h> +#include <linux/dma-mapping.h> +#include <linux/jiffies.h> +#include <linux/kernel.h> +#include <linux/mutex.h> +#include <linux/slab.h> +#include <linux/string.h> +#include <linux/workqueue.h> + +#include "adf_admin.h" +#include "adf_accel_devices.h" +#include "adf_common_drv.h" +#include "adf_telemetry.h" + +#define TL_IS_ZERO(input) ((input) == 0) + +static bool is_tl_supported(struct adf_accel_dev *accel_dev) +{ + u16 fw_caps = GET_HW_DATA(accel_dev)->fw_capabilities; + + return fw_caps & TL_CAPABILITY_BIT; +} + +static int validate_tl_data(struct adf_tl_hw_data *tl_data) +{ + if (!tl_data->dev_counters || + TL_IS_ZERO(tl_data->num_dev_counters) || + !tl_data->sl_util_counters || + !tl_data->sl_exec_counters || + !tl_data->rp_counters || + TL_IS_ZERO(tl_data->num_rp_counters)) + return -EOPNOTSUPP; + + return 0; +} + +static int adf_tl_alloc_mem(struct adf_accel_dev *accel_dev) +{ + struct adf_tl_hw_data *tl_data = &GET_TL_DATA(accel_dev); + struct device *dev = &GET_DEV(accel_dev); + size_t regs_sz = tl_data->layout_sz; + struct adf_telemetry *telemetry; + int node = dev_to_node(dev); + void *tl_data_regs; + unsigned int i; + + telemetry = kzalloc_node(sizeof(*telemetry), GFP_KERNEL, node); + if (!telemetry) + return -ENOMEM; + + telemetry->rp_num_indexes = kmalloc_array(tl_data->max_rp, + sizeof(*telemetry->rp_num_indexes), + GFP_KERNEL); + if (!telemetry->rp_num_indexes) + goto err_free_tl; + + telemetry->regs_hist_buff = kmalloc_array(tl_data->num_hbuff, + sizeof(*telemetry->regs_hist_buff), + GFP_KERNEL); + if (!telemetry->regs_hist_buff) + goto err_free_rp_indexes; + + telemetry->regs_data = dma_alloc_coherent(dev, regs_sz, + &telemetry->regs_data_p, + GFP_KERNEL); + if (!telemetry->regs_data) + goto err_free_regs_hist_buff; + + for (i = 0; i < tl_data->num_hbuff; i++) { + tl_data_regs = kzalloc_node(regs_sz, GFP_KERNEL, node); + if (!tl_data_regs) + goto err_free_dma; + + telemetry->regs_hist_buff[i] = tl_data_regs; + } + + accel_dev->telemetry = telemetry; + + return 0; + +err_free_dma: + dma_free_coherent(dev, regs_sz, telemetry->regs_data, + telemetry->regs_data_p); + + while (i--) + kfree(telemetry->regs_hist_buff[i]); + +err_free_regs_hist_buff: + kfree(telemetry->regs_hist_buff); +err_free_rp_indexes: + kfree(telemetry->rp_num_indexes); +err_free_tl: + kfree(telemetry); + + return -ENOMEM; +} + +static void adf_tl_free_mem(struct adf_accel_dev *accel_dev) +{ + struct adf_tl_hw_data *tl_data = &GET_TL_DATA(accel_dev); + struct adf_telemetry *telemetry = accel_dev->telemetry; + struct device *dev = &GET_DEV(accel_dev); + size_t regs_sz = tl_data->layout_sz; + unsigned int i; + + for (i = 0; i < tl_data->num_hbuff; i++) + kfree(telemetry->regs_hist_buff[i]); + + dma_free_coherent(dev, regs_sz, telemetry->regs_data, + telemetry->regs_data_p); + + kfree(telemetry->regs_hist_buff); + kfree(telemetry->rp_num_indexes); + kfree(telemetry); + accel_dev->telemetry = NULL; +} + +static unsigned long get_next_timeout(void) +{ + return msecs_to_jiffies(ADF_TL_TIMER_INT_MS); +} + +static void snapshot_regs(struct adf_telemetry *telemetry, size_t size) +{ + void *dst = telemetry->regs_hist_buff[telemetry->hb_num]; + void *src = telemetry->regs_data; + + memcpy(dst, src, size); +} + +static void tl_work_handler(struct work_struct *work) +{ + struct delayed_work *delayed_work; + struct adf_telemetry *telemetry; + struct adf_tl_hw_data *tl_data; + u32 msg_cnt, old_msg_cnt; + size_t layout_sz; + u32 *regs_data; + size_t id; + + delayed_work = to_delayed_work(work); + telemetry = container_of(delayed_work, struct adf_telemetry, work_ctx); + tl_data = &GET_TL_DATA(telemetry->accel_dev); + regs_data = telemetry->regs_data; + + id = tl_data->msg_cnt_off / sizeof(*regs_data); + layout_sz = tl_data->layout_sz; + + if (!atomic_read(&telemetry->state)) { + cancel_delayed_work_sync(&telemetry->work_ctx); + return; + } + + msg_cnt = regs_data[id]; + old_msg_cnt = msg_cnt; + if (msg_cnt == telemetry->msg_cnt) + goto out; + + mutex_lock(&telemetry->regs_hist_lock); + + snapshot_regs(telemetry, layout_sz); + + /* Check if data changed while updating it */ + msg_cnt = regs_data[id]; + if (old_msg_cnt != msg_cnt) + snapshot_regs(telemetry, layout_sz); + + telemetry->msg_cnt = msg_cnt; + telemetry->hb_num++; + telemetry->hb_num %= telemetry->hbuffs; + + mutex_unlock(&telemetry->regs_hist_lock); + +out: + adf_misc_wq_queue_delayed_work(&telemetry->work_ctx, get_next_timeout()); +} + +int adf_tl_halt(struct adf_accel_dev *accel_dev) +{ + struct adf_telemetry *telemetry = accel_dev->telemetry; + struct device *dev = &GET_DEV(accel_dev); + int ret; + + cancel_delayed_work_sync(&telemetry->work_ctx); + atomic_set(&telemetry->state, 0); + + ret = adf_send_admin_tl_stop(accel_dev); + if (ret) + dev_err(dev, "failed to stop telemetry\n"); + + return ret; +} + +int adf_tl_run(struct adf_accel_dev *accel_dev, int state) +{ + struct adf_tl_hw_data *tl_data = &GET_TL_DATA(accel_dev); + struct adf_telemetry *telemetry = accel_dev->telemetry; + struct device *dev = &GET_DEV(accel_dev); + size_t layout_sz = tl_data->layout_sz; + int ret; + + ret = adf_send_admin_tl_start(accel_dev, telemetry->regs_data_p, + layout_sz, telemetry->rp_num_indexes, + &telemetry->slice_cnt); + if (ret) { + dev_err(dev, "failed to start telemetry\n"); + return ret; + } + + telemetry->hbuffs = state; + atomic_set(&telemetry->state, state); + + adf_misc_wq_queue_delayed_work(&telemetry->work_ctx, get_next_timeout()); + + return 0; +} + +int adf_tl_init(struct adf_accel_dev *accel_dev) +{ + struct adf_tl_hw_data *tl_data = &GET_TL_DATA(accel_dev); + u8 max_rp = GET_TL_DATA(accel_dev).max_rp; + struct device *dev = &GET_DEV(accel_dev); + struct adf_telemetry *telemetry; + unsigned int i; + int ret; + + ret = validate_tl_data(tl_data); + if (ret) + return ret; + + ret = adf_tl_alloc_mem(accel_dev); + if (ret) { + dev_err(dev, "failed to initialize: %d\n", ret); + return ret; + } + + telemetry = accel_dev->telemetry; + telemetry->accel_dev = accel_dev; + + mutex_init(&telemetry->wr_lock); + mutex_init(&telemetry->regs_hist_lock); + INIT_DELAYED_WORK(&telemetry->work_ctx, tl_work_handler); + + for (i = 0; i < max_rp; i++) + telemetry->rp_num_indexes[i] = ADF_TL_RP_REGS_DISABLED; + + return 0; +} + +int adf_tl_start(struct adf_accel_dev *accel_dev) +{ + struct device *dev = &GET_DEV(accel_dev); + + if (!accel_dev->telemetry) + return -EOPNOTSUPP; + + if (!is_tl_supported(accel_dev)) { + dev_info(dev, "feature not supported by FW\n"); + adf_tl_free_mem(accel_dev); + return -EOPNOTSUPP; + } + + return 0; +} + +void adf_tl_stop(struct adf_accel_dev *accel_dev) +{ + if (!accel_dev->telemetry) + return; + + if (atomic_read(&accel_dev->telemetry->state)) + adf_tl_halt(accel_dev); +} + +void adf_tl_shutdown(struct adf_accel_dev *accel_dev) +{ + if (!accel_dev->telemetry) + return; + + adf_tl_free_mem(accel_dev); +} diff --git a/drivers/crypto/intel/qat/qat_common/adf_telemetry.h b/drivers/crypto/intel/qat/qat_common/adf_telemetry.h new file mode 100644 index 000000000000..9be81cd3b886 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_telemetry.h @@ -0,0 +1,99 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (c) 2023 Intel Corporation. */ +#ifndef ADF_TELEMETRY_H +#define ADF_TELEMETRY_H + +#include <linux/bits.h> +#include <linux/mutex.h> +#include <linux/types.h> +#include <linux/workqueue.h> + +#include "icp_qat_fw_init_admin.h" + +struct adf_accel_dev; +struct adf_tl_dbg_counter; +struct dentry; + +#define ADF_TL_SL_CNT_COUNT \ + (sizeof(struct icp_qat_fw_init_admin_slice_cnt) / sizeof(__u8)) + +#define TL_CAPABILITY_BIT BIT(1) +/* Interval within device writes data to DMA region. Value in milliseconds. */ +#define ADF_TL_DATA_WR_INTERVAL_MS 1000 +/* Interval within timer interrupt should be handled. Value in milliseconds. */ +#define ADF_TL_TIMER_INT_MS (ADF_TL_DATA_WR_INTERVAL_MS / 2) + +#define ADF_TL_RP_REGS_DISABLED (0xff) + +struct adf_tl_hw_data { + size_t layout_sz; + size_t slice_reg_sz; + size_t rp_reg_sz; + size_t msg_cnt_off; + const struct adf_tl_dbg_counter *dev_counters; + const struct adf_tl_dbg_counter *sl_util_counters; + const struct adf_tl_dbg_counter *sl_exec_counters; + const struct adf_tl_dbg_counter *rp_counters; + u8 num_hbuff; + u8 cpp_ns_per_cycle; + u8 bw_units_to_bytes; + u8 num_dev_counters; + u8 num_rp_counters; + u8 max_rp; +}; + +struct adf_telemetry { + struct adf_accel_dev *accel_dev; + atomic_t state; + u32 hbuffs; + int hb_num; + u32 msg_cnt; + dma_addr_t regs_data_p; /* bus address for DMA mapping */ + void *regs_data; /* virtual address for DMA mapping */ + /** + * @regs_hist_buff: array of pointers to copies of the last @hbuffs + * values of @regs_data + */ + void **regs_hist_buff; + struct dentry *dbg_dir; + u8 *rp_num_indexes; + /** + * @regs_hist_lock: protects from race conditions between write and read + * to the copies referenced by @regs_hist_buff + */ + struct mutex regs_hist_lock; + /** + * @wr_lock: protects from concurrent writes to debugfs telemetry files + */ + struct mutex wr_lock; + struct delayed_work work_ctx; + struct icp_qat_fw_init_admin_slice_cnt slice_cnt; +}; + +#ifdef CONFIG_DEBUG_FS +int adf_tl_init(struct adf_accel_dev *accel_dev); +int adf_tl_start(struct adf_accel_dev *accel_dev); +void adf_tl_stop(struct adf_accel_dev *accel_dev); +void adf_tl_shutdown(struct adf_accel_dev *accel_dev); +int adf_tl_run(struct adf_accel_dev *accel_dev, int state); +int adf_tl_halt(struct adf_accel_dev *accel_dev); +#else +static inline int adf_tl_init(struct adf_accel_dev *accel_dev) +{ + return 0; +} + +static inline int adf_tl_start(struct adf_accel_dev *accel_dev) +{ + return 0; +} + +static inline void adf_tl_stop(struct adf_accel_dev *accel_dev) +{ +} + +static inline void adf_tl_shutdown(struct adf_accel_dev *accel_dev) +{ +} +#endif /* CONFIG_DEBUG_FS */ +#endif /* ADF_TELEMETRY_H */ diff --git a/drivers/crypto/intel/qat/qat_common/adf_tl_debugfs.c b/drivers/crypto/intel/qat/qat_common/adf_tl_debugfs.c new file mode 100644 index 000000000000..c8241f5a0a26 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_tl_debugfs.c @@ -0,0 +1,710 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (c) 2023 Intel Corporation. */ +#define dev_fmt(fmt) "Telemetry debugfs: " fmt + +#include <linux/atomic.h> +#include <linux/debugfs.h> +#include <linux/dev_printk.h> +#include <linux/dcache.h> +#include <linux/file.h> +#include <linux/kernel.h> +#include <linux/math64.h> +#include <linux/mutex.h> +#include <linux/seq_file.h> +#include <linux/slab.h> +#include <linux/units.h> + +#include "adf_accel_devices.h" +#include "adf_cfg_strings.h" +#include "adf_telemetry.h" +#include "adf_tl_debugfs.h" + +#define TL_VALUE_MIN_PADDING 20 +#define TL_KEY_MIN_PADDING 23 +#define TL_RP_SRV_UNKNOWN "Unknown" + +static int tl_collect_values_u32(struct adf_telemetry *telemetry, + size_t counter_offset, u64 *arr) +{ + unsigned int samples, hb_idx, i; + u32 *regs_hist_buff; + u32 counter_val; + + samples = min(telemetry->msg_cnt, telemetry->hbuffs); + hb_idx = telemetry->hb_num + telemetry->hbuffs - samples; + + mutex_lock(&telemetry->regs_hist_lock); + + for (i = 0; i < samples; i++) { + regs_hist_buff = telemetry->regs_hist_buff[hb_idx % telemetry->hbuffs]; + counter_val = regs_hist_buff[counter_offset / sizeof(counter_val)]; + arr[i] = counter_val; + hb_idx++; + } + + mutex_unlock(&telemetry->regs_hist_lock); + + return samples; +} + +static int tl_collect_values_u64(struct adf_telemetry *telemetry, + size_t counter_offset, u64 *arr) +{ + unsigned int samples, hb_idx, i; + u64 *regs_hist_buff; + u64 counter_val; + + samples = min(telemetry->msg_cnt, telemetry->hbuffs); + hb_idx = telemetry->hb_num + telemetry->hbuffs - samples; + + mutex_lock(&telemetry->regs_hist_lock); + + for (i = 0; i < samples; i++) { + regs_hist_buff = telemetry->regs_hist_buff[hb_idx % telemetry->hbuffs]; + counter_val = regs_hist_buff[counter_offset / sizeof(counter_val)]; + arr[i] = counter_val; + hb_idx++; + } + + mutex_unlock(&telemetry->regs_hist_lock); + + return samples; +} + +/** + * avg_array() - Return average of values within an array. + * @array: Array of values. + * @len: Number of elements. + * + * This algorithm computes average of an array without running into overflow. + * + * Return: average of values. + */ +#define avg_array(array, len) ( \ +{ \ + typeof(&(array)[0]) _array = (array); \ + __unqual_scalar_typeof(_array[0]) _x = 0; \ + __unqual_scalar_typeof(_array[0]) _y = 0; \ + __unqual_scalar_typeof(_array[0]) _a, _b; \ + typeof(len) _len = (len); \ + size_t _i; \ + \ + for (_i = 0; _i < _len; _i++) { \ + _a = _array[_i]; \ + _b = do_div(_a, _len); \ + _x += _a; \ + if (_y >= _len - _b) { \ + _x++; \ + _y -= _len - _b; \ + } else { \ + _y += _b; \ + } \ + } \ + do_div(_y, _len); \ + (_x + _y); \ +}) + +/* Calculation function for simple counter. */ +static int tl_calc_count(struct adf_telemetry *telemetry, + const struct adf_tl_dbg_counter *ctr, + struct adf_tl_dbg_aggr_values *vals) +{ + struct adf_tl_hw_data *tl_data = &GET_TL_DATA(telemetry->accel_dev); + u64 *hist_vals; + int sample_cnt; + int ret = 0; + + hist_vals = kmalloc_array(tl_data->num_hbuff, sizeof(*hist_vals), + GFP_KERNEL); + if (!hist_vals) + return -ENOMEM; + + memset(vals, 0, sizeof(*vals)); + sample_cnt = tl_collect_values_u32(telemetry, ctr->offset1, hist_vals); + if (!sample_cnt) + goto out_free_hist_vals; + + vals->curr = hist_vals[sample_cnt - 1]; + vals->min = min_array(hist_vals, sample_cnt); + vals->max = max_array(hist_vals, sample_cnt); + vals->avg = avg_array(hist_vals, sample_cnt); + +out_free_hist_vals: + kfree(hist_vals); + return ret; +} + +/* Convert CPP bus cycles to ns. */ +static int tl_cycles_to_ns(struct adf_telemetry *telemetry, + const struct adf_tl_dbg_counter *ctr, + struct adf_tl_dbg_aggr_values *vals) +{ + struct adf_tl_hw_data *tl_data = &GET_TL_DATA(telemetry->accel_dev); + u8 cpp_ns_per_cycle = tl_data->cpp_ns_per_cycle; + int ret; + + ret = tl_calc_count(telemetry, ctr, vals); + if (ret) + return ret; + + vals->curr *= cpp_ns_per_cycle; + vals->min *= cpp_ns_per_cycle; + vals->max *= cpp_ns_per_cycle; + vals->avg *= cpp_ns_per_cycle; + + return 0; +} + +/* + * Compute latency cumulative average with division of accumulated value + * by sample count. Returned value is in ns. + */ +static int tl_lat_acc_avg(struct adf_telemetry *telemetry, + const struct adf_tl_dbg_counter *ctr, + struct adf_tl_dbg_aggr_values *vals) +{ + struct adf_tl_hw_data *tl_data = &GET_TL_DATA(telemetry->accel_dev); + u8 cpp_ns_per_cycle = tl_data->cpp_ns_per_cycle; + u8 num_hbuff = tl_data->num_hbuff; + int sample_cnt, i; + u64 *hist_vals; + u64 *hist_cnt; + int ret = 0; + + hist_vals = kmalloc_array(num_hbuff, sizeof(*hist_vals), GFP_KERNEL); + if (!hist_vals) + return -ENOMEM; + + hist_cnt = kmalloc_array(num_hbuff, sizeof(*hist_cnt), GFP_KERNEL); + if (!hist_cnt) { + ret = -ENOMEM; + goto out_free_hist_vals; + } + + memset(vals, 0, sizeof(*vals)); + sample_cnt = tl_collect_values_u64(telemetry, ctr->offset1, hist_vals); + if (!sample_cnt) + goto out_free_hist_cnt; + + tl_collect_values_u32(telemetry, ctr->offset2, hist_cnt); + + for (i = 0; i < sample_cnt; i++) { + /* Avoid division by 0 if count is 0. */ + if (hist_cnt[i]) + hist_vals[i] = div_u64(hist_vals[i] * cpp_ns_per_cycle, + hist_cnt[i]); + else + hist_vals[i] = 0; + } + + vals->curr = hist_vals[sample_cnt - 1]; + vals->min = min_array(hist_vals, sample_cnt); + vals->max = max_array(hist_vals, sample_cnt); + vals->avg = avg_array(hist_vals, sample_cnt); + +out_free_hist_cnt: + kfree(hist_cnt); +out_free_hist_vals: + kfree(hist_vals); + return ret; +} + +/* Convert HW raw bandwidth units to Mbps. */ +static int tl_bw_hw_units_to_mbps(struct adf_telemetry *telemetry, + const struct adf_tl_dbg_counter *ctr, + struct adf_tl_dbg_aggr_values *vals) +{ + struct adf_tl_hw_data *tl_data = &GET_TL_DATA(telemetry->accel_dev); + u16 bw_hw_2_bits = tl_data->bw_units_to_bytes * BITS_PER_BYTE; + u64 *hist_vals; + int sample_cnt; + int ret = 0; + + hist_vals = kmalloc_array(tl_data->num_hbuff, sizeof(*hist_vals), + GFP_KERNEL); + if (!hist_vals) + return -ENOMEM; + + memset(vals, 0, sizeof(*vals)); + sample_cnt = tl_collect_values_u32(telemetry, ctr->offset1, hist_vals); + if (!sample_cnt) + goto out_free_hist_vals; + + vals->curr = div_u64(hist_vals[sample_cnt - 1] * bw_hw_2_bits, MEGA); + vals->min = div_u64(min_array(hist_vals, sample_cnt) * bw_hw_2_bits, MEGA); + vals->max = div_u64(max_array(hist_vals, sample_cnt) * bw_hw_2_bits, MEGA); + vals->avg = div_u64(avg_array(hist_vals, sample_cnt) * bw_hw_2_bits, MEGA); + +out_free_hist_vals: + kfree(hist_vals); + return ret; +} + +static void tl_seq_printf_counter(struct adf_telemetry *telemetry, + struct seq_file *s, const char *name, + struct adf_tl_dbg_aggr_values *vals) +{ + seq_printf(s, "%-*s", TL_KEY_MIN_PADDING, name); + seq_printf(s, "%*llu", TL_VALUE_MIN_PADDING, vals->curr); + if (atomic_read(&telemetry->state) > 1) { + seq_printf(s, "%*llu", TL_VALUE_MIN_PADDING, vals->min); + seq_printf(s, "%*llu", TL_VALUE_MIN_PADDING, vals->max); + seq_printf(s, "%*llu", TL_VALUE_MIN_PADDING, vals->avg); + } + seq_puts(s, "\n"); +} + +static int tl_calc_and_print_counter(struct adf_telemetry *telemetry, + struct seq_file *s, + const struct adf_tl_dbg_counter *ctr, + const char *name) +{ + const char *counter_name = name ? name : ctr->name; + enum adf_tl_counter_type type = ctr->type; + struct adf_tl_dbg_aggr_values vals; + int ret; + + switch (type) { + case ADF_TL_SIMPLE_COUNT: + ret = tl_calc_count(telemetry, ctr, &vals); + break; + case ADF_TL_COUNTER_NS: + ret = tl_cycles_to_ns(telemetry, ctr, &vals); + break; + case ADF_TL_COUNTER_NS_AVG: + ret = tl_lat_acc_avg(telemetry, ctr, &vals); + break; + case ADF_TL_COUNTER_MBPS: + ret = tl_bw_hw_units_to_mbps(telemetry, ctr, &vals); + break; + default: + return -EINVAL; + } + + if (ret) + return ret; + + tl_seq_printf_counter(telemetry, s, counter_name, &vals); + + return 0; +} + +static int tl_print_sl_counter(struct adf_telemetry *telemetry, + const struct adf_tl_dbg_counter *ctr, + struct seq_file *s, u8 cnt_id) +{ + size_t sl_regs_sz = GET_TL_DATA(telemetry->accel_dev).slice_reg_sz; + struct adf_tl_dbg_counter slice_ctr; + size_t offset_inc = cnt_id * sl_regs_sz; + char cnt_name[MAX_COUNT_NAME_SIZE]; + + snprintf(cnt_name, MAX_COUNT_NAME_SIZE, "%s%d", ctr->name, cnt_id); + slice_ctr = *ctr; + slice_ctr.offset1 += offset_inc; + + return tl_calc_and_print_counter(telemetry, s, &slice_ctr, cnt_name); +} + +static int tl_calc_and_print_sl_counters(struct adf_accel_dev *accel_dev, + struct seq_file *s, u8 cnt_type, u8 cnt_id) +{ + struct adf_tl_hw_data *tl_data = &GET_TL_DATA(accel_dev); + struct adf_telemetry *telemetry = accel_dev->telemetry; + const struct adf_tl_dbg_counter *sl_tl_util_counters; + const struct adf_tl_dbg_counter *sl_tl_exec_counters; + const struct adf_tl_dbg_counter *ctr; + int ret; + + sl_tl_util_counters = tl_data->sl_util_counters; + sl_tl_exec_counters = tl_data->sl_exec_counters; + + ctr = &sl_tl_util_counters[cnt_type]; + + ret = tl_print_sl_counter(telemetry, ctr, s, cnt_id); + if (ret) { + dev_notice(&GET_DEV(accel_dev), + "invalid slice utilization counter type\n"); + return ret; + } + + ctr = &sl_tl_exec_counters[cnt_type]; + + ret = tl_print_sl_counter(telemetry, ctr, s, cnt_id); + if (ret) { + dev_notice(&GET_DEV(accel_dev), + "invalid slice execution counter type\n"); + return ret; + } + + return 0; +} + +static void tl_print_msg_cnt(struct seq_file *s, u32 msg_cnt) +{ + seq_printf(s, "%-*s", TL_KEY_MIN_PADDING, SNAPSHOT_CNT_MSG); + seq_printf(s, "%*u\n", TL_VALUE_MIN_PADDING, msg_cnt); +} + +static int tl_print_dev_data(struct adf_accel_dev *accel_dev, + struct seq_file *s) +{ + struct adf_tl_hw_data *tl_data = &GET_TL_DATA(accel_dev); + struct adf_telemetry *telemetry = accel_dev->telemetry; + const struct adf_tl_dbg_counter *dev_tl_counters; + u8 num_dev_counters = tl_data->num_dev_counters; + u8 *sl_cnt = (u8 *)&telemetry->slice_cnt; + const struct adf_tl_dbg_counter *ctr; + unsigned int i; + int ret; + u8 j; + + if (!atomic_read(&telemetry->state)) { + dev_info(&GET_DEV(accel_dev), "not enabled\n"); + return -EPERM; + } + + dev_tl_counters = tl_data->dev_counters; + + tl_print_msg_cnt(s, telemetry->msg_cnt); + + /* Print device level telemetry. */ + for (i = 0; i < num_dev_counters; i++) { + ctr = &dev_tl_counters[i]; + ret = tl_calc_and_print_counter(telemetry, s, ctr, NULL); + if (ret) { + dev_notice(&GET_DEV(accel_dev), + "invalid counter type\n"); + return ret; + } + } + + /* Print per slice telemetry. */ + for (i = 0; i < ADF_TL_SL_CNT_COUNT; i++) { + for (j = 0; j < sl_cnt[i]; j++) { + ret = tl_calc_and_print_sl_counters(accel_dev, s, i, j); + if (ret) + return ret; + } + } + + return 0; +} + +static int tl_dev_data_show(struct seq_file *s, void *unused) +{ + struct adf_accel_dev *accel_dev = s->private; + + if (!accel_dev) + return -EINVAL; + + return tl_print_dev_data(accel_dev, s); +} +DEFINE_SHOW_ATTRIBUTE(tl_dev_data); + +static int tl_control_show(struct seq_file *s, void *unused) +{ + struct adf_accel_dev *accel_dev = s->private; + + if (!accel_dev) + return -EINVAL; + + seq_printf(s, "%d\n", atomic_read(&accel_dev->telemetry->state)); + + return 0; +} + +static ssize_t tl_control_write(struct file *file, const char __user *userbuf, + size_t count, loff_t *ppos) +{ + struct seq_file *seq_f = file->private_data; + struct adf_accel_dev *accel_dev; + struct adf_telemetry *telemetry; + struct adf_tl_hw_data *tl_data; + struct device *dev; + u32 input; + int ret; + + accel_dev = seq_f->private; + if (!accel_dev) + return -EINVAL; + + tl_data = &GET_TL_DATA(accel_dev); + telemetry = accel_dev->telemetry; + dev = &GET_DEV(accel_dev); + + mutex_lock(&telemetry->wr_lock); + + ret = kstrtou32_from_user(userbuf, count, 10, &input); + if (ret) + goto unlock_and_exit; + + if (input > tl_data->num_hbuff) { + dev_info(dev, "invalid control input\n"); + ret = -EINVAL; + goto unlock_and_exit; + } + + /* If input is 0, just stop telemetry. */ + if (!input) { + ret = adf_tl_halt(accel_dev); + if (!ret) + ret = count; + + goto unlock_and_exit; + } + + /* If TL is already enabled, stop it. */ + if (atomic_read(&telemetry->state)) { + dev_info(dev, "already enabled, restarting.\n"); + ret = adf_tl_halt(accel_dev); + if (ret) + goto unlock_and_exit; + } + + ret = adf_tl_run(accel_dev, input); + if (ret) + goto unlock_and_exit; + + ret = count; + +unlock_and_exit: + mutex_unlock(&telemetry->wr_lock); + return ret; +} +DEFINE_SHOW_STORE_ATTRIBUTE(tl_control); + +static int get_rp_index_from_file(const struct file *f, u8 *rp_id, u8 rp_num) +{ + char alpha; + u8 index; + int ret; + + ret = sscanf(f->f_path.dentry->d_name.name, ADF_TL_RP_REGS_FNAME, &alpha); + if (ret != 1) + return -EINVAL; + + index = ADF_TL_DBG_RP_INDEX_ALPHA(alpha); + *rp_id = index; + + return 0; +} + +static int adf_tl_dbg_change_rp_index(struct adf_accel_dev *accel_dev, + unsigned int new_rp_num, + unsigned int rp_regs_index) +{ + struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev); + struct adf_telemetry *telemetry = accel_dev->telemetry; + struct device *dev = &GET_DEV(accel_dev); + unsigned int i; + u8 curr_state; + int ret; + + if (new_rp_num >= hw_data->num_rps) { + dev_info(dev, "invalid Ring Pair number selected\n"); + return -EINVAL; + } + + for (i = 0; i < hw_data->tl_data.max_rp; i++) { + if (telemetry->rp_num_indexes[i] == new_rp_num) { + dev_info(dev, "RP nr: %d is already selected in slot rp_%c_data\n", + new_rp_num, ADF_TL_DBG_RP_ALPHA_INDEX(i)); + return 0; + } + } + + dev_dbg(dev, "selecting RP nr %u into slot rp_%c_data\n", + new_rp_num, ADF_TL_DBG_RP_ALPHA_INDEX(rp_regs_index)); + + curr_state = atomic_read(&telemetry->state); + + if (curr_state) { + ret = adf_tl_halt(accel_dev); + if (ret) + return ret; + + telemetry->rp_num_indexes[rp_regs_index] = new_rp_num; + + ret = adf_tl_run(accel_dev, curr_state); + if (ret) + return ret; + } else { + telemetry->rp_num_indexes[rp_regs_index] = new_rp_num; + } + + return 0; +} + +static void tl_print_rp_srv(struct adf_accel_dev *accel_dev, struct seq_file *s, + u8 rp_idx) +{ + u32 banks_per_vf = GET_HW_DATA(accel_dev)->num_banks_per_vf; + enum adf_cfg_service_type svc; + + seq_printf(s, "%-*s", TL_KEY_MIN_PADDING, RP_SERVICE_TYPE); + + svc = GET_SRV_TYPE(accel_dev, rp_idx % banks_per_vf); + switch (svc) { + case COMP: + seq_printf(s, "%*s\n", TL_VALUE_MIN_PADDING, ADF_CFG_DC); + break; + case SYM: + seq_printf(s, "%*s\n", TL_VALUE_MIN_PADDING, ADF_CFG_SYM); + break; + case ASYM: + seq_printf(s, "%*s\n", TL_VALUE_MIN_PADDING, ADF_CFG_ASYM); + break; + default: + seq_printf(s, "%*s\n", TL_VALUE_MIN_PADDING, TL_RP_SRV_UNKNOWN); + break; + } +} + +static int tl_print_rp_data(struct adf_accel_dev *accel_dev, struct seq_file *s, + u8 rp_regs_index) +{ + struct adf_tl_hw_data *tl_data = &GET_TL_DATA(accel_dev); + struct adf_telemetry *telemetry = accel_dev->telemetry; + const struct adf_tl_dbg_counter *rp_tl_counters; + u8 num_rp_counters = tl_data->num_rp_counters; + size_t rp_regs_sz = tl_data->rp_reg_sz; + struct adf_tl_dbg_counter ctr; + unsigned int i; + u8 rp_idx; + int ret; + + if (!atomic_read(&telemetry->state)) { + dev_info(&GET_DEV(accel_dev), "not enabled\n"); + return -EPERM; + } + + rp_tl_counters = tl_data->rp_counters; + rp_idx = telemetry->rp_num_indexes[rp_regs_index]; + + if (rp_idx == ADF_TL_RP_REGS_DISABLED) { + dev_info(&GET_DEV(accel_dev), "no RP number selected in rp_%c_data\n", + ADF_TL_DBG_RP_ALPHA_INDEX(rp_regs_index)); + return -EPERM; + } + + tl_print_msg_cnt(s, telemetry->msg_cnt); + seq_printf(s, "%-*s", TL_KEY_MIN_PADDING, RP_NUM_INDEX); + seq_printf(s, "%*d\n", TL_VALUE_MIN_PADDING, rp_idx); + tl_print_rp_srv(accel_dev, s, rp_idx); + + for (i = 0; i < num_rp_counters; i++) { + ctr = rp_tl_counters[i]; + ctr.offset1 += rp_regs_sz * rp_regs_index; + ctr.offset2 += rp_regs_sz * rp_regs_index; + ret = tl_calc_and_print_counter(telemetry, s, &ctr, NULL); + if (ret) { + dev_dbg(&GET_DEV(accel_dev), + "invalid RP counter type\n"); + return ret; + } + } + + return 0; +} + +static int tl_rp_data_show(struct seq_file *s, void *unused) +{ + struct adf_accel_dev *accel_dev = s->private; + u8 rp_regs_index; + u8 max_rp; + int ret; + + if (!accel_dev) + return -EINVAL; + + max_rp = GET_TL_DATA(accel_dev).max_rp; + ret = get_rp_index_from_file(s->file, &rp_regs_index, max_rp); + if (ret) { + dev_dbg(&GET_DEV(accel_dev), "invalid RP data file name\n"); + return ret; + } + + return tl_print_rp_data(accel_dev, s, rp_regs_index); +} + +static ssize_t tl_rp_data_write(struct file *file, const char __user *userbuf, + size_t count, loff_t *ppos) +{ + struct seq_file *seq_f = file->private_data; + struct adf_accel_dev *accel_dev; + struct adf_telemetry *telemetry; + unsigned int new_rp_num; + u8 rp_regs_index; + u8 max_rp; + int ret; + + accel_dev = seq_f->private; + if (!accel_dev) + return -EINVAL; + + telemetry = accel_dev->telemetry; + max_rp = GET_TL_DATA(accel_dev).max_rp; + + mutex_lock(&telemetry->wr_lock); + + ret = get_rp_index_from_file(file, &rp_regs_index, max_rp); + if (ret) { + dev_dbg(&GET_DEV(accel_dev), "invalid RP data file name\n"); + goto unlock_and_exit; + } + + ret = kstrtou32_from_user(userbuf, count, 10, &new_rp_num); + if (ret) + goto unlock_and_exit; + + ret = adf_tl_dbg_change_rp_index(accel_dev, new_rp_num, rp_regs_index); + if (ret) + goto unlock_and_exit; + + ret = count; + +unlock_and_exit: + mutex_unlock(&telemetry->wr_lock); + return ret; +} +DEFINE_SHOW_STORE_ATTRIBUTE(tl_rp_data); + +void adf_tl_dbgfs_add(struct adf_accel_dev *accel_dev) +{ + struct adf_telemetry *telemetry = accel_dev->telemetry; + struct dentry *parent = accel_dev->debugfs_dir; + u8 max_rp = GET_TL_DATA(accel_dev).max_rp; + char name[ADF_TL_RP_REGS_FNAME_SIZE]; + struct dentry *dir; + unsigned int i; + + if (!telemetry) + return; + + dir = debugfs_create_dir("telemetry", parent); + accel_dev->telemetry->dbg_dir = dir; + debugfs_create_file("device_data", 0444, dir, accel_dev, &tl_dev_data_fops); + debugfs_create_file("control", 0644, dir, accel_dev, &tl_control_fops); + + for (i = 0; i < max_rp; i++) { + snprintf(name, sizeof(name), ADF_TL_RP_REGS_FNAME, + ADF_TL_DBG_RP_ALPHA_INDEX(i)); + debugfs_create_file(name, 0644, dir, accel_dev, &tl_rp_data_fops); + } +} + +void adf_tl_dbgfs_rm(struct adf_accel_dev *accel_dev) +{ + struct adf_telemetry *telemetry = accel_dev->telemetry; + struct dentry *dbg_dir; + + if (!telemetry) + return; + + dbg_dir = telemetry->dbg_dir; + + debugfs_remove_recursive(dbg_dir); + + if (atomic_read(&telemetry->state)) + adf_tl_halt(accel_dev); +} diff --git a/drivers/crypto/intel/qat/qat_common/adf_tl_debugfs.h b/drivers/crypto/intel/qat/qat_common/adf_tl_debugfs.h new file mode 100644 index 000000000000..11cc9eae19b3 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_tl_debugfs.h @@ -0,0 +1,117 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (c) 2023 Intel Corporation. */ +#ifndef ADF_TL_DEBUGFS_H +#define ADF_TL_DEBUGFS_H + +#include <linux/types.h> + +struct adf_accel_dev; + +#define MAX_COUNT_NAME_SIZE 32 +#define SNAPSHOT_CNT_MSG "sample_cnt" +#define RP_NUM_INDEX "rp_num" +#define PCI_TRANS_CNT_NAME "pci_trans_cnt" +#define MAX_RD_LAT_NAME "max_rd_lat" +#define RD_LAT_ACC_NAME "rd_lat_acc_avg" +#define MAX_LAT_NAME "max_gp_lat" +#define LAT_ACC_NAME "gp_lat_acc_avg" +#define BW_IN_NAME "bw_in" +#define BW_OUT_NAME "bw_out" +#define PAGE_REQ_LAT_NAME "at_page_req_lat_avg" +#define AT_TRANS_LAT_NAME "at_trans_lat_avg" +#define AT_MAX_UTLB_USED_NAME "at_max_tlb_used" +#define AT_GLOB_DTLB_HIT_NAME "at_glob_devtlb_hit" +#define AT_GLOB_DTLB_MISS_NAME "at_glob_devtlb_miss" +#define AT_PAYLD_DTLB_HIT_NAME "tl_at_payld_devtlb_hit" +#define AT_PAYLD_DTLB_MISS_NAME "tl_at_payld_devtlb_miss" +#define RP_SERVICE_TYPE "service_type" + +#define ADF_TL_DBG_RP_ALPHA_INDEX(index) ((index) + 'A') +#define ADF_TL_DBG_RP_INDEX_ALPHA(alpha) ((alpha) - 'A') + +#define ADF_TL_RP_REGS_FNAME "rp_%c_data" +#define ADF_TL_RP_REGS_FNAME_SIZE 16 + +#define ADF_TL_DATA_REG_OFF(reg, qat_gen) \ + offsetof(struct adf_##qat_gen##_tl_layout, reg) + +#define ADF_TL_DEV_REG_OFF(reg, qat_gen) \ + (ADF_TL_DATA_REG_OFF(tl_device_data_regs, qat_gen) + \ + offsetof(struct adf_##qat_gen##_tl_device_data_regs, reg)) + +#define ADF_TL_SLICE_REG_OFF(slice, reg, qat_gen) \ + (ADF_TL_DEV_REG_OFF(slice##_slices[0], qat_gen) + \ + offsetof(struct adf_##qat_gen##_tl_slice_data_regs, reg)) + +#define ADF_TL_RP_REG_OFF(reg, qat_gen) \ + (ADF_TL_DATA_REG_OFF(tl_ring_pairs_data_regs[0], qat_gen) + \ + offsetof(struct adf_##qat_gen##_tl_ring_pair_data_regs, reg)) + +/** + * enum adf_tl_counter_type - telemetry counter types + * @ADF_TL_COUNTER_UNSUPPORTED: unsupported counter + * @ADF_TL_SIMPLE_COUNT: simple counter + * @ADF_TL_COUNTER_NS: latency counter, value in ns + * @ADF_TL_COUNTER_NS_AVG: accumulated average latency counter, value in ns + * @ADF_TL_COUNTER_MBPS: bandwidth, value in MBps + */ +enum adf_tl_counter_type { + ADF_TL_COUNTER_UNSUPPORTED, + ADF_TL_SIMPLE_COUNT, + ADF_TL_COUNTER_NS, + ADF_TL_COUNTER_NS_AVG, + ADF_TL_COUNTER_MBPS, +}; + +/** + * struct adf_tl_dbg_counter - telemetry counter definition + * @name: name of the counter as printed in the report + * @adf_tl_counter_type: type of the counter + * @offset1: offset of 1st register + * @offset2: offset of 2nd optional register + */ +struct adf_tl_dbg_counter { + const char *name; + enum adf_tl_counter_type type; + size_t offset1; + size_t offset2; +}; + +#define ADF_TL_COUNTER(_name, _type, _offset) \ +{ .name = _name, \ + .type = _type, \ + .offset1 = _offset \ +} + +#define ADF_TL_COUNTER_LATENCY(_name, _type, _offset1, _offset2) \ +{ .name = _name, \ + .type = _type, \ + .offset1 = _offset1, \ + .offset2 = _offset2 \ +} + +/* Telemetry counter aggregated values. */ +struct adf_tl_dbg_aggr_values { + u64 curr; + u64 min; + u64 max; + u64 avg; +}; + +/** + * adf_tl_dbgfs_add() - Add telemetry's debug fs entries. + * @accel_dev: Pointer to acceleration device. + * + * Creates telemetry's debug fs folder and attributes in QAT debug fs root. + */ +void adf_tl_dbgfs_add(struct adf_accel_dev *accel_dev); + +/** + * adf_tl_dbgfs_rm() - Remove telemetry's debug fs entries. + * @accel_dev: Pointer to acceleration device. + * + * Removes telemetry's debug fs folder and attributes from QAT debug fs root. + */ +void adf_tl_dbgfs_rm(struct adf_accel_dev *accel_dev); + +#endif /* ADF_TL_DEBUGFS_H */ diff --git a/drivers/crypto/intel/qat/qat_common/icp_qat_fw_init_admin.h b/drivers/crypto/intel/qat/qat_common/icp_qat_fw_init_admin.h index cd418b51d9f3..63cf18e2a4e5 100644 --- a/drivers/crypto/intel/qat/qat_common/icp_qat_fw_init_admin.h +++ b/drivers/crypto/intel/qat/qat_common/icp_qat_fw_init_admin.h @@ -29,6 +29,8 @@ enum icp_qat_fw_init_admin_cmd_id { ICP_QAT_FW_RL_ADD = 134, ICP_QAT_FW_RL_UPDATE = 135, ICP_QAT_FW_RL_REMOVE = 136, + ICP_QAT_FW_TL_START = 137, + ICP_QAT_FW_TL_STOP = 138, }; enum icp_qat_fw_init_admin_resp_status { @@ -36,6 +38,13 @@ enum icp_qat_fw_init_admin_resp_status { ICP_QAT_FW_INIT_RESP_STATUS_FAIL }; +struct icp_qat_fw_init_admin_tl_rp_indexes { + __u8 rp_num_index_0; + __u8 rp_num_index_1; + __u8 rp_num_index_2; + __u8 rp_num_index_3; +}; + struct icp_qat_fw_init_admin_slice_cnt { __u8 cpr_cnt; __u8 xlt_cnt; @@ -87,6 +96,7 @@ struct icp_qat_fw_init_admin_req { __u8 rp_count; }; __u32 idle_filter; + struct icp_qat_fw_init_admin_tl_rp_indexes rp_indexes; }; __u32 resrvd4; diff --git a/drivers/crypto/intel/qat/qat_common/icp_qat_hw.h b/drivers/crypto/intel/qat/qat_common/icp_qat_hw.h index eb2ef225bcee..b8f1c4ffb8b5 100644 --- a/drivers/crypto/intel/qat/qat_common/icp_qat_hw.h +++ b/drivers/crypto/intel/qat/qat_common/icp_qat_hw.h @@ -18,7 +18,12 @@ enum icp_qat_hw_ae_id { ICP_QAT_HW_AE_9 = 9, ICP_QAT_HW_AE_10 = 10, ICP_QAT_HW_AE_11 = 11, - ICP_QAT_HW_AE_DELIMITER = 12 + ICP_QAT_HW_AE_12 = 12, + ICP_QAT_HW_AE_13 = 13, + ICP_QAT_HW_AE_14 = 14, + ICP_QAT_HW_AE_15 = 15, + ICP_QAT_HW_AE_16 = 16, + ICP_QAT_HW_AE_DELIMITER = 17 }; enum icp_qat_hw_qat_id { @@ -95,7 +100,7 @@ enum icp_qat_capabilities_mask { /* Bits 10-11 are currently reserved */ ICP_ACCEL_CAPABILITIES_HKDF = BIT(12), ICP_ACCEL_CAPABILITIES_ECEDMONT = BIT(13), - /* Bit 14 is currently reserved */ + ICP_ACCEL_CAPABILITIES_EXT_ALGCHAIN = BIT(14), ICP_ACCEL_CAPABILITIES_SHA3_EXT = BIT(15), ICP_ACCEL_CAPABILITIES_AESGCM_SPC = BIT(16), ICP_ACCEL_CAPABILITIES_CHACHA_POLY = BIT(17), @@ -107,7 +112,10 @@ enum icp_qat_capabilities_mask { ICP_ACCEL_CAPABILITIES_CNV_INTEGRITY64 = BIT(23), ICP_ACCEL_CAPABILITIES_LZ4_COMPRESSION = BIT(24), ICP_ACCEL_CAPABILITIES_LZ4S_COMPRESSION = BIT(25), - ICP_ACCEL_CAPABILITIES_AES_V2 = BIT(26) + ICP_ACCEL_CAPABILITIES_AES_V2 = BIT(26), + /* Bits 27-28 are currently reserved */ + ICP_ACCEL_CAPABILITIES_ZUC_256 = BIT(29), + ICP_ACCEL_CAPABILITIES_WIRELESS_CRYPTO_EXT = BIT(30), }; #define QAT_AUTH_MODE_BITPOS 4 diff --git a/drivers/crypto/intel/qat/qat_common/icp_qat_uclo.h b/drivers/crypto/intel/qat/qat_common/icp_qat_uclo.h index 69482abdb8b9..e28241bdd0f4 100644 --- a/drivers/crypto/intel/qat/qat_common/icp_qat_uclo.h +++ b/drivers/crypto/intel/qat/qat_common/icp_qat_uclo.h @@ -7,7 +7,7 @@ #define ICP_QAT_AC_C62X_DEV_TYPE 0x01000000 #define ICP_QAT_AC_C3XXX_DEV_TYPE 0x02000000 #define ICP_QAT_AC_4XXX_A_DEV_TYPE 0x08000000 -#define ICP_QAT_UCLO_MAX_AE 12 +#define ICP_QAT_UCLO_MAX_AE 17 #define ICP_QAT_UCLO_MAX_CTX 8 #define ICP_QAT_UCLO_MAX_UIMAGE (ICP_QAT_UCLO_MAX_AE * ICP_QAT_UCLO_MAX_CTX) #define ICP_QAT_UCLO_MAX_USTORE 0x4000 diff --git a/drivers/crypto/intel/qat/qat_common/qat_hal.c b/drivers/crypto/intel/qat/qat_common/qat_hal.c index cbb946a80076..317cafa9d11f 100644 --- a/drivers/crypto/intel/qat/qat_common/qat_hal.c +++ b/drivers/crypto/intel/qat/qat_common/qat_hal.c @@ -697,12 +697,16 @@ static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle, case ADF_4XXX_PCI_DEVICE_ID: case ADF_401XX_PCI_DEVICE_ID: case ADF_402XX_PCI_DEVICE_ID: + case ADF_420XX_PCI_DEVICE_ID: handle->chip_info->mmp_sram_size = 0; handle->chip_info->nn = false; handle->chip_info->lm2lm3 = true; handle->chip_info->lm_size = ICP_QAT_UCLO_MAX_LMEM_REG_2X; handle->chip_info->icp_rst_csr = ICP_RESET_CPP0; - handle->chip_info->icp_rst_mask = 0x100015; + if (handle->pci_dev->device == ADF_420XX_PCI_DEVICE_ID) + handle->chip_info->icp_rst_mask = 0x100155; + else + handle->chip_info->icp_rst_mask = 0x100015; handle->chip_info->glb_clk_enable_csr = ICP_GLOBAL_CLK_ENABLE_CPP0; handle->chip_info->misc_ctl_csr = MISC_CONTROL_C4XXX; handle->chip_info->wakeup_event_val = 0x80000000; diff --git a/drivers/crypto/intel/qat/qat_common/qat_uclo.c b/drivers/crypto/intel/qat/qat_common/qat_uclo.c index e27ea7e28c51..ad2c64af7427 100644 --- a/drivers/crypto/intel/qat/qat_common/qat_uclo.c +++ b/drivers/crypto/intel/qat/qat_common/qat_uclo.c @@ -733,6 +733,7 @@ qat_uclo_get_dev_type(struct icp_qat_fw_loader_handle *handle) case ADF_4XXX_PCI_DEVICE_ID: case ADF_401XX_PCI_DEVICE_ID: case ADF_402XX_PCI_DEVICE_ID: + case ADF_420XX_PCI_DEVICE_ID: return ICP_QAT_AC_4XXX_A_DEV_TYPE; default: pr_err("QAT: unsupported device 0x%x\n", diff --git a/drivers/crypto/marvell/cesa/cesa.c b/drivers/crypto/marvell/cesa/cesa.c index 5744df30c838..5fd31ba715c2 100644 --- a/drivers/crypto/marvell/cesa/cesa.c +++ b/drivers/crypto/marvell/cesa/cesa.c @@ -488,7 +488,7 @@ static int mv_cesa_probe(struct platform_device *pdev) for (i = 0; i < caps->nengines; i++) { struct mv_cesa_engine *engine = &cesa->engines[i]; - char res_name[7]; + char res_name[16]; engine->id = i; spin_lock_init(&engine->lock); @@ -509,7 +509,7 @@ static int mv_cesa_probe(struct platform_device *pdev) * Not all platforms can gate the CESA clocks: do not complain * if the clock does not exist. */ - snprintf(res_name, sizeof(res_name), "cesa%d", i); + snprintf(res_name, sizeof(res_name), "cesa%u", i); engine->clk = devm_clk_get(dev, res_name); if (IS_ERR(engine->clk)) { engine->clk = devm_clk_get(dev, NULL); @@ -517,7 +517,7 @@ static int mv_cesa_probe(struct platform_device *pdev) engine->clk = NULL; } - snprintf(res_name, sizeof(res_name), "cesaz%d", i); + snprintf(res_name, sizeof(res_name), "cesaz%u", i); engine->zclk = devm_clk_get(dev, res_name); if (IS_ERR(engine->zclk)) engine->zclk = NULL; diff --git a/drivers/crypto/marvell/octeontx/otx_cptvf_algs.c b/drivers/crypto/marvell/octeontx/otx_cptvf_algs.c index 1c2c870e887a..3c5d577d8f0d 100644 --- a/drivers/crypto/marvell/octeontx/otx_cptvf_algs.c +++ b/drivers/crypto/marvell/octeontx/otx_cptvf_algs.c @@ -473,12 +473,6 @@ static int otx_cpt_skcipher_ecb_aes_setkey(struct crypto_skcipher *tfm, return cpt_aes_setkey(tfm, key, keylen, OTX_CPT_AES_ECB); } -static int otx_cpt_skcipher_cfb_aes_setkey(struct crypto_skcipher *tfm, - const u8 *key, u32 keylen) -{ - return cpt_aes_setkey(tfm, key, keylen, OTX_CPT_AES_CFB); -} - static int otx_cpt_skcipher_cbc_des3_setkey(struct crypto_skcipher *tfm, const u8 *key, u32 keylen) { @@ -1352,23 +1346,6 @@ static struct skcipher_alg otx_cpt_skciphers[] = { { .encrypt = otx_cpt_skcipher_encrypt, .decrypt = otx_cpt_skcipher_decrypt, }, { - .base.cra_name = "cfb(aes)", - .base.cra_driver_name = "cpt_cfb_aes", - .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, - .base.cra_blocksize = AES_BLOCK_SIZE, - .base.cra_ctxsize = sizeof(struct otx_cpt_enc_ctx), - .base.cra_alignmask = 7, - .base.cra_priority = 4001, - .base.cra_module = THIS_MODULE, - - .init = otx_cpt_enc_dec_init, - .ivsize = AES_BLOCK_SIZE, - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .setkey = otx_cpt_skcipher_cfb_aes_setkey, - .encrypt = otx_cpt_skcipher_encrypt, - .decrypt = otx_cpt_skcipher_decrypt, -}, { .base.cra_name = "cbc(des3_ede)", .base.cra_driver_name = "cpt_cbc_des3_ede", .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY, diff --git a/drivers/crypto/marvell/octeontx2/cn10k_cpt.c b/drivers/crypto/marvell/octeontx2/cn10k_cpt.c index 93d22b328991..79b4e74804f6 100644 --- a/drivers/crypto/marvell/octeontx2/cn10k_cpt.c +++ b/drivers/crypto/marvell/octeontx2/cn10k_cpt.c @@ -14,12 +14,14 @@ static struct cpt_hw_ops otx2_hw_ops = { .send_cmd = otx2_cpt_send_cmd, .cpt_get_compcode = otx2_cpt_get_compcode, .cpt_get_uc_compcode = otx2_cpt_get_uc_compcode, + .cpt_sg_info_create = otx2_sg_info_create, }; static struct cpt_hw_ops cn10k_hw_ops = { .send_cmd = cn10k_cpt_send_cmd, .cpt_get_compcode = cn10k_cpt_get_compcode, .cpt_get_uc_compcode = cn10k_cpt_get_uc_compcode, + .cpt_sg_info_create = otx2_sg_info_create, }; static void cn10k_cpt_send_cmd(union otx2_cpt_inst_s *cptinst, u32 insts_num, @@ -78,12 +80,9 @@ int cn10k_cptvf_lmtst_init(struct otx2_cptvf_dev *cptvf) struct pci_dev *pdev = cptvf->pdev; resource_size_t offset, size; - if (!test_bit(CN10K_LMTST, &cptvf->cap_flag)) { - cptvf->lfs.ops = &otx2_hw_ops; + if (!test_bit(CN10K_LMTST, &cptvf->cap_flag)) return 0; - } - cptvf->lfs.ops = &cn10k_hw_ops; offset = pci_resource_start(pdev, PCI_MBOX_BAR_NUM); size = pci_resource_len(pdev, PCI_MBOX_BAR_NUM); /* Map VF LMILINE region */ @@ -96,3 +95,82 @@ int cn10k_cptvf_lmtst_init(struct otx2_cptvf_dev *cptvf) return 0; } EXPORT_SYMBOL_NS_GPL(cn10k_cptvf_lmtst_init, CRYPTO_DEV_OCTEONTX2_CPT); + +void cn10k_cpt_hw_ctx_clear(struct pci_dev *pdev, + struct cn10k_cpt_errata_ctx *er_ctx) +{ + u64 cptr_dma; + + if (!is_dev_cn10ka_ax(pdev)) + return; + + cptr_dma = er_ctx->cptr_dma & ~(BIT_ULL(60)); + cn10k_cpt_ctx_flush(pdev, cptr_dma, true); + dma_unmap_single(&pdev->dev, cptr_dma, CN10K_CPT_HW_CTX_SIZE, + DMA_BIDIRECTIONAL); + kfree(er_ctx->hw_ctx); +} +EXPORT_SYMBOL_NS_GPL(cn10k_cpt_hw_ctx_clear, CRYPTO_DEV_OCTEONTX2_CPT); + +void cn10k_cpt_hw_ctx_set(union cn10k_cpt_hw_ctx *hctx, u16 ctx_sz) +{ + hctx->w0.aop_valid = 1; + hctx->w0.ctx_hdr_sz = 0; + hctx->w0.ctx_sz = ctx_sz; + hctx->w0.ctx_push_sz = 1; +} +EXPORT_SYMBOL_NS_GPL(cn10k_cpt_hw_ctx_set, CRYPTO_DEV_OCTEONTX2_CPT); + +int cn10k_cpt_hw_ctx_init(struct pci_dev *pdev, + struct cn10k_cpt_errata_ctx *er_ctx) +{ + union cn10k_cpt_hw_ctx *hctx; + u64 cptr_dma; + + er_ctx->cptr_dma = 0; + er_ctx->hw_ctx = NULL; + + if (!is_dev_cn10ka_ax(pdev)) + return 0; + + hctx = kmalloc(CN10K_CPT_HW_CTX_SIZE, GFP_KERNEL); + if (unlikely(!hctx)) + return -ENOMEM; + cptr_dma = dma_map_single(&pdev->dev, hctx, CN10K_CPT_HW_CTX_SIZE, + DMA_BIDIRECTIONAL); + + cn10k_cpt_hw_ctx_set(hctx, 1); + er_ctx->hw_ctx = hctx; + er_ctx->cptr_dma = cptr_dma | BIT_ULL(60); + + return 0; +} +EXPORT_SYMBOL_NS_GPL(cn10k_cpt_hw_ctx_init, CRYPTO_DEV_OCTEONTX2_CPT); + +void cn10k_cpt_ctx_flush(struct pci_dev *pdev, u64 cptr, bool inval) +{ + struct otx2_cptvf_dev *cptvf = pci_get_drvdata(pdev); + struct otx2_cptlfs_info *lfs = &cptvf->lfs; + u64 reg; + + reg = (uintptr_t)cptr >> 7; + if (inval) + reg = reg | BIT_ULL(46); + + otx2_cpt_write64(lfs->reg_base, lfs->blkaddr, lfs->lf[0].slot, + OTX2_CPT_LF_CTX_FLUSH, reg); + /* Make sure that the FLUSH operation is complete */ + wmb(); + otx2_cpt_read64(lfs->reg_base, lfs->blkaddr, lfs->lf[0].slot, + OTX2_CPT_LF_CTX_ERR); +} +EXPORT_SYMBOL_NS_GPL(cn10k_cpt_ctx_flush, CRYPTO_DEV_OCTEONTX2_CPT); + +void cptvf_hw_ops_get(struct otx2_cptvf_dev *cptvf) +{ + if (test_bit(CN10K_LMTST, &cptvf->cap_flag)) + cptvf->lfs.ops = &cn10k_hw_ops; + else + cptvf->lfs.ops = &otx2_hw_ops; +} +EXPORT_SYMBOL_NS_GPL(cptvf_hw_ops_get, CRYPTO_DEV_OCTEONTX2_CPT); diff --git a/drivers/crypto/marvell/octeontx2/cn10k_cpt.h b/drivers/crypto/marvell/octeontx2/cn10k_cpt.h index aaefc7e38e06..92be3ecf570f 100644 --- a/drivers/crypto/marvell/octeontx2/cn10k_cpt.h +++ b/drivers/crypto/marvell/octeontx2/cn10k_cpt.h @@ -8,6 +8,26 @@ #include "otx2_cptpf.h" #include "otx2_cptvf.h" +#define CN10K_CPT_HW_CTX_SIZE 256 + +union cn10k_cpt_hw_ctx { + u64 u; + struct { + u64 reserved_0_47:48; + u64 ctx_push_sz:7; + u64 reserved_55:1; + u64 ctx_hdr_sz:2; + u64 aop_valid:1; + u64 reserved_59:1; + u64 ctx_sz:4; + } w0; +}; + +struct cn10k_cpt_errata_ctx { + union cn10k_cpt_hw_ctx *hw_ctx; + u64 cptr_dma; +}; + static inline u8 cn10k_cpt_get_compcode(union otx2_cpt_res_s *result) { return ((struct cn10k_cpt_res_s *)result)->compcode; @@ -30,5 +50,12 @@ static inline u8 otx2_cpt_get_uc_compcode(union otx2_cpt_res_s *result) int cn10k_cptpf_lmtst_init(struct otx2_cptpf_dev *cptpf); int cn10k_cptvf_lmtst_init(struct otx2_cptvf_dev *cptvf); +void cn10k_cpt_ctx_flush(struct pci_dev *pdev, u64 cptr, bool inval); +int cn10k_cpt_hw_ctx_init(struct pci_dev *pdev, + struct cn10k_cpt_errata_ctx *er_ctx); +void cn10k_cpt_hw_ctx_clear(struct pci_dev *pdev, + struct cn10k_cpt_errata_ctx *er_ctx); +void cn10k_cpt_hw_ctx_set(union cn10k_cpt_hw_ctx *hctx, u16 ctx_sz); +void cptvf_hw_ops_get(struct otx2_cptvf_dev *cptvf); #endif /* __CN10K_CPTLF_H */ diff --git a/drivers/crypto/marvell/octeontx2/otx2_cpt_common.h b/drivers/crypto/marvell/octeontx2/otx2_cpt_common.h index 46b778bbbee4..c5b7c57574ef 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cpt_common.h +++ b/drivers/crypto/marvell/octeontx2/otx2_cpt_common.h @@ -56,7 +56,11 @@ struct otx2_cpt_rx_inline_lf_cfg { u16 param2; u16 opcode; u32 credit; + u32 credit_th; + u16 bpid; u32 reserved; + u8 ctx_ilen_valid : 1; + u8 ctx_ilen : 7; }; /* @@ -102,7 +106,10 @@ union otx2_cpt_eng_caps { u64 kasumi:1; u64 des:1; u64 crc:1; - u64 reserved_14_63:50; + u64 mmul:1; + u64 reserved_15_33:19; + u64 pdcp_chain:1; + u64 reserved_35_63:29; }; }; @@ -145,6 +152,35 @@ static inline bool is_dev_otx2(struct pci_dev *pdev) return false; } +static inline bool is_dev_cn10ka(struct pci_dev *pdev) +{ + return pdev->subsystem_device == CPT_PCI_SUBSYS_DEVID_CN10K_A; +} + +static inline bool is_dev_cn10ka_ax(struct pci_dev *pdev) +{ + if (pdev->subsystem_device == CPT_PCI_SUBSYS_DEVID_CN10K_A && + ((pdev->revision & 0xFF) == 4 || (pdev->revision & 0xFF) == 0x50 || + (pdev->revision & 0xff) == 0x51)) + return true; + + return false; +} + +static inline bool is_dev_cn10kb(struct pci_dev *pdev) +{ + return pdev->subsystem_device == CPT_PCI_SUBSYS_DEVID_CN10K_B; +} + +static inline bool is_dev_cn10ka_b0(struct pci_dev *pdev) +{ + if (pdev->subsystem_device == CPT_PCI_SUBSYS_DEVID_CN10K_A && + (pdev->revision & 0xFF) == 0x54) + return true; + + return false; +} + static inline void otx2_cpt_set_hw_caps(struct pci_dev *pdev, unsigned long *cap_flag) { @@ -154,6 +190,21 @@ static inline void otx2_cpt_set_hw_caps(struct pci_dev *pdev, } } +static inline bool cpt_is_errata_38550_exists(struct pci_dev *pdev) +{ + if (is_dev_otx2(pdev) || is_dev_cn10ka_ax(pdev)) + return true; + + return false; +} + +static inline bool cpt_feature_sgv2(struct pci_dev *pdev) +{ + if (!is_dev_otx2(pdev) && !is_dev_cn10ka_ax(pdev)) + return true; + + return false; +} int otx2_cpt_send_ready_msg(struct otx2_mbox *mbox, struct pci_dev *pdev); int otx2_cpt_send_mbox_msg(struct otx2_mbox *mbox, struct pci_dev *pdev); @@ -171,5 +222,6 @@ int otx2_cpt_attach_rscrs_msg(struct otx2_cptlfs_info *lfs); int otx2_cpt_detach_rsrcs_msg(struct otx2_cptlfs_info *lfs); int otx2_cpt_msix_offset_msg(struct otx2_cptlfs_info *lfs); int otx2_cpt_sync_mbox_msg(struct otx2_mbox *mbox); +int otx2_cpt_lf_reset_msg(struct otx2_cptlfs_info *lfs, int slot); #endif /* __OTX2_CPT_COMMON_H */ diff --git a/drivers/crypto/marvell/octeontx2/otx2_cpt_devlink.c b/drivers/crypto/marvell/octeontx2/otx2_cpt_devlink.c index a2aba0b0d68a..d2b8d26db968 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cpt_devlink.c +++ b/drivers/crypto/marvell/octeontx2/otx2_cpt_devlink.c @@ -24,10 +24,45 @@ static int otx2_cpt_dl_egrp_delete(struct devlink *dl, u32 id, static int otx2_cpt_dl_uc_info(struct devlink *dl, u32 id, struct devlink_param_gset_ctx *ctx) { + ctx->val.vstr[0] = '\0'; + + return 0; +} + +static int otx2_cpt_dl_t106_mode_get(struct devlink *dl, u32 id, + struct devlink_param_gset_ctx *ctx) +{ struct otx2_cpt_devlink *cpt_dl = devlink_priv(dl); struct otx2_cptpf_dev *cptpf = cpt_dl->cptpf; + struct pci_dev *pdev = cptpf->pdev; + u64 reg_val = 0; + + otx2_cpt_read_af_reg(&cptpf->afpf_mbox, pdev, CPT_AF_CTL, ®_val, + BLKADDR_CPT0); + ctx->val.vu8 = (reg_val >> 18) & 0x1; + + return 0; +} - otx2_cpt_print_uc_dbg_info(cptpf); +static int otx2_cpt_dl_t106_mode_set(struct devlink *dl, u32 id, + struct devlink_param_gset_ctx *ctx) +{ + struct otx2_cpt_devlink *cpt_dl = devlink_priv(dl); + struct otx2_cptpf_dev *cptpf = cpt_dl->cptpf; + struct pci_dev *pdev = cptpf->pdev; + u64 reg_val = 0; + + if (cptpf->enabled_vfs != 0 || cptpf->eng_grps.is_grps_created) + return -EPERM; + + if (cpt_feature_sgv2(pdev)) { + otx2_cpt_read_af_reg(&cptpf->afpf_mbox, pdev, CPT_AF_CTL, + ®_val, BLKADDR_CPT0); + reg_val &= ~(0x1ULL << 18); + reg_val |= ((u64)ctx->val.vu8 & 0x1) << 18; + return otx2_cpt_write_af_reg(&cptpf->afpf_mbox, pdev, + CPT_AF_CTL, reg_val, BLKADDR_CPT0); + } return 0; } @@ -36,6 +71,7 @@ enum otx2_cpt_dl_param_id { OTX2_CPT_DEVLINK_PARAM_ID_BASE = DEVLINK_PARAM_GENERIC_ID_MAX, OTX2_CPT_DEVLINK_PARAM_ID_EGRP_CREATE, OTX2_CPT_DEVLINK_PARAM_ID_EGRP_DELETE, + OTX2_CPT_DEVLINK_PARAM_ID_T106_MODE, }; static const struct devlink_param otx2_cpt_dl_params[] = { @@ -49,6 +85,11 @@ static const struct devlink_param otx2_cpt_dl_params[] = { BIT(DEVLINK_PARAM_CMODE_RUNTIME), otx2_cpt_dl_uc_info, otx2_cpt_dl_egrp_delete, NULL), + DEVLINK_PARAM_DRIVER(OTX2_CPT_DEVLINK_PARAM_ID_T106_MODE, + "t106_mode", DEVLINK_PARAM_TYPE_U8, + BIT(DEVLINK_PARAM_CMODE_RUNTIME), + otx2_cpt_dl_t106_mode_get, otx2_cpt_dl_t106_mode_set, + NULL), }; static int otx2_cpt_dl_info_firmware_version_put(struct devlink_info_req *req, @@ -120,7 +161,6 @@ int otx2_cpt_register_dl(struct otx2_cptpf_dev *cptpf) devlink_free(dl); return ret; } - devlink_register(dl); return 0; diff --git a/drivers/crypto/marvell/octeontx2/otx2_cpt_hw_types.h b/drivers/crypto/marvell/octeontx2/otx2_cpt_hw_types.h index 6f947978e4e8..7e746a4def86 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cpt_hw_types.h +++ b/drivers/crypto/marvell/octeontx2/otx2_cpt_hw_types.h @@ -13,6 +13,9 @@ #define CN10K_CPT_PCI_PF_DEVICE_ID 0xA0F2 #define CN10K_CPT_PCI_VF_DEVICE_ID 0xA0F3 +#define CPT_PCI_SUBSYS_DEVID_CN10K_A 0xB900 +#define CPT_PCI_SUBSYS_DEVID_CN10K_B 0xBD00 + /* Mailbox interrupts offset */ #define OTX2_CPT_PF_MBOX_INT 6 #define OTX2_CPT_PF_INT_VEC_E_MBOXX(x, a) ((x) + (a)) @@ -99,6 +102,9 @@ #define OTX2_CPT_LF_Q_INST_PTR (0x110) #define OTX2_CPT_LF_Q_GRP_PTR (0x120) #define OTX2_CPT_LF_NQX(a) (0x400 | (a) << 3) +#define OTX2_CPT_LF_CTX_CTL (0x500) +#define OTX2_CPT_LF_CTX_FLUSH (0x510) +#define OTX2_CPT_LF_CTX_ERR (0x520) #define OTX2_CPT_RVU_FUNC_BLKADDR_SHIFT 20 /* LMT LF registers */ #define OTX2_CPT_LMT_LFBASE BIT_ULL(OTX2_CPT_RVU_FUNC_BLKADDR_SHIFT) @@ -467,7 +473,8 @@ union otx2_cptx_af_lf_ctrl { u64 cont_err:1; u64 reserved_11_15:5; u64 nixtx_en:1; - u64 reserved_17_47:31; + u64 ctx_ilen:3; + u64 reserved_17_47:28; u64 grp:8; u64 reserved_56_63:8; } s; diff --git a/drivers/crypto/marvell/octeontx2/otx2_cpt_mbox_common.c b/drivers/crypto/marvell/octeontx2/otx2_cpt_mbox_common.c index 273ee5352a50..5be0103c1fb8 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cpt_mbox_common.c +++ b/drivers/crypto/marvell/octeontx2/otx2_cpt_mbox_common.c @@ -229,3 +229,29 @@ int otx2_cpt_sync_mbox_msg(struct otx2_mbox *mbox) return otx2_mbox_check_rsp_msgs(mbox, 0); } EXPORT_SYMBOL_NS_GPL(otx2_cpt_sync_mbox_msg, CRYPTO_DEV_OCTEONTX2_CPT); + +int otx2_cpt_lf_reset_msg(struct otx2_cptlfs_info *lfs, int slot) +{ + struct otx2_mbox *mbox = lfs->mbox; + struct pci_dev *pdev = lfs->pdev; + struct cpt_lf_rst_req *req; + int ret; + + req = (struct cpt_lf_rst_req *)otx2_mbox_alloc_msg_rsp(mbox, 0, sizeof(*req), + sizeof(struct msg_rsp)); + if (!req) { + dev_err(&pdev->dev, "RVU MBOX failed to get message.\n"); + return -EFAULT; + } + + req->hdr.id = MBOX_MSG_CPT_LF_RESET; + req->hdr.sig = OTX2_MBOX_REQ_SIG; + req->hdr.pcifunc = 0; + req->slot = slot; + ret = otx2_cpt_send_mbox_msg(mbox, pdev); + if (ret) + return ret; + + return ret; +} +EXPORT_SYMBOL_NS_GPL(otx2_cpt_lf_reset_msg, CRYPTO_DEV_OCTEONTX2_CPT); diff --git a/drivers/crypto/marvell/octeontx2/otx2_cpt_reqmgr.h b/drivers/crypto/marvell/octeontx2/otx2_cpt_reqmgr.h index dbb1ee746f4c..e27e849b01df 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cpt_reqmgr.h +++ b/drivers/crypto/marvell/octeontx2/otx2_cpt_reqmgr.h @@ -27,6 +27,13 @@ #define OTX2_CPT_MAX_REQ_SIZE 65535 +#define SG_COMPS_MAX 4 +#define SGV2_COMPS_MAX 3 + +#define SG_COMP_3 3 +#define SG_COMP_2 2 +#define SG_COMP_1 1 + union otx2_cpt_opcode { u16 flags; struct { @@ -40,6 +47,8 @@ struct otx2_cptvf_request { u32 param2; u16 dlen; union otx2_cpt_opcode opcode; + dma_addr_t cptr_dma; + void *cptr; }; /* @@ -143,6 +152,8 @@ struct otx2_cpt_inst_info { unsigned long time_in; u32 dlen; u32 dma_len; + u64 gthr_sz; + u64 sctr_sz; u8 extra_time; }; @@ -157,6 +168,16 @@ struct otx2_cpt_sglist_component { __be64 ptr3; }; +struct cn10kb_cpt_sglist_component { + u16 len0; + u16 len1; + u16 len2; + u16 valid_segs; + u64 ptr0; + u64 ptr1; + u64 ptr2; +}; + static inline void otx2_cpt_info_destroy(struct pci_dev *pdev, struct otx2_cpt_inst_info *info) { @@ -188,6 +209,283 @@ static inline void otx2_cpt_info_destroy(struct pci_dev *pdev, kfree(info); } +static inline int setup_sgio_components(struct pci_dev *pdev, + struct otx2_cpt_buf_ptr *list, + int buf_count, u8 *buffer) +{ + struct otx2_cpt_sglist_component *sg_ptr; + int components; + int i, j; + + if (unlikely(!list)) { + dev_err(&pdev->dev, "Input list pointer is NULL\n"); + return -EINVAL; + } + + for (i = 0; i < buf_count; i++) { + if (unlikely(!list[i].vptr)) + continue; + list[i].dma_addr = dma_map_single(&pdev->dev, list[i].vptr, + list[i].size, + DMA_BIDIRECTIONAL); + if (unlikely(dma_mapping_error(&pdev->dev, list[i].dma_addr))) { + dev_err(&pdev->dev, "Dma mapping failed\n"); + goto sg_cleanup; + } + } + components = buf_count / SG_COMPS_MAX; + sg_ptr = (struct otx2_cpt_sglist_component *)buffer; + for (i = 0; i < components; i++) { + sg_ptr->len0 = cpu_to_be16(list[i * SG_COMPS_MAX + 0].size); + sg_ptr->len1 = cpu_to_be16(list[i * SG_COMPS_MAX + 1].size); + sg_ptr->len2 = cpu_to_be16(list[i * SG_COMPS_MAX + 2].size); + sg_ptr->len3 = cpu_to_be16(list[i * SG_COMPS_MAX + 3].size); + sg_ptr->ptr0 = cpu_to_be64(list[i * SG_COMPS_MAX + 0].dma_addr); + sg_ptr->ptr1 = cpu_to_be64(list[i * SG_COMPS_MAX + 1].dma_addr); + sg_ptr->ptr2 = cpu_to_be64(list[i * SG_COMPS_MAX + 2].dma_addr); + sg_ptr->ptr3 = cpu_to_be64(list[i * SG_COMPS_MAX + 3].dma_addr); + sg_ptr++; + } + components = buf_count % SG_COMPS_MAX; + + switch (components) { + case SG_COMP_3: + sg_ptr->len2 = cpu_to_be16(list[i * SG_COMPS_MAX + 2].size); + sg_ptr->ptr2 = cpu_to_be64(list[i * SG_COMPS_MAX + 2].dma_addr); + fallthrough; + case SG_COMP_2: + sg_ptr->len1 = cpu_to_be16(list[i * SG_COMPS_MAX + 1].size); + sg_ptr->ptr1 = cpu_to_be64(list[i * SG_COMPS_MAX + 1].dma_addr); + fallthrough; + case SG_COMP_1: + sg_ptr->len0 = cpu_to_be16(list[i * SG_COMPS_MAX + 0].size); + sg_ptr->ptr0 = cpu_to_be64(list[i * SG_COMPS_MAX + 0].dma_addr); + break; + default: + break; + } + return 0; + +sg_cleanup: + for (j = 0; j < i; j++) { + if (list[j].dma_addr) { + dma_unmap_single(&pdev->dev, list[j].dma_addr, + list[j].size, DMA_BIDIRECTIONAL); + } + + list[j].dma_addr = 0; + } + return -EIO; +} + +static inline int sgv2io_components_setup(struct pci_dev *pdev, + struct otx2_cpt_buf_ptr *list, + int buf_count, u8 *buffer) +{ + struct cn10kb_cpt_sglist_component *sg_ptr; + int components; + int i, j; + + if (unlikely(!list)) { + dev_err(&pdev->dev, "Input list pointer is NULL\n"); + return -EFAULT; + } + + for (i = 0; i < buf_count; i++) { + if (unlikely(!list[i].vptr)) + continue; + list[i].dma_addr = dma_map_single(&pdev->dev, list[i].vptr, + list[i].size, + DMA_BIDIRECTIONAL); + if (unlikely(dma_mapping_error(&pdev->dev, list[i].dma_addr))) { + dev_err(&pdev->dev, "Dma mapping failed\n"); + goto sg_cleanup; + } + } + components = buf_count / SGV2_COMPS_MAX; + sg_ptr = (struct cn10kb_cpt_sglist_component *)buffer; + for (i = 0; i < components; i++) { + sg_ptr->len0 = list[i * SGV2_COMPS_MAX + 0].size; + sg_ptr->len1 = list[i * SGV2_COMPS_MAX + 1].size; + sg_ptr->len2 = list[i * SGV2_COMPS_MAX + 2].size; + sg_ptr->ptr0 = list[i * SGV2_COMPS_MAX + 0].dma_addr; + sg_ptr->ptr1 = list[i * SGV2_COMPS_MAX + 1].dma_addr; + sg_ptr->ptr2 = list[i * SGV2_COMPS_MAX + 2].dma_addr; + sg_ptr->valid_segs = SGV2_COMPS_MAX; + sg_ptr++; + } + components = buf_count % SGV2_COMPS_MAX; + + sg_ptr->valid_segs = components; + switch (components) { + case SG_COMP_2: + sg_ptr->len1 = list[i * SGV2_COMPS_MAX + 1].size; + sg_ptr->ptr1 = list[i * SGV2_COMPS_MAX + 1].dma_addr; + fallthrough; + case SG_COMP_1: + sg_ptr->len0 = list[i * SGV2_COMPS_MAX + 0].size; + sg_ptr->ptr0 = list[i * SGV2_COMPS_MAX + 0].dma_addr; + break; + default: + break; + } + return 0; + +sg_cleanup: + for (j = 0; j < i; j++) { + if (list[j].dma_addr) { + dma_unmap_single(&pdev->dev, list[j].dma_addr, + list[j].size, DMA_BIDIRECTIONAL); + } + + list[j].dma_addr = 0; + } + return -EIO; +} + +static inline struct otx2_cpt_inst_info * +cn10k_sgv2_info_create(struct pci_dev *pdev, struct otx2_cpt_req_info *req, + gfp_t gfp) +{ + u32 dlen = 0, g_len, sg_len, info_len; + int align = OTX2_CPT_DMA_MINALIGN; + struct otx2_cpt_inst_info *info; + u16 g_sz_bytes, s_sz_bytes; + u32 total_mem_len; + int i; + + g_sz_bytes = ((req->in_cnt + 2) / 3) * + sizeof(struct cn10kb_cpt_sglist_component); + s_sz_bytes = ((req->out_cnt + 2) / 3) * + sizeof(struct cn10kb_cpt_sglist_component); + + g_len = ALIGN(g_sz_bytes, align); + sg_len = ALIGN(g_len + s_sz_bytes, align); + info_len = ALIGN(sizeof(*info), align); + total_mem_len = sg_len + info_len + sizeof(union otx2_cpt_res_s); + + info = kzalloc(total_mem_len, gfp); + if (unlikely(!info)) + return NULL; + + for (i = 0; i < req->in_cnt; i++) + dlen += req->in[i].size; + + info->dlen = dlen; + info->in_buffer = (u8 *)info + info_len; + info->gthr_sz = req->in_cnt; + info->sctr_sz = req->out_cnt; + + /* Setup gather (input) components */ + if (sgv2io_components_setup(pdev, req->in, req->in_cnt, + info->in_buffer)) { + dev_err(&pdev->dev, "Failed to setup gather list\n"); + goto destroy_info; + } + + if (sgv2io_components_setup(pdev, req->out, req->out_cnt, + &info->in_buffer[g_len])) { + dev_err(&pdev->dev, "Failed to setup scatter list\n"); + goto destroy_info; + } + + info->dma_len = total_mem_len - info_len; + info->dptr_baddr = dma_map_single(&pdev->dev, info->in_buffer, + info->dma_len, DMA_BIDIRECTIONAL); + if (unlikely(dma_mapping_error(&pdev->dev, info->dptr_baddr))) { + dev_err(&pdev->dev, "DMA Mapping failed for cpt req\n"); + goto destroy_info; + } + info->rptr_baddr = info->dptr_baddr + g_len; + /* + * Get buffer for union otx2_cpt_res_s response + * structure and its physical address + */ + info->completion_addr = info->in_buffer + sg_len; + info->comp_baddr = info->dptr_baddr + sg_len; + + return info; + +destroy_info: + otx2_cpt_info_destroy(pdev, info); + return NULL; +} + +/* SG list header size in bytes */ +#define SG_LIST_HDR_SIZE 8 +static inline struct otx2_cpt_inst_info * +otx2_sg_info_create(struct pci_dev *pdev, struct otx2_cpt_req_info *req, + gfp_t gfp) +{ + int align = OTX2_CPT_DMA_MINALIGN; + struct otx2_cpt_inst_info *info; + u32 dlen, align_dlen, info_len; + u16 g_sz_bytes, s_sz_bytes; + u32 total_mem_len; + + if (unlikely(req->in_cnt > OTX2_CPT_MAX_SG_IN_CNT || + req->out_cnt > OTX2_CPT_MAX_SG_OUT_CNT)) { + dev_err(&pdev->dev, "Error too many sg components\n"); + return NULL; + } + + g_sz_bytes = ((req->in_cnt + 3) / 4) * + sizeof(struct otx2_cpt_sglist_component); + s_sz_bytes = ((req->out_cnt + 3) / 4) * + sizeof(struct otx2_cpt_sglist_component); + + dlen = g_sz_bytes + s_sz_bytes + SG_LIST_HDR_SIZE; + align_dlen = ALIGN(dlen, align); + info_len = ALIGN(sizeof(*info), align); + total_mem_len = align_dlen + info_len + sizeof(union otx2_cpt_res_s); + + info = kzalloc(total_mem_len, gfp); + if (unlikely(!info)) + return NULL; + + info->dlen = dlen; + info->in_buffer = (u8 *)info + info_len; + + ((u16 *)info->in_buffer)[0] = req->out_cnt; + ((u16 *)info->in_buffer)[1] = req->in_cnt; + ((u16 *)info->in_buffer)[2] = 0; + ((u16 *)info->in_buffer)[3] = 0; + cpu_to_be64s((u64 *)info->in_buffer); + + /* Setup gather (input) components */ + if (setup_sgio_components(pdev, req->in, req->in_cnt, + &info->in_buffer[8])) { + dev_err(&pdev->dev, "Failed to setup gather list\n"); + goto destroy_info; + } + + if (setup_sgio_components(pdev, req->out, req->out_cnt, + &info->in_buffer[8 + g_sz_bytes])) { + dev_err(&pdev->dev, "Failed to setup scatter list\n"); + goto destroy_info; + } + + info->dma_len = total_mem_len - info_len; + info->dptr_baddr = dma_map_single(&pdev->dev, info->in_buffer, + info->dma_len, DMA_BIDIRECTIONAL); + if (unlikely(dma_mapping_error(&pdev->dev, info->dptr_baddr))) { + dev_err(&pdev->dev, "DMA Mapping failed for cpt req\n"); + goto destroy_info; + } + /* + * Get buffer for union otx2_cpt_res_s response + * structure and its physical address + */ + info->completion_addr = info->in_buffer + align_dlen; + info->comp_baddr = info->dptr_baddr + align_dlen; + + return info; + +destroy_info: + otx2_cpt_info_destroy(pdev, info); + return NULL; +} + struct otx2_cptlf_wqe; int otx2_cpt_do_request(struct pci_dev *pdev, struct otx2_cpt_req_info *req, int cpu_num); diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptlf.c b/drivers/crypto/marvell/octeontx2/otx2_cptlf.c index 6edd27ff8c4e..b52728e3c0d1 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cptlf.c +++ b/drivers/crypto/marvell/octeontx2/otx2_cptlf.c @@ -106,6 +106,32 @@ static int cptlf_set_grp_and_pri(struct otx2_cptlfs_info *lfs, return ret; } +static int cptlf_set_ctx_ilen(struct otx2_cptlfs_info *lfs, int ctx_ilen) +{ + union otx2_cptx_af_lf_ctrl lf_ctrl; + struct otx2_cptlf_info *lf; + int slot, ret = 0; + + for (slot = 0; slot < lfs->lfs_num; slot++) { + lf = &lfs->lf[slot]; + + ret = otx2_cpt_read_af_reg(lfs->mbox, lfs->pdev, + CPT_AF_LFX_CTL(lf->slot), + &lf_ctrl.u, lfs->blkaddr); + if (ret) + return ret; + + lf_ctrl.s.ctx_ilen = ctx_ilen; + + ret = otx2_cpt_write_af_reg(lfs->mbox, lfs->pdev, + CPT_AF_LFX_CTL(lf->slot), + lf_ctrl.u, lfs->blkaddr); + if (ret) + return ret; + } + return ret; +} + static void cptlf_hw_init(struct otx2_cptlfs_info *lfs) { /* Disable instruction queues */ @@ -151,26 +177,14 @@ static void cptlf_set_misc_intrs(struct otx2_cptlfs_info *lfs, u8 enable) irq_misc.u); } -static void cptlf_enable_intrs(struct otx2_cptlfs_info *lfs) -{ - int slot; - - /* Enable done interrupts */ - for (slot = 0; slot < lfs->lfs_num; slot++) - otx2_cpt_write64(lfs->reg_base, lfs->blkaddr, slot, - OTX2_CPT_LF_DONE_INT_ENA_W1S, 0x1); - /* Enable Misc interrupts */ - cptlf_set_misc_intrs(lfs, true); -} - -static void cptlf_disable_intrs(struct otx2_cptlfs_info *lfs) +static void cptlf_set_done_intrs(struct otx2_cptlfs_info *lfs, u8 enable) { + u64 reg = enable ? OTX2_CPT_LF_DONE_INT_ENA_W1S : + OTX2_CPT_LF_DONE_INT_ENA_W1C; int slot; for (slot = 0; slot < lfs->lfs_num; slot++) - otx2_cpt_write64(lfs->reg_base, lfs->blkaddr, slot, - OTX2_CPT_LF_DONE_INT_ENA_W1C, 0x1); - cptlf_set_misc_intrs(lfs, false); + otx2_cpt_write64(lfs->reg_base, lfs->blkaddr, slot, reg, 0x1); } static inline int cptlf_read_done_cnt(struct otx2_cptlf_info *lf) @@ -257,24 +271,44 @@ static irqreturn_t cptlf_done_intr_handler(int irq, void *arg) return IRQ_HANDLED; } -void otx2_cptlf_unregister_interrupts(struct otx2_cptlfs_info *lfs) +void otx2_cptlf_unregister_misc_interrupts(struct otx2_cptlfs_info *lfs) { - int i, offs, vector; + int i, irq_offs, vector; + irq_offs = OTX2_CPT_LF_INT_VEC_E_MISC; for (i = 0; i < lfs->lfs_num; i++) { - for (offs = 0; offs < OTX2_CPT_LF_MSIX_VECTORS; offs++) { - if (!lfs->lf[i].is_irq_reg[offs]) - continue; + if (!lfs->lf[i].is_irq_reg[irq_offs]) + continue; - vector = pci_irq_vector(lfs->pdev, - lfs->lf[i].msix_offset + offs); - free_irq(vector, &lfs->lf[i]); - lfs->lf[i].is_irq_reg[offs] = false; - } + vector = pci_irq_vector(lfs->pdev, + lfs->lf[i].msix_offset + irq_offs); + free_irq(vector, &lfs->lf[i]); + lfs->lf[i].is_irq_reg[irq_offs] = false; + } + + cptlf_set_misc_intrs(lfs, false); +} +EXPORT_SYMBOL_NS_GPL(otx2_cptlf_unregister_misc_interrupts, + CRYPTO_DEV_OCTEONTX2_CPT); + +void otx2_cptlf_unregister_done_interrupts(struct otx2_cptlfs_info *lfs) +{ + int i, irq_offs, vector; + + irq_offs = OTX2_CPT_LF_INT_VEC_E_DONE; + for (i = 0; i < lfs->lfs_num; i++) { + if (!lfs->lf[i].is_irq_reg[irq_offs]) + continue; + + vector = pci_irq_vector(lfs->pdev, + lfs->lf[i].msix_offset + irq_offs); + free_irq(vector, &lfs->lf[i]); + lfs->lf[i].is_irq_reg[irq_offs] = false; } - cptlf_disable_intrs(lfs); + + cptlf_set_done_intrs(lfs, false); } -EXPORT_SYMBOL_NS_GPL(otx2_cptlf_unregister_interrupts, +EXPORT_SYMBOL_NS_GPL(otx2_cptlf_unregister_done_interrupts, CRYPTO_DEV_OCTEONTX2_CPT); static int cptlf_do_register_interrrupts(struct otx2_cptlfs_info *lfs, @@ -296,34 +330,53 @@ static int cptlf_do_register_interrrupts(struct otx2_cptlfs_info *lfs, return ret; } -int otx2_cptlf_register_interrupts(struct otx2_cptlfs_info *lfs) +int otx2_cptlf_register_misc_interrupts(struct otx2_cptlfs_info *lfs) { + bool is_cpt1 = (lfs->blkaddr == BLKADDR_CPT1); int irq_offs, ret, i; + irq_offs = OTX2_CPT_LF_INT_VEC_E_MISC; for (i = 0; i < lfs->lfs_num; i++) { - irq_offs = OTX2_CPT_LF_INT_VEC_E_MISC; - snprintf(lfs->lf[i].irq_name[irq_offs], 32, "CPTLF Misc%d", i); + snprintf(lfs->lf[i].irq_name[irq_offs], 32, "CPT%dLF Misc%d", + is_cpt1, i); ret = cptlf_do_register_interrrupts(lfs, i, irq_offs, cptlf_misc_intr_handler); if (ret) goto free_irq; + } + cptlf_set_misc_intrs(lfs, true); + return 0; - irq_offs = OTX2_CPT_LF_INT_VEC_E_DONE; - snprintf(lfs->lf[i].irq_name[irq_offs], 32, "OTX2_CPTLF Done%d", - i); +free_irq: + otx2_cptlf_unregister_misc_interrupts(lfs); + return ret; +} +EXPORT_SYMBOL_NS_GPL(otx2_cptlf_register_misc_interrupts, + CRYPTO_DEV_OCTEONTX2_CPT); + +int otx2_cptlf_register_done_interrupts(struct otx2_cptlfs_info *lfs) +{ + bool is_cpt1 = (lfs->blkaddr == BLKADDR_CPT1); + int irq_offs, ret, i; + + irq_offs = OTX2_CPT_LF_INT_VEC_E_DONE; + for (i = 0; i < lfs->lfs_num; i++) { + snprintf(lfs->lf[i].irq_name[irq_offs], 32, + "OTX2_CPT%dLF Done%d", is_cpt1, i); ret = cptlf_do_register_interrrupts(lfs, i, irq_offs, cptlf_done_intr_handler); if (ret) goto free_irq; } - cptlf_enable_intrs(lfs); + cptlf_set_done_intrs(lfs, true); return 0; free_irq: - otx2_cptlf_unregister_interrupts(lfs); + otx2_cptlf_unregister_done_interrupts(lfs); return ret; } -EXPORT_SYMBOL_NS_GPL(otx2_cptlf_register_interrupts, CRYPTO_DEV_OCTEONTX2_CPT); +EXPORT_SYMBOL_NS_GPL(otx2_cptlf_register_done_interrupts, + CRYPTO_DEV_OCTEONTX2_CPT); void otx2_cptlf_free_irqs_affinity(struct otx2_cptlfs_info *lfs) { @@ -416,11 +469,17 @@ int otx2_cptlf_init(struct otx2_cptlfs_info *lfs, u8 eng_grp_mask, int pri, if (ret) goto free_iq; + if (lfs->ctx_ilen_ovrd) { + ret = cptlf_set_ctx_ilen(lfs, lfs->ctx_ilen); + if (ret) + goto free_iq; + } + return 0; free_iq: - otx2_cpt_free_instruction_queues(lfs); cptlf_hw_cleanup(lfs); + otx2_cpt_free_instruction_queues(lfs); detach_rsrcs: otx2_cpt_detach_rsrcs_msg(lfs); clear_lfs_num: @@ -431,11 +490,13 @@ EXPORT_SYMBOL_NS_GPL(otx2_cptlf_init, CRYPTO_DEV_OCTEONTX2_CPT); void otx2_cptlf_shutdown(struct otx2_cptlfs_info *lfs) { - lfs->lfs_num = 0; /* Cleanup LFs hardware side */ cptlf_hw_cleanup(lfs); + /* Free instruction queues */ + otx2_cpt_free_instruction_queues(lfs); /* Send request to detach LFs */ otx2_cpt_detach_rsrcs_msg(lfs); + lfs->lfs_num = 0; } EXPORT_SYMBOL_NS_GPL(otx2_cptlf_shutdown, CRYPTO_DEV_OCTEONTX2_CPT); diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptlf.h b/drivers/crypto/marvell/octeontx2/otx2_cptlf.h index 5302fe3d0e6f..bd8604be2952 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cptlf.h +++ b/drivers/crypto/marvell/octeontx2/otx2_cptlf.h @@ -5,6 +5,7 @@ #define __OTX2_CPTLF_H #include <linux/soc/marvell/octeontx2/asm.h> +#include <linux/bitfield.h> #include <mbox.h> #include <rvu.h> #include "otx2_cpt_common.h" @@ -99,6 +100,9 @@ struct cpt_hw_ops { struct otx2_cptlf_info *lf); u8 (*cpt_get_compcode)(union otx2_cpt_res_s *result); u8 (*cpt_get_uc_compcode)(union otx2_cpt_res_s *result); + struct otx2_cpt_inst_info * + (*cpt_sg_info_create)(struct pci_dev *pdev, struct otx2_cpt_req_info *req, + gfp_t gfp); }; struct otx2_cptlfs_info { @@ -116,6 +120,9 @@ struct otx2_cptlfs_info { u8 kvf_limits; /* Kernel crypto limits */ atomic_t state; /* LF's state. started/reset */ int blkaddr; /* CPT blkaddr: BLKADDR_CPT0/BLKADDR_CPT1 */ + int global_slot; /* Global slot across the blocks */ + u8 ctx_ilen; + u8 ctx_ilen_ovrd; }; static inline void otx2_cpt_free_instruction_queues( @@ -203,48 +210,71 @@ static inline void otx2_cptlf_set_iqueues_size(struct otx2_cptlfs_info *lfs) otx2_cptlf_do_set_iqueue_size(&lfs->lf[slot]); } +#define INFLIGHT GENMASK_ULL(8, 0) +#define GRB_CNT GENMASK_ULL(39, 32) +#define GWB_CNT GENMASK_ULL(47, 40) +#define XQ_XOR GENMASK_ULL(63, 63) +#define DQPTR GENMASK_ULL(19, 0) +#define NQPTR GENMASK_ULL(51, 32) + static inline void otx2_cptlf_do_disable_iqueue(struct otx2_cptlf_info *lf) { - union otx2_cptx_lf_ctl lf_ctl = { .u = 0x0 }; - union otx2_cptx_lf_inprog lf_inprog; + void __iomem *reg_base = lf->lfs->reg_base; + struct pci_dev *pdev = lf->lfs->pdev; u8 blkaddr = lf->lfs->blkaddr; - int timeout = 20; + int timeout = 1000000; + u64 inprog, inst_ptr; + u64 slot = lf->slot; + u64 qsize, pending; + int i = 0; /* Disable instructions enqueuing */ - otx2_cpt_write64(lf->lfs->reg_base, blkaddr, lf->slot, - OTX2_CPT_LF_CTL, lf_ctl.u); + otx2_cpt_write64(reg_base, blkaddr, slot, OTX2_CPT_LF_CTL, 0x0); + + inprog = otx2_cpt_read64(reg_base, blkaddr, slot, OTX2_CPT_LF_INPROG); + inprog |= BIT_ULL(16); + otx2_cpt_write64(reg_base, blkaddr, slot, OTX2_CPT_LF_INPROG, inprog); - /* Wait for instruction queue to become empty */ + qsize = otx2_cpt_read64(reg_base, blkaddr, slot, OTX2_CPT_LF_Q_SIZE) & 0x7FFF; + do { + inst_ptr = otx2_cpt_read64(reg_base, blkaddr, slot, OTX2_CPT_LF_Q_INST_PTR); + pending = (FIELD_GET(XQ_XOR, inst_ptr) * qsize * 40) + + FIELD_GET(NQPTR, inst_ptr) - FIELD_GET(DQPTR, inst_ptr); + udelay(1); + timeout--; + } while ((pending != 0) && (timeout != 0)); + + if (timeout == 0) + dev_warn(&pdev->dev, "TIMEOUT: CPT poll on pending instructions\n"); + + timeout = 1000000; + /* Wait for CPT queue to become execution-quiescent */ do { - lf_inprog.u = otx2_cpt_read64(lf->lfs->reg_base, blkaddr, - lf->slot, OTX2_CPT_LF_INPROG); - if (!lf_inprog.s.inflight) - break; - - usleep_range(10000, 20000); - if (timeout-- < 0) { - dev_err(&lf->lfs->pdev->dev, - "Error LF %d is still busy.\n", lf->slot); - break; + inprog = otx2_cpt_read64(reg_base, blkaddr, slot, OTX2_CPT_LF_INPROG); + + if ((FIELD_GET(INFLIGHT, inprog) == 0) && + (FIELD_GET(GRB_CNT, inprog) == 0)) { + i++; + } else { + i = 0; + timeout--; } + } while ((timeout != 0) && (i < 10)); - } while (1); - - /* - * Disable executions in the LF's queue, - * the queue should be empty at this point - */ - lf_inprog.s.eena = 0x0; - otx2_cpt_write64(lf->lfs->reg_base, blkaddr, lf->slot, - OTX2_CPT_LF_INPROG, lf_inprog.u); + if (timeout == 0) + dev_warn(&pdev->dev, "TIMEOUT: CPT poll on inflight count\n"); + /* Wait for 2 us to flush all queue writes to memory */ + udelay(2); } static inline void otx2_cptlf_disable_iqueues(struct otx2_cptlfs_info *lfs) { int slot; - for (slot = 0; slot < lfs->lfs_num; slot++) + for (slot = 0; slot < lfs->lfs_num; slot++) { otx2_cptlf_do_disable_iqueue(&lfs->lf[slot]); + otx2_cpt_lf_reset_msg(lfs, lfs->global_slot + slot); + } } static inline void otx2_cptlf_set_iqueue_enq(struct otx2_cptlf_info *lf, @@ -282,6 +312,19 @@ static inline void otx2_cptlf_set_iqueue_exec(struct otx2_cptlf_info *lf, OTX2_CPT_LF_INPROG, lf_inprog.u); } +static inline void otx2_cptlf_set_ctx_flr_flush(struct otx2_cptlf_info *lf) +{ + u8 blkaddr = lf->lfs->blkaddr; + u64 val; + + val = otx2_cpt_read64(lf->lfs->reg_base, blkaddr, lf->slot, + OTX2_CPT_LF_CTX_CTL); + val |= BIT_ULL(0); + + otx2_cpt_write64(lf->lfs->reg_base, blkaddr, lf->slot, + OTX2_CPT_LF_CTX_CTL, val); +} + static inline void otx2_cptlf_enable_iqueue_exec(struct otx2_cptlf_info *lf) { otx2_cptlf_set_iqueue_exec(lf, true); @@ -297,6 +340,10 @@ static inline void otx2_cptlf_enable_iqueues(struct otx2_cptlfs_info *lfs) int slot; for (slot = 0; slot < lfs->lfs_num; slot++) { + /* Enable flush on FLR for Errata */ + if (is_dev_cn10kb(lfs->pdev)) + otx2_cptlf_set_ctx_flr_flush(&lfs->lf[slot]); + otx2_cptlf_enable_iqueue_exec(&lfs->lf[slot]); otx2_cptlf_enable_iqueue_enq(&lfs->lf[slot]); } @@ -382,8 +429,10 @@ static inline void otx2_cptlf_set_dev_info(struct otx2_cptlfs_info *lfs, int otx2_cptlf_init(struct otx2_cptlfs_info *lfs, u8 eng_grp_msk, int pri, int lfs_num); void otx2_cptlf_shutdown(struct otx2_cptlfs_info *lfs); -int otx2_cptlf_register_interrupts(struct otx2_cptlfs_info *lfs); -void otx2_cptlf_unregister_interrupts(struct otx2_cptlfs_info *lfs); +int otx2_cptlf_register_misc_interrupts(struct otx2_cptlfs_info *lfs); +int otx2_cptlf_register_done_interrupts(struct otx2_cptlfs_info *lfs); +void otx2_cptlf_unregister_misc_interrupts(struct otx2_cptlfs_info *lfs); +void otx2_cptlf_unregister_done_interrupts(struct otx2_cptlfs_info *lfs); void otx2_cptlf_free_irqs_affinity(struct otx2_cptlfs_info *lfs); int otx2_cptlf_set_irqs_affinity(struct otx2_cptlfs_info *lfs); diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptpf.h b/drivers/crypto/marvell/octeontx2/otx2_cptpf.h index a209ec5af381..e5859a1e1c60 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cptpf.h +++ b/drivers/crypto/marvell/octeontx2/otx2_cptpf.h @@ -71,4 +71,8 @@ void otx2_cptpf_afpf_mbox_up_handler(struct work_struct *work); irqreturn_t otx2_cptpf_vfpf_mbox_intr(int irq, void *arg); void otx2_cptpf_vfpf_mbox_handler(struct work_struct *work); +int otx2_inline_cptlf_setup(struct otx2_cptpf_dev *cptpf, + struct otx2_cptlfs_info *lfs, u8 egrp, int num_lfs); +void otx2_inline_cptlf_cleanup(struct otx2_cptlfs_info *lfs); + #endif /* __OTX2_CPTPF_H */ diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptpf_main.c b/drivers/crypto/marvell/octeontx2/otx2_cptpf_main.c index e34223daa327..400e36d9908f 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cptpf_main.c +++ b/drivers/crypto/marvell/octeontx2/otx2_cptpf_main.c @@ -14,6 +14,8 @@ #define OTX2_CPT_DRV_STRING "Marvell RVU CPT Physical Function Driver" #define CPT_UC_RID_CN9K_B0 1 +#define CPT_UC_RID_CN10K_A 4 +#define CPT_UC_RID_CN10K_B 5 static void cptpf_enable_vfpf_mbox_intr(struct otx2_cptpf_dev *cptpf, int num_vfs) @@ -587,43 +589,22 @@ static int cpt_is_pf_usable(struct otx2_cptpf_dev *cptpf) return 0; } -static int cptx_device_reset(struct otx2_cptpf_dev *cptpf, int blkaddr) +static void cptpf_get_rid(struct pci_dev *pdev, struct otx2_cptpf_dev *cptpf) { - int timeout = 10, ret; - u64 reg = 0; + struct otx2_cpt_eng_grps *eng_grps = &cptpf->eng_grps; + u64 reg_val = 0x0; - ret = otx2_cpt_write_af_reg(&cptpf->afpf_mbox, cptpf->pdev, - CPT_AF_BLK_RST, 0x1, blkaddr); - if (ret) - return ret; - - do { - ret = otx2_cpt_read_af_reg(&cptpf->afpf_mbox, cptpf->pdev, - CPT_AF_BLK_RST, ®, blkaddr); - if (ret) - return ret; - - if (!((reg >> 63) & 0x1)) - break; - - usleep_range(10000, 20000); - if (timeout-- < 0) - return -EBUSY; - } while (1); - - return ret; -} - -static int cptpf_device_reset(struct otx2_cptpf_dev *cptpf) -{ - int ret = 0; - - if (cptpf->has_cpt1) { - ret = cptx_device_reset(cptpf, BLKADDR_CPT1); - if (ret) - return ret; + if (is_dev_otx2(pdev)) { + eng_grps->rid = pdev->revision; + return; } - return cptx_device_reset(cptpf, BLKADDR_CPT0); + otx2_cpt_read_af_reg(&cptpf->afpf_mbox, pdev, CPT_AF_CTL, ®_val, + BLKADDR_CPT0); + if ((cpt_feature_sgv2(pdev) && (reg_val & BIT_ULL(18))) || + is_dev_cn10ka_ax(pdev)) + eng_grps->rid = CPT_UC_RID_CN10K_A; + else if (cpt_feature_sgv2(pdev)) + eng_grps->rid = CPT_UC_RID_CN10K_B; } static void cptpf_check_block_implemented(struct otx2_cptpf_dev *cptpf) @@ -643,10 +624,6 @@ static int cptpf_device_init(struct otx2_cptpf_dev *cptpf) /* check if 'implemented' bit is set for block BLKADDR_CPT1 */ cptpf_check_block_implemented(cptpf); - /* Reset the CPT PF device */ - ret = cptpf_device_reset(cptpf); - if (ret) - return ret; /* Get number of SE, IE and AE engines */ ret = otx2_cpt_read_af_reg(&cptpf->afpf_mbox, cptpf->pdev, @@ -701,6 +678,7 @@ static int cptpf_sriov_enable(struct pci_dev *pdev, int num_vfs) if (ret) goto destroy_flr; + cptpf_get_rid(pdev, cptpf); /* Get CPT HW capabilities using LOAD_FVC operation. */ ret = otx2_cpt_discover_eng_capabilities(cptpf); if (ret) @@ -744,7 +722,7 @@ static int otx2_cptpf_probe(struct pci_dev *pdev, { struct device *dev = &pdev->dev; struct otx2_cptpf_dev *cptpf; - int err; + int err, num_vec; cptpf = devm_kzalloc(dev, sizeof(*cptpf), GFP_KERNEL); if (!cptpf) @@ -779,8 +757,13 @@ static int otx2_cptpf_probe(struct pci_dev *pdev, if (err) goto clear_drvdata; - err = pci_alloc_irq_vectors(pdev, RVU_PF_INT_VEC_CNT, - RVU_PF_INT_VEC_CNT, PCI_IRQ_MSIX); + num_vec = pci_msix_vec_count(cptpf->pdev); + if (num_vec <= 0) { + err = -EINVAL; + goto clear_drvdata; + } + + err = pci_alloc_irq_vectors(pdev, num_vec, num_vec, PCI_IRQ_MSIX); if (err < 0) { dev_err(dev, "Request for %d msix vectors failed\n", RVU_PF_INT_VEC_CNT); @@ -797,6 +780,7 @@ static int otx2_cptpf_probe(struct pci_dev *pdev, goto destroy_afpf_mbox; cptpf->max_vfs = pci_sriov_get_totalvfs(pdev); + cptpf->kvf_limits = 1; err = cn10k_cptpf_lmtst_init(cptpf); if (err) @@ -844,6 +828,14 @@ static void otx2_cptpf_remove(struct pci_dev *pdev) cptpf_sriov_disable(pdev); otx2_cpt_unregister_dl(cptpf); + + /* Cleanup Inline CPT LF's if attached */ + if (cptpf->lfs.lfs_num) + otx2_inline_cptlf_cleanup(&cptpf->lfs); + + if (cptpf->cpt1_lfs.lfs_num) + otx2_inline_cptlf_cleanup(&cptpf->cpt1_lfs); + /* Delete sysfs entry created for kernel VF limits */ sysfs_remove_group(&pdev->dev.kobj, &cptpf_sysfs_group); /* Cleanup engine groups */ diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptpf_mbox.c b/drivers/crypto/marvell/octeontx2/otx2_cptpf_mbox.c index 480b3720f15a..ec1ac7e836a3 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cptpf_mbox.c +++ b/drivers/crypto/marvell/octeontx2/otx2_cptpf_mbox.c @@ -78,7 +78,7 @@ static int handle_msg_get_caps(struct otx2_cptpf_dev *cptpf, rsp->hdr.sig = OTX2_MBOX_RSP_SIG; rsp->hdr.pcifunc = req->pcifunc; rsp->cpt_pf_drv_version = OTX2_CPT_PF_DRV_VERSION; - rsp->cpt_revision = cptpf->pdev->revision; + rsp->cpt_revision = cptpf->eng_grps.rid; memcpy(&rsp->eng_caps, &cptpf->eng_caps, sizeof(rsp->eng_caps)); return 0; @@ -171,6 +171,8 @@ static int rx_inline_ipsec_lf_cfg(struct otx2_cptpf_dev *cptpf, u8 egrp, nix_req->hdr.id = MBOX_MSG_NIX_INLINE_IPSEC_CFG; nix_req->hdr.sig = OTX2_MBOX_REQ_SIG; nix_req->enable = 1; + nix_req->credit_th = req->credit_th; + nix_req->bpid = req->bpid; if (!req->credit || req->credit > OTX2_CPT_INST_QLEN_MSGS) nix_req->cpt_credit = OTX2_CPT_INST_QLEN_MSGS - 1; else @@ -197,12 +199,53 @@ static int rx_inline_ipsec_lf_cfg(struct otx2_cptpf_dev *cptpf, u8 egrp, return send_inline_ipsec_inbound_msg(cptpf, req->sso_pf_func, 0); } +int +otx2_inline_cptlf_setup(struct otx2_cptpf_dev *cptpf, + struct otx2_cptlfs_info *lfs, u8 egrp, int num_lfs) +{ + int ret; + + ret = otx2_cptlf_init(lfs, 1 << egrp, OTX2_CPT_QUEUE_HI_PRIO, 1); + if (ret) { + dev_err(&cptpf->pdev->dev, + "LF configuration failed for RX inline ipsec.\n"); + return ret; + } + + /* Get msix offsets for attached LFs */ + ret = otx2_cpt_msix_offset_msg(lfs); + if (ret) + goto cleanup_lf; + + /* Register for CPT LF Misc interrupts */ + ret = otx2_cptlf_register_misc_interrupts(lfs); + if (ret) + goto free_irq; + + return 0; +free_irq: + otx2_cptlf_unregister_misc_interrupts(lfs); +cleanup_lf: + otx2_cptlf_shutdown(lfs); + return ret; +} + +void +otx2_inline_cptlf_cleanup(struct otx2_cptlfs_info *lfs) +{ + /* Unregister misc interrupt */ + otx2_cptlf_unregister_misc_interrupts(lfs); + + /* Cleanup LFs */ + otx2_cptlf_shutdown(lfs); +} + static int handle_msg_rx_inline_ipsec_lf_cfg(struct otx2_cptpf_dev *cptpf, struct mbox_msghdr *req) { struct otx2_cpt_rx_inline_lf_cfg *cfg_req; + int num_lfs = 1, ret; u8 egrp; - int ret; cfg_req = (struct otx2_cpt_rx_inline_lf_cfg *)req; if (cptpf->lfs.lfs_num) { @@ -223,11 +266,13 @@ static int handle_msg_rx_inline_ipsec_lf_cfg(struct otx2_cptpf_dev *cptpf, otx2_cptlf_set_dev_info(&cptpf->lfs, cptpf->pdev, cptpf->reg_base, &cptpf->afpf_mbox, BLKADDR_CPT0); - ret = otx2_cptlf_init(&cptpf->lfs, 1 << egrp, OTX2_CPT_QUEUE_HI_PRIO, - 1); + cptpf->lfs.global_slot = 0; + cptpf->lfs.ctx_ilen_ovrd = cfg_req->ctx_ilen_valid; + cptpf->lfs.ctx_ilen = cfg_req->ctx_ilen; + + ret = otx2_inline_cptlf_setup(cptpf, &cptpf->lfs, egrp, num_lfs); if (ret) { - dev_err(&cptpf->pdev->dev, - "LF configuration failed for RX inline ipsec.\n"); + dev_err(&cptpf->pdev->dev, "Inline-Ipsec CPT0 LF setup failed.\n"); return ret; } @@ -236,11 +281,13 @@ static int handle_msg_rx_inline_ipsec_lf_cfg(struct otx2_cptpf_dev *cptpf, otx2_cptlf_set_dev_info(&cptpf->cpt1_lfs, cptpf->pdev, cptpf->reg_base, &cptpf->afpf_mbox, BLKADDR_CPT1); - ret = otx2_cptlf_init(&cptpf->cpt1_lfs, 1 << egrp, - OTX2_CPT_QUEUE_HI_PRIO, 1); + cptpf->cpt1_lfs.global_slot = num_lfs; + cptpf->cpt1_lfs.ctx_ilen_ovrd = cfg_req->ctx_ilen_valid; + cptpf->cpt1_lfs.ctx_ilen = cfg_req->ctx_ilen; + ret = otx2_inline_cptlf_setup(cptpf, &cptpf->cpt1_lfs, egrp, + num_lfs); if (ret) { - dev_err(&cptpf->pdev->dev, - "LF configuration failed for RX inline ipsec.\n"); + dev_err(&cptpf->pdev->dev, "Inline CPT1 LF setup failed.\n"); goto lf_cleanup; } cptpf->rsrc_req_blkaddr = 0; @@ -253,9 +300,9 @@ static int handle_msg_rx_inline_ipsec_lf_cfg(struct otx2_cptpf_dev *cptpf, return 0; lf1_cleanup: - otx2_cptlf_shutdown(&cptpf->cpt1_lfs); + otx2_inline_cptlf_cleanup(&cptpf->cpt1_lfs); lf_cleanup: - otx2_cptlf_shutdown(&cptpf->lfs); + otx2_inline_cptlf_cleanup(&cptpf->lfs); return ret; } @@ -410,6 +457,8 @@ static void process_afpf_mbox_msg(struct otx2_cptpf_dev *cptpf, struct otx2_cptlfs_info *lfs = &cptpf->lfs; struct device *dev = &cptpf->pdev->dev; struct cpt_rd_wr_reg_msg *rsp_rd_wr; + struct msix_offset_rsp *rsp_msix; + int i; if (msg->id >= MBOX_MSG_MAX) { dev_err(dev, "MBOX msg with unknown ID %d\n", msg->id); @@ -428,6 +477,14 @@ static void process_afpf_mbox_msg(struct otx2_cptpf_dev *cptpf, cptpf->pf_id = (msg->pcifunc >> RVU_PFVF_PF_SHIFT) & RVU_PFVF_PF_MASK; break; + case MBOX_MSG_MSIX_OFFSET: + rsp_msix = (struct msix_offset_rsp *) msg; + for (i = 0; i < rsp_msix->cptlfs; i++) + lfs->lf[i].msix_offset = rsp_msix->cptlf_msixoff[i]; + + for (i = 0; i < rsp_msix->cpt1_lfs; i++) + lfs->lf[i].msix_offset = rsp_msix->cpt1_lf_msixoff[i]; + break; case MBOX_MSG_CPT_RD_WR_REGISTER: rsp_rd_wr = (struct cpt_rd_wr_reg_msg *)msg; if (msg->rc) { @@ -449,6 +506,7 @@ static void process_afpf_mbox_msg(struct otx2_cptpf_dev *cptpf, break; case MBOX_MSG_CPT_INLINE_IPSEC_CFG: case MBOX_MSG_NIX_INLINE_IPSEC_CFG: + case MBOX_MSG_CPT_LF_RESET: break; default: diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c b/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c index 1958b797a421..5c9484646172 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c +++ b/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c @@ -16,7 +16,11 @@ #define LOADFVC_MAJOR_OP 0x01 #define LOADFVC_MINOR_OP 0x08 -#define CTX_FLUSH_TIMER_CNT 0xFFFFFF +/* + * Interval to flush dirty data for next CTX entry. The interval is measured + * in increments of 10ns(interval time = CTX_FLUSH_TIMER_COUNT * 10ns). + */ +#define CTX_FLUSH_TIMER_CNT 0x2FAF0 struct fw_info_t { struct list_head ucodes; @@ -117,12 +121,10 @@ static char *get_ucode_type_str(int ucode_type) static int get_ucode_type(struct device *dev, struct otx2_cpt_ucode_hdr *ucode_hdr, - int *ucode_type) + int *ucode_type, u16 rid) { - struct otx2_cptpf_dev *cptpf = dev_get_drvdata(dev); char ver_str_prefix[OTX2_CPT_UCODE_VER_STR_SZ]; char tmp_ver_str[OTX2_CPT_UCODE_VER_STR_SZ]; - struct pci_dev *pdev = cptpf->pdev; int i, val = 0; u8 nn; @@ -130,7 +132,7 @@ static int get_ucode_type(struct device *dev, for (i = 0; i < strlen(tmp_ver_str); i++) tmp_ver_str[i] = tolower(tmp_ver_str[i]); - sprintf(ver_str_prefix, "ocpt-%02d", pdev->revision); + sprintf(ver_str_prefix, "ocpt-%02d", rid); if (!strnstr(tmp_ver_str, ver_str_prefix, OTX2_CPT_UCODE_VER_STR_SZ)) return -EINVAL; @@ -359,7 +361,7 @@ static int cpt_attach_and_enable_cores(struct otx2_cpt_eng_grp_info *eng_grp, } static int load_fw(struct device *dev, struct fw_info_t *fw_info, - char *filename) + char *filename, u16 rid) { struct otx2_cpt_ucode_hdr *ucode_hdr; struct otx2_cpt_uc_info_t *uc_info; @@ -375,7 +377,7 @@ static int load_fw(struct device *dev, struct fw_info_t *fw_info, goto free_uc_info; ucode_hdr = (struct otx2_cpt_ucode_hdr *)uc_info->fw->data; - ret = get_ucode_type(dev, ucode_hdr, &ucode_type); + ret = get_ucode_type(dev, ucode_hdr, &ucode_type, rid); if (ret) goto release_fw; @@ -389,6 +391,7 @@ static int load_fw(struct device *dev, struct fw_info_t *fw_info, set_ucode_filename(&uc_info->ucode, filename); memcpy(uc_info->ucode.ver_str, ucode_hdr->ver_str, OTX2_CPT_UCODE_VER_STR_SZ); + uc_info->ucode.ver_str[OTX2_CPT_UCODE_VER_STR_SZ] = 0; uc_info->ucode.ver_num = ucode_hdr->ver_num; uc_info->ucode.type = ucode_type; uc_info->ucode.size = ucode_size; @@ -448,7 +451,8 @@ static void print_uc_info(struct fw_info_t *fw_info) } } -static int cpt_ucode_load_fw(struct pci_dev *pdev, struct fw_info_t *fw_info) +static int cpt_ucode_load_fw(struct pci_dev *pdev, struct fw_info_t *fw_info, + u16 rid) { char filename[OTX2_CPT_NAME_LENGTH]; char eng_type[8] = {0}; @@ -462,9 +466,9 @@ static int cpt_ucode_load_fw(struct pci_dev *pdev, struct fw_info_t *fw_info) eng_type[i] = tolower(eng_type[i]); snprintf(filename, sizeof(filename), "mrvl/cpt%02d/%s.out", - pdev->revision, eng_type); + rid, eng_type); /* Request firmware for each engine type */ - ret = load_fw(&pdev->dev, fw_info, filename); + ret = load_fw(&pdev->dev, fw_info, filename, rid); if (ret) goto release_fw; } @@ -1155,7 +1159,7 @@ int otx2_cpt_create_eng_grps(struct otx2_cptpf_dev *cptpf, if (eng_grps->is_grps_created) goto unlock; - ret = cpt_ucode_load_fw(pdev, &fw_info); + ret = cpt_ucode_load_fw(pdev, &fw_info, eng_grps->rid); if (ret) goto unlock; @@ -1230,14 +1234,16 @@ int otx2_cpt_create_eng_grps(struct otx2_cptpf_dev *cptpf, */ rnm_to_cpt_errata_fixup(&pdev->dev); + otx2_cpt_read_af_reg(&cptpf->afpf_mbox, pdev, CPT_AF_CTL, ®_val, + BLKADDR_CPT0); /* * Configure engine group mask to allow context prefetching * for the groups and enable random number request, to enable * CPT to request random numbers from RNM. */ + reg_val |= OTX2_CPT_ALL_ENG_GRPS_MASK << 3 | BIT_ULL(16); otx2_cpt_write_af_reg(&cptpf->afpf_mbox, pdev, CPT_AF_CTL, - OTX2_CPT_ALL_ENG_GRPS_MASK << 3 | BIT_ULL(16), - BLKADDR_CPT0); + reg_val, BLKADDR_CPT0); /* * Set interval to periodically flush dirty data for the next * CTX cache entry. Set the interval count to maximum supported @@ -1252,10 +1258,12 @@ int otx2_cpt_create_eng_grps(struct otx2_cptpf_dev *cptpf, * encounters a fault/poison, a rare case may result in * unpredictable data being delivered to a CPT engine. */ - otx2_cpt_read_af_reg(&cptpf->afpf_mbox, pdev, CPT_AF_DIAG, ®_val, - BLKADDR_CPT0); - otx2_cpt_write_af_reg(&cptpf->afpf_mbox, pdev, CPT_AF_DIAG, - reg_val | BIT_ULL(24), BLKADDR_CPT0); + if (cpt_is_errata_38550_exists(pdev)) { + otx2_cpt_read_af_reg(&cptpf->afpf_mbox, pdev, CPT_AF_DIAG, + ®_val, BLKADDR_CPT0); + otx2_cpt_write_af_reg(&cptpf->afpf_mbox, pdev, CPT_AF_DIAG, + reg_val | BIT_ULL(24), BLKADDR_CPT0); + } mutex_unlock(&eng_grps->lock); return 0; @@ -1412,7 +1420,7 @@ static int create_eng_caps_discovery_grps(struct pci_dev *pdev, int ret; mutex_lock(&eng_grps->lock); - ret = cpt_ucode_load_fw(pdev, &fw_info); + ret = cpt_ucode_load_fw(pdev, &fw_info, eng_grps->rid); if (ret) { mutex_unlock(&eng_grps->lock); return ret; @@ -1686,13 +1694,14 @@ int otx2_cpt_dl_custom_egrp_create(struct otx2_cptpf_dev *cptpf, goto err_unlock; } INIT_LIST_HEAD(&fw_info.ucodes); - ret = load_fw(dev, &fw_info, ucode_filename[0]); + + ret = load_fw(dev, &fw_info, ucode_filename[0], eng_grps->rid); if (ret) { dev_err(dev, "Unable to load firmware %s\n", ucode_filename[0]); goto err_unlock; } if (ucode_idx > 1) { - ret = load_fw(dev, &fw_info, ucode_filename[1]); + ret = load_fw(dev, &fw_info, ucode_filename[1], eng_grps->rid); if (ret) { dev_err(dev, "Unable to load firmware %s\n", ucode_filename[1]); diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.h b/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.h index e69320a54b5d..365fe8943bd9 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.h +++ b/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.h @@ -73,7 +73,7 @@ struct otx2_cpt_ucode_hdr { }; struct otx2_cpt_ucode { - u8 ver_str[OTX2_CPT_UCODE_VER_STR_SZ];/* + u8 ver_str[OTX2_CPT_UCODE_VER_STR_SZ + 1];/* * ucode version in readable * format */ @@ -150,6 +150,7 @@ struct otx2_cpt_eng_grps { int engs_num; /* total number of engines supported */ u8 eng_ref_cnt[OTX2_CPT_MAX_ENGINES];/* engines reference count */ bool is_grps_created; /* Is the engine groups are already created */ + u16 rid; }; struct otx2_cptpf_dev; int otx2_cpt_init_eng_grps(struct pci_dev *pdev, diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptvf.h b/drivers/crypto/marvell/octeontx2/otx2_cptvf.h index 994291e90da1..11ab9af1df15 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cptvf.h +++ b/drivers/crypto/marvell/octeontx2/otx2_cptvf.h @@ -22,6 +22,7 @@ struct otx2_cptvf_dev { int blkaddr; void *bbuf_base; unsigned long cap_flag; + u64 eng_caps[OTX2_CPT_MAX_ENG_TYPES]; }; irqreturn_t otx2_cptvf_pfvf_mbox_intr(int irq, void *arg); @@ -29,5 +30,6 @@ void otx2_cptvf_pfvf_mbox_handler(struct work_struct *work); int otx2_cptvf_send_eng_grp_num_msg(struct otx2_cptvf_dev *cptvf, int eng_type); int otx2_cptvf_send_kvf_limits_msg(struct otx2_cptvf_dev *cptvf); int otx2_cpt_mbox_bbuf_init(struct otx2_cptvf_dev *cptvf, struct pci_dev *pdev); +int otx2_cptvf_send_caps_msg(struct otx2_cptvf_dev *cptvf); #endif /* __OTX2_CPTVF_H */ diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.c b/drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.c index e27ddd3c4e55..1604fc58dc13 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.c +++ b/drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.c @@ -17,6 +17,7 @@ #include "otx2_cptvf.h" #include "otx2_cptvf_algs.h" #include "otx2_cpt_reqmgr.h" +#include "cn10k_cpt.h" /* Size of salt in AES GCM mode */ #define AES_GCM_SALT_SIZE 4 @@ -384,6 +385,9 @@ static inline int cpt_enc_dec(struct skcipher_request *req, u32 enc) req_info->is_trunc_hmac = false; req_info->ctrl.s.grp = otx2_cpt_get_kcrypto_eng_grp_num(pdev); + req_info->req.cptr = ctx->er_ctx.hw_ctx; + req_info->req.cptr_dma = ctx->er_ctx.cptr_dma; + /* * We perform an asynchronous send and once * the request is completed the driver would @@ -530,6 +534,8 @@ static int otx2_cpt_enc_dec_init(struct crypto_skcipher *stfm) struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(stfm); struct crypto_tfm *tfm = crypto_skcipher_tfm(stfm); struct crypto_alg *alg = tfm->__crt_alg; + struct pci_dev *pdev; + int ret, cpu_num; memset(ctx, 0, sizeof(*ctx)); /* @@ -541,6 +547,15 @@ static int otx2_cpt_enc_dec_init(struct crypto_skcipher *stfm) stfm, sizeof(struct otx2_cpt_req_ctx) + sizeof(struct skcipher_request)); + ret = get_se_device(&pdev, &cpu_num); + if (ret) + return ret; + + ctx->pdev = pdev; + ret = cn10k_cpt_hw_ctx_init(pdev, &ctx->er_ctx); + if (ret) + return ret; + return cpt_skcipher_fallback_init(ctx, alg); } @@ -552,6 +567,7 @@ static void otx2_cpt_skcipher_exit(struct crypto_skcipher *tfm) crypto_free_skcipher(ctx->fbk_cipher); ctx->fbk_cipher = NULL; } + cn10k_cpt_hw_ctx_clear(ctx->pdev, &ctx->er_ctx); } static int cpt_aead_fallback_init(struct otx2_cpt_aead_ctx *ctx, @@ -576,6 +592,8 @@ static int cpt_aead_init(struct crypto_aead *atfm, u8 cipher_type, u8 mac_type) struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(atfm); struct crypto_tfm *tfm = crypto_aead_tfm(atfm); struct crypto_alg *alg = tfm->__crt_alg; + struct pci_dev *pdev; + int ret, cpu_num; ctx->cipher_type = cipher_type; ctx->mac_type = mac_type; @@ -632,6 +650,15 @@ static int cpt_aead_init(struct crypto_aead *atfm, u8 cipher_type, u8 mac_type) } crypto_aead_set_reqsize_dma(atfm, sizeof(struct otx2_cpt_req_ctx)); + ret = get_se_device(&pdev, &cpu_num); + if (ret) + return ret; + + ctx->pdev = pdev; + ret = cn10k_cpt_hw_ctx_init(pdev, &ctx->er_ctx); + if (ret) + return ret; + return cpt_aead_fallback_init(ctx, alg); } @@ -694,6 +721,7 @@ static void otx2_cpt_aead_exit(struct crypto_aead *tfm) crypto_free_aead(ctx->fbk_cipher); ctx->fbk_cipher = NULL; } + cn10k_cpt_hw_ctx_clear(ctx->pdev, &ctx->er_ctx); } static int otx2_cpt_aead_gcm_set_authsize(struct crypto_aead *tfm, @@ -1299,6 +1327,9 @@ static int cpt_aead_enc_dec(struct aead_request *req, u8 reg_type, u8 enc) req_info->is_enc = enc; req_info->is_trunc_hmac = false; + req_info->req.cptr = ctx->er_ctx.hw_ctx; + req_info->req.cptr_dma = ctx->er_ctx.cptr_dma; + switch (reg_type) { case OTX2_CPT_AEAD_ENC_DEC_REQ: status = create_aead_input_list(req, enc); diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.h b/drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.h index f04184bd1744..d29f84f01cee 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.h +++ b/drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.h @@ -9,6 +9,7 @@ #include <crypto/skcipher.h> #include <crypto/aead.h> #include "otx2_cpt_common.h" +#include "cn10k_cpt.h" #define OTX2_CPT_MAX_ENC_KEY_SIZE 32 #define OTX2_CPT_MAX_HASH_KEY_SIZE 64 @@ -123,6 +124,8 @@ struct otx2_cpt_enc_ctx { u8 key_type; u8 enc_align_len; struct crypto_skcipher *fbk_cipher; + struct pci_dev *pdev; + struct cn10k_cpt_errata_ctx er_ctx; }; union otx2_cpt_offset_ctrl { @@ -161,6 +164,8 @@ struct otx2_cpt_aead_ctx { struct crypto_shash *hashalg; struct otx2_cpt_sdesc *sdesc; struct crypto_aead *fbk_cipher; + struct cn10k_cpt_errata_ctx er_ctx; + struct pci_dev *pdev; u8 *ipad; u8 *opad; u32 enc_key_len; diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptvf_main.c b/drivers/crypto/marvell/octeontx2/otx2_cptvf_main.c index bac729c885f9..527d34cc258b 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cptvf_main.c +++ b/drivers/crypto/marvell/octeontx2/otx2_cptvf_main.c @@ -246,11 +246,15 @@ static void cptvf_lf_shutdown(struct otx2_cptlfs_info *lfs) /* Unregister crypto algorithms */ otx2_cpt_crypto_exit(lfs->pdev, THIS_MODULE); /* Unregister LFs interrupts */ - otx2_cptlf_unregister_interrupts(lfs); + otx2_cptlf_unregister_misc_interrupts(lfs); + otx2_cptlf_unregister_done_interrupts(lfs); /* Cleanup LFs software side */ lf_sw_cleanup(lfs); + /* Free instruction queues */ + otx2_cpt_free_instruction_queues(lfs); /* Send request to detach LFs */ otx2_cpt_detach_rsrcs_msg(lfs); + lfs->lfs_num = 0; } static int cptvf_lf_init(struct otx2_cptvf_dev *cptvf) @@ -277,8 +281,7 @@ static int cptvf_lf_init(struct otx2_cptvf_dev *cptvf) if (ret) return ret; - lfs_num = cptvf->lfs.kvf_limits ? cptvf->lfs.kvf_limits : - num_online_cpus(); + lfs_num = cptvf->lfs.kvf_limits; otx2_cptlf_set_dev_info(lfs, cptvf->pdev, cptvf->reg_base, &cptvf->pfvf_mbox, cptvf->blkaddr); @@ -298,7 +301,11 @@ static int cptvf_lf_init(struct otx2_cptvf_dev *cptvf) goto cleanup_lf; /* Register LFs interrupts */ - ret = otx2_cptlf_register_interrupts(lfs); + ret = otx2_cptlf_register_misc_interrupts(lfs); + if (ret) + goto cleanup_lf_sw; + + ret = otx2_cptlf_register_done_interrupts(lfs); if (ret) goto cleanup_lf_sw; @@ -319,7 +326,8 @@ static int cptvf_lf_init(struct otx2_cptvf_dev *cptvf) disable_irqs: otx2_cptlf_free_irqs_affinity(lfs); unregister_intr: - otx2_cptlf_unregister_interrupts(lfs); + otx2_cptlf_unregister_misc_interrupts(lfs); + otx2_cptlf_unregister_done_interrupts(lfs); cleanup_lf_sw: lf_sw_cleanup(lfs); cleanup_lf: @@ -380,6 +388,17 @@ static int otx2_cptvf_probe(struct pci_dev *pdev, goto destroy_pfvf_mbox; cptvf->blkaddr = BLKADDR_CPT0; + + cptvf_hw_ops_get(cptvf); + + ret = otx2_cptvf_send_caps_msg(cptvf); + if (ret) { + dev_err(&pdev->dev, "Couldn't get CPT engine capabilities.\n"); + goto unregister_interrupts; + } + if (cptvf->eng_caps[OTX2_CPT_SE_TYPES] & BIT_ULL(35)) + cptvf->lfs.ops->cpt_sg_info_create = cn10k_sgv2_info_create; + /* Initialize CPT LFs */ ret = cptvf_lf_init(cptvf); if (ret) diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptvf_mbox.c b/drivers/crypto/marvell/octeontx2/otx2_cptvf_mbox.c index 75c403f2b1d9..d9fa5f6e204d 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cptvf_mbox.c +++ b/drivers/crypto/marvell/octeontx2/otx2_cptvf_mbox.c @@ -72,6 +72,7 @@ static void process_pfvf_mbox_mbox_msg(struct otx2_cptvf_dev *cptvf, struct otx2_cptlfs_info *lfs = &cptvf->lfs; struct otx2_cpt_kvf_limits_rsp *rsp_limits; struct otx2_cpt_egrp_num_rsp *rsp_grp; + struct otx2_cpt_caps_rsp *eng_caps; struct cpt_rd_wr_reg_msg *rsp_reg; struct msix_offset_rsp *rsp_msix; int i; @@ -127,6 +128,13 @@ static void process_pfvf_mbox_mbox_msg(struct otx2_cptvf_dev *cptvf, rsp_limits = (struct otx2_cpt_kvf_limits_rsp *) msg; cptvf->lfs.kvf_limits = rsp_limits->kvf_limits; break; + case MBOX_MSG_GET_CAPS: + eng_caps = (struct otx2_cpt_caps_rsp *)msg; + memcpy(cptvf->eng_caps, eng_caps->eng_caps, + sizeof(cptvf->eng_caps)); + break; + case MBOX_MSG_CPT_LF_RESET: + break; default: dev_err(&cptvf->pdev->dev, "Unsupported msg %d received.\n", msg->id); @@ -205,3 +213,23 @@ int otx2_cptvf_send_kvf_limits_msg(struct otx2_cptvf_dev *cptvf) return otx2_cpt_send_mbox_msg(mbox, pdev); } + +int otx2_cptvf_send_caps_msg(struct otx2_cptvf_dev *cptvf) +{ + struct otx2_mbox *mbox = &cptvf->pfvf_mbox; + struct pci_dev *pdev = cptvf->pdev; + struct mbox_msghdr *req; + + req = (struct mbox_msghdr *) + otx2_mbox_alloc_msg_rsp(mbox, 0, sizeof(*req), + sizeof(struct otx2_cpt_caps_rsp)); + if (!req) { + dev_err(&pdev->dev, "RVU MBOX failed to get message.\n"); + return -EFAULT; + } + req->id = MBOX_MSG_GET_CAPS; + req->sig = OTX2_MBOX_REQ_SIG; + req->pcifunc = OTX2_CPT_RVU_PFFUNC(cptvf->vf_id, 0); + + return otx2_cpt_send_mbox_msg(mbox, pdev); +} diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptvf_reqmgr.c b/drivers/crypto/marvell/octeontx2/otx2_cptvf_reqmgr.c index 811ded72ce5f..5387c68f3c9d 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cptvf_reqmgr.c +++ b/drivers/crypto/marvell/octeontx2/otx2_cptvf_reqmgr.c @@ -4,9 +4,6 @@ #include "otx2_cptvf.h" #include "otx2_cpt_common.h" -/* SG list header size in bytes */ -#define SG_LIST_HDR_SIZE 8 - /* Default timeout when waiting for free pending entry in us */ #define CPT_PENTRY_TIMEOUT 1000 #define CPT_PENTRY_STEP 50 @@ -26,9 +23,9 @@ static void otx2_cpt_dump_sg_list(struct pci_dev *pdev, pr_debug("Gather list size %d\n", req->in_cnt); for (i = 0; i < req->in_cnt; i++) { - pr_debug("Buffer %d size %d, vptr 0x%p, dmaptr 0x%p\n", i, + pr_debug("Buffer %d size %d, vptr 0x%p, dmaptr 0x%llx\n", i, req->in[i].size, req->in[i].vptr, - (void *) req->in[i].dma_addr); + req->in[i].dma_addr); pr_debug("Buffer hexdump (%d bytes)\n", req->in[i].size); print_hex_dump_debug("", DUMP_PREFIX_NONE, 16, 1, @@ -36,9 +33,9 @@ static void otx2_cpt_dump_sg_list(struct pci_dev *pdev, } pr_debug("Scatter list size %d\n", req->out_cnt); for (i = 0; i < req->out_cnt; i++) { - pr_debug("Buffer %d size %d, vptr 0x%p, dmaptr 0x%p\n", i, + pr_debug("Buffer %d size %d, vptr 0x%p, dmaptr 0x%llx\n", i, req->out[i].size, req->out[i].vptr, - (void *) req->out[i].dma_addr); + req->out[i].dma_addr); pr_debug("Buffer hexdump (%d bytes)\n", req->out[i].size); print_hex_dump_debug("", DUMP_PREFIX_NONE, 16, 1, req->out[i].vptr, req->out[i].size, false); @@ -84,149 +81,6 @@ static inline void free_pentry(struct otx2_cpt_pending_entry *pentry) pentry->busy = false; } -static inline int setup_sgio_components(struct pci_dev *pdev, - struct otx2_cpt_buf_ptr *list, - int buf_count, u8 *buffer) -{ - struct otx2_cpt_sglist_component *sg_ptr = NULL; - int ret = 0, i, j; - int components; - - if (unlikely(!list)) { - dev_err(&pdev->dev, "Input list pointer is NULL\n"); - return -EFAULT; - } - - for (i = 0; i < buf_count; i++) { - if (unlikely(!list[i].vptr)) - continue; - list[i].dma_addr = dma_map_single(&pdev->dev, list[i].vptr, - list[i].size, - DMA_BIDIRECTIONAL); - if (unlikely(dma_mapping_error(&pdev->dev, list[i].dma_addr))) { - dev_err(&pdev->dev, "Dma mapping failed\n"); - ret = -EIO; - goto sg_cleanup; - } - } - components = buf_count / 4; - sg_ptr = (struct otx2_cpt_sglist_component *)buffer; - for (i = 0; i < components; i++) { - sg_ptr->len0 = cpu_to_be16(list[i * 4 + 0].size); - sg_ptr->len1 = cpu_to_be16(list[i * 4 + 1].size); - sg_ptr->len2 = cpu_to_be16(list[i * 4 + 2].size); - sg_ptr->len3 = cpu_to_be16(list[i * 4 + 3].size); - sg_ptr->ptr0 = cpu_to_be64(list[i * 4 + 0].dma_addr); - sg_ptr->ptr1 = cpu_to_be64(list[i * 4 + 1].dma_addr); - sg_ptr->ptr2 = cpu_to_be64(list[i * 4 + 2].dma_addr); - sg_ptr->ptr3 = cpu_to_be64(list[i * 4 + 3].dma_addr); - sg_ptr++; - } - components = buf_count % 4; - - switch (components) { - case 3: - sg_ptr->len2 = cpu_to_be16(list[i * 4 + 2].size); - sg_ptr->ptr2 = cpu_to_be64(list[i * 4 + 2].dma_addr); - fallthrough; - case 2: - sg_ptr->len1 = cpu_to_be16(list[i * 4 + 1].size); - sg_ptr->ptr1 = cpu_to_be64(list[i * 4 + 1].dma_addr); - fallthrough; - case 1: - sg_ptr->len0 = cpu_to_be16(list[i * 4 + 0].size); - sg_ptr->ptr0 = cpu_to_be64(list[i * 4 + 0].dma_addr); - break; - default: - break; - } - return ret; - -sg_cleanup: - for (j = 0; j < i; j++) { - if (list[j].dma_addr) { - dma_unmap_single(&pdev->dev, list[j].dma_addr, - list[j].size, DMA_BIDIRECTIONAL); - } - - list[j].dma_addr = 0; - } - return ret; -} - -static inline struct otx2_cpt_inst_info *info_create(struct pci_dev *pdev, - struct otx2_cpt_req_info *req, - gfp_t gfp) -{ - int align = OTX2_CPT_DMA_MINALIGN; - struct otx2_cpt_inst_info *info; - u32 dlen, align_dlen, info_len; - u16 g_sz_bytes, s_sz_bytes; - u32 total_mem_len; - - if (unlikely(req->in_cnt > OTX2_CPT_MAX_SG_IN_CNT || - req->out_cnt > OTX2_CPT_MAX_SG_OUT_CNT)) { - dev_err(&pdev->dev, "Error too many sg components\n"); - return NULL; - } - - g_sz_bytes = ((req->in_cnt + 3) / 4) * - sizeof(struct otx2_cpt_sglist_component); - s_sz_bytes = ((req->out_cnt + 3) / 4) * - sizeof(struct otx2_cpt_sglist_component); - - dlen = g_sz_bytes + s_sz_bytes + SG_LIST_HDR_SIZE; - align_dlen = ALIGN(dlen, align); - info_len = ALIGN(sizeof(*info), align); - total_mem_len = align_dlen + info_len + sizeof(union otx2_cpt_res_s); - - info = kzalloc(total_mem_len, gfp); - if (unlikely(!info)) - return NULL; - - info->dlen = dlen; - info->in_buffer = (u8 *)info + info_len; - - ((u16 *)info->in_buffer)[0] = req->out_cnt; - ((u16 *)info->in_buffer)[1] = req->in_cnt; - ((u16 *)info->in_buffer)[2] = 0; - ((u16 *)info->in_buffer)[3] = 0; - cpu_to_be64s((u64 *)info->in_buffer); - - /* Setup gather (input) components */ - if (setup_sgio_components(pdev, req->in, req->in_cnt, - &info->in_buffer[8])) { - dev_err(&pdev->dev, "Failed to setup gather list\n"); - goto destroy_info; - } - - if (setup_sgio_components(pdev, req->out, req->out_cnt, - &info->in_buffer[8 + g_sz_bytes])) { - dev_err(&pdev->dev, "Failed to setup scatter list\n"); - goto destroy_info; - } - - info->dma_len = total_mem_len - info_len; - info->dptr_baddr = dma_map_single(&pdev->dev, info->in_buffer, - info->dma_len, DMA_BIDIRECTIONAL); - if (unlikely(dma_mapping_error(&pdev->dev, info->dptr_baddr))) { - dev_err(&pdev->dev, "DMA Mapping failed for cpt req\n"); - goto destroy_info; - } - /* - * Get buffer for union otx2_cpt_res_s response - * structure and its physical address - */ - info->completion_addr = info->in_buffer + align_dlen; - info->comp_baddr = info->dptr_baddr + align_dlen; - - return info; - -destroy_info: - otx2_cpt_info_destroy(pdev, info); - return NULL; -} - static int process_request(struct pci_dev *pdev, struct otx2_cpt_req_info *req, struct otx2_cpt_pending_queue *pqueue, struct otx2_cptlf_info *lf) @@ -247,7 +101,7 @@ static int process_request(struct pci_dev *pdev, struct otx2_cpt_req_info *req, if (unlikely(!otx2_cptlf_started(lf->lfs))) return -ENODEV; - info = info_create(pdev, req, gfp); + info = lf->lfs->ops->cpt_sg_info_create(pdev, req, gfp); if (unlikely(!info)) { dev_err(&pdev->dev, "Setting up cpt inst info failed"); return -ENOMEM; @@ -303,9 +157,9 @@ static int process_request(struct pci_dev *pdev, struct otx2_cpt_req_info *req, /* 64-bit swap for microcode data reads, not needed for addresses*/ cpu_to_be64s(&iq_cmd.cmd.u); - iq_cmd.dptr = info->dptr_baddr; - iq_cmd.rptr = 0; - iq_cmd.cptr.u = 0; + iq_cmd.dptr = info->dptr_baddr | info->gthr_sz << 60; + iq_cmd.rptr = info->rptr_baddr | info->sctr_sz << 60; + iq_cmd.cptr.s.cptr = cpt_req->cptr_dma; iq_cmd.cptr.s.grp = ctrl->s.grp; /* Fill in the CPT_INST_S type command for HW interpretation */ diff --git a/drivers/crypto/n2_core.c b/drivers/crypto/n2_core.c index caea98622c33..7a3083debc2b 100644 --- a/drivers/crypto/n2_core.c +++ b/drivers/crypto/n2_core.c @@ -1121,19 +1121,6 @@ static const struct n2_skcipher_tmpl skcipher_tmpls[] = { .decrypt = n2_decrypt_chaining, }, }, - { .name = "cfb(des)", - .drv_name = "cfb-des", - .block_size = DES_BLOCK_SIZE, - .enc_type = (ENC_TYPE_ALG_DES | - ENC_TYPE_CHAINING_CFB), - .skcipher = { - .min_keysize = DES_KEY_SIZE, - .max_keysize = DES_KEY_SIZE, - .setkey = n2_des_setkey, - .encrypt = n2_encrypt_chaining, - .decrypt = n2_decrypt_chaining, - }, - }, /* 3DES: ECB CBC and CFB are supported */ { .name = "ecb(des3_ede)", @@ -1163,19 +1150,7 @@ static const struct n2_skcipher_tmpl skcipher_tmpls[] = { .decrypt = n2_decrypt_chaining, }, }, - { .name = "cfb(des3_ede)", - .drv_name = "cfb-3des", - .block_size = DES_BLOCK_SIZE, - .enc_type = (ENC_TYPE_ALG_3DES | - ENC_TYPE_CHAINING_CFB), - .skcipher = { - .min_keysize = 3 * DES_KEY_SIZE, - .max_keysize = 3 * DES_KEY_SIZE, - .setkey = n2_3des_setkey, - .encrypt = n2_encrypt_chaining, - .decrypt = n2_decrypt_chaining, - }, - }, + /* AES: ECB CBC and CTR are supported */ { .name = "ecb(aes)", .drv_name = "ecb-aes", @@ -1382,8 +1357,12 @@ static int __n2_register_one_hmac(struct n2_ahash_alg *n2ahash) ahash->setkey = n2_hmac_async_setkey; base = &ahash->halg.base; - snprintf(base->cra_name, CRYPTO_MAX_ALG_NAME, "hmac(%s)", p->child_alg); - snprintf(base->cra_driver_name, CRYPTO_MAX_ALG_NAME, "hmac-%s-n2", p->child_alg); + if (snprintf(base->cra_name, CRYPTO_MAX_ALG_NAME, "hmac(%s)", + p->child_alg) >= CRYPTO_MAX_ALG_NAME) + goto out_free_p; + if (snprintf(base->cra_driver_name, CRYPTO_MAX_ALG_NAME, "hmac-%s-n2", + p->child_alg) >= CRYPTO_MAX_ALG_NAME) + goto out_free_p; base->cra_ctxsize = sizeof(struct n2_hmac_ctx); base->cra_init = n2_hmac_cra_init; @@ -1394,6 +1373,7 @@ static int __n2_register_one_hmac(struct n2_ahash_alg *n2ahash) if (err) { pr_err("%s alg registration failed\n", base->cra_name); list_del(&p->derived.entry); +out_free_p: kfree(p); } else { pr_info("%s alg registered\n", base->cra_name); diff --git a/drivers/crypto/rockchip/rk3288_crypto_skcipher.c b/drivers/crypto/rockchip/rk3288_crypto_skcipher.c index da95747d973f..9393e10671c2 100644 --- a/drivers/crypto/rockchip/rk3288_crypto_skcipher.c +++ b/drivers/crypto/rockchip/rk3288_crypto_skcipher.c @@ -445,8 +445,8 @@ static int rk_cipher_tfm_init(struct crypto_skcipher *tfm) return PTR_ERR(ctx->fallback_tfm); } - tfm->reqsize = sizeof(struct rk_cipher_rctx) + - crypto_skcipher_reqsize(ctx->fallback_tfm); + crypto_skcipher_set_reqsize(tfm, sizeof(struct rk_cipher_rctx) + + crypto_skcipher_reqsize(ctx->fallback_tfm)); return 0; } diff --git a/drivers/crypto/sa2ul.c b/drivers/crypto/sa2ul.c index 6846a8429574..78a4930c6480 100644 --- a/drivers/crypto/sa2ul.c +++ b/drivers/crypto/sa2ul.c @@ -1869,9 +1869,8 @@ static int sa_aead_setkey(struct crypto_aead *authenc, crypto_aead_set_flags(ctx->fallback.aead, crypto_aead_get_flags(authenc) & CRYPTO_TFM_REQ_MASK); - crypto_aead_setkey(ctx->fallback.aead, key, keylen); - return 0; + return crypto_aead_setkey(ctx->fallback.aead, key, keylen); } static int sa_aead_setauthsize(struct crypto_aead *tfm, unsigned int authsize) diff --git a/drivers/crypto/sahara.c b/drivers/crypto/sahara.c index 02065131c300..3423b5cde1c7 100644 --- a/drivers/crypto/sahara.c +++ b/drivers/crypto/sahara.c @@ -15,6 +15,7 @@ #include <crypto/internal/hash.h> #include <crypto/internal/skcipher.h> #include <crypto/scatterwalk.h> +#include <crypto/engine.h> #include <crypto/sha1.h> #include <crypto/sha2.h> @@ -24,106 +25,101 @@ #include <linux/io.h> #include <linux/irq.h> #include <linux/kernel.h> -#include <linux/kthread.h> #include <linux/module.h> #include <linux/of.h> #include <linux/platform_device.h> #include <linux/spinlock.h> -#define SHA_BUFFER_LEN PAGE_SIZE -#define SAHARA_MAX_SHA_BLOCK_SIZE SHA256_BLOCK_SIZE - -#define SAHARA_NAME "sahara" -#define SAHARA_VERSION_3 3 -#define SAHARA_VERSION_4 4 -#define SAHARA_TIMEOUT_MS 1000 -#define SAHARA_MAX_HW_DESC 2 -#define SAHARA_MAX_HW_LINK 20 - -#define FLAGS_MODE_MASK 0x000f -#define FLAGS_ENCRYPT BIT(0) -#define FLAGS_CBC BIT(1) -#define FLAGS_NEW_KEY BIT(3) - -#define SAHARA_HDR_BASE 0x00800000 -#define SAHARA_HDR_SKHA_ALG_AES 0 -#define SAHARA_HDR_SKHA_OP_ENC (1 << 2) -#define SAHARA_HDR_SKHA_MODE_ECB (0 << 3) -#define SAHARA_HDR_SKHA_MODE_CBC (1 << 3) -#define SAHARA_HDR_FORM_DATA (5 << 16) -#define SAHARA_HDR_FORM_KEY (8 << 16) -#define SAHARA_HDR_LLO (1 << 24) -#define SAHARA_HDR_CHA_SKHA (1 << 28) -#define SAHARA_HDR_CHA_MDHA (2 << 28) -#define SAHARA_HDR_PARITY_BIT (1 << 31) - -#define SAHARA_HDR_MDHA_SET_MODE_MD_KEY 0x20880000 -#define SAHARA_HDR_MDHA_SET_MODE_HASH 0x208D0000 -#define SAHARA_HDR_MDHA_HASH 0xA0850000 -#define SAHARA_HDR_MDHA_STORE_DIGEST 0x20820000 -#define SAHARA_HDR_MDHA_ALG_SHA1 0 -#define SAHARA_HDR_MDHA_ALG_MD5 1 -#define SAHARA_HDR_MDHA_ALG_SHA256 2 -#define SAHARA_HDR_MDHA_ALG_SHA224 3 -#define SAHARA_HDR_MDHA_PDATA (1 << 2) -#define SAHARA_HDR_MDHA_HMAC (1 << 3) -#define SAHARA_HDR_MDHA_INIT (1 << 5) -#define SAHARA_HDR_MDHA_IPAD (1 << 6) -#define SAHARA_HDR_MDHA_OPAD (1 << 7) -#define SAHARA_HDR_MDHA_SWAP (1 << 8) -#define SAHARA_HDR_MDHA_MAC_FULL (1 << 9) -#define SAHARA_HDR_MDHA_SSL (1 << 10) - -/* SAHARA can only process one request at a time */ -#define SAHARA_QUEUE_LENGTH 1 - -#define SAHARA_REG_VERSION 0x00 -#define SAHARA_REG_DAR 0x04 -#define SAHARA_REG_CONTROL 0x08 -#define SAHARA_CONTROL_SET_THROTTLE(x) (((x) & 0xff) << 24) -#define SAHARA_CONTROL_SET_MAXBURST(x) (((x) & 0xff) << 16) -#define SAHARA_CONTROL_RNG_AUTORSD (1 << 7) -#define SAHARA_CONTROL_ENABLE_INT (1 << 4) -#define SAHARA_REG_CMD 0x0C -#define SAHARA_CMD_RESET (1 << 0) -#define SAHARA_CMD_CLEAR_INT (1 << 8) -#define SAHARA_CMD_CLEAR_ERR (1 << 9) -#define SAHARA_CMD_SINGLE_STEP (1 << 10) -#define SAHARA_CMD_MODE_BATCH (1 << 16) -#define SAHARA_CMD_MODE_DEBUG (1 << 18) -#define SAHARA_REG_STATUS 0x10 -#define SAHARA_STATUS_GET_STATE(x) ((x) & 0x7) -#define SAHARA_STATE_IDLE 0 -#define SAHARA_STATE_BUSY 1 -#define SAHARA_STATE_ERR 2 -#define SAHARA_STATE_FAULT 3 -#define SAHARA_STATE_COMPLETE 4 -#define SAHARA_STATE_COMP_FLAG (1 << 2) -#define SAHARA_STATUS_DAR_FULL (1 << 3) -#define SAHARA_STATUS_ERROR (1 << 4) -#define SAHARA_STATUS_SECURE (1 << 5) -#define SAHARA_STATUS_FAIL (1 << 6) -#define SAHARA_STATUS_INIT (1 << 7) -#define SAHARA_STATUS_RNG_RESEED (1 << 8) -#define SAHARA_STATUS_ACTIVE_RNG (1 << 9) -#define SAHARA_STATUS_ACTIVE_MDHA (1 << 10) -#define SAHARA_STATUS_ACTIVE_SKHA (1 << 11) -#define SAHARA_STATUS_MODE_BATCH (1 << 16) -#define SAHARA_STATUS_MODE_DEDICATED (1 << 17) -#define SAHARA_STATUS_MODE_DEBUG (1 << 18) -#define SAHARA_STATUS_GET_ISTATE(x) (((x) >> 24) & 0xff) -#define SAHARA_REG_ERRSTATUS 0x14 -#define SAHARA_ERRSTATUS_GET_SOURCE(x) ((x) & 0xf) -#define SAHARA_ERRSOURCE_CHA 14 -#define SAHARA_ERRSOURCE_DMA 15 -#define SAHARA_ERRSTATUS_DMA_DIR (1 << 8) -#define SAHARA_ERRSTATUS_GET_DMASZ(x)(((x) >> 9) & 0x3) -#define SAHARA_ERRSTATUS_GET_DMASRC(x) (((x) >> 13) & 0x7) -#define SAHARA_ERRSTATUS_GET_CHASRC(x) (((x) >> 16) & 0xfff) -#define SAHARA_ERRSTATUS_GET_CHAERR(x) (((x) >> 28) & 0x3) -#define SAHARA_REG_FADDR 0x18 -#define SAHARA_REG_CDAR 0x1C -#define SAHARA_REG_IDAR 0x20 +#define SHA_BUFFER_LEN PAGE_SIZE +#define SAHARA_MAX_SHA_BLOCK_SIZE SHA256_BLOCK_SIZE + +#define SAHARA_NAME "sahara" +#define SAHARA_VERSION_3 3 +#define SAHARA_VERSION_4 4 +#define SAHARA_TIMEOUT_MS 1000 +#define SAHARA_MAX_HW_DESC 2 +#define SAHARA_MAX_HW_LINK 20 + +#define FLAGS_MODE_MASK 0x000f +#define FLAGS_ENCRYPT BIT(0) +#define FLAGS_CBC BIT(1) + +#define SAHARA_HDR_BASE 0x00800000 +#define SAHARA_HDR_SKHA_ALG_AES 0 +#define SAHARA_HDR_SKHA_MODE_ECB 0 +#define SAHARA_HDR_SKHA_OP_ENC BIT(2) +#define SAHARA_HDR_SKHA_MODE_CBC BIT(3) +#define SAHARA_HDR_FORM_DATA (5 << 16) +#define SAHARA_HDR_FORM_KEY BIT(19) +#define SAHARA_HDR_LLO BIT(24) +#define SAHARA_HDR_CHA_SKHA BIT(28) +#define SAHARA_HDR_CHA_MDHA BIT(29) +#define SAHARA_HDR_PARITY_BIT BIT(31) + +#define SAHARA_HDR_MDHA_SET_MODE_MD_KEY 0x20880000 +#define SAHARA_HDR_MDHA_SET_MODE_HASH 0x208D0000 +#define SAHARA_HDR_MDHA_HASH 0xA0850000 +#define SAHARA_HDR_MDHA_STORE_DIGEST 0x20820000 +#define SAHARA_HDR_MDHA_ALG_SHA1 0 +#define SAHARA_HDR_MDHA_ALG_MD5 1 +#define SAHARA_HDR_MDHA_ALG_SHA256 2 +#define SAHARA_HDR_MDHA_ALG_SHA224 3 +#define SAHARA_HDR_MDHA_PDATA BIT(2) +#define SAHARA_HDR_MDHA_HMAC BIT(3) +#define SAHARA_HDR_MDHA_INIT BIT(5) +#define SAHARA_HDR_MDHA_IPAD BIT(6) +#define SAHARA_HDR_MDHA_OPAD BIT(7) +#define SAHARA_HDR_MDHA_SWAP BIT(8) +#define SAHARA_HDR_MDHA_MAC_FULL BIT(9) +#define SAHARA_HDR_MDHA_SSL BIT(10) + +#define SAHARA_REG_VERSION 0x00 +#define SAHARA_REG_DAR 0x04 +#define SAHARA_REG_CONTROL 0x08 +#define SAHARA_CONTROL_SET_THROTTLE(x) (((x) & 0xff) << 24) +#define SAHARA_CONTROL_SET_MAXBURST(x) (((x) & 0xff) << 16) +#define SAHARA_CONTROL_RNG_AUTORSD BIT(7) +#define SAHARA_CONTROL_ENABLE_INT BIT(4) +#define SAHARA_REG_CMD 0x0C +#define SAHARA_CMD_RESET BIT(0) +#define SAHARA_CMD_CLEAR_INT BIT(8) +#define SAHARA_CMD_CLEAR_ERR BIT(9) +#define SAHARA_CMD_SINGLE_STEP BIT(10) +#define SAHARA_CMD_MODE_BATCH BIT(16) +#define SAHARA_CMD_MODE_DEBUG BIT(18) +#define SAHARA_REG_STATUS 0x10 +#define SAHARA_STATUS_GET_STATE(x) ((x) & 0x7) +#define SAHARA_STATE_IDLE 0 +#define SAHARA_STATE_BUSY 1 +#define SAHARA_STATE_ERR 2 +#define SAHARA_STATE_FAULT 3 +#define SAHARA_STATE_COMPLETE 4 +#define SAHARA_STATE_COMP_FLAG BIT(2) +#define SAHARA_STATUS_DAR_FULL BIT(3) +#define SAHARA_STATUS_ERROR BIT(4) +#define SAHARA_STATUS_SECURE BIT(5) +#define SAHARA_STATUS_FAIL BIT(6) +#define SAHARA_STATUS_INIT BIT(7) +#define SAHARA_STATUS_RNG_RESEED BIT(8) +#define SAHARA_STATUS_ACTIVE_RNG BIT(9) +#define SAHARA_STATUS_ACTIVE_MDHA BIT(10) +#define SAHARA_STATUS_ACTIVE_SKHA BIT(11) +#define SAHARA_STATUS_MODE_BATCH BIT(16) +#define SAHARA_STATUS_MODE_DEDICATED BIT(17) +#define SAHARA_STATUS_MODE_DEBUG BIT(18) +#define SAHARA_STATUS_GET_ISTATE(x) (((x) >> 24) & 0xff) +#define SAHARA_REG_ERRSTATUS 0x14 +#define SAHARA_ERRSTATUS_GET_SOURCE(x) ((x) & 0xf) +#define SAHARA_ERRSOURCE_CHA 14 +#define SAHARA_ERRSOURCE_DMA 15 +#define SAHARA_ERRSTATUS_DMA_DIR BIT(8) +#define SAHARA_ERRSTATUS_GET_DMASZ(x) (((x) >> 9) & 0x3) +#define SAHARA_ERRSTATUS_GET_DMASRC(x) (((x) >> 13) & 0x7) +#define SAHARA_ERRSTATUS_GET_CHASRC(x) (((x) >> 16) & 0xfff) +#define SAHARA_ERRSTATUS_GET_CHAERR(x) (((x) >> 28) & 0x3) +#define SAHARA_REG_FADDR 0x18 +#define SAHARA_REG_CDAR 0x1C +#define SAHARA_REG_IDAR 0x20 struct sahara_hw_desc { u32 hdr; @@ -141,8 +137,6 @@ struct sahara_hw_link { }; struct sahara_ctx { - unsigned long flags; - /* AES-specific context */ int keylen; u8 key[AES_KEYSIZE_128]; @@ -151,6 +145,7 @@ struct sahara_ctx { struct sahara_aes_reqctx { unsigned long mode; + u8 iv_out[AES_BLOCK_SIZE]; struct skcipher_request fallback_req; // keep at the end }; @@ -170,7 +165,6 @@ struct sahara_aes_reqctx { * @total: total number of bytes for transfer * @last: is this the last block * @first: is this the first block - * @active: inside a transfer */ struct sahara_sha_reqctx { u8 buf[SAHARA_MAX_SHA_BLOCK_SIZE]; @@ -186,7 +180,6 @@ struct sahara_sha_reqctx { size_t total; unsigned int last; unsigned int first; - unsigned int active; }; struct sahara_dev { @@ -195,12 +188,9 @@ struct sahara_dev { void __iomem *regs_base; struct clk *clk_ipg; struct clk *clk_ahb; - spinlock_t queue_spinlock; - struct task_struct *kthread; struct completion dma_completion; struct sahara_ctx *ctx; - struct crypto_queue queue; unsigned long flags; struct sahara_hw_desc *hw_desc[SAHARA_MAX_HW_DESC]; @@ -224,7 +214,7 @@ struct sahara_dev { struct scatterlist *out_sg; int nb_out_sg; - u32 error; + struct crypto_engine *engine; }; static struct sahara_dev *dev_ptr; @@ -446,27 +436,24 @@ static int sahara_hw_descriptor_create(struct sahara_dev *dev) int ret; int i, j; int idx = 0; + u32 len; - /* Copy new key if necessary */ - if (ctx->flags & FLAGS_NEW_KEY) { - memcpy(dev->key_base, ctx->key, ctx->keylen); - ctx->flags &= ~FLAGS_NEW_KEY; + memcpy(dev->key_base, ctx->key, ctx->keylen); - if (dev->flags & FLAGS_CBC) { - dev->hw_desc[idx]->len1 = AES_BLOCK_SIZE; - dev->hw_desc[idx]->p1 = dev->iv_phys_base; - } else { - dev->hw_desc[idx]->len1 = 0; - dev->hw_desc[idx]->p1 = 0; - } - dev->hw_desc[idx]->len2 = ctx->keylen; - dev->hw_desc[idx]->p2 = dev->key_phys_base; - dev->hw_desc[idx]->next = dev->hw_phys_desc[1]; + if (dev->flags & FLAGS_CBC) { + dev->hw_desc[idx]->len1 = AES_BLOCK_SIZE; + dev->hw_desc[idx]->p1 = dev->iv_phys_base; + } else { + dev->hw_desc[idx]->len1 = 0; + dev->hw_desc[idx]->p1 = 0; + } + dev->hw_desc[idx]->len2 = ctx->keylen; + dev->hw_desc[idx]->p2 = dev->key_phys_base; + dev->hw_desc[idx]->next = dev->hw_phys_desc[1]; + dev->hw_desc[idx]->hdr = sahara_aes_key_hdr(dev); - dev->hw_desc[idx]->hdr = sahara_aes_key_hdr(dev); + idx++; - idx++; - } dev->nb_in_sg = sg_nents_for_len(dev->in_sg, dev->total); if (dev->nb_in_sg < 0) { @@ -488,24 +475,27 @@ static int sahara_hw_descriptor_create(struct sahara_dev *dev) DMA_TO_DEVICE); if (!ret) { dev_err(dev->device, "couldn't map in sg\n"); - goto unmap_in; + return -EINVAL; } + ret = dma_map_sg(dev->device, dev->out_sg, dev->nb_out_sg, DMA_FROM_DEVICE); if (!ret) { dev_err(dev->device, "couldn't map out sg\n"); - goto unmap_out; + goto unmap_in; } /* Create input links */ dev->hw_desc[idx]->p1 = dev->hw_phys_link[0]; sg = dev->in_sg; + len = dev->total; for (i = 0; i < dev->nb_in_sg; i++) { - dev->hw_link[i]->len = sg->length; + dev->hw_link[i]->len = min(len, sg->length); dev->hw_link[i]->p = sg->dma_address; if (i == (dev->nb_in_sg - 1)) { dev->hw_link[i]->next = 0; } else { + len -= min(len, sg->length); dev->hw_link[i]->next = dev->hw_phys_link[i + 1]; sg = sg_next(sg); } @@ -514,12 +504,14 @@ static int sahara_hw_descriptor_create(struct sahara_dev *dev) /* Create output links */ dev->hw_desc[idx]->p2 = dev->hw_phys_link[i]; sg = dev->out_sg; + len = dev->total; for (j = i; j < dev->nb_out_sg + i; j++) { - dev->hw_link[j]->len = sg->length; + dev->hw_link[j]->len = min(len, sg->length); dev->hw_link[j]->p = sg->dma_address; if (j == (dev->nb_out_sg + i - 1)) { dev->hw_link[j]->next = 0; } else { + len -= min(len, sg->length); dev->hw_link[j]->next = dev->hw_phys_link[j + 1]; sg = sg_next(sg); } @@ -538,9 +530,6 @@ static int sahara_hw_descriptor_create(struct sahara_dev *dev) return 0; -unmap_out: - dma_unmap_sg(dev->device, dev->out_sg, dev->nb_out_sg, - DMA_FROM_DEVICE); unmap_in: dma_unmap_sg(dev->device, dev->in_sg, dev->nb_in_sg, DMA_TO_DEVICE); @@ -548,8 +537,24 @@ unmap_in: return -EINVAL; } +static void sahara_aes_cbc_update_iv(struct skcipher_request *req) +{ + struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); + struct sahara_aes_reqctx *rctx = skcipher_request_ctx(req); + unsigned int ivsize = crypto_skcipher_ivsize(skcipher); + + /* Update IV buffer to contain the last ciphertext block */ + if (rctx->mode & FLAGS_ENCRYPT) { + sg_pcopy_to_buffer(req->dst, sg_nents(req->dst), req->iv, + ivsize, req->cryptlen - ivsize); + } else { + memcpy(req->iv, rctx->iv_out, ivsize); + } +} + static int sahara_aes_process(struct skcipher_request *req) { + struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); struct sahara_dev *dev = dev_ptr; struct sahara_ctx *ctx; struct sahara_aes_reqctx *rctx; @@ -571,8 +576,17 @@ static int sahara_aes_process(struct skcipher_request *req) rctx->mode &= FLAGS_MODE_MASK; dev->flags = (dev->flags & ~FLAGS_MODE_MASK) | rctx->mode; - if ((dev->flags & FLAGS_CBC) && req->iv) - memcpy(dev->iv_base, req->iv, AES_KEYSIZE_128); + if ((dev->flags & FLAGS_CBC) && req->iv) { + unsigned int ivsize = crypto_skcipher_ivsize(skcipher); + + memcpy(dev->iv_base, req->iv, ivsize); + + if (!(dev->flags & FLAGS_ENCRYPT)) { + sg_pcopy_to_buffer(req->src, sg_nents(req->src), + rctx->iv_out, ivsize, + req->cryptlen - ivsize); + } + } /* assign new context to device */ dev->ctx = ctx; @@ -585,16 +599,20 @@ static int sahara_aes_process(struct skcipher_request *req) timeout = wait_for_completion_timeout(&dev->dma_completion, msecs_to_jiffies(SAHARA_TIMEOUT_MS)); - if (!timeout) { - dev_err(dev->device, "AES timeout\n"); - return -ETIMEDOUT; - } dma_unmap_sg(dev->device, dev->out_sg, dev->nb_out_sg, DMA_FROM_DEVICE); dma_unmap_sg(dev->device, dev->in_sg, dev->nb_in_sg, DMA_TO_DEVICE); + if (!timeout) { + dev_err(dev->device, "AES timeout\n"); + return -ETIMEDOUT; + } + + if ((dev->flags & FLAGS_CBC) && req->iv) + sahara_aes_cbc_update_iv(req); + return 0; } @@ -608,7 +626,6 @@ static int sahara_aes_setkey(struct crypto_skcipher *tfm, const u8 *key, /* SAHARA only supports 128bit keys */ if (keylen == AES_KEYSIZE_128) { memcpy(ctx->key, key, keylen); - ctx->flags |= FLAGS_NEW_KEY; return 0; } @@ -624,109 +641,67 @@ static int sahara_aes_setkey(struct crypto_skcipher *tfm, const u8 *key, return crypto_skcipher_setkey(ctx->fallback, key, keylen); } +static int sahara_aes_fallback(struct skcipher_request *req, unsigned long mode) +{ + struct sahara_aes_reqctx *rctx = skcipher_request_ctx(req); + struct sahara_ctx *ctx = crypto_skcipher_ctx( + crypto_skcipher_reqtfm(req)); + + skcipher_request_set_tfm(&rctx->fallback_req, ctx->fallback); + skcipher_request_set_callback(&rctx->fallback_req, + req->base.flags, + req->base.complete, + req->base.data); + skcipher_request_set_crypt(&rctx->fallback_req, req->src, + req->dst, req->cryptlen, req->iv); + + if (mode & FLAGS_ENCRYPT) + return crypto_skcipher_encrypt(&rctx->fallback_req); + + return crypto_skcipher_decrypt(&rctx->fallback_req); +} + static int sahara_aes_crypt(struct skcipher_request *req, unsigned long mode) { struct sahara_aes_reqctx *rctx = skcipher_request_ctx(req); + struct sahara_ctx *ctx = crypto_skcipher_ctx( + crypto_skcipher_reqtfm(req)); struct sahara_dev *dev = dev_ptr; - int err = 0; + + if (!req->cryptlen) + return 0; + + if (unlikely(ctx->keylen != AES_KEYSIZE_128)) + return sahara_aes_fallback(req, mode); dev_dbg(dev->device, "nbytes: %d, enc: %d, cbc: %d\n", req->cryptlen, !!(mode & FLAGS_ENCRYPT), !!(mode & FLAGS_CBC)); - if (!IS_ALIGNED(req->cryptlen, AES_BLOCK_SIZE)) { - dev_err(dev->device, - "request size is not exact amount of AES blocks\n"); + if (!IS_ALIGNED(req->cryptlen, AES_BLOCK_SIZE)) return -EINVAL; - } rctx->mode = mode; - spin_lock_bh(&dev->queue_spinlock); - err = crypto_enqueue_request(&dev->queue, &req->base); - spin_unlock_bh(&dev->queue_spinlock); - - wake_up_process(dev->kthread); - - return err; + return crypto_transfer_skcipher_request_to_engine(dev->engine, req); } static int sahara_aes_ecb_encrypt(struct skcipher_request *req) { - struct sahara_aes_reqctx *rctx = skcipher_request_ctx(req); - struct sahara_ctx *ctx = crypto_skcipher_ctx( - crypto_skcipher_reqtfm(req)); - - if (unlikely(ctx->keylen != AES_KEYSIZE_128)) { - skcipher_request_set_tfm(&rctx->fallback_req, ctx->fallback); - skcipher_request_set_callback(&rctx->fallback_req, - req->base.flags, - req->base.complete, - req->base.data); - skcipher_request_set_crypt(&rctx->fallback_req, req->src, - req->dst, req->cryptlen, req->iv); - return crypto_skcipher_encrypt(&rctx->fallback_req); - } - return sahara_aes_crypt(req, FLAGS_ENCRYPT); } static int sahara_aes_ecb_decrypt(struct skcipher_request *req) { - struct sahara_aes_reqctx *rctx = skcipher_request_ctx(req); - struct sahara_ctx *ctx = crypto_skcipher_ctx( - crypto_skcipher_reqtfm(req)); - - if (unlikely(ctx->keylen != AES_KEYSIZE_128)) { - skcipher_request_set_tfm(&rctx->fallback_req, ctx->fallback); - skcipher_request_set_callback(&rctx->fallback_req, - req->base.flags, - req->base.complete, - req->base.data); - skcipher_request_set_crypt(&rctx->fallback_req, req->src, - req->dst, req->cryptlen, req->iv); - return crypto_skcipher_decrypt(&rctx->fallback_req); - } - return sahara_aes_crypt(req, 0); } static int sahara_aes_cbc_encrypt(struct skcipher_request *req) { - struct sahara_aes_reqctx *rctx = skcipher_request_ctx(req); - struct sahara_ctx *ctx = crypto_skcipher_ctx( - crypto_skcipher_reqtfm(req)); - - if (unlikely(ctx->keylen != AES_KEYSIZE_128)) { - skcipher_request_set_tfm(&rctx->fallback_req, ctx->fallback); - skcipher_request_set_callback(&rctx->fallback_req, - req->base.flags, - req->base.complete, - req->base.data); - skcipher_request_set_crypt(&rctx->fallback_req, req->src, - req->dst, req->cryptlen, req->iv); - return crypto_skcipher_encrypt(&rctx->fallback_req); - } - return sahara_aes_crypt(req, FLAGS_ENCRYPT | FLAGS_CBC); } static int sahara_aes_cbc_decrypt(struct skcipher_request *req) { - struct sahara_aes_reqctx *rctx = skcipher_request_ctx(req); - struct sahara_ctx *ctx = crypto_skcipher_ctx( - crypto_skcipher_reqtfm(req)); - - if (unlikely(ctx->keylen != AES_KEYSIZE_128)) { - skcipher_request_set_tfm(&rctx->fallback_req, ctx->fallback); - skcipher_request_set_callback(&rctx->fallback_req, - req->base.flags, - req->base.complete, - req->base.data); - skcipher_request_set_crypt(&rctx->fallback_req, req->src, - req->dst, req->cryptlen, req->iv); - return crypto_skcipher_decrypt(&rctx->fallback_req); - } - return sahara_aes_crypt(req, FLAGS_CBC); } @@ -783,6 +758,7 @@ static int sahara_sha_hw_links_create(struct sahara_dev *dev, int start) { struct scatterlist *sg; + unsigned int len; unsigned int i; int ret; @@ -804,12 +780,14 @@ static int sahara_sha_hw_links_create(struct sahara_dev *dev, if (!ret) return -EFAULT; + len = rctx->total; for (i = start; i < dev->nb_in_sg + start; i++) { - dev->hw_link[i]->len = sg->length; + dev->hw_link[i]->len = min(len, sg->length); dev->hw_link[i]->p = sg->dma_address; if (i == (dev->nb_in_sg + start - 1)) { dev->hw_link[i]->next = 0; } else { + len -= min(len, sg->length); dev->hw_link[i]->next = dev->hw_phys_link[i + 1]; sg = sg_next(sg); } @@ -890,24 +868,6 @@ static int sahara_sha_hw_context_descriptor_create(struct sahara_dev *dev, return 0; } -static int sahara_walk_and_recalc(struct scatterlist *sg, unsigned int nbytes) -{ - if (!sg || !sg->length) - return nbytes; - - while (nbytes && sg) { - if (nbytes <= sg->length) { - sg->length = nbytes; - sg_mark_end(sg); - break; - } - nbytes -= sg->length; - sg = sg_next(sg); - } - - return nbytes; -} - static int sahara_sha_prepare_request(struct ahash_request *req) { struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); @@ -944,36 +904,20 @@ static int sahara_sha_prepare_request(struct ahash_request *req) hash_later, 0); } - /* nbytes should now be multiple of blocksize */ - req->nbytes = req->nbytes - hash_later; - - sahara_walk_and_recalc(req->src, req->nbytes); - + rctx->total = len - hash_later; /* have data from previous operation and current */ if (rctx->buf_cnt && req->nbytes) { sg_init_table(rctx->in_sg_chain, 2); sg_set_buf(rctx->in_sg_chain, rctx->rembuf, rctx->buf_cnt); - sg_chain(rctx->in_sg_chain, 2, req->src); - - rctx->total = req->nbytes + rctx->buf_cnt; rctx->in_sg = rctx->in_sg_chain; - - req->src = rctx->in_sg_chain; /* only data from previous operation */ } else if (rctx->buf_cnt) { - if (req->src) - rctx->in_sg = req->src; - else - rctx->in_sg = rctx->in_sg_chain; - /* buf was copied into rembuf above */ + rctx->in_sg = rctx->in_sg_chain; sg_init_one(rctx->in_sg, rctx->rembuf, rctx->buf_cnt); - rctx->total = rctx->buf_cnt; /* no data from previous operation */ } else { rctx->in_sg = req->src; - rctx->total = req->nbytes; - req->src = rctx->in_sg; } /* on next call, we only have the remaining data in the buffer */ @@ -994,7 +938,10 @@ static int sahara_sha_process(struct ahash_request *req) return ret; if (rctx->first) { - sahara_sha_hw_data_descriptor_create(dev, rctx, req, 0); + ret = sahara_sha_hw_data_descriptor_create(dev, rctx, req, 0); + if (ret) + return ret; + dev->hw_desc[0]->next = 0; rctx->first = 0; } else { @@ -1002,7 +949,10 @@ static int sahara_sha_process(struct ahash_request *req) sahara_sha_hw_context_descriptor_create(dev, rctx, req, 0); dev->hw_desc[0]->next = dev->hw_phys_desc[1]; - sahara_sha_hw_data_descriptor_create(dev, rctx, req, 1); + ret = sahara_sha_hw_data_descriptor_create(dev, rctx, req, 1); + if (ret) + return ret; + dev->hw_desc[1]->next = 0; } @@ -1015,62 +965,44 @@ static int sahara_sha_process(struct ahash_request *req) timeout = wait_for_completion_timeout(&dev->dma_completion, msecs_to_jiffies(SAHARA_TIMEOUT_MS)); - if (!timeout) { - dev_err(dev->device, "SHA timeout\n"); - return -ETIMEDOUT; - } if (rctx->sg_in_idx) dma_unmap_sg(dev->device, dev->in_sg, dev->nb_in_sg, DMA_TO_DEVICE); + if (!timeout) { + dev_err(dev->device, "SHA timeout\n"); + return -ETIMEDOUT; + } + memcpy(rctx->context, dev->context_base, rctx->context_size); - if (req->result) + if (req->result && rctx->last) memcpy(req->result, rctx->context, rctx->digest_size); return 0; } -static int sahara_queue_manage(void *data) +static int sahara_do_one_request(struct crypto_engine *engine, void *areq) { - struct sahara_dev *dev = data; - struct crypto_async_request *async_req; - struct crypto_async_request *backlog; - int ret = 0; - - do { - __set_current_state(TASK_INTERRUPTIBLE); - - spin_lock_bh(&dev->queue_spinlock); - backlog = crypto_get_backlog(&dev->queue); - async_req = crypto_dequeue_request(&dev->queue); - spin_unlock_bh(&dev->queue_spinlock); - - if (backlog) - crypto_request_complete(backlog, -EINPROGRESS); - - if (async_req) { - if (crypto_tfm_alg_type(async_req->tfm) == - CRYPTO_ALG_TYPE_AHASH) { - struct ahash_request *req = - ahash_request_cast(async_req); - - ret = sahara_sha_process(req); - } else { - struct skcipher_request *req = - skcipher_request_cast(async_req); - - ret = sahara_aes_process(req); - } + struct crypto_async_request *async_req = areq; + int err; - crypto_request_complete(async_req, ret); + if (crypto_tfm_alg_type(async_req->tfm) == CRYPTO_ALG_TYPE_AHASH) { + struct ahash_request *req = ahash_request_cast(async_req); - continue; - } + err = sahara_sha_process(req); + local_bh_disable(); + crypto_finalize_hash_request(engine, req, err); + local_bh_enable(); + } else { + struct skcipher_request *req = skcipher_request_cast(async_req); - schedule(); - } while (!kthread_should_stop()); + err = sahara_aes_process(skcipher_request_cast(async_req)); + local_bh_disable(); + crypto_finalize_skcipher_request(engine, req, err); + local_bh_enable(); + } return 0; } @@ -1079,25 +1011,13 @@ static int sahara_sha_enqueue(struct ahash_request *req, int last) { struct sahara_sha_reqctx *rctx = ahash_request_ctx(req); struct sahara_dev *dev = dev_ptr; - int ret; if (!req->nbytes && !last) return 0; rctx->last = last; - if (!rctx->active) { - rctx->active = 1; - rctx->first = 1; - } - - spin_lock_bh(&dev->queue_spinlock); - ret = crypto_enqueue_request(&dev->queue, &req->base); - spin_unlock_bh(&dev->queue_spinlock); - - wake_up_process(dev->kthread); - - return ret; + return crypto_transfer_hash_request_to_engine(dev->engine, req); } static int sahara_sha_init(struct ahash_request *req) @@ -1121,7 +1041,7 @@ static int sahara_sha_init(struct ahash_request *req) } rctx->context_size = rctx->digest_size + 4; - rctx->active = 0; + rctx->first = 1; return 0; } @@ -1170,100 +1090,119 @@ static int sahara_sha_import(struct ahash_request *req, const void *in) static int sahara_sha_cra_init(struct crypto_tfm *tfm) { crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), - sizeof(struct sahara_sha_reqctx) + - SHA_BUFFER_LEN + SHA256_BLOCK_SIZE); + sizeof(struct sahara_sha_reqctx)); return 0; } -static struct skcipher_alg aes_algs[] = { +static struct skcipher_engine_alg aes_algs[] = { { - .base.cra_name = "ecb(aes)", - .base.cra_driver_name = "sahara-ecb-aes", - .base.cra_priority = 300, - .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, - .base.cra_blocksize = AES_BLOCK_SIZE, - .base.cra_ctxsize = sizeof(struct sahara_ctx), - .base.cra_alignmask = 0x0, - .base.cra_module = THIS_MODULE, - - .init = sahara_aes_init_tfm, - .exit = sahara_aes_exit_tfm, - .min_keysize = AES_MIN_KEY_SIZE , - .max_keysize = AES_MAX_KEY_SIZE, - .setkey = sahara_aes_setkey, - .encrypt = sahara_aes_ecb_encrypt, - .decrypt = sahara_aes_ecb_decrypt, + .base = { + .base.cra_name = "ecb(aes)", + .base.cra_driver_name = "sahara-ecb-aes", + .base.cra_priority = 300, + .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct sahara_ctx), + .base.cra_alignmask = 0x0, + .base.cra_module = THIS_MODULE, + + .init = sahara_aes_init_tfm, + .exit = sahara_aes_exit_tfm, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .setkey = sahara_aes_setkey, + .encrypt = sahara_aes_ecb_encrypt, + .decrypt = sahara_aes_ecb_decrypt, + }, + .op = { + .do_one_request = sahara_do_one_request, + }, }, { - .base.cra_name = "cbc(aes)", - .base.cra_driver_name = "sahara-cbc-aes", - .base.cra_priority = 300, - .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, - .base.cra_blocksize = AES_BLOCK_SIZE, - .base.cra_ctxsize = sizeof(struct sahara_ctx), - .base.cra_alignmask = 0x0, - .base.cra_module = THIS_MODULE, - - .init = sahara_aes_init_tfm, - .exit = sahara_aes_exit_tfm, - .min_keysize = AES_MIN_KEY_SIZE , - .max_keysize = AES_MAX_KEY_SIZE, - .ivsize = AES_BLOCK_SIZE, - .setkey = sahara_aes_setkey, - .encrypt = sahara_aes_cbc_encrypt, - .decrypt = sahara_aes_cbc_decrypt, + .base = { + .base.cra_name = "cbc(aes)", + .base.cra_driver_name = "sahara-cbc-aes", + .base.cra_priority = 300, + .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct sahara_ctx), + .base.cra_alignmask = 0x0, + .base.cra_module = THIS_MODULE, + + .init = sahara_aes_init_tfm, + .exit = sahara_aes_exit_tfm, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = sahara_aes_setkey, + .encrypt = sahara_aes_cbc_encrypt, + .decrypt = sahara_aes_cbc_decrypt, + }, + .op = { + .do_one_request = sahara_do_one_request, + }, } }; -static struct ahash_alg sha_v3_algs[] = { +static struct ahash_engine_alg sha_v3_algs[] = { { - .init = sahara_sha_init, - .update = sahara_sha_update, - .final = sahara_sha_final, - .finup = sahara_sha_finup, - .digest = sahara_sha_digest, - .export = sahara_sha_export, - .import = sahara_sha_import, - .halg.digestsize = SHA1_DIGEST_SIZE, - .halg.statesize = sizeof(struct sahara_sha_reqctx), - .halg.base = { - .cra_name = "sha1", - .cra_driver_name = "sahara-sha1", - .cra_priority = 300, - .cra_flags = CRYPTO_ALG_ASYNC | - CRYPTO_ALG_NEED_FALLBACK, - .cra_blocksize = SHA1_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct sahara_ctx), - .cra_alignmask = 0, - .cra_module = THIS_MODULE, - .cra_init = sahara_sha_cra_init, - } + .base = { + .init = sahara_sha_init, + .update = sahara_sha_update, + .final = sahara_sha_final, + .finup = sahara_sha_finup, + .digest = sahara_sha_digest, + .export = sahara_sha_export, + .import = sahara_sha_import, + .halg.digestsize = SHA1_DIGEST_SIZE, + .halg.statesize = sizeof(struct sahara_sha_reqctx), + .halg.base = { + .cra_name = "sha1", + .cra_driver_name = "sahara-sha1", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA1_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct sahara_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = sahara_sha_cra_init, + } + }, + .op = { + .do_one_request = sahara_do_one_request, + }, }, }; -static struct ahash_alg sha_v4_algs[] = { +static struct ahash_engine_alg sha_v4_algs[] = { { - .init = sahara_sha_init, - .update = sahara_sha_update, - .final = sahara_sha_final, - .finup = sahara_sha_finup, - .digest = sahara_sha_digest, - .export = sahara_sha_export, - .import = sahara_sha_import, - .halg.digestsize = SHA256_DIGEST_SIZE, - .halg.statesize = sizeof(struct sahara_sha_reqctx), - .halg.base = { - .cra_name = "sha256", - .cra_driver_name = "sahara-sha256", - .cra_priority = 300, - .cra_flags = CRYPTO_ALG_ASYNC | - CRYPTO_ALG_NEED_FALLBACK, - .cra_blocksize = SHA256_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct sahara_ctx), - .cra_alignmask = 0, - .cra_module = THIS_MODULE, - .cra_init = sahara_sha_cra_init, - } + .base = { + .init = sahara_sha_init, + .update = sahara_sha_update, + .final = sahara_sha_final, + .finup = sahara_sha_finup, + .digest = sahara_sha_digest, + .export = sahara_sha_export, + .import = sahara_sha_import, + .halg.digestsize = SHA256_DIGEST_SIZE, + .halg.statesize = sizeof(struct sahara_sha_reqctx), + .halg.base = { + .cra_name = "sha256", + .cra_driver_name = "sahara-sha256", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct sahara_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = sahara_sha_cra_init, + } + }, + .op = { + .do_one_request = sahara_do_one_request, + }, }, }; @@ -1278,14 +1217,11 @@ static irqreturn_t sahara_irq_handler(int irq, void *data) sahara_decode_status(dev, stat); - if (SAHARA_STATUS_GET_STATE(stat) == SAHARA_STATE_BUSY) { + if (SAHARA_STATUS_GET_STATE(stat) == SAHARA_STATE_BUSY) return IRQ_NONE; - } else if (SAHARA_STATUS_GET_STATE(stat) == SAHARA_STATE_COMPLETE) { - dev->error = 0; - } else { + + if (SAHARA_STATUS_GET_STATE(stat) != SAHARA_STATE_COMPLETE) sahara_decode_error(dev, err); - dev->error = -EINVAL; - } complete(&dev->dma_completion); @@ -1296,57 +1232,42 @@ static irqreturn_t sahara_irq_handler(int irq, void *data) static int sahara_register_algs(struct sahara_dev *dev) { int err; - unsigned int i, j, k, l; - for (i = 0; i < ARRAY_SIZE(aes_algs); i++) { - err = crypto_register_skcipher(&aes_algs[i]); - if (err) - goto err_aes_algs; - } + err = crypto_engine_register_skciphers(aes_algs, ARRAY_SIZE(aes_algs)); + if (err) + return err; + + err = crypto_engine_register_ahashes(sha_v3_algs, + ARRAY_SIZE(sha_v3_algs)); + if (err) + goto err_aes_algs; - for (k = 0; k < ARRAY_SIZE(sha_v3_algs); k++) { - err = crypto_register_ahash(&sha_v3_algs[k]); + if (dev->version > SAHARA_VERSION_3) { + err = crypto_engine_register_ahashes(sha_v4_algs, + ARRAY_SIZE(sha_v4_algs)); if (err) goto err_sha_v3_algs; } - if (dev->version > SAHARA_VERSION_3) - for (l = 0; l < ARRAY_SIZE(sha_v4_algs); l++) { - err = crypto_register_ahash(&sha_v4_algs[l]); - if (err) - goto err_sha_v4_algs; - } - return 0; -err_sha_v4_algs: - for (j = 0; j < l; j++) - crypto_unregister_ahash(&sha_v4_algs[j]); - err_sha_v3_algs: - for (j = 0; j < k; j++) - crypto_unregister_ahash(&sha_v3_algs[j]); + crypto_engine_unregister_ahashes(sha_v3_algs, ARRAY_SIZE(sha_v3_algs)); err_aes_algs: - for (j = 0; j < i; j++) - crypto_unregister_skcipher(&aes_algs[j]); + crypto_engine_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs)); return err; } static void sahara_unregister_algs(struct sahara_dev *dev) { - unsigned int i; - - for (i = 0; i < ARRAY_SIZE(aes_algs); i++) - crypto_unregister_skcipher(&aes_algs[i]); - - for (i = 0; i < ARRAY_SIZE(sha_v3_algs); i++) - crypto_unregister_ahash(&sha_v3_algs[i]); + crypto_engine_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs)); + crypto_engine_unregister_ahashes(sha_v3_algs, ARRAY_SIZE(sha_v3_algs)); if (dev->version > SAHARA_VERSION_3) - for (i = 0; i < ARRAY_SIZE(sha_v4_algs); i++) - crypto_unregister_ahash(&sha_v4_algs[i]); + crypto_engine_unregister_ahashes(sha_v4_algs, + ARRAY_SIZE(sha_v4_algs)); } static const struct of_device_id sahara_dt_ids[] = { @@ -1383,32 +1304,27 @@ static int sahara_probe(struct platform_device *pdev) err = devm_request_irq(&pdev->dev, irq, sahara_irq_handler, 0, dev_name(&pdev->dev), dev); - if (err) { - dev_err(&pdev->dev, "failed to request irq\n"); - return err; - } + if (err) + return dev_err_probe(&pdev->dev, err, + "failed to request irq\n"); /* clocks */ - dev->clk_ipg = devm_clk_get(&pdev->dev, "ipg"); - if (IS_ERR(dev->clk_ipg)) { - dev_err(&pdev->dev, "Could not get ipg clock\n"); - return PTR_ERR(dev->clk_ipg); - } + dev->clk_ipg = devm_clk_get_enabled(&pdev->dev, "ipg"); + if (IS_ERR(dev->clk_ipg)) + return dev_err_probe(&pdev->dev, PTR_ERR(dev->clk_ipg), + "Could not get ipg clock\n"); - dev->clk_ahb = devm_clk_get(&pdev->dev, "ahb"); - if (IS_ERR(dev->clk_ahb)) { - dev_err(&pdev->dev, "Could not get ahb clock\n"); - return PTR_ERR(dev->clk_ahb); - } + dev->clk_ahb = devm_clk_get_enabled(&pdev->dev, "ahb"); + if (IS_ERR(dev->clk_ahb)) + return dev_err_probe(&pdev->dev, PTR_ERR(dev->clk_ahb), + "Could not get ahb clock\n"); /* Allocate HW descriptors */ dev->hw_desc[0] = dmam_alloc_coherent(&pdev->dev, SAHARA_MAX_HW_DESC * sizeof(struct sahara_hw_desc), &dev->hw_phys_desc[0], GFP_KERNEL); - if (!dev->hw_desc[0]) { - dev_err(&pdev->dev, "Could not allocate hw descriptors\n"); + if (!dev->hw_desc[0]) return -ENOMEM; - } dev->hw_desc[1] = dev->hw_desc[0] + 1; dev->hw_phys_desc[1] = dev->hw_phys_desc[0] + sizeof(struct sahara_hw_desc); @@ -1416,10 +1332,8 @@ static int sahara_probe(struct platform_device *pdev) /* Allocate space for iv and key */ dev->key_base = dmam_alloc_coherent(&pdev->dev, 2 * AES_KEYSIZE_128, &dev->key_phys_base, GFP_KERNEL); - if (!dev->key_base) { - dev_err(&pdev->dev, "Could not allocate memory for key\n"); + if (!dev->key_base) return -ENOMEM; - } dev->iv_base = dev->key_base + AES_KEYSIZE_128; dev->iv_phys_base = dev->key_phys_base + AES_KEYSIZE_128; @@ -1427,45 +1341,36 @@ static int sahara_probe(struct platform_device *pdev) dev->context_base = dmam_alloc_coherent(&pdev->dev, SHA256_DIGEST_SIZE + 4, &dev->context_phys_base, GFP_KERNEL); - if (!dev->context_base) { - dev_err(&pdev->dev, "Could not allocate memory for MDHA context\n"); + if (!dev->context_base) return -ENOMEM; - } /* Allocate space for HW links */ dev->hw_link[0] = dmam_alloc_coherent(&pdev->dev, SAHARA_MAX_HW_LINK * sizeof(struct sahara_hw_link), &dev->hw_phys_link[0], GFP_KERNEL); - if (!dev->hw_link[0]) { - dev_err(&pdev->dev, "Could not allocate hw links\n"); + if (!dev->hw_link[0]) return -ENOMEM; - } for (i = 1; i < SAHARA_MAX_HW_LINK; i++) { dev->hw_phys_link[i] = dev->hw_phys_link[i - 1] + sizeof(struct sahara_hw_link); dev->hw_link[i] = dev->hw_link[i - 1] + 1; } - crypto_init_queue(&dev->queue, SAHARA_QUEUE_LENGTH); - - spin_lock_init(&dev->queue_spinlock); - dev_ptr = dev; - dev->kthread = kthread_run(sahara_queue_manage, dev, "sahara_crypto"); - if (IS_ERR(dev->kthread)) { - return PTR_ERR(dev->kthread); + dev->engine = crypto_engine_alloc_init(&pdev->dev, true); + if (!dev->engine) + return -ENOMEM; + + err = crypto_engine_start(dev->engine); + if (err) { + crypto_engine_exit(dev->engine); + return dev_err_probe(&pdev->dev, err, + "Could not start crypto engine\n"); } init_completion(&dev->dma_completion); - err = clk_prepare_enable(dev->clk_ipg); - if (err) - return err; - err = clk_prepare_enable(dev->clk_ahb); - if (err) - goto clk_ipg_disable; - version = sahara_read(dev, SAHARA_REG_VERSION); if (of_device_is_compatible(pdev->dev.of_node, "fsl,imx27-sahara")) { if (version != SAHARA_VERSION_3) @@ -1477,8 +1382,8 @@ static int sahara_probe(struct platform_device *pdev) version = (version >> 8) & 0xff; } if (err == -ENODEV) { - dev_err(&pdev->dev, "SAHARA version %d not supported\n", - version); + dev_err_probe(&pdev->dev, err, + "SAHARA version %d not supported\n", version); goto err_algs; } @@ -1501,11 +1406,7 @@ static int sahara_probe(struct platform_device *pdev) return 0; err_algs: - kthread_stop(dev->kthread); - dev_ptr = NULL; - clk_disable_unprepare(dev->clk_ahb); -clk_ipg_disable: - clk_disable_unprepare(dev->clk_ipg); + crypto_engine_exit(dev->engine); return err; } @@ -1514,14 +1415,8 @@ static void sahara_remove(struct platform_device *pdev) { struct sahara_dev *dev = platform_get_drvdata(pdev); - kthread_stop(dev->kthread); - + crypto_engine_exit(dev->engine); sahara_unregister_algs(dev); - - clk_disable_unprepare(dev->clk_ipg); - clk_disable_unprepare(dev->clk_ahb); - - dev_ptr = NULL; } static struct platform_driver sahara_driver = { diff --git a/drivers/crypto/starfive/Kconfig b/drivers/crypto/starfive/Kconfig index 2cb192502c1b..cb59357b58b2 100644 --- a/drivers/crypto/starfive/Kconfig +++ b/drivers/crypto/starfive/Kconfig @@ -4,7 +4,7 @@ config CRYPTO_DEV_JH7110 tristate "StarFive JH7110 cryptographic engine driver" - depends on SOC_STARFIVE || AMBA_PL08X || COMPILE_TEST + depends on (SOC_STARFIVE && AMBA_PL08X) || COMPILE_TEST depends on HAS_DMA select CRYPTO_ENGINE select CRYPTO_HMAC diff --git a/drivers/crypto/starfive/jh7110-aes.c b/drivers/crypto/starfive/jh7110-aes.c index 9378e6682f0e..1ac15cc4ef3c 100644 --- a/drivers/crypto/starfive/jh7110-aes.c +++ b/drivers/crypto/starfive/jh7110-aes.c @@ -262,12 +262,7 @@ static int starfive_aes_hw_init(struct starfive_cryp_ctx *ctx) rctx->csr.aes.mode = hw_mode; rctx->csr.aes.cmode = !is_encrypt(cryp); rctx->csr.aes.ie = 1; - - if (hw_mode == STARFIVE_AES_MODE_CFB || - hw_mode == STARFIVE_AES_MODE_OFB) - rctx->csr.aes.stmode = STARFIVE_AES_MODE_XFB_128; - else - rctx->csr.aes.stmode = STARFIVE_AES_MODE_XFB_1; + rctx->csr.aes.stmode = STARFIVE_AES_MODE_XFB_1; if (cryp->side_chan) { rctx->csr.aes.delay_aes = 1; @@ -294,8 +289,6 @@ static int starfive_aes_hw_init(struct starfive_cryp_ctx *ctx) starfive_aes_ccm_init(ctx); starfive_aes_aead_hw_start(ctx, hw_mode); break; - case STARFIVE_AES_MODE_OFB: - case STARFIVE_AES_MODE_CFB: case STARFIVE_AES_MODE_CBC: case STARFIVE_AES_MODE_CTR: starfive_aes_write_iv(ctx, (void *)cryp->req.sreq->iv); @@ -500,7 +493,7 @@ static int starfive_aes_prepare_req(struct skcipher_request *req, scatterwalk_start(&cryp->out_walk, rctx->out_sg); if (cryp->assoclen) { - rctx->adata = kzalloc(ALIGN(cryp->assoclen, AES_BLOCK_SIZE), GFP_KERNEL); + rctx->adata = kzalloc(cryp->assoclen + AES_BLOCK_SIZE, GFP_KERNEL); if (!rctx->adata) return dev_err_probe(cryp->dev, -ENOMEM, "Failed to alloc memory for adata"); @@ -569,7 +562,7 @@ static int starfive_aes_aead_do_one_req(struct crypto_engine *engine, void *areq struct starfive_cryp_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); struct starfive_cryp_dev *cryp = ctx->cryp; - struct starfive_cryp_request_ctx *rctx = ctx->rctx; + struct starfive_cryp_request_ctx *rctx; u32 block[AES_BLOCK_32]; u32 stat; int err; @@ -579,6 +572,8 @@ static int starfive_aes_aead_do_one_req(struct crypto_engine *engine, void *areq if (err) return err; + rctx = ctx->rctx; + if (!cryp->assoclen) goto write_text; @@ -783,26 +778,6 @@ static int starfive_aes_cbc_decrypt(struct skcipher_request *req) return starfive_aes_crypt(req, STARFIVE_AES_MODE_CBC); } -static int starfive_aes_cfb_encrypt(struct skcipher_request *req) -{ - return starfive_aes_crypt(req, STARFIVE_AES_MODE_CFB | FLG_ENCRYPT); -} - -static int starfive_aes_cfb_decrypt(struct skcipher_request *req) -{ - return starfive_aes_crypt(req, STARFIVE_AES_MODE_CFB); -} - -static int starfive_aes_ofb_encrypt(struct skcipher_request *req) -{ - return starfive_aes_crypt(req, STARFIVE_AES_MODE_OFB | FLG_ENCRYPT); -} - -static int starfive_aes_ofb_decrypt(struct skcipher_request *req) -{ - return starfive_aes_crypt(req, STARFIVE_AES_MODE_OFB); -} - static int starfive_aes_ctr_encrypt(struct skcipher_request *req) { return starfive_aes_crypt(req, STARFIVE_AES_MODE_CTR | FLG_ENCRYPT); @@ -908,48 +883,6 @@ static struct skcipher_engine_alg skcipher_algs[] = { .op = { .do_one_request = starfive_aes_do_one_req, }, -}, { - .base.init = starfive_aes_init_tfm, - .base.setkey = starfive_aes_setkey, - .base.encrypt = starfive_aes_cfb_encrypt, - .base.decrypt = starfive_aes_cfb_decrypt, - .base.min_keysize = AES_MIN_KEY_SIZE, - .base.max_keysize = AES_MAX_KEY_SIZE, - .base.ivsize = AES_BLOCK_SIZE, - .base.base = { - .cra_name = "cfb(aes)", - .cra_driver_name = "starfive-cfb-aes", - .cra_priority = 200, - .cra_flags = CRYPTO_ALG_ASYNC, - .cra_blocksize = 1, - .cra_ctxsize = sizeof(struct starfive_cryp_ctx), - .cra_alignmask = 0xf, - .cra_module = THIS_MODULE, - }, - .op = { - .do_one_request = starfive_aes_do_one_req, - }, -}, { - .base.init = starfive_aes_init_tfm, - .base.setkey = starfive_aes_setkey, - .base.encrypt = starfive_aes_ofb_encrypt, - .base.decrypt = starfive_aes_ofb_decrypt, - .base.min_keysize = AES_MIN_KEY_SIZE, - .base.max_keysize = AES_MAX_KEY_SIZE, - .base.ivsize = AES_BLOCK_SIZE, - .base.base = { - .cra_name = "ofb(aes)", - .cra_driver_name = "starfive-ofb-aes", - .cra_priority = 200, - .cra_flags = CRYPTO_ALG_ASYNC, - .cra_blocksize = 1, - .cra_ctxsize = sizeof(struct starfive_cryp_ctx), - .cra_alignmask = 0xf, - .cra_module = THIS_MODULE, - }, - .op = { - .do_one_request = starfive_aes_do_one_req, - }, }, }; diff --git a/drivers/crypto/starfive/jh7110-cryp.c b/drivers/crypto/starfive/jh7110-cryp.c index 08e974e0dd12..425fddf3a8ab 100644 --- a/drivers/crypto/starfive/jh7110-cryp.c +++ b/drivers/crypto/starfive/jh7110-cryp.c @@ -109,12 +109,6 @@ static irqreturn_t starfive_cryp_irq(int irq, void *priv) tasklet_schedule(&cryp->hash_done); } - if (status & STARFIVE_IE_FLAG_PKA_DONE) { - mask |= STARFIVE_IE_MASK_PKA_DONE; - writel(mask, cryp->base + STARFIVE_IE_MASK_OFFSET); - complete(&cryp->pka_done); - } - return IRQ_HANDLED; } @@ -159,8 +153,6 @@ static int starfive_cryp_probe(struct platform_device *pdev) return dev_err_probe(&pdev->dev, PTR_ERR(cryp->rst), "Error getting hardware reset line\n"); - init_completion(&cryp->pka_done); - irq = platform_get_irq(pdev, 0); if (irq < 0) return irq; @@ -168,7 +160,7 @@ static int starfive_cryp_probe(struct platform_device *pdev) ret = devm_request_irq(&pdev->dev, irq, starfive_cryp_irq, 0, pdev->name, (void *)cryp); if (ret) - return dev_err_probe(&pdev->dev, irq, + return dev_err_probe(&pdev->dev, ret, "Failed to register interrupt handler\n"); clk_prepare_enable(cryp->hclk); @@ -180,12 +172,8 @@ static int starfive_cryp_probe(struct platform_device *pdev) spin_unlock(&dev_list.lock); ret = starfive_dma_init(cryp); - if (ret) { - if (ret == -EPROBE_DEFER) - goto err_probe_defer; - else - goto err_dma_init; - } + if (ret) + goto err_dma_init; /* Initialize crypto engine */ cryp->engine = crypto_engine_alloc_init(&pdev->dev, 1); @@ -233,7 +221,7 @@ err_dma_init: tasklet_kill(&cryp->aes_done); tasklet_kill(&cryp->hash_done); -err_probe_defer: + return ret; } diff --git a/drivers/crypto/starfive/jh7110-cryp.h b/drivers/crypto/starfive/jh7110-cryp.h index fe011d50473d..6cdf6db5d904 100644 --- a/drivers/crypto/starfive/jh7110-cryp.h +++ b/drivers/crypto/starfive/jh7110-cryp.h @@ -50,8 +50,6 @@ union starfive_aes_csr { u32 ccm_start :1; #define STARFIVE_AES_MODE_ECB 0x0 #define STARFIVE_AES_MODE_CBC 0x1 -#define STARFIVE_AES_MODE_CFB 0x2 -#define STARFIVE_AES_MODE_OFB 0x3 #define STARFIVE_AES_MODE_CTR 0x4 #define STARFIVE_AES_MODE_CCM 0x5 #define STARFIVE_AES_MODE_GCM 0x6 @@ -125,6 +123,15 @@ union starfive_pka_cacr { }; }; +union starfive_pka_casr { + u32 v; + struct { +#define STARFIVE_PKA_DONE BIT(0) + u32 done :1; + u32 rsvd_0 :31; + }; +}; + struct starfive_rsa_key { u8 *n; u8 *e; @@ -183,7 +190,6 @@ struct starfive_cryp_dev { struct crypto_engine *engine; struct tasklet_struct aes_done; struct tasklet_struct hash_done; - struct completion pka_done; size_t assoclen; size_t total_in; size_t total_out; diff --git a/drivers/crypto/starfive/jh7110-rsa.c b/drivers/crypto/starfive/jh7110-rsa.c index f31bbd825f88..cf8bda7f0855 100644 --- a/drivers/crypto/starfive/jh7110-rsa.c +++ b/drivers/crypto/starfive/jh7110-rsa.c @@ -6,13 +6,7 @@ */ #include <linux/crypto.h> -#include <linux/delay.h> -#include <linux/device.h> -#include <linux/dma-direct.h> -#include <linux/interrupt.h> #include <linux/iopoll.h> -#include <linux/io.h> -#include <linux/mod_devicetable.h> #include <crypto/akcipher.h> #include <crypto/algapi.h> #include <crypto/internal/akcipher.h> @@ -28,13 +22,13 @@ #define STARFIVE_PKA_CAER_OFFSET (STARFIVE_PKA_REGS_OFFSET + 0x108) #define STARFIVE_PKA_CANR_OFFSET (STARFIVE_PKA_REGS_OFFSET + 0x208) -// R^2 mod N and N0' +/* R ^ 2 mod N and N0' */ #define CRYPTO_CMD_PRE 0x0 -// A * R mod N ==> A +/* A * R mod N ==> A */ #define CRYPTO_CMD_ARN 0x5 -// A * E * R mod N ==> A +/* A * E * R mod N ==> A */ #define CRYPTO_CMD_AERN 0x6 -// A * A * R mod N ==> A +/* A * A * R mod N ==> A */ #define CRYPTO_CMD_AARN 0x7 #define STARFIVE_RSA_MAX_KEYSZ 256 @@ -43,31 +37,17 @@ static inline int starfive_pka_wait_done(struct starfive_cryp_ctx *ctx) { struct starfive_cryp_dev *cryp = ctx->cryp; + u32 status; - return wait_for_completion_timeout(&cryp->pka_done, - usecs_to_jiffies(100000)); -} - -static inline void starfive_pka_irq_mask_clear(struct starfive_cryp_ctx *ctx) -{ - struct starfive_cryp_dev *cryp = ctx->cryp; - u32 stat; - - stat = readl(cryp->base + STARFIVE_IE_MASK_OFFSET); - stat &= ~STARFIVE_IE_MASK_PKA_DONE; - writel(stat, cryp->base + STARFIVE_IE_MASK_OFFSET); - - reinit_completion(&cryp->pka_done); + return readl_relaxed_poll_timeout(cryp->base + STARFIVE_PKA_CASR_OFFSET, status, + status & STARFIVE_PKA_DONE, 10, 100000); } static void starfive_rsa_free_key(struct starfive_rsa_key *key) { - if (key->d) - kfree_sensitive(key->d); - if (key->e) - kfree_sensitive(key->e); - if (key->n) - kfree_sensitive(key->n); + kfree_sensitive(key->d); + kfree_sensitive(key->e); + kfree_sensitive(key->n); memset(key, 0, sizeof(*key)); } @@ -114,10 +94,9 @@ static int starfive_rsa_montgomery_form(struct starfive_cryp_ctx *ctx, rctx->csr.pka.not_r2 = 1; rctx->csr.pka.ie = 1; - starfive_pka_irq_mask_clear(ctx); writel(rctx->csr.pka.v, cryp->base + STARFIVE_PKA_CACR_OFFSET); - if (!starfive_pka_wait_done(ctx)) + if (starfive_pka_wait_done(ctx)) return -ETIMEDOUT; for (loop = 0; loop <= opsize; loop++) @@ -136,10 +115,9 @@ static int starfive_rsa_montgomery_form(struct starfive_cryp_ctx *ctx, rctx->csr.pka.start = 1; rctx->csr.pka.ie = 1; - starfive_pka_irq_mask_clear(ctx); writel(rctx->csr.pka.v, cryp->base + STARFIVE_PKA_CACR_OFFSET); - if (!starfive_pka_wait_done(ctx)) + if (starfive_pka_wait_done(ctx)) return -ETIMEDOUT; } else { rctx->csr.pka.v = 0; @@ -151,10 +129,9 @@ static int starfive_rsa_montgomery_form(struct starfive_cryp_ctx *ctx, rctx->csr.pka.pre_expf = 1; rctx->csr.pka.ie = 1; - starfive_pka_irq_mask_clear(ctx); writel(rctx->csr.pka.v, cryp->base + STARFIVE_PKA_CACR_OFFSET); - if (!starfive_pka_wait_done(ctx)) + if (starfive_pka_wait_done(ctx)) return -ETIMEDOUT; for (loop = 0; loop <= count; loop++) @@ -172,10 +149,9 @@ static int starfive_rsa_montgomery_form(struct starfive_cryp_ctx *ctx, rctx->csr.pka.start = 1; rctx->csr.pka.ie = 1; - starfive_pka_irq_mask_clear(ctx); writel(rctx->csr.pka.v, cryp->base + STARFIVE_PKA_CACR_OFFSET); - if (!starfive_pka_wait_done(ctx)) + if (starfive_pka_wait_done(ctx)) return -ETIMEDOUT; } @@ -226,11 +202,10 @@ static int starfive_rsa_cpu_start(struct starfive_cryp_ctx *ctx, u32 *result, rctx->csr.pka.start = 1; rctx->csr.pka.ie = 1; - starfive_pka_irq_mask_clear(ctx); writel(rctx->csr.pka.v, cryp->base + STARFIVE_PKA_CACR_OFFSET); ret = -ETIMEDOUT; - if (!starfive_pka_wait_done(ctx)) + if (starfive_pka_wait_done(ctx)) goto rsa_err; if (mlen) { @@ -242,10 +217,9 @@ static int starfive_rsa_cpu_start(struct starfive_cryp_ctx *ctx, u32 *result, rctx->csr.pka.start = 1; rctx->csr.pka.ie = 1; - starfive_pka_irq_mask_clear(ctx); writel(rctx->csr.pka.v, cryp->base + STARFIVE_PKA_CACR_OFFSET); - if (!starfive_pka_wait_done(ctx)) + if (starfive_pka_wait_done(ctx)) goto rsa_err; } } diff --git a/drivers/crypto/stm32/stm32-crc32.c b/drivers/crypto/stm32/stm32-crc32.c index b2d5c8921ab3..b0cf6d2fd352 100644 --- a/drivers/crypto/stm32/stm32-crc32.c +++ b/drivers/crypto/stm32/stm32-crc32.c @@ -104,7 +104,7 @@ static struct stm32_crc *stm32_crc_get_next_crc(void) struct stm32_crc *crc; spin_lock_bh(&crc_list.lock); - crc = list_first_entry(&crc_list.dev_list, struct stm32_crc, list); + crc = list_first_entry_or_null(&crc_list.dev_list, struct stm32_crc, list); if (crc) list_move_tail(&crc->list, &crc_list.dev_list); spin_unlock_bh(&crc_list.lock); diff --git a/drivers/crypto/stm32/stm32-cryp.c b/drivers/crypto/stm32/stm32-cryp.c index c3cbc2673338..11ad4ffdce0d 100644 --- a/drivers/crypto/stm32/stm32-cryp.c +++ b/drivers/crypto/stm32/stm32-cryp.c @@ -838,7 +838,7 @@ static int stm32_cryp_aead_one_req(struct crypto_engine *engine, void *areq); static int stm32_cryp_aes_aead_init(struct crypto_aead *tfm) { - tfm->reqsize = sizeof(struct stm32_cryp_reqctx); + crypto_aead_set_reqsize(tfm, sizeof(struct stm32_cryp_reqctx)); return 0; } diff --git a/drivers/crypto/virtio/virtio_crypto_common.h b/drivers/crypto/virtio/virtio_crypto_common.h index 154590e1f764..7059bbe5a2eb 100644 --- a/drivers/crypto/virtio/virtio_crypto_common.h +++ b/drivers/crypto/virtio/virtio_crypto_common.h @@ -10,6 +10,7 @@ #include <linux/virtio.h> #include <linux/crypto.h> #include <linux/spinlock.h> +#include <linux/interrupt.h> #include <crypto/aead.h> #include <crypto/aes.h> #include <crypto/engine.h> @@ -28,6 +29,7 @@ struct data_queue { char name[32]; struct crypto_engine *engine; + struct tasklet_struct done_task; }; struct virtio_crypto { diff --git a/drivers/crypto/virtio/virtio_crypto_core.c b/drivers/crypto/virtio/virtio_crypto_core.c index 43a0838d31ff..b909c6a2bf1c 100644 --- a/drivers/crypto/virtio/virtio_crypto_core.c +++ b/drivers/crypto/virtio/virtio_crypto_core.c @@ -72,27 +72,28 @@ int virtio_crypto_ctrl_vq_request(struct virtio_crypto *vcrypto, struct scatterl return 0; } -static void virtcrypto_dataq_callback(struct virtqueue *vq) +static void virtcrypto_done_task(unsigned long data) { - struct virtio_crypto *vcrypto = vq->vdev->priv; + struct data_queue *data_vq = (struct data_queue *)data; + struct virtqueue *vq = data_vq->vq; struct virtio_crypto_request *vc_req; - unsigned long flags; unsigned int len; - unsigned int qid = vq->index; - spin_lock_irqsave(&vcrypto->data_vq[qid].lock, flags); do { virtqueue_disable_cb(vq); while ((vc_req = virtqueue_get_buf(vq, &len)) != NULL) { - spin_unlock_irqrestore( - &vcrypto->data_vq[qid].lock, flags); if (vc_req->alg_cb) vc_req->alg_cb(vc_req, len); - spin_lock_irqsave( - &vcrypto->data_vq[qid].lock, flags); } } while (!virtqueue_enable_cb(vq)); - spin_unlock_irqrestore(&vcrypto->data_vq[qid].lock, flags); +} + +static void virtcrypto_dataq_callback(struct virtqueue *vq) +{ + struct virtio_crypto *vcrypto = vq->vdev->priv; + struct data_queue *dq = &vcrypto->data_vq[vq->index]; + + tasklet_schedule(&dq->done_task); } static int virtcrypto_find_vqs(struct virtio_crypto *vi) @@ -150,6 +151,8 @@ static int virtcrypto_find_vqs(struct virtio_crypto *vi) ret = -ENOMEM; goto err_engine; } + tasklet_init(&vi->data_vq[i].done_task, virtcrypto_done_task, + (unsigned long)&vi->data_vq[i]); } kfree(names); @@ -497,12 +500,15 @@ static void virtcrypto_free_unused_reqs(struct virtio_crypto *vcrypto) static void virtcrypto_remove(struct virtio_device *vdev) { struct virtio_crypto *vcrypto = vdev->priv; + int i; dev_info(&vdev->dev, "Start virtcrypto_remove.\n"); flush_work(&vcrypto->config_work); if (virtcrypto_dev_started(vcrypto)) virtcrypto_dev_stop(vcrypto); + for (i = 0; i < vcrypto->max_data_queues; i++) + tasklet_kill(&vcrypto->data_vq[i].done_task); virtio_reset_device(vdev); virtcrypto_free_unused_reqs(vcrypto); virtcrypto_clear_crypto_engines(vcrypto); diff --git a/drivers/dma/idxd/Makefile b/drivers/dma/idxd/Makefile index c5e679070e46..2b4a0d406e1e 100644 --- a/drivers/dma/idxd/Makefile +++ b/drivers/dma/idxd/Makefile @@ -4,7 +4,7 @@ obj-$(CONFIG_INTEL_IDXD_BUS) += idxd_bus.o idxd_bus-y := bus.o obj-$(CONFIG_INTEL_IDXD) += idxd.o -idxd-y := init.o irq.o device.o sysfs.o submit.o dma.o cdev.o debugfs.o +idxd-y := init.o irq.o device.o sysfs.o submit.o dma.o cdev.o debugfs.o defaults.o idxd-$(CONFIG_INTEL_IDXD_PERFMON) += perfmon.o diff --git a/drivers/dma/idxd/bus.c b/drivers/dma/idxd/bus.c index 6f84621053c6..0c9e689a2e77 100644 --- a/drivers/dma/idxd/bus.c +++ b/drivers/dma/idxd/bus.c @@ -67,11 +67,17 @@ static void idxd_config_bus_remove(struct device *dev) idxd_drv->remove(idxd_dev); } +static int idxd_bus_uevent(const struct device *dev, struct kobj_uevent_env *env) +{ + return add_uevent_var(env, "MODALIAS=" IDXD_DEVICES_MODALIAS_FMT, 0); +} + struct bus_type dsa_bus_type = { .name = "dsa", .match = idxd_config_bus_match, .probe = idxd_config_bus_probe, .remove = idxd_config_bus_remove, + .uevent = idxd_bus_uevent, }; EXPORT_SYMBOL_GPL(dsa_bus_type); diff --git a/drivers/dma/idxd/cdev.c b/drivers/dma/idxd/cdev.c index 0423655f5a88..1d918d45d9f6 100644 --- a/drivers/dma/idxd/cdev.c +++ b/drivers/dma/idxd/cdev.c @@ -550,7 +550,7 @@ static int idxd_user_drv_probe(struct idxd_dev *idxd_dev) } wq->type = IDXD_WQT_USER; - rc = drv_enable_wq(wq); + rc = idxd_drv_enable_wq(wq); if (rc < 0) goto err; @@ -565,7 +565,7 @@ static int idxd_user_drv_probe(struct idxd_dev *idxd_dev) return 0; err_cdev: - drv_disable_wq(wq); + idxd_drv_disable_wq(wq); err: destroy_workqueue(wq->wq); wq->type = IDXD_WQT_NONE; @@ -580,7 +580,7 @@ static void idxd_user_drv_remove(struct idxd_dev *idxd_dev) mutex_lock(&wq->wq_lock); idxd_wq_del_cdev(wq); - drv_disable_wq(wq); + idxd_drv_disable_wq(wq); wq->type = IDXD_WQT_NONE; destroy_workqueue(wq->wq); wq->wq = NULL; diff --git a/drivers/dma/idxd/defaults.c b/drivers/dma/idxd/defaults.c new file mode 100644 index 000000000000..c607ae8dd12c --- /dev/null +++ b/drivers/dma/idxd/defaults.c @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright(c) 2023 Intel Corporation. All rights rsvd. */ +#include <linux/kernel.h> +#include "idxd.h" + +int idxd_load_iaa_device_defaults(struct idxd_device *idxd) +{ + struct idxd_engine *engine; + struct idxd_group *group; + struct idxd_wq *wq; + + if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) + return 0; + + wq = idxd->wqs[0]; + + if (wq->state != IDXD_WQ_DISABLED) + return -EPERM; + + /* set mode to "dedicated" */ + set_bit(WQ_FLAG_DEDICATED, &wq->flags); + wq->threshold = 0; + + /* only setting up 1 wq, so give it all the wq space */ + wq->size = idxd->max_wq_size; + + /* set priority to 10 */ + wq->priority = 10; + + /* set type to "kernel" */ + wq->type = IDXD_WQT_KERNEL; + + /* set wq group to 0 */ + group = idxd->groups[0]; + wq->group = group; + group->num_wqs++; + + /* set name to "iaa_crypto" */ + memset(wq->name, 0, WQ_NAME_SIZE + 1); + strscpy(wq->name, "iaa_crypto", WQ_NAME_SIZE + 1); + + /* set driver_name to "crypto" */ + memset(wq->driver_name, 0, DRIVER_NAME_SIZE + 1); + strscpy(wq->driver_name, "crypto", DRIVER_NAME_SIZE + 1); + + engine = idxd->engines[0]; + + /* set engine group to 0 */ + engine->group = idxd->groups[0]; + engine->group->num_engines++; + + return 0; +} diff --git a/drivers/dma/idxd/device.c b/drivers/dma/idxd/device.c index 8f754f922217..f43d81128b96 100644 --- a/drivers/dma/idxd/device.c +++ b/drivers/dma/idxd/device.c @@ -161,6 +161,7 @@ int idxd_wq_alloc_resources(struct idxd_wq *wq) free_hw_descs(wq); return rc; } +EXPORT_SYMBOL_NS_GPL(idxd_wq_alloc_resources, IDXD); void idxd_wq_free_resources(struct idxd_wq *wq) { @@ -174,6 +175,7 @@ void idxd_wq_free_resources(struct idxd_wq *wq) dma_free_coherent(dev, wq->compls_size, wq->compls, wq->compls_addr); sbitmap_queue_free(&wq->sbq); } +EXPORT_SYMBOL_NS_GPL(idxd_wq_free_resources, IDXD); int idxd_wq_enable(struct idxd_wq *wq) { @@ -405,6 +407,7 @@ int idxd_wq_init_percpu_ref(struct idxd_wq *wq) reinit_completion(&wq->wq_resurrect); return 0; } +EXPORT_SYMBOL_NS_GPL(idxd_wq_init_percpu_ref, IDXD); void __idxd_wq_quiesce(struct idxd_wq *wq) { @@ -414,6 +417,7 @@ void __idxd_wq_quiesce(struct idxd_wq *wq) complete_all(&wq->wq_resurrect); wait_for_completion(&wq->wq_dead); } +EXPORT_SYMBOL_NS_GPL(__idxd_wq_quiesce, IDXD); void idxd_wq_quiesce(struct idxd_wq *wq) { @@ -421,6 +425,7 @@ void idxd_wq_quiesce(struct idxd_wq *wq) __idxd_wq_quiesce(wq); mutex_unlock(&wq->wq_lock); } +EXPORT_SYMBOL_NS_GPL(idxd_wq_quiesce, IDXD); /* Device control bits */ static inline bool idxd_is_enabled(struct idxd_device *idxd) @@ -1266,7 +1271,7 @@ static void idxd_flush_pending_descs(struct idxd_irq_entry *ie) tx = &desc->txd; tx->callback = NULL; tx->callback_result = NULL; - idxd_dma_complete_txd(desc, ctype, true); + idxd_dma_complete_txd(desc, ctype, true, NULL, NULL); } } @@ -1350,7 +1355,7 @@ err_irq: return rc; } -int drv_enable_wq(struct idxd_wq *wq) +int idxd_drv_enable_wq(struct idxd_wq *wq) { struct idxd_device *idxd = wq->idxd; struct device *dev = &idxd->pdev->dev; @@ -1482,8 +1487,9 @@ err_map_portal: err: return rc; } +EXPORT_SYMBOL_NS_GPL(idxd_drv_enable_wq, IDXD); -void drv_disable_wq(struct idxd_wq *wq) +void idxd_drv_disable_wq(struct idxd_wq *wq) { struct idxd_device *idxd = wq->idxd; struct device *dev = &idxd->pdev->dev; @@ -1503,6 +1509,7 @@ void drv_disable_wq(struct idxd_wq *wq) wq->type = IDXD_WQT_NONE; wq->client_count = 0; } +EXPORT_SYMBOL_NS_GPL(idxd_drv_disable_wq, IDXD); int idxd_device_drv_probe(struct idxd_dev *idxd_dev) { diff --git a/drivers/dma/idxd/dma.c b/drivers/dma/idxd/dma.c index 47a01893cfdb..cd835eabd31b 100644 --- a/drivers/dma/idxd/dma.c +++ b/drivers/dma/idxd/dma.c @@ -22,7 +22,7 @@ static inline struct idxd_wq *to_idxd_wq(struct dma_chan *c) void idxd_dma_complete_txd(struct idxd_desc *desc, enum idxd_complete_type comp_type, - bool free_desc) + bool free_desc, void *ctx, u32 *status) { struct idxd_device *idxd = desc->wq->idxd; struct dma_async_tx_descriptor *tx; @@ -314,7 +314,7 @@ static int idxd_dmaengine_drv_probe(struct idxd_dev *idxd_dev) wq->type = IDXD_WQT_KERNEL; - rc = drv_enable_wq(wq); + rc = idxd_drv_enable_wq(wq); if (rc < 0) { dev_dbg(dev, "Enable wq %d failed: %d\n", wq->id, rc); rc = -ENXIO; @@ -333,7 +333,7 @@ static int idxd_dmaengine_drv_probe(struct idxd_dev *idxd_dev) return 0; err_dma: - drv_disable_wq(wq); + idxd_drv_disable_wq(wq); err: wq->type = IDXD_WQT_NONE; mutex_unlock(&wq->wq_lock); @@ -347,7 +347,7 @@ static void idxd_dmaengine_drv_remove(struct idxd_dev *idxd_dev) mutex_lock(&wq->wq_lock); __idxd_wq_quiesce(wq); idxd_unregister_dma_channel(wq); - drv_disable_wq(wq); + idxd_drv_disable_wq(wq); mutex_unlock(&wq->wq_lock); } @@ -359,6 +359,7 @@ static enum idxd_dev_type dev_types[] = { struct idxd_device_driver idxd_dmaengine_drv = { .probe = idxd_dmaengine_drv_probe, .remove = idxd_dmaengine_drv_remove, + .desc_complete = idxd_dma_complete_txd, .name = "dmaengine", .type = dev_types, }; diff --git a/drivers/dma/idxd/idxd.h b/drivers/dma/idxd/idxd.h index 1e89c80a07fc..47de3f93ff1e 100644 --- a/drivers/dma/idxd/idxd.h +++ b/drivers/dma/idxd/idxd.h @@ -13,6 +13,7 @@ #include <linux/bitmap.h> #include <linux/perf_event.h> #include <linux/iommu.h> +#include <linux/crypto.h> #include <uapi/linux/idxd.h> #include "registers.h" @@ -57,11 +58,23 @@ enum idxd_type { #define IDXD_ENQCMDS_RETRIES 32 #define IDXD_ENQCMDS_MAX_RETRIES 64 +enum idxd_complete_type { + IDXD_COMPLETE_NORMAL = 0, + IDXD_COMPLETE_ABORT, + IDXD_COMPLETE_DEV_FAIL, +}; + +struct idxd_desc; + struct idxd_device_driver { const char *name; enum idxd_dev_type *type; int (*probe)(struct idxd_dev *idxd_dev); void (*remove)(struct idxd_dev *idxd_dev); + void (*desc_complete)(struct idxd_desc *desc, + enum idxd_complete_type comp_type, + bool free_desc, + void *ctx, u32 *status); struct device_driver drv; }; @@ -174,12 +187,6 @@ enum idxd_op_type { IDXD_OP_NONBLOCK = 1, }; -enum idxd_complete_type { - IDXD_COMPLETE_NORMAL = 0, - IDXD_COMPLETE_ABORT, - IDXD_COMPLETE_DEV_FAIL, -}; - struct idxd_dma_chan { struct dma_chan chan; struct idxd_wq *wq; @@ -270,6 +277,8 @@ struct idxd_dma_dev { struct dma_device dma; }; +typedef int (*load_device_defaults_fn_t) (struct idxd_device *idxd); + struct idxd_driver_data { const char *name_prefix; enum idxd_type type; @@ -279,6 +288,7 @@ struct idxd_driver_data { int evl_cr_off; int cr_status_off; int cr_result_off; + load_device_defaults_fn_t load_device_defaults; }; struct idxd_evl { @@ -378,6 +388,14 @@ static inline unsigned int evl_size(struct idxd_device *idxd) return idxd->evl->size * evl_ent_size(idxd); } +struct crypto_ctx { + struct acomp_req *req; + struct crypto_tfm *tfm; + dma_addr_t src_addr; + dma_addr_t dst_addr; + bool compress; +}; + /* IDXD software descriptor */ struct idxd_desc { union { @@ -390,7 +408,10 @@ struct idxd_desc { struct iax_completion_record *iax_completion; }; dma_addr_t compl_dma; - struct dma_async_tx_descriptor txd; + union { + struct dma_async_tx_descriptor txd; + struct crypto_ctx crypto; + }; struct llist_node llnode; struct list_head list; int id; @@ -417,6 +438,15 @@ enum idxd_completion_status { #define idxd_dev_to_idxd(idxd_dev) container_of(idxd_dev, struct idxd_device, idxd_dev) #define idxd_dev_to_wq(idxd_dev) container_of(idxd_dev, struct idxd_wq, idxd_dev) +static inline struct idxd_device_driver *wq_to_idxd_drv(struct idxd_wq *wq) +{ + struct device *dev = wq_confdev(wq); + struct idxd_device_driver *idxd_drv = + container_of(dev->driver, struct idxd_device_driver, drv); + + return idxd_drv; +} + static inline struct idxd_device *confdev_to_idxd(struct device *dev) { struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev); @@ -618,6 +648,16 @@ static inline int idxd_wq_refcount(struct idxd_wq *wq) return wq->client_count; }; +static inline void idxd_wq_set_private(struct idxd_wq *wq, void *private) +{ + dev_set_drvdata(wq_confdev(wq), private); +} + +static inline void *idxd_wq_get_private(struct idxd_wq *wq) +{ + return dev_get_drvdata(wq_confdev(wq)); +} + /* * Intel IAA does not support batch processing. * The max batch size of device, max batch size of wq and @@ -655,6 +695,9 @@ static inline int idxd_wq_driver_name_match(struct idxd_wq *wq, struct device *d return (strncmp(wq->driver_name, dev->driver->name, strlen(dev->driver->name)) == 0); } +#define MODULE_ALIAS_IDXD_DEVICE(type) MODULE_ALIAS("idxd:t" __stringify(type) "*") +#define IDXD_DEVICES_MODALIAS_FMT "idxd:t%d" + int __must_check __idxd_driver_register(struct idxd_device_driver *idxd_drv, struct module *module, const char *mod_name); #define idxd_driver_register(driver) \ @@ -665,6 +708,24 @@ void idxd_driver_unregister(struct idxd_device_driver *idxd_drv); #define module_idxd_driver(__idxd_driver) \ module_driver(__idxd_driver, idxd_driver_register, idxd_driver_unregister) +void idxd_free_desc(struct idxd_wq *wq, struct idxd_desc *desc); +void idxd_dma_complete_txd(struct idxd_desc *desc, + enum idxd_complete_type comp_type, + bool free_desc, void *ctx, u32 *status); + +static inline void idxd_desc_complete(struct idxd_desc *desc, + enum idxd_complete_type comp_type, + bool free_desc) +{ + struct idxd_device_driver *drv; + u32 status; + + drv = wq_to_idxd_drv(desc->wq); + if (drv->desc_complete) + drv->desc_complete(desc, comp_type, free_desc, + &desc->txd, &status); +} + int idxd_register_bus_type(void); void idxd_unregister_bus_type(void); int idxd_register_devices(struct idxd_device *idxd); @@ -672,6 +733,7 @@ void idxd_unregister_devices(struct idxd_device *idxd); void idxd_wqs_quiesce(struct idxd_device *idxd); bool idxd_queue_int_handle_resubmit(struct idxd_desc *desc); void multi_u64_to_bmap(unsigned long *bmap, u64 *val, int count); +int idxd_load_iaa_device_defaults(struct idxd_device *idxd); /* device interrupt control */ irqreturn_t idxd_misc_thread(int vec, void *data); @@ -682,8 +744,8 @@ void idxd_unmask_error_interrupts(struct idxd_device *idxd); /* device control */ int idxd_device_drv_probe(struct idxd_dev *idxd_dev); void idxd_device_drv_remove(struct idxd_dev *idxd_dev); -int drv_enable_wq(struct idxd_wq *wq); -void drv_disable_wq(struct idxd_wq *wq); +int idxd_drv_enable_wq(struct idxd_wq *wq); +void idxd_drv_disable_wq(struct idxd_wq *wq); int idxd_device_init_reset(struct idxd_device *idxd); int idxd_device_enable(struct idxd_device *idxd); int idxd_device_disable(struct idxd_device *idxd); @@ -718,14 +780,11 @@ int idxd_wq_request_irq(struct idxd_wq *wq); /* submission */ int idxd_submit_desc(struct idxd_wq *wq, struct idxd_desc *desc); struct idxd_desc *idxd_alloc_desc(struct idxd_wq *wq, enum idxd_op_type optype); -void idxd_free_desc(struct idxd_wq *wq, struct idxd_desc *desc); int idxd_enqcmds(struct idxd_wq *wq, void __iomem *portal, const void *desc); /* dmaengine */ int idxd_register_dma_device(struct idxd_device *idxd); void idxd_unregister_dma_device(struct idxd_device *idxd); -void idxd_dma_complete_txd(struct idxd_desc *desc, - enum idxd_complete_type comp_type, bool free_desc); /* cdev */ int idxd_cdev_register(void); diff --git a/drivers/dma/idxd/init.c b/drivers/dma/idxd/init.c index 0eb1c827a215..14df1f1347a8 100644 --- a/drivers/dma/idxd/init.c +++ b/drivers/dma/idxd/init.c @@ -59,6 +59,7 @@ static struct idxd_driver_data idxd_driver_data[] = { .evl_cr_off = offsetof(struct iax_evl_entry, cr), .cr_status_off = offsetof(struct iax_completion_record, status), .cr_result_off = offsetof(struct iax_completion_record, error_code), + .load_device_defaults = idxd_load_iaa_device_defaults, }, }; @@ -745,6 +746,12 @@ static int idxd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) goto err; } + if (data->load_device_defaults) { + rc = data->load_device_defaults(idxd); + if (rc) + dev_warn(dev, "IDXD loading device defaults failed\n"); + } + rc = idxd_register_devices(idxd); if (rc) { dev_err(dev, "IDXD sysfs setup failed\n"); diff --git a/drivers/dma/idxd/irq.c b/drivers/dma/idxd/irq.c index 2183d7f9cdbd..c8a0aa874b11 100644 --- a/drivers/dma/idxd/irq.c +++ b/drivers/dma/idxd/irq.c @@ -123,7 +123,7 @@ static void idxd_abort_invalid_int_handle_descs(struct idxd_irq_entry *ie) list_for_each_entry_safe(d, t, &flist, list) { list_del(&d->list); - idxd_dma_complete_txd(d, IDXD_COMPLETE_ABORT, true); + idxd_desc_complete(d, IDXD_COMPLETE_ABORT, true); } } @@ -534,7 +534,7 @@ static void idxd_int_handle_resubmit_work(struct work_struct *work) */ if (rc != -EAGAIN) { desc->completion->status = IDXD_COMP_DESC_ABORT; - idxd_dma_complete_txd(desc, IDXD_COMPLETE_ABORT, false); + idxd_desc_complete(desc, IDXD_COMPLETE_ABORT, false); } idxd_free_desc(wq, desc); } @@ -575,11 +575,11 @@ static void irq_process_pending_llist(struct idxd_irq_entry *irq_entry) * and 0xff, which DSA_COMP_STATUS_MASK can mask out. */ if (unlikely(desc->completion->status == IDXD_COMP_DESC_ABORT)) { - idxd_dma_complete_txd(desc, IDXD_COMPLETE_ABORT, true); + idxd_desc_complete(desc, IDXD_COMPLETE_ABORT, true); continue; } - idxd_dma_complete_txd(desc, IDXD_COMPLETE_NORMAL, true); + idxd_desc_complete(desc, IDXD_COMPLETE_NORMAL, true); } else { spin_lock(&irq_entry->list_lock); list_add_tail(&desc->list, @@ -618,11 +618,11 @@ static void irq_process_work_list(struct idxd_irq_entry *irq_entry) * and 0xff, which DSA_COMP_STATUS_MASK can mask out. */ if (unlikely(desc->completion->status == IDXD_COMP_DESC_ABORT)) { - idxd_dma_complete_txd(desc, IDXD_COMPLETE_ABORT, true); + idxd_desc_complete(desc, IDXD_COMPLETE_ABORT, true); continue; } - idxd_dma_complete_txd(desc, IDXD_COMPLETE_NORMAL, true); + idxd_desc_complete(desc, IDXD_COMPLETE_NORMAL, true); } } diff --git a/drivers/dma/idxd/submit.c b/drivers/dma/idxd/submit.c index 3f922518e3a5..817a564413b0 100644 --- a/drivers/dma/idxd/submit.c +++ b/drivers/dma/idxd/submit.c @@ -61,6 +61,7 @@ struct idxd_desc *idxd_alloc_desc(struct idxd_wq *wq, enum idxd_op_type optype) return __get_desc(wq, idx, cpu); } +EXPORT_SYMBOL_NS_GPL(idxd_alloc_desc, IDXD); void idxd_free_desc(struct idxd_wq *wq, struct idxd_desc *desc) { @@ -69,6 +70,7 @@ void idxd_free_desc(struct idxd_wq *wq, struct idxd_desc *desc) desc->cpu = -1; sbitmap_queue_clear(&wq->sbq, desc->id, cpu); } +EXPORT_SYMBOL_NS_GPL(idxd_free_desc, IDXD); static struct idxd_desc *list_abort_desc(struct idxd_wq *wq, struct idxd_irq_entry *ie, struct idxd_desc *desc) @@ -125,7 +127,8 @@ static void llist_abort_desc(struct idxd_wq *wq, struct idxd_irq_entry *ie, spin_unlock(&ie->list_lock); if (found) - idxd_dma_complete_txd(found, IDXD_COMPLETE_ABORT, false); + idxd_dma_complete_txd(found, IDXD_COMPLETE_ABORT, false, + NULL, NULL); /* * completing the descriptor will return desc to allocator and @@ -135,7 +138,8 @@ static void llist_abort_desc(struct idxd_wq *wq, struct idxd_irq_entry *ie, */ list_for_each_entry_safe(d, t, &flist, list) { list_del_init(&d->list); - idxd_dma_complete_txd(found, IDXD_COMPLETE_ABORT, true); + idxd_dma_complete_txd(found, IDXD_COMPLETE_ABORT, true, + NULL, NULL); } } @@ -215,3 +219,4 @@ int idxd_submit_desc(struct idxd_wq *wq, struct idxd_desc *desc) percpu_ref_put(&wq->wq_active); return 0; } +EXPORT_SYMBOL_NS_GPL(idxd_submit_desc, IDXD); diff --git a/include/crypto/hash.h b/include/crypto/hash.h index c7bdbece27cc..5d61f576cfc8 100644 --- a/include/crypto/hash.h +++ b/include/crypto/hash.h @@ -212,13 +212,9 @@ struct shash_desc { * This is a counterpart to @init_tfm, used to remove * various changes set in @init_tfm. * @clone_tfm: Copy transform into new object, may allocate memory. - * @digestsize: see struct ahash_alg - * @statesize: see struct ahash_alg * @descsize: Size of the operational state for the message digest. This state * size is the memory size that needs to be allocated for * shash_desc.__ctx - * @stat: Statistics for hash algorithm. - * @base: internally used * @halg: see struct hash_alg_common * @HASH_ALG_COMMON: see struct hash_alg_common */ diff --git a/include/crypto/if_alg.h b/include/crypto/if_alg.h index ef8ce86b1f78..78ecaf5db04c 100644 --- a/include/crypto/if_alg.h +++ b/include/crypto/if_alg.h @@ -121,6 +121,7 @@ struct af_alg_async_req { * * @tsgl_list: Link to TX SGL * @iv: IV for cipher operation + * @state: Existing state for continuing operation * @aead_assoclen: Length of AAD for AEAD cipher operations * @completion: Work queue for synchronous operation * @used: TX bytes sent to kernel. This variable is used to @@ -136,11 +137,13 @@ struct af_alg_async_req { * recvmsg is invoked. * @init: True if metadata has been sent. * @len: Length of memory allocated for this data structure. + * @inflight: Non-zero when AIO requests are in flight. */ struct af_alg_ctx { struct list_head tsgl_list; void *iv; + void *state; size_t aead_assoclen; struct crypto_wait wait; @@ -154,6 +157,8 @@ struct af_alg_ctx { bool init; unsigned int len; + + unsigned int inflight; }; int af_alg_register_type(const struct af_alg_type *type); diff --git a/include/crypto/skcipher.h b/include/crypto/skcipher.h index ea18af48346b..c8857d7bdb37 100644 --- a/include/crypto/skcipher.h +++ b/include/crypto/skcipher.h @@ -15,6 +15,17 @@ #include <linux/string.h> #include <linux/types.h> +/* Set this bit if the lskcipher operation is a continuation. */ +#define CRYPTO_LSKCIPHER_FLAG_CONT 0x00000001 +/* Set this bit if the lskcipher operation is final. */ +#define CRYPTO_LSKCIPHER_FLAG_FINAL 0x00000002 +/* The bit CRYPTO_TFM_REQ_MAY_SLEEP can also be set if needed. */ + +/* Set this bit if the skcipher operation is a continuation. */ +#define CRYPTO_SKCIPHER_REQ_CONT 0x00000001 +/* Set this bit if the skcipher operation is not final. */ +#define CRYPTO_SKCIPHER_REQ_NOTFINAL 0x00000002 + struct scatterlist; /** @@ -91,6 +102,7 @@ struct crypto_istat_cipher { * IV of exactly that size to perform the encrypt or decrypt operation. * @chunksize: Equal to the block size except for stream ciphers such as * CTR where it is set to the underlying block size. + * @statesize: Size of the internal state for the algorithm. * @stat: Statistics for cipher algorithm * @base: Definition of a generic crypto algorithm. */ @@ -99,6 +111,7 @@ struct crypto_istat_cipher { unsigned int max_keysize; \ unsigned int ivsize; \ unsigned int chunksize; \ + unsigned int statesize; \ \ SKCIPHER_ALG_COMMON_STAT \ \ @@ -108,16 +121,6 @@ struct skcipher_alg_common SKCIPHER_ALG_COMMON; /** * struct skcipher_alg - symmetric key cipher definition - * @min_keysize: Minimum key size supported by the transformation. This is the - * smallest key length supported by this transformation algorithm. - * This must be set to one of the pre-defined values as this is - * not hardware specific. Possible values for this field can be - * found via git grep "_MIN_KEY_SIZE" include/crypto/ - * @max_keysize: Maximum key size supported by the transformation. This is the - * largest key length supported by this transformation algorithm. - * This must be set to one of the pre-defined values as this is - * not hardware specific. Possible values for this field can be - * found via git grep "_MAX_KEY_SIZE" include/crypto/ * @setkey: Set key for the transformation. This function is used to either * program a supplied key into the hardware or store the key in the * transformation context for programming it later. Note that this @@ -141,6 +144,17 @@ struct skcipher_alg_common SKCIPHER_ALG_COMMON; * be called in parallel with the same transformation object. * @decrypt: Decrypt a single block. This is a reverse counterpart to @encrypt * and the conditions are exactly the same. + * @export: Export partial state of the transformation. This function dumps the + * entire state of the ongoing transformation into a provided block of + * data so it can be @import 'ed back later on. This is useful in case + * you want to save partial result of the transformation after + * processing certain amount of data and reload this partial result + * multiple times later on for multiple re-use. No data processing + * happens at this point. + * @import: Import partial state of the transformation. This function loads the + * entire state of the ongoing transformation from a provided block of + * data so the transformation can continue from this point onward. No + * data processing happens at this point. * @init: Initialize the cryptographic transformation object. This function * is used to initialize the cryptographic transformation object. * This function is called only once at the instantiation time, right @@ -152,15 +166,9 @@ struct skcipher_alg_common SKCIPHER_ALG_COMMON; * @exit: Deinitialize the cryptographic transformation object. This is a * counterpart to @init, used to remove various changes set in * @init. - * @ivsize: IV size applicable for transformation. The consumer must provide an - * IV of exactly that size to perform the encrypt or decrypt operation. - * @chunksize: Equal to the block size except for stream ciphers such as - * CTR where it is set to the underlying block size. * @walksize: Equal to the chunk size except in cases where the algorithm is * considerably more efficient if it can operate on multiple chunks * in parallel. Should be a multiple of chunksize. - * @stat: Statistics for cipher algorithm - * @base: Definition of a generic crypto algorithm. * @co: see struct skcipher_alg_common * * All fields except @ivsize are mandatory and must be filled. @@ -170,6 +178,8 @@ struct skcipher_alg { unsigned int keylen); int (*encrypt)(struct skcipher_request *req); int (*decrypt)(struct skcipher_request *req); + int (*export)(struct skcipher_request *req, void *out); + int (*import)(struct skcipher_request *req, const void *in); int (*init)(struct crypto_skcipher *tfm); void (*exit)(struct crypto_skcipher *tfm); @@ -200,6 +210,9 @@ struct skcipher_alg { * may be left over if length is not a multiple of blocks * and there is more to come (final == false). The number of * left-over bytes should be returned in case of success. + * The siv field shall be as long as ivsize + statesize with + * the IV placed at the front. The state will be used by the + * algorithm internally. * @decrypt: Decrypt a number of bytes. This is a reverse counterpart to * @encrypt and the conditions are exactly the same. * @init: Initialize the cryptographic transformation object. This function @@ -215,9 +228,9 @@ struct lskcipher_alg { int (*setkey)(struct crypto_lskcipher *tfm, const u8 *key, unsigned int keylen); int (*encrypt)(struct crypto_lskcipher *tfm, const u8 *src, - u8 *dst, unsigned len, u8 *iv, bool final); + u8 *dst, unsigned len, u8 *siv, u32 flags); int (*decrypt)(struct crypto_lskcipher *tfm, const u8 *src, - u8 *dst, unsigned len, u8 *iv, bool final); + u8 *dst, unsigned len, u8 *siv, u32 flags); int (*init)(struct crypto_lskcipher *tfm); void (*exit)(struct crypto_lskcipher *tfm); @@ -496,6 +509,40 @@ static inline unsigned int crypto_lskcipher_chunksize( return crypto_lskcipher_alg(tfm)->co.chunksize; } +/** + * crypto_skcipher_statesize() - obtain state size + * @tfm: cipher handle + * + * Some algorithms cannot be chained with the IV alone. They carry + * internal state which must be replicated if data is to be processed + * incrementally. The size of that state can be obtained with this + * function. + * + * Return: state size in bytes + */ +static inline unsigned int crypto_skcipher_statesize( + struct crypto_skcipher *tfm) +{ + return crypto_skcipher_alg_common(tfm)->statesize; +} + +/** + * crypto_lskcipher_statesize() - obtain state size + * @tfm: cipher handle + * + * Some algorithms cannot be chained with the IV alone. They carry + * internal state which must be replicated if data is to be processed + * incrementally. The size of that state can be obtained with this + * function. + * + * Return: state size in bytes + */ +static inline unsigned int crypto_lskcipher_statesize( + struct crypto_lskcipher *tfm) +{ + return crypto_lskcipher_alg(tfm)->co.statesize; +} + static inline unsigned int crypto_sync_skcipher_blocksize( struct crypto_sync_skcipher *tfm) { @@ -684,14 +731,48 @@ int crypto_skcipher_encrypt(struct skcipher_request *req); int crypto_skcipher_decrypt(struct skcipher_request *req); /** + * crypto_skcipher_export() - export partial state + * @req: reference to the skcipher_request handle that holds all information + * needed to perform the operation + * @out: output buffer of sufficient size that can hold the state + * + * Export partial state of the transformation. This function dumps the + * entire state of the ongoing transformation into a provided block of + * data so it can be @import 'ed back later on. This is useful in case + * you want to save partial result of the transformation after + * processing certain amount of data and reload this partial result + * multiple times later on for multiple re-use. No data processing + * happens at this point. + * + * Return: 0 if the cipher operation was successful; < 0 if an error occurred + */ +int crypto_skcipher_export(struct skcipher_request *req, void *out); + +/** + * crypto_skcipher_import() - import partial state + * @req: reference to the skcipher_request handle that holds all information + * needed to perform the operation + * @in: buffer holding the state + * + * Import partial state of the transformation. This function loads the + * entire state of the ongoing transformation from a provided block of + * data so the transformation can continue from this point onward. No + * data processing happens at this point. + * + * Return: 0 if the cipher operation was successful; < 0 if an error occurred + */ +int crypto_skcipher_import(struct skcipher_request *req, const void *in); + +/** * crypto_lskcipher_encrypt() - encrypt plaintext * @tfm: lskcipher handle * @src: source buffer * @dst: destination buffer * @len: number of bytes to process - * @iv: IV for the cipher operation which must comply with the IV size defined - * by crypto_lskcipher_ivsize - * + * @siv: IV + state for the cipher operation. The length of the IV must + * comply with the IV size defined by crypto_lskcipher_ivsize. The + * IV is then followed with a buffer with the length as specified by + * crypto_lskcipher_statesize. * Encrypt plaintext data using the lskcipher handle. * * Return: >=0 if the cipher operation was successful, if positive @@ -699,7 +780,7 @@ int crypto_skcipher_decrypt(struct skcipher_request *req); * < 0 if an error occurred */ int crypto_lskcipher_encrypt(struct crypto_lskcipher *tfm, const u8 *src, - u8 *dst, unsigned len, u8 *iv); + u8 *dst, unsigned len, u8 *siv); /** * crypto_lskcipher_decrypt() - decrypt ciphertext @@ -707,8 +788,10 @@ int crypto_lskcipher_encrypt(struct crypto_lskcipher *tfm, const u8 *src, * @src: source buffer * @dst: destination buffer * @len: number of bytes to process - * @iv: IV for the cipher operation which must comply with the IV size defined - * by crypto_lskcipher_ivsize + * @siv: IV + state for the cipher operation. The length of the IV must + * comply with the IV size defined by crypto_lskcipher_ivsize. The + * IV is then followed with a buffer with the length as specified by + * crypto_lskcipher_statesize. * * Decrypt ciphertext data using the lskcipher handle. * @@ -717,7 +800,7 @@ int crypto_lskcipher_encrypt(struct crypto_lskcipher *tfm, const u8 *src, * < 0 if an error occurred */ int crypto_lskcipher_decrypt(struct crypto_lskcipher *tfm, const u8 *src, - u8 *dst, unsigned len, u8 *iv); + u8 *dst, unsigned len, u8 *siv); /** * DOC: Symmetric Key Cipher Request Handle diff --git a/include/linux/hisi_acc_qm.h b/include/linux/hisi_acc_qm.h index ddc7ebb70523..5f4c74facf6a 100644 --- a/include/linux/hisi_acc_qm.h +++ b/include/linux/hisi_acc_qm.h @@ -108,17 +108,13 @@ enum qm_stop_reason { }; enum qm_state { - QM_INIT = 0, - QM_START, - QM_CLOSE, + QM_WORK = 0, QM_STOP, }; enum qp_state { - QP_INIT = 1, - QP_START, + QP_START = 1, QP_STOP, - QP_CLOSE, }; enum qm_hw_ver { @@ -160,6 +156,11 @@ enum qm_cap_bits { QM_SUPPORT_RPM, }; +struct qm_dev_alg { + u64 alg_msk; + const char *alg; +}; + struct dfx_diff_registers { u32 *regs; u32 reg_offset; @@ -265,6 +266,16 @@ struct hisi_qm_cap_info { u32 v3_val; }; +struct hisi_qm_cap_record { + u32 type; + u32 cap_val; +}; + +struct hisi_qm_cap_tables { + struct hisi_qm_cap_record *qm_cap_table; + struct hisi_qm_cap_record *dev_cap_table; +}; + struct hisi_qm_list { struct mutex lock; struct list_head list; @@ -365,7 +376,6 @@ struct hisi_qm { struct work_struct rst_work; struct work_struct cmd_process; - const char *algs; bool use_sva; resource_size_t phys_base; @@ -376,6 +386,8 @@ struct hisi_qm { u32 mb_qos; u32 type_rate; struct qm_err_isolate isolate_data; + + struct hisi_qm_cap_tables cap_tables; }; struct hisi_qp_status { @@ -563,6 +575,8 @@ void hisi_qm_regs_dump(struct seq_file *s, struct debugfs_regset32 *regset); u32 hisi_qm_get_hw_info(struct hisi_qm *qm, const struct hisi_qm_cap_info *info_table, u32 index, bool is_read); +int hisi_qm_set_algs(struct hisi_qm *qm, u64 alg_msk, const struct qm_dev_alg *dev_algs, + u32 dev_algs_size); /* Used by VFIO ACC live migration driver */ struct pci_driver *hisi_sec_get_pf_driver(void); diff --git a/lib/crypto/aesgcm.c b/lib/crypto/aesgcm.c index c632d6e17af8..6bba6473fdf3 100644 --- a/lib/crypto/aesgcm.c +++ b/lib/crypto/aesgcm.c @@ -73,6 +73,19 @@ static void aesgcm_ghash(be128 *ghash, const be128 *key, const void *src, } } +/** + * aesgcm_mac - Generates the authentication tag using AES-GCM algorithm. + * @ctx: The data structure that will hold the AES-GCM key schedule + * @src: The input source data. + * @src_len: Length of the source data. + * @assoc: Points to the associated data. + * @assoc_len: Length of the associated data values. + * @ctr: Points to the counter value. + * @authtag: The output buffer for the authentication tag. + * + * It takes in the AES-GCM context, source data, associated data, counter value, + * and an output buffer for the authentication tag. + */ static void aesgcm_mac(const struct aesgcm_ctx *ctx, const u8 *src, int src_len, const u8 *assoc, int assoc_len, __be32 *ctr, u8 *authtag) { diff --git a/lib/crypto/mpi/ec.c b/lib/crypto/mpi/ec.c index 40f5908e57a4..e16dca1e23d5 100644 --- a/lib/crypto/mpi/ec.c +++ b/lib/crypto/mpi/ec.c @@ -584,6 +584,9 @@ void mpi_ec_init(struct mpi_ec_ctx *ctx, enum gcry_mpi_ec_models model, ctx->a = mpi_copy(a); ctx->b = mpi_copy(b); + ctx->d = NULL; + ctx->t.two_inv_p = NULL; + ctx->t.p_barrett = use_barrett > 0 ? mpi_barrett_init(ctx->p, 0) : NULL; mpi_ec_get_reset(ctx); diff --git a/tools/crypto/tcrypt/tcrypt_speed_compare.py b/tools/crypto/tcrypt/tcrypt_speed_compare.py new file mode 100755 index 000000000000..f3f5783cdc06 --- /dev/null +++ b/tools/crypto/tcrypt/tcrypt_speed_compare.py @@ -0,0 +1,190 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +# +# Copyright (C) xFusion Digital Technologies Co., Ltd., 2023 +# +# Author: Wang Jinchao <wangjinchao@xfusion.com> +# +""" +A tool for comparing tcrypt speed test logs. + +Please note that for such a comparison, stability depends +on whether we allow frequency to float or pin the frequency. + +Both support tests for operations within one second and +cycles of operation. +For example, use it in the bash script below. + +```bash +#!/bin/bash + +# log file prefix +seq_num=0 + +# When sec=0, it will perform cycle tests; +# otherwise, it indicates the duration of a single test +sec=0 +num_mb=8 +mode=211 + +# base speed test +lsmod | grep pcrypt && modprobe -r pcrypt +dmesg -C +modprobe tcrypt alg="pcrypt(rfc4106(gcm(aes)))" type=3 +modprobe tcrypt mode=${mode} sec=${sec} num_mb=${num_mb} +dmesg > ${seq_num}_base_dmesg.log + +# new speed test +lsmod | grep pcrypt && modprobe -r pcrypt +dmesg -C +modprobe tcrypt alg="pcrypt(rfc4106(gcm(aes)))" type=3 +modprobe tcrypt mode=${mode} sec=${sec} num_mb=${num_mb} +dmesg > ${seq_num}_new_dmesg.log +lsmod | grep pcrypt && modprobe -r pcrypt + +tools/crypto/tcrypt/tcrypt_speed_compare.py \ + ${seq_num}_base_dmesg.log \ + ${seq_num}_new_dmesg.log \ + >${seq_num}_compare.log +grep 'average' -A2 -B0 --group-separator="" ${seq_num}_compare.log +``` +""" + +import sys +import re + + +def parse_title(line): + pattern = r'tcrypt: testing speed of (.*?) (encryption|decryption)' + match = re.search(pattern, line) + if match: + alg = match.group(1) + op = match.group(2) + return alg, op + else: + return "", "" + + +def parse_item(line): + pattern_operations = r'\((\d+) bit key, (\d+) byte blocks\): (\d+) operations' + pattern_cycles = r'\((\d+) bit key, (\d+) byte blocks\): 1 operation in (\d+) cycles' + match = re.search(pattern_operations, line) + if match: + res = { + "bit_key": int(match.group(1)), + "byte_blocks": int(match.group(2)), + "operations": int(match.group(3)), + } + return res + + match = re.search(pattern_cycles, line) + if match: + res = { + "bit_key": int(match.group(1)), + "byte_blocks": int(match.group(2)), + "cycles": int(match.group(3)), + } + return res + + return None + + +def parse(filepath): + result = {} + alg, op = "", "" + with open(filepath, 'r') as file: + for line in file: + if not line: + continue + _alg, _op = parse_title(line) + if _alg: + alg, op = _alg, _op + if alg not in result: + result[alg] = {} + if op not in result[alg]: + result[alg][op] = [] + continue + parsed_result = parse_item(line) + if parsed_result: + result[alg][op].append(parsed_result) + return result + + +def merge(base, new): + merged = {} + for alg in base.keys(): + merged[alg] = {} + for op in base[alg].keys(): + if op not in merged[alg]: + merged[alg][op] = [] + for index in range(len(base[alg][op])): + merged_item = { + "bit_key": base[alg][op][index]["bit_key"], + "byte_blocks": base[alg][op][index]["byte_blocks"], + } + if "operations" in base[alg][op][index].keys(): + merged_item["base_ops"] = base[alg][op][index]["operations"] + merged_item["new_ops"] = new[alg][op][index]["operations"] + else: + merged_item["base_cycles"] = base[alg][op][index]["cycles"] + merged_item["new_cycles"] = new[alg][op][index]["cycles"] + + merged[alg][op].append(merged_item) + return merged + + +def format(merged): + for alg in merged.keys(): + for op in merged[alg].keys(): + base_sum = 0 + new_sum = 0 + differ_sum = 0 + differ_cnt = 0 + print() + hlen = 80 + print("="*hlen) + print(f"{alg}") + print(f"{' '*(len(alg)//3) + op}") + print("-"*hlen) + key = "" + if "base_ops" in merged[alg][op][0]: + key = "ops" + print(f"bit key | byte blocks | base ops | new ops | differ(%)") + else: + key = "cycles" + print(f"bit key | byte blocks | base cycles | new cycles | differ(%)") + for index in range(len(merged[alg][op])): + item = merged[alg][op][index] + base_cnt = item[f"base_{key}"] + new_cnt = item[f"new_{key}"] + base_sum += base_cnt + new_sum += new_cnt + differ = round((new_cnt - base_cnt)*100/base_cnt, 2) + differ_sum += differ + differ_cnt += 1 + bit_key = item["bit_key"] + byte_blocks = item["byte_blocks"] + print( + f"{bit_key:<7} | {byte_blocks:<11} | {base_cnt:<11} | {new_cnt:<11} | {differ:<8}") + average_speed_up = "{:.2f}".format(differ_sum/differ_cnt) + ops_total_speed_up = "{:.2f}".format( + (base_sum - new_sum) * 100 / base_sum) + print('-'*hlen) + print(f"average differ(%s) | total_differ(%)") + print('-'*hlen) + print(f"{average_speed_up:<21} | {ops_total_speed_up:<10}") + print('='*hlen) + + +def main(base_log, new_log): + base = parse(base_log) + new = parse(new_log) + merged = merge(base, new) + format(merged) + + +if __name__ == "__main__": + if len(sys.argv) != 3: + print(f"usage: {sys.argv[0]} base_log new_log") + exit(-1) + main(sys.argv[1], sys.argv[2]) |